Got it! Needed an intermediate posts loop before listing out individual posts
<ul>
{% for category in site.categories %}
<li><a name="{{ category | first }}">{{ category | first }}</a>
<ul>
{% for post in category.last %}
<li><a href="https://stackoverflow.com/questions/20872861/{{ post.url }}">{{ post.title }}</a></li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>