reworked cache with redis + improved fetching

12 files changed, 350 insertions, 117 deletions

diff --git a/backend/auth/oauth.go b/backend/auth/oauth.go
index 9be1da0..a1b0419 100644
--- a/backend/auth/oauth.go
+++ b/backend/auth/oauth.go
@@ -44,6 +44,18 @@ type TokenResponse struct {
 	ExpiresIn    int    `json:"expires_in"`
 }
 
+func (s *OAuthState) Copy() *OAuthState {
+	return &OAuthState{
+		State:        s.State,
+		DID:          s.DID,
+		PDS:          s.PDS,
+		AccessToken:  s.AccessToken,
+		RefreshToken: s.RefreshToken,
+		ExpiresAt:    s.ExpiresAt,
+		KeyPair:      s.KeyPair,
+	}
+}
+
 func GenerateRandomToken() (string, error) {
 	b := make([]byte, 32)
 	_, err := rand.Read(b)
diff --git a/backend/fetchers/app.bsky.actor.go b/backend/fetchers/app.bsky.actor.go
index 0752223..a56fe0b 100644
--- a/backend/fetchers/app.bsky.actor.go
+++ b/backend/fetchers/app.bsky.actor.go
@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"time"
 )
 
 type DIDDocument struct {
@@ -136,12 +137,13 @@ func FetchAppBskyActorProfile(logger *slog.Logger, did, pdsURL, slingshotURL str
 	uri := fmt.Sprintf("at://%s/app.bsky.actor.profile/self", did)
 
 	// try slingshot first
-	if slingshotURL != "" {
+	if slingshotURL != "" && general.SlingshotAvailable() {
 		rawRecord, err := general.FetchRawRecordByURI(logger, uri, slingshotURL, session)
 		if err != nil {
 			logger.Warn("slingshot profile fetch failed", "did", did, "error", err)
+			general.SetSlingshotBackoff(5 * time.Minute)
 		} else {
-			return parseAppBskyActorProfile(did, rawRecord, cache)
+			return parseAppBskyActorProfile(did, pdsURL, rawRecord, cache)
 		}
 	}
 
@@ -151,7 +153,7 @@ func FetchAppBskyActorProfile(logger *slog.Logger, did, pdsURL, slingshotURL str
 		return nil, err
 	}
 
-	return parseAppBskyActorProfile(did, rawRecord, cache)
+	return parseAppBskyActorProfile(did, pdsURL, rawRecord, cache)
 }
 
 func FetchAppBskyActorProfiles(logger *slog.Logger, dids []string, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache) map[string]notes.Author {
@@ -165,7 +167,7 @@ func FetchAppBskyActorProfiles(logger *slog.Logger, dids []string, pdsURL, sling
 	return authors
 }
 
-func parseAppBskyActorProfile(did string, rawRecord *general.RawRecord, cache *Cache) (*notes.Author, error) {
+func parseAppBskyActorProfile(did, pds string, rawRecord *general.RawRecord, cache *Cache) (*notes.Author, error) {
 	var profile notes.AppBskyActorProfileRecord
 	if err := json.Unmarshal(rawRecord.Value, &profile); err != nil {
 		return nil, err
@@ -176,7 +178,7 @@ func parseAppBskyActorProfile(did string, rawRecord *general.RawRecord, cache *C
 		handle = "handle.invalid"
 	}
 
-	author, err := notes.AuthorFromAppBskyActorProfileRecord(profile, did, handle)
+	author, err := notes.AuthorFromAppBskyActorProfileRecord(profile, did, handle, pds)
 
 	cache.SetAuthor(did, author)
 	return author, nil
diff --git a/backend/fetchers/app.bsky.feed.go b/backend/fetchers/app.bsky.feed.go
index 16e93c5..f490efb 100644
--- a/backend/fetchers/app.bsky.feed.go
+++ b/backend/fetchers/app.bsky.feed.go
@@ -7,11 +7,12 @@ import (
 	"llostre/backend/general"
 	"llostre/backend/notes"
 	"log/slog"
+	"time"
 )
 
 func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache) ([]notes.Note, error) {
 	// get author of the note
-	author, found := cache.GetProfile(did)
+	author, found := cache.GetAuthor(did)
 	var err error
 	if !found {
 		author, err = FetchAppBskyActorProfile(logger, did, pdsURL, slingshotURL, session, cache)
@@ -21,10 +22,11 @@ func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string
 	}
 
 	// try slingshot first if given
-	if slingshotURL != "" {
+	if slingshotURL != "" && general.SlingshotAvailable() {
 		records, err := general.FetchRawRecordCollection(logger, did, "app.bsky.feed.post", slingshotURL, session)
 		if err != nil {
-			logger.Error("Fetching from slingshot failed")
+			logger.Error("fetching from slingshot failed, falling back to PDS")
+			general.SetSlingshotBackoff(5 * time.Minute)
 		} else {
 			return parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
 		}
@@ -38,6 +40,39 @@ func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string
 	return parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
 }
 
+func FetchAppBskyFeedPostsWithCursor(logger *slog.Logger, did, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache, cursor string, limit int) ([]notes.Note, error) {
+	author, found := cache.GetAuthor(did)
+	var err error
+	if !found {
+		author, err = FetchAppBskyActorProfile(logger, did, pdsURL, slingshotURL, session, cache)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if slingshotURL != "" && general.SlingshotAvailable() {
+		records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(logger, did, "app.bsky.feed.post", slingshotURL, session, cursor, limit)
+		if err != nil {
+			logger.Error("fetching from slingshot failed, falling back to PDS")
+			general.SetSlingshotBackoff(5 * time.Minute)
+		} else {
+			notes, err := parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
+			cache.SetCursor(did, nextCursor)
+			return notes, err
+		}
+	}
+
+	records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(logger, did, "app.bsky.feed.post", pdsURL, session, cursor, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	notes, err := parseAppBskyFeedPostCollection(logger, pdsURL, session, records, author, cache)
+
+	cache.SetCursor(did, nextCursor)
+	return notes, err
+}
+
 func GetAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, session *auth.OAuthState, cache *Cache) (*notes.Note, error) {
 	// check cache, return if it was cached without auth or if it was cached with auth and we have auth
 	logger.Debug("Getting app.bsky.feed.post", "uri", uri)
@@ -68,7 +103,7 @@ func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, se
 	logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL)
 
 	// get author of the note
-	author, found := cache.GetProfile(repo)
+	author, found := cache.GetAuthor(repo)
 	if !found {
 		author, err = FetchAppBskyActorProfile(logger, repo, pdsURL, slingshotURL, session, cache)
 	}
@@ -77,10 +112,11 @@ func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, se
 	}
 
 	// try slingshot first if given
-	if slingshotURL != "" {
+	if slingshotURL != "" && general.SlingshotAvailable() {
 		rawRecord, err := general.FetchRawRecordByURI(logger, uri, slingshotURL, session)
 		if err != nil {
 			logger.Error("fetching from slingshot failed", "error", err)
+			general.SetSlingshotBackoff(5 * time.Minute)
 		} else {
 			return parseAppBskyFeedPostRecord(rawRecord, author, cache)
 		}
@@ -157,7 +193,7 @@ func FetchParents(logger *slog.Logger, replyToURI string, slingshotURL string, s
 		}
 
 		// preemptively fetch posts from author to avoid hammering servers if there's a back and forth
-		if _, found := cache.GetProfile(note.AuthorDID); !found {
+		if _, found := cache.GetAuthor(note.AuthorDID); !found {
 			logger.Debug("caching posts from thread participant", "did", note.AuthorDID)
 			authorPDS, err := FetchPDS(note.AuthorDID)
 			if err != nil {
diff --git a/backend/fetchers/cache.go b/backend/fetchers/cache.go
index 01c37fe..26be070 100644
--- a/backend/fetchers/cache.go
+++ b/backend/fetchers/cache.go
@@ -1,109 +1,167 @@
 package fetchers
 
 import (
-	"llostre/backend/notes"
-	"sync"
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
 	"time"
-)
 
-type CacheEntry struct {
-	Note      *notes.Note
-	ExpiresAt time.Time
-}
+	"llostre/backend/notes"
+
+	"github.com/redis/go-redis/v9"
+)
 
 type Cache struct {
-	posts    map[string]*CacheEntry
-	profiles map[string]*AuthorCacheEntry
-	mu       sync.RWMutex
+	client     *redis.Client
+	ctx        context.Context
+	expiryTime time.Duration
 }
 
-func NewCache() *Cache {
-	return &Cache{
-		posts:    make(map[string]*CacheEntry),
-		profiles: make(map[string]*AuthorCacheEntry),
+func NewCache(addr, password string, expiryMinutes int) (*Cache, error) {
+	client := redis.NewClient(&redis.Options{
+		Addr:     addr,
+		Password: password,
+		DB:       0,
+	})
+
+	ctx := context.Background()
+	if err := client.Ping(ctx).Err(); err != nil {
+		return nil, fmt.Errorf("redis connection failed: %w", err)
 	}
+
+	return &Cache{
+		client:     client,
+		ctx:        ctx,
+		expiryTime: time.Duration(expiryMinutes) * time.Minute,
+	}, nil
 }
 
 func (c *Cache) Get(uri string) (*notes.Note, bool) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
+	key := fmt.Sprintf("post:%s", uri)
+	data, err := c.client.Get(c.ctx, key).Bytes()
+	if err != nil {
+		return nil, false
+	}
+
+	var note notes.Note
+	if err := json.Unmarshal(data, &note); err != nil {
+		return nil, false
+	}
+
+	return &note, true
+}
 
-	entry, exists := c.posts[uri]
+func (c *Cache) Set(uri string, note *notes.Note) {
+	key := fmt.Sprintf("post:%s", uri)
+	data, err := json.Marshal(note)
+	if err != nil {
+		return
+	}
+
+	c.client.Set(c.ctx, key, data, c.expiryTime)
+}
 
-	if !exists {
+func (c *Cache) GetAuthor(did string) (*notes.Author, bool) {
+	key := fmt.Sprintf("profile:%s", did)
+	data, err := c.client.Get(c.ctx, key).Bytes()
+	if err != nil {
 		return nil, false
 	}
 
-	if time.Now().After(entry.ExpiresAt) {
-		delete(c.posts, uri)
+	var author notes.Author
+	if err := json.Unmarshal(data, &author); err != nil {
 		return nil, false
 	}
 
-	return entry.Note, true
+	return &author, true
 }
 
-func (c *Cache) Set(uri string, note *notes.Note) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
+func (c *Cache) SetAuthor(did string, author *notes.Author) {
+	key := fmt.Sprintf("profile:%s", did)
+	data, err := json.Marshal(author)
+	if err != nil {
+		return
+	}
+
+	c.client.Set(c.ctx, key, data, c.expiryTime)
+}
 
-	c.posts[uri] = &CacheEntry{
-		Note:      note,
-		ExpiresAt: time.Now().Add(15 * time.Minute),
+func (c *Cache) GetCursor(did string) (string, bool) {
+	key := fmt.Sprintf("cursor:%s", did)
+	cursor, err := c.client.Get(c.ctx, key).Result()
+	if err != nil {
+		return "", false
 	}
+	return cursor, true
 }
 
-func (c *Cache) GetMany(uris []string) map[string]*notes.Note {
-	c.mu.Lock()
-	defer c.mu.Unlock()
+func (c *Cache) SetCursor(did, cursor string) {
+	key := fmt.Sprintf("cursor:%s", did)
+	if cursor == "" {
+		c.client.Del(c.ctx, key)
+		return
+	}
+	c.client.Set(c.ctx, key, cursor, 24*time.Hour)
+}
 
-	result := make(map[string]*notes.Note)
-	now := time.Now()
+func (c *Cache) GetPostsByDID(did string) ([]notes.Note, error) {
+	pattern := fmt.Sprintf("post:at://%s/*", did)
+	var posts []notes.Note
 
-	for _, uri := range uris {
-		entry, exists := c.posts[uri]
-		if !exists {
+	iter := c.client.Scan(c.ctx, 0, pattern, 0).Iterator()
+	for iter.Next(c.ctx) {
+		data, err := c.client.Get(c.ctx, iter.Val()).Bytes()
+		if err != nil {
 			continue
 		}
-
-		if now.After(entry.ExpiresAt) {
-			delete(c.posts, uri)
+		var note notes.Note
+		if err := json.Unmarshal(data, &note); err != nil {
 			continue
 		}
+		posts = append(posts, note)
+	}
 
-		result[uri] = entry.Note
+	if err := iter.Err(); err != nil {
+		return nil, err
 	}
 
-	return result
+	sort.Slice(posts, func(i, j int) bool {
+		return posts[i].CreatedAt.After(posts[j].CreatedAt)
+	})
+
+	return posts, nil
 }
 
-type AuthorCacheEntry struct {
-	Author    *notes.Author
-	ExpiresAt time.Time
+// track oldest sent post per profile
+func (c *Cache) GetOldestSent(did string) (time.Time, bool) {
+	key := fmt.Sprintf("oldest_sent:%s", did)
+	ts, err := c.client.Get(c.ctx, key).Int64()
+	if err != nil {
+		return time.Time{}, false
+	}
+	return time.Unix(ts, 0), true
 }
 
-func (c *Cache) GetProfile(did string) (*notes.Author, bool) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
+func (c *Cache) SetOldestSent(did string, timestamp time.Time) {
+	key := fmt.Sprintf("oldest_sent:%s", did)
+	c.client.Set(c.ctx, key, timestamp.Unix(), 24*time.Hour)
+}
 
-	entry, exists := c.profiles[did]
-	if !exists {
-		return nil, false
+// count posts older than timestamp for a profile
+func (c *Cache) CountPostsOlderThan(did string, timestamp time.Time) (int, error) {
+	posts, err := c.GetPostsByDID(did)
+	if err != nil {
+		return 0, err
 	}
-
-	if time.Now().After(entry.ExpiresAt) {
-		delete(c.profiles, did)
-		return nil, false
+	for i, p := range posts {
+		if p.CreatedAt.Before(timestamp) {
+			return len(posts) - i, nil
+		}
 	}
-
-	return entry.Author, true
+	return 0, nil
 }
 
-func (c *Cache) SetAuthor(did string, author *notes.Author) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	c.profiles[did] = &AuthorCacheEntry{
-		Author:    author,
-		ExpiresAt: time.Now().Add(15 * time.Minute),
-	}
+func (c *Cache) Close() error {
+	return c.client.Close()
 }
diff --git a/backend/general/records.go b/backend/general/records.go
index ea2b308..d82b289 100644
--- a/backend/general/records.go
+++ b/backend/general/records.go
@@ -55,6 +55,53 @@ func FetchRawRecordCollection(logger *slog.Logger, did, collection, baseURL stri
 	return result.Records, nil
 }
 
+func FetchRawRecordCollectionWithCursor(logger *slog.Logger, did, collection, baseURL string, session *auth.OAuthState, cursor string, limit int) ([]RawRecord, string, error) {
+	url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s", baseURL, did, collection)
+	if cursor != "" {
+		url += "&cursor=" + cursor
+	}
+	if limit > 0 {
+		url += fmt.Sprintf("&limit=%d", limit)
+	}
+
+	logger.Debug("fetching raw record collection", "did", did, "endpoint", baseURL, "collection", collection, "cursor", cursor)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, "", err
+	}
+
+	if session != nil {
+		dpopProof, err := session.KeyPair.GenerateDPoPProof("GET", url, "")
+		if err != nil {
+			return nil, "", err
+		}
+		req.Header.Set("Authorization", "Bearer "+session.AccessToken)
+		req.Header.Set("DPoP", dpopProof)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, "", fmt.Errorf("server returned %d: %s", resp.StatusCode, string(body))
+	}
+
+	var result struct {
+		Records []RawRecord `json:"records"`
+		Cursor  string      `json:"cursor"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, "", err
+	}
+
+	return result.Records, result.Cursor, nil
+}
+
 func FetchRawRecordByURI(logger *slog.Logger, uri, baseURL string, session *auth.OAuthState) (*RawRecord, error) {
 	repo, collection, rkey, err := ParseURI(uri)
 	if err != nil {
diff --git a/backend/general/utils.go b/backend/general/utils.go
index fe0854a..ac27443 100644
--- a/backend/general/utils.go
+++ b/backend/general/utils.go
@@ -3,6 +3,13 @@ package general
 import (
 	"fmt"
 	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	slingshotBackoff   time.Time
+	slingshotBackoffMu sync.Mutex
 )
 
 func ParseURI(uri string) (repo, collection, rkey string, err error) {
@@ -14,3 +21,15 @@ func ParseURI(uri string) (repo, collection, rkey string, err error) {
 
 	return parts[0], parts[1], parts[2], nil
 }
+
+func SlingshotAvailable() bool {
+	slingshotBackoffMu.Lock()
+	defer slingshotBackoffMu.Unlock()
+	return time.Now().After(slingshotBackoff)
+}
+
+func SetSlingshotBackoff(d time.Duration) {
+	slingshotBackoffMu.Lock()
+	defer slingshotBackoffMu.Unlock()
+	slingshotBackoff = time.Now().Add(d)
+}
diff --git a/backend/go.mod b/backend/go.mod
index 5eb59a3..7f2db06 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -6,3 +6,10 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/joho/godotenv v1.5.1
 )
+
+require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/redis/go-redis/v9 v9.18.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+)
diff --git a/backend/go.sum b/backend/go.sum
index a829f15..1d779d1 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -1,4 +1,12 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
+github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
diff --git a/backend/main.go b/backend/main.go
index eb6d70b..27767b0 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -5,11 +5,12 @@ import (
 	"fmt"
 	"llostre/backend/auth"
 	"llostre/backend/fetchers"
+	"llostre/backend/notes"
 	"log/slog"
 	"net/http"
 	"net/url"
 	"os"
-	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -25,6 +26,7 @@ var (
 	sessionsMutex sync.RWMutex
 	sessions      = make(map[string]*auth.OAuthState)
 	cache         *fetchers.Cache
+	pageSize      int
 )
 
 func init() {
@@ -57,8 +59,28 @@ func init() {
 	}))
 
 	slingshotURL = os.Getenv("SLINGSHOT_URL")
+	ttlMin := os.Getenv("TTL_MIN")
 
-	cache = fetchers.NewCache()
+	var expiryMinutes int
+	if ttlMin != "" {
+		var err error
+		expiryMinutes, err = strconv.Atoi(ttlMin)
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		expiryMinutes = 15
+	}
+	redisAddr := os.Getenv("REDIS_ADDR")
+	redisPassword := os.Getenv("REDIS_PASSWORD")
+	postCache, err := fetchers.NewCache(redisAddr, redisPassword, expiryMinutes)
+	if err != nil {
+		logger.Error("failed to connect to redis", "error", err)
+		panic(err)
+	}
+
+	pageSize = 50
+	cache = postCache
 }
 
 func loginHandler(w http.ResponseWriter, r *http.Request) {
@@ -249,32 +271,42 @@ func sessionHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
-func refreshTokens(session *auth.OAuthState, cookieValue string) error {
+func getSession(cookieValue string) (*auth.OAuthState, error) {
 	// Check for access token refresh
-	needsRefresh := time.Now().After(session.ExpiresAt)
-	pds := session.PDS
-	refreshToken := session.RefreshToken
-	keypair := session.KeyPair
+	sessionsMutex.RLock()
+	session, exists := sessions[cookieValue]
+	if !exists {
+		return nil, fmt.Errorf("no session found")
+	}
+	sessionRead := session.Copy()
 	sessionsMutex.RUnlock()
 
+	logger.Debug("session found", "did", session.DID, "pds", session.PDS, "access token length", len(session.AccessToken), "session keypair", session.KeyPair)
+	if session.KeyPair != nil {
+		logger.Debug("session public key", "key", session.KeyPair.PublicKey)
+	}
+
+	needsRefresh := time.Now().After(sessionRead.ExpiresAt)
+
 	if needsRefresh {
-		newTokens, err := auth.RefreshAccessToken(pds, refreshToken, backendURL+"/callback", keypair)
+		newTokens, err := auth.RefreshAccessToken(sessionRead.PDS, sessionRead.RefreshToken, backendURL+"/callback", sessionRead.KeyPair)
 		if err != nil {
 			sessionsMutex.Lock()
 			delete(sessions, cookieValue)
 			sessionsMutex.Unlock()
 			logger.Error("no authorization servers found for token refresh")
-			return fmt.Errorf("no authorization servers found")
+			return nil, fmt.Errorf("no authorization servers found")
 		}
 
 		sessionsMutex.Lock()
+		session, _ = sessions[cookieValue]
 		session.AccessToken = newTokens.AccessToken
 		session.RefreshToken = newTokens.RefreshToken
 		session.ExpiresAt = time.Now().Add(time.Duration(newTokens.ExpiresIn) * time.Second)
 		sessionsMutex.Unlock()
 	}
 
-	return nil
+	return session, nil
 }
 
 func postsHandler(w http.ResponseWriter, r *http.Request) {
@@ -289,33 +321,50 @@ func postsHandler(w http.ResponseWriter, r *http.Request) {
 
 	logger.Debug("cookie received", "value", cookie.Value)
 
-	sessionsMutex.RLock()
-	session, exists := sessions[cookie.Value]
-	if !exists {
-		sessionsMutex.RUnlock()
+	session, err := getSession(cookie.Value)
+	if err != nil {
 		http.Error(w, "invalid session", http.StatusUnauthorized)
 		return
 	}
 
-	logger.Debug("session found", "did", session.DID, "pds", session.PDS, "access token length", len(session.AccessToken), "session keypair", session.KeyPair)
-	if session.KeyPair != nil {
-		logger.Debug("session public key", "key", session.KeyPair.PublicKey)
-	}
-
-	refreshTokens(session, cookie.Value)
-
 	// Fetch app.bsky.feed.post records
-	posts, err := fetchers.FetchAppBskyFeedPosts(logger, session.DID, session.PDS, slingshotURL, session, cache)
+	var posts []notes.Note
+
+	cached, err := cache.GetPostsByDID(session.DID)
+	logger.Debug("cache read", "count", len(cached), "err", err)
 	if err != nil {
-		logger.Error("fetching app.bsky.feed.posts failed", "error", err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+		logger.Warn("cache read failed, falling back to fetch", "error", err)
 	}
 
-	sort.Slice(posts, func(i, j int) bool {
-		return posts[i].CreatedAt.After(posts[j].CreatedAt)
-	})
+	if len(cached) > 0 {
+		posts = notes.DeduplicateThreads(cached)
+	} else {
+		posts, err = fetchers.FetchAppBskyFeedPostsWithCursor(logger, session.DID, session.PDS, slingshotURL, session, cache, "", pageSize)
+		if err != nil {
+			logger.Error("fetching app.bsky.feed.posts failed", "error", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+	logger.Debug("posts ready", "count", len(posts))
+
+	// background refetch if cache is running low
+	oldestSent, hasOldest := cache.GetOldestSent(session.DID)
+	if !hasOldest || len(posts) > 0 {
+		oldest := posts[len(posts)-1].CreatedAt
+		count, _ := cache.CountPostsOlderThan(session.DID, oldest)
+		cursor, hasCursor := cache.GetCursor(session.DID)
+		logger.Debug("refetch check", "count", count, "pageSize", pageSize, "hasCursor", hasCursor, "oldest", oldest)
+		if count < pageSize && hasCursor {
+			sessionCopy := session.Copy()
+			go fetchers.FetchAppBskyFeedPostsWithCursor(logger, sessionCopy.DID, sessionCopy.PDS, slingshotURL, sessionCopy, cache, cursor, pageSize)
+		}
+		if !hasOldest || oldest.Before(oldestSent) {
+			cache.SetOldestSent(session.DID, oldest)
+		}
+	}
 
+	// return posts
 	logger.Debug("successfully fetched posts", "number", len(posts))
 
 	w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -342,15 +391,11 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		logger.Warn("unauthorized")
 	} else {
-		sessionsMutex.RLock()
-		session, exists := sessions[cookie.Value]
-		sessionsMutex.RUnlock()
+		session, err := getSession(cookie.Value)
 
-		if !exists {
-			logger.Warn("invalid session")
+		if err != nil {
+			logger.Warn("invalid session", "error", err)
 		} else {
-			refreshTokens(session, cookie.Value)
-
 			currentSession = session
 		}
 	}
@@ -366,6 +411,8 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
+	defer cache.Close()
+
 	// serve static files
 	fs := http.FileServer(http.Dir("../frontend/dist"))
 	http.Handle("/", fs)
diff --git a/backend/notes/app.bsky.actor.go b/backend/notes/app.bsky.actor.go
index 797a8e8..00a1ff1 100644
--- a/backend/notes/app.bsky.actor.go
+++ b/backend/notes/app.bsky.actor.go
@@ -25,13 +25,13 @@ type AppBskyActorProfileRecord struct {
 	} `json:"labels,omitempty"`
 }
 
-func AuthorFromAppBskyActorProfileRecord(record AppBskyActorProfileRecord, did, handle string) (*Author, error) {
+func AuthorFromAppBskyActorProfileRecord(record AppBskyActorProfileRecord, did, handle, pds string) (*Author, error) {
 	var avatarURL, bannerURL string
 	if record.Avatar != nil {
-		avatarURL = fmt.Sprintf("https://cdn.bsky.app/img/avatar/plain/%s/%s@jpeg", did, record.Avatar.Ref.Link)
+		avatarURL = fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pds, did, record.Avatar.Ref.Link)
 	}
 	if record.Banner != nil {
-		bannerURL = fmt.Sprintf("https://cdn.bsky.app/img/banner/plain/%s/%s@jpeg", did, record.Banner.Ref.Link)
+		bannerURL = fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pds, did, record.Banner.Ref.Link)
 	}
 
 	// check for !no-unauthenticated label
@@ -45,6 +45,7 @@ func AuthorFromAppBskyActorProfileRecord(record AppBskyActorProfileRecord, did,
 	}
 
 	author := &Author{
+		PDS:         pds,
 		Handle:      handle,
 		DisplayName: record.DisplayName,
 		AvatarURL:   avatarURL,
diff --git a/backend/notes/author.go b/backend/notes/author.go
index 644a10c..faf4f7b 100644
--- a/backend/notes/author.go
+++ b/backend/notes/author.go
@@ -1,6 +1,7 @@
 package notes
 
 type Author struct {
+	PDS         string
 	Handle      string
 	DisplayName string
 	AvatarURL   string
diff --git a/backend/notes/note.go b/backend/notes/note.go
index 7ef537f..72c885d 100644
--- a/backend/notes/note.go
+++ b/backend/notes/note.go
@@ -1,7 +1,6 @@
 package notes
 
 import (
-	"sort"
 	"time"
 )
 
@@ -54,10 +53,6 @@ func isChainContained(smallChainPost, bigChainPost Note) bool {
 }
 
 func DeduplicateThreads(posts []Note) []Note {
-	sort.Slice(posts, func(i, j int) bool {
-		return len(posts[i].Parents) > len(posts[j].Parents)
-	})
-
 	filtered := []Note{}
 	for _, post := range posts {
 		isRedundant := false