reworked cache with redis + improved fetching

1 files changed, 42 insertions, 6 deletions

diff --git a/backend/fetchers/app.bsky.feed.go b/backend/fetchers/app.bsky.feed.go
index 16e93c5..f490efb 100644
--- a/backend/fetchers/app.bsky.feed.go
+++ b/backend/fetchers/app.bsky.feed.go
@@ -7,11 +7,12 @@ import (
 	"llostre/backend/general"
 	"llostre/backend/notes"
 	"log/slog"
+	"time"
 )
 
 func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache) ([]notes.Note, error) {
 	// get author of the note
-	author, found := cache.GetProfile(did)
+	author, found := cache.GetAuthor(did)
 	var err error
 	if !found {
 		author, err = FetchAppBskyActorProfile(logger, did, pdsURL, slingshotURL, session, cache)
@@ -21,10 +22,11 @@ func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string
 	}
 
 	// try slingshot first if given
-	if slingshotURL != "" {
+	if slingshotURL != "" && general.SlingshotAvailable() {
 		records, err := general.FetchRawRecordCollection(logger, did, "app.bsky.feed.post", slingshotURL, session)
 		if err != nil {
-			logger.Error("Fetching from slingshot failed")
+			logger.Error("fetching from slingshot failed, falling back to PDS")
+			general.SetSlingshotBackoff(5 * time.Minute)
 		} else {
 			return parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
 		}
@@ -38,6 +40,39 @@ func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string
 	return parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
 }
 
+func FetchAppBskyFeedPostsWithCursor(logger *slog.Logger, did, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache, cursor string, limit int) ([]notes.Note, error) {
+	author, found := cache.GetAuthor(did)
+	var err error
+	if !found {
+		author, err = FetchAppBskyActorProfile(logger, did, pdsURL, slingshotURL, session, cache)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if slingshotURL != "" && general.SlingshotAvailable() {
+		records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(logger, did, "app.bsky.feed.post", slingshotURL, session, cursor, limit)
+		if err != nil {
+			logger.Error("fetching from slingshot failed, falling back to PDS")
+			general.SetSlingshotBackoff(5 * time.Minute)
+		} else {
+			notes, err := parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
+			cache.SetCursor(did, nextCursor)
+			return notes, err
+		}
+	}
+
+	records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(logger, did, "app.bsky.feed.post", pdsURL, session, cursor, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	notes, err := parseAppBskyFeedPostCollection(logger, pdsURL, session, records, author, cache)
+
+	cache.SetCursor(did, nextCursor)
+	return notes, err
+}
+
 func GetAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, session *auth.OAuthState, cache *Cache) (*notes.Note, error) {
 	// check cache, return if it was cached without auth or if it was cached with auth and we have auth
 	logger.Debug("Getting app.bsky.feed.post", "uri", uri)
@@ -68,7 +103,7 @@ func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, se
 	logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL)
 
 	// get author of the note
-	author, found := cache.GetProfile(repo)
+	author, found := cache.GetAuthor(repo)
 	if !found {
 		author, err = FetchAppBskyActorProfile(logger, repo, pdsURL, slingshotURL, session, cache)
 	}
@@ -77,10 +112,11 @@ func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, se
 	}
 
 	// try slingshot first if given
-	if slingshotURL != "" {
+	if slingshotURL != "" && general.SlingshotAvailable() {
 		rawRecord, err := general.FetchRawRecordByURI(logger, uri, slingshotURL, session)
 		if err != nil {
 			logger.Error("fetching from slingshot failed", "error", err)
+			general.SetSlingshotBackoff(5 * time.Minute)
 		} else {
 			return parseAppBskyFeedPostRecord(rawRecord, author, cache)
 		}
@@ -157,7 +193,7 @@ func FetchParents(logger *slog.Logger, replyToURI string, slingshotURL string, s
 		}
 
 		// preemptively fetch posts from author to avoid hammering servers if there's a back and forth
-		if _, found := cache.GetProfile(note.AuthorDID); !found {
+		if _, found := cache.GetAuthor(note.AuthorDID); !found {
 			logger.Debug("caching posts from thread participant", "did", note.AuthorDID)
 			authorPDS, err := FetchPDS(note.AuthorDID)
 			if err != nil {