1 files changed, 217 insertions, 0 deletions
diff --git a/fetchers/app.bsky.feed.go b/fetchers/app.bsky.feed.go
new file mode 100644
index 0000000..5eebec9
--- /dev/null
+++ b/fetchers/app.bsky.feed.go
@@ -0,0 +1,217 @@
+package fetchers
+
+import (
+	"encoding/json"
+	"fmt"
+	"llostre/auth"
+	"llostre/general"
+	"llostre/notes"
+	"time"
+)
+
+func FetchAppBskyFeedPosts(conf *Config, did, pdsURL string, session *auth.OAuthState) ([]notes.Note, error) {
+	// get author of the note
+	author, found := conf.Cache.GetAuthor(did)
+	var err error
+	if !found {
+		author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// try slingshot first if given
+	if conf.SlingshotURL != "" && general.SlingshotAvailable() {
+		records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session)
+		if err != nil {
+			conf.Logger.Error("fetching from slingshot failed, falling back to PDS")
+			general.SetSlingshotBackoff(5 * time.Minute)
+		} else {
+			return parseAppBskyFeedPostCollection(conf, session, records, author)
+		}
+	}
+
+	records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", pdsURL, session)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseAppBskyFeedPostCollection(conf, session, records, author)
+}
+
+func FetchAppBskyFeedPostsWithCursor(conf *Config, did, pdsURL string, session *auth.OAuthState, cursor string) ([]notes.Note, error) {
+	author, found := conf.Cache.GetAuthor(did)
+	var err error
+	if !found {
+		author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if conf.SlingshotURL != "" && general.SlingshotAvailable() {
+		records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session, cursor, conf.PageSize)
+		if err != nil {
+			conf.Logger.Error("fetching from slingshot failed, falling back to PDS")
+			general.SetSlingshotBackoff(5 * time.Minute)
+		} else {
+			notes, err := parseAppBskyFeedPostCollection(conf, session, records, author)
+			conf.Cache.SetCursor(did, nextCursor)
+			return notes, err
+		}
+	}
+
+	records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", pdsURL, session, cursor, conf.PageSize)
+	if err != nil {
+		return nil, err
+	}
+
+	notes, err := parseAppBskyFeedPostCollection(conf, session, records, author)
+
+	conf.Cache.SetCursor(did, nextCursor)
+	return notes, err
+}
+
+func GetAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) {
+	// check cache, return if it was cached without auth or if it was cached with auth and we have auth
+	conf.Logger.Debug("Getting app.bsky.feed.post", "uri", uri)
+	if cached, found := conf.Cache.Get(uri); found {
+		conf.Logger.Debug("Found in the cache", "public", !cached.AuthorNoUnauth)
+		if !cached.AuthorNoUnauth {
+			return cached, nil
+		}
+
+		// author requires auth - if we have a session, serve it
+		// if the token was invalid, trust that it would have been caught by the "refresh token" call in main
+		if session != nil {
+			return cached, nil
+		}
+	}
+
+	return FetchAppBskyFeedPostByURI(conf, uri, session)
+}
+
+func FetchAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) {
+	repo, _, _, err := general.ParseURI(uri)
+
+	// need to resolve DID to PDS
+	pdsURL, err := FetchPDS(repo)
+	if err != nil {
+		return nil, err
+	}
+	conf.Logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL)
+
+	// get author of the note
+	author, found := conf.Cache.GetAuthor(repo)
+	if !found {
+		author, err = FetchAppBskyActorProfile(conf, repo, pdsURL, session)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// try slingshot first if given
+	if conf.SlingshotURL != "" && general.SlingshotAvailable() {
+		rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, conf.SlingshotURL, session)
+		if err != nil {
+			conf.Logger.Error("fetching from slingshot failed", "error", err)
+			general.SetSlingshotBackoff(5 * time.Minute)
+		} else {
+			return parseAppBskyFeedPostRecord(rawRecord, author, conf)
+		}
+	}
+
+	rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, pdsURL, session)
+	if err != nil {
+		conf.Logger.Error("Fetching from PDS failed")
+		return nil, err
+	}
+
+	return parseAppBskyFeedPostRecord(rawRecord, author, conf)
+}
+
+func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, author *notes.Author, conf *Config) (*notes.Note, error) {
+	if rawRecord == nil {
+		return nil, fmt.Errorf("unable to parse nil record")
+	}
+
+	record := notes.AppBskyFeedPostRecord{
+		URI: rawRecord.URI,
+	}
+	err := json.Unmarshal(rawRecord.Value, &record.Value)
+	if err != nil {
+		return nil, err
+	}
+
+	note, err := notes.NoteFromAppBskyFeedPostRecord(record, author)
+	if err != nil {
+		return nil, err
+	}
+
+	// cache it
+	conf.Cache.Set(note.URI, note)
+
+	conf.Logger.Debug("parsed note", "uri", note.URI, "media_count", len(note.Media), "embed", record.Value.Embed)
+
+	return note, nil
+}
+
+func parseAppBskyFeedPostCollection(conf *Config, session *auth.OAuthState, records []general.RawRecord, author *notes.Author) ([]notes.Note, error) {
+	posts := make([]notes.Note, 0, len(records))
+	for _, record := range records {
+		note, err := parseAppBskyFeedPostRecord(&record, author, conf)
+		if err != nil {
+			conf.Logger.Error("failed to parse post record", "error", err)
+			continue
+		}
+		if note.ReplyTo != nil {
+			parents, err := FetchParents(conf, *note.ReplyTo, session, 10)
+			if err != nil {
+				conf.Logger.Warn("failed to fetch parent chain", "uri", note.URI, "error", err)
+			}
+			note.Parents = parents
+		}
+		conf.Cache.Set(note.URI, note)
+		posts = append(posts, *note)
+	}
+
+	return notes.DeduplicateThreads(posts), nil
+}
+
+func FetchParents(conf *Config, replyToURI string, session *auth.OAuthState, maxDepth int) ([]notes.Note, error) {
+	parents := []notes.Note{}
+	currentURI := replyToURI
+
+	for currentURI != "" {
+		note, err := GetAppBskyFeedPostByURI(conf, currentURI, session)
+		if err != nil {
+			conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err)
+			note = notes.MissingNote(currentURI)
+
+			conf.Cache.Set(currentURI, note)
+			parents = append(parents, *note)
+			break
+		}
+
+		// preemptively fetch posts from author to avoid hammering servers if there's a back and forth
+		if _, found := conf.Cache.GetAuthor(note.AuthorDID); !found {
+			conf.Logger.Debug("caching posts from thread participant", "did", note.AuthorDID)
+			authorPDS, err := FetchPDS(note.AuthorDID)
+			if err != nil {
+				conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err)
+				break
+			}
+			go FetchAppBskyFeedPostsWithCursor(conf, note.AuthorDID, authorPDS, session, "")
+		}
+
+		parents = append(parents, *note)
+
+		if note.ReplyTo == nil {
+			break
+		}
+
+		currentURI = *note.ReplyTo
+	}
+
+	return parents, nil
+}