package fetchers import ( "encoding/json" "fmt" "llostre/auth" "llostre/general" "llostre/notes" "time" ) func FetchAppBskyFeedPosts(conf *Config, did, pdsURL string, session *auth.OAuthState) ([]notes.Note, error) { // get author of the note author, found := conf.Cache.GetAuthor(did) var err error if !found { author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session) } if err != nil { return nil, err } // try slingshot first if given if notes.IsPublic(author) && conf.SlingshotURL != "" && general.SlingshotAvailable() { records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session) if err != nil { conf.Logger.Error("fetching from slingshot failed, falling back to PDS") general.SetSlingshotBackoff(5 * time.Minute) } else { return parseAppBskyFeedPostCollection(conf, session, records, author) } } records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", pdsURL, session) if err != nil { return nil, err } return parseAppBskyFeedPostCollection(conf, session, records, author) } func FetchAppBskyFeedPostsWithCursor(conf *Config, did, pdsURL string, session *auth.OAuthState, cursor string) ([]notes.Note, error) { author, found := conf.Cache.GetAuthor(did) var err error if !found { author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session) } if err != nil { return nil, err } if conf.SlingshotURL != "" && general.SlingshotAvailable() { records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session, cursor, conf.PageSize) if err != nil { conf.Logger.Error("fetching from slingshot failed, falling back to PDS") general.SetSlingshotBackoff(5 * time.Minute) } else { notes, err := parseAppBskyFeedPostCollection(conf, session, records, author) conf.Cache.SetCursor(did, nextCursor) updatePostCount(conf, did, cursor, len(notes)) return notes, err } } records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", pdsURL, session, cursor, conf.PageSize) if err != nil { return nil, err } notes, err := parseAppBskyFeedPostCollection(conf, session, records, author) conf.Cache.SetCursor(did, nextCursor) updatePostCount(conf, did, cursor, len(notes)) return notes, err } func updatePostCount(conf *Config, did, cursor string, fetchedCount int) { if cursor == "" { conf.Cache.SetPostCount(did, fetchedCount) } else { conf.Cache.AddPostCount(did, fetchedCount) } } func GetAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) { // check cache, return if it was cached without auth or if it was cached with auth and we have auth conf.Logger.Debug("Getting app.bsky.feed.post", "uri", uri) if cached, found := conf.Cache.Get(uri); found { conf.Logger.Debug("Found in the cache", "public", !cached.AuthorNoUnauth) if !cached.AuthorNoUnauth { return cached, nil } // author requires auth // if we have a session, serve it // if the token was invalid, trust that it would have been caught by the "refresh token" call in main if session != nil { return cached, nil } } return FetchAppBskyFeedPostByURI(conf, uri, session) } func FetchAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) { repo, _, _, err := general.ParseURI(uri) // need to resolve DID to PDS pdsURL, err := FetchPDS(repo) if err != nil { return nil, err } conf.Logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL) // get author of the note author, found := conf.Cache.GetAuthor(repo) if !found { author, err = FetchAppBskyActorProfile(conf, repo, pdsURL, session) } if err != nil { return nil, err } // try slingshot first if given if conf.SlingshotURL != "" && general.SlingshotAvailable() { rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, conf.SlingshotURL, session) if err != nil { conf.Logger.Error("fetching from slingshot failed", "error", err) general.SetSlingshotBackoff(5 * time.Minute) } else { return parseAppBskyFeedPostRecord(rawRecord, author, conf) } } rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, pdsURL, session) if err != nil { conf.Logger.Error("Fetching from PDS failed") return nil, err } return parseAppBskyFeedPostRecord(rawRecord, author, conf) } func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, author *notes.Author, conf *Config) (*notes.Note, error) { if rawRecord == nil { return nil, fmt.Errorf("unable to parse nil record") } record := notes.AppBskyFeedPostRecord{ URI: rawRecord.URI, } err := json.Unmarshal(rawRecord.Value, &record.Value) if err != nil { return nil, err } note, err := notes.NoteFromAppBskyFeedPostRecord(record, author, conf.PorxieURL) if err != nil { return nil, err } // cache it conf.Cache.Set(note.URI, note) if len(note.Media) > 0 { conf.Logger.Debug("parsed note", "uri", note.URI, "media_count", len(note.Media), "embed", record.Value.Embed) } return note, nil } func parseAppBskyFeedPostCollection(conf *Config, session *auth.OAuthState, records []general.RawRecord, author *notes.Author) ([]notes.Note, error) { posts := make([]notes.Note, 0, len(records)) for _, record := range records { note, err := parseAppBskyFeedPostRecord(&record, author, conf) if err != nil { conf.Logger.Error("failed to parse post record", "error", err) continue } if note.ReplyTo != nil { parents, err := FetchParents(conf, *note.ReplyTo, session, 10) if err != nil { conf.Logger.Warn("failed to fetch parent chain", "uri", note.URI, "error", err) } note.Parents = parents } conf.Cache.Set(note.URI, note) posts = append(posts, *note) } return posts, nil } func FetchParents(conf *Config, replyToURI string, session *auth.OAuthState, maxDepth int) ([]notes.Note, error) { parents := []notes.Note{} currentURI := replyToURI for currentURI != "" { note, err := GetAppBskyFeedPostByURI(conf, currentURI, session) if err != nil { conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err) note = notes.MissingNote(currentURI) conf.Cache.Set(currentURI, note) parents = append(parents, *note) break } // preemptively fetch posts from author to avoid hammering servers if there's a back and forth if _, found := conf.Cache.GetAuthor(note.AuthorDID); !found { conf.Logger.Debug("caching posts from thread participant", "did", note.AuthorDID) authorPDS, err := FetchPDS(note.AuthorDID) if err != nil { conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err) break } go FetchAppBskyFeedPostsWithCursor(conf, note.AuthorDID, authorPDS, session, "") } parents = append(parents, *note) if note.ReplyTo == nil { break } currentURI = *note.ReplyTo } return parents, nil }