aboutsummaryrefslogtreecommitdiff
path: root/backend/fetchers/app.bsky.feed.go
diff options
context:
space:
mode:
Diffstat (limited to 'backend/fetchers/app.bsky.feed.go')
-rw-r--r--backend/fetchers/app.bsky.feed.go107
1 files changed, 54 insertions, 53 deletions
diff --git a/backend/fetchers/app.bsky.feed.go b/backend/fetchers/app.bsky.feed.go
index f490efb..8562a9c 100644
--- a/backend/fetchers/app.bsky.feed.go
+++ b/backend/fetchers/app.bsky.feed.go
@@ -6,78 +6,77 @@ import (
"llostre/backend/auth"
"llostre/backend/general"
"llostre/backend/notes"
- "log/slog"
"time"
)
-func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache) ([]notes.Note, error) {
+func FetchAppBskyFeedPosts(conf *Config, did, pdsURL string, session *auth.OAuthState) ([]notes.Note, error) {
// get author of the note
- author, found := cache.GetAuthor(did)
+ author, found := conf.Cache.GetAuthor(did)
var err error
if !found {
- author, err = FetchAppBskyActorProfile(logger, did, pdsURL, slingshotURL, session, cache)
+ author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session)
}
if err != nil {
return nil, err
}
// try slingshot first if given
- if slingshotURL != "" && general.SlingshotAvailable() {
- records, err := general.FetchRawRecordCollection(logger, did, "app.bsky.feed.post", slingshotURL, session)
+ if conf.SlingshotURL != "" && general.SlingshotAvailable() {
+ records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session)
if err != nil {
- logger.Error("fetching from slingshot failed, falling back to PDS")
+ conf.Logger.Error("fetching from slingshot failed, falling back to PDS")
general.SetSlingshotBackoff(5 * time.Minute)
} else {
- return parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
+ return parseAppBskyFeedPostCollection(conf, session, records, author)
}
}
- records, err := general.FetchRawRecordCollection(logger, did, "app.bsky.feed.post", pdsURL, session)
+ records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", pdsURL, session)
if err != nil {
return nil, err
}
- return parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
+ return parseAppBskyFeedPostCollection(conf, session, records, author)
}
-func FetchAppBskyFeedPostsWithCursor(logger *slog.Logger, did, pdsURL, slingshotURL string, session *auth.OAuthState, cache *Cache, cursor string, limit int) ([]notes.Note, error) {
- author, found := cache.GetAuthor(did)
+func FetchAppBskyFeedPostsWithCursor(conf *Config, did, pdsURL string, session *auth.OAuthState, cursor string) ([]notes.Note, error) {
+ author, found := conf.Cache.GetAuthor(did)
var err error
if !found {
- author, err = FetchAppBskyActorProfile(logger, did, pdsURL, slingshotURL, session, cache)
+ author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session)
}
if err != nil {
return nil, err
}
- if slingshotURL != "" && general.SlingshotAvailable() {
- records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(logger, did, "app.bsky.feed.post", slingshotURL, session, cursor, limit)
+ if conf.SlingshotURL != "" && general.SlingshotAvailable() {
+ records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session, cursor, conf.PageSize)
if err != nil {
- logger.Error("fetching from slingshot failed, falling back to PDS")
+ conf.Logger.Error("fetching from slingshot failed, falling back to PDS")
general.SetSlingshotBackoff(5 * time.Minute)
} else {
- notes, err := parseAppBskyFeedPostCollection(logger, slingshotURL, session, records, author, cache)
- cache.SetCursor(did, nextCursor)
+ notes, err := parseAppBskyFeedPostCollection(conf, session, records, author)
+ conf.Cache.SetCursor(did, nextCursor)
return notes, err
}
}
- records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(logger, did, "app.bsky.feed.post", pdsURL, session, cursor, limit)
+ records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", pdsURL, session, cursor, conf.PageSize)
if err != nil {
return nil, err
}
- notes, err := parseAppBskyFeedPostCollection(logger, pdsURL, session, records, author, cache)
+ notes, err := parseAppBskyFeedPostCollection(conf, session, records, author)
- cache.SetCursor(did, nextCursor)
+ conf.Cache.SetCursor(did, nextCursor)
return notes, err
}
-func GetAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, session *auth.OAuthState, cache *Cache) (*notes.Note, error) {
+func GetAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) {
// check cache, return if it was cached without auth or if it was cached with auth and we have auth
- logger.Debug("Getting app.bsky.feed.post", "uri", uri)
- if cached, found := cache.Get(uri); found {
- logger.Debug("Found in the cache", "public", !cached.AuthorNoUnauth)
+ conf.Logger.Debug("Getting app.bsky.feed.post", "uri", uri)
+ if cached, found := conf.Cache.Get(uri); found {
+ conf.Logger.Debug("Found in the cache", "public", !cached.AuthorNoUnauth)
if !cached.AuthorNoUnauth {
return cached, nil
}
@@ -89,10 +88,10 @@ func GetAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, sess
}
}
- return FetchAppBskyFeedPostByURI(logger, uri, slingshotURL, session, cache)
+ return FetchAppBskyFeedPostByURI(conf, uri, session)
}
-func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, session *auth.OAuthState, cache *Cache) (*notes.Note, error) {
+func FetchAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) {
repo, _, _, err := general.ParseURI(uri)
// need to resolve DID to PDS
@@ -100,38 +99,38 @@ func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, se
if err != nil {
return nil, err
}
- logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL)
+ conf.Logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL)
// get author of the note
- author, found := cache.GetAuthor(repo)
+ author, found := conf.Cache.GetAuthor(repo)
if !found {
- author, err = FetchAppBskyActorProfile(logger, repo, pdsURL, slingshotURL, session, cache)
+ author, err = FetchAppBskyActorProfile(conf, repo, pdsURL, session)
}
if err != nil {
return nil, err
}
// try slingshot first if given
- if slingshotURL != "" && general.SlingshotAvailable() {
- rawRecord, err := general.FetchRawRecordByURI(logger, uri, slingshotURL, session)
+ if conf.SlingshotURL != "" && general.SlingshotAvailable() {
+ rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, conf.SlingshotURL, session)
if err != nil {
- logger.Error("fetching from slingshot failed", "error", err)
+ conf.Logger.Error("fetching from slingshot failed", "error", err)
general.SetSlingshotBackoff(5 * time.Minute)
} else {
- return parseAppBskyFeedPostRecord(rawRecord, author, cache)
+ return parseAppBskyFeedPostRecord(rawRecord, author, conf)
}
}
- rawRecord, err := general.FetchRawRecordByURI(logger, uri, pdsURL, session)
+ rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, pdsURL, session)
if err != nil {
- logger.Error("Fetching from PDS failed")
+ conf.Logger.Error("Fetching from PDS failed")
return nil, err
}
- return parseAppBskyFeedPostRecord(rawRecord, author, cache)
+ return parseAppBskyFeedPostRecord(rawRecord, author, conf)
}
-func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, author *notes.Author, cache *Cache) (*notes.Note, error) {
+func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, author *notes.Author, conf *Config) (*notes.Note, error) {
if rawRecord == nil {
return nil, fmt.Errorf("unable to parse nil record")
}
@@ -150,57 +149,59 @@ func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, author *notes.Auth
}
// cache it
- cache.Set(note.URI, note)
+ conf.Cache.Set(note.URI, note)
+
+ conf.Logger.Debug("parsed note", "uri", note.URI, "media_count", len(note.Media), "embed", record.Value.Embed)
return note, nil
}
-func parseAppBskyFeedPostCollection(logger *slog.Logger, slingshotURL string, session *auth.OAuthState, records []general.RawRecord, author *notes.Author, cache *Cache) ([]notes.Note, error) {
+func parseAppBskyFeedPostCollection(conf *Config, session *auth.OAuthState, records []general.RawRecord, author *notes.Author) ([]notes.Note, error) {
posts := make([]notes.Note, 0, len(records))
for _, record := range records {
- note, err := parseAppBskyFeedPostRecord(&record, author, cache)
+ note, err := parseAppBskyFeedPostRecord(&record, author, conf)
if err != nil {
- logger.Error("failed to parse post record", "error", err)
+ conf.Logger.Error("failed to parse post record", "error", err)
continue
}
if note.ReplyTo != nil {
- parents, err := FetchParents(logger, *note.ReplyTo, slingshotURL, session, cache, 10)
+ parents, err := FetchParents(conf, *note.ReplyTo, session, 10)
if err != nil {
- logger.Warn("failed to fetch parent chain", "uri", note.URI, "error", err)
+ conf.Logger.Warn("failed to fetch parent chain", "uri", note.URI, "error", err)
}
note.Parents = parents
}
- cache.Set(note.URI, note)
+ conf.Cache.Set(note.URI, note)
posts = append(posts, *note)
}
return notes.DeduplicateThreads(posts), nil
}
-func FetchParents(logger *slog.Logger, replyToURI string, slingshotURL string, session *auth.OAuthState, cache *Cache, maxDepth int) ([]notes.Note, error) {
+func FetchParents(conf *Config, replyToURI string, session *auth.OAuthState, maxDepth int) ([]notes.Note, error) {
parents := []notes.Note{}
currentURI := replyToURI
for currentURI != "" {
- note, err := GetAppBskyFeedPostByURI(logger, currentURI, slingshotURL, session, cache)
+ note, err := GetAppBskyFeedPostByURI(conf, currentURI, session)
if err != nil {
- logger.Warn("failed to fetch parent", "uri", currentURI, "error", err)
+ conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err)
note = notes.MissingNote(currentURI)
- cache.Set(currentURI, note)
+ conf.Cache.Set(currentURI, note)
parents = append(parents, *note)
break
}
// preemptively fetch posts from author to avoid hammering servers if there's a back and forth
- if _, found := cache.GetAuthor(note.AuthorDID); !found {
- logger.Debug("caching posts from thread participant", "did", note.AuthorDID)
+ if _, found := conf.Cache.GetAuthor(note.AuthorDID); !found {
+ conf.Logger.Debug("caching posts from thread participant", "did", note.AuthorDID)
authorPDS, err := FetchPDS(note.AuthorDID)
if err != nil {
- logger.Warn("failed to fetch parent", "uri", currentURI, "error", err)
+ conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err)
break
}
- go FetchAppBskyFeedPosts(logger, note.AuthorDID, authorPDS, slingshotURL, session, cache)
+ go FetchAppBskyFeedPostsWithCursor(conf, note.AuthorDID, authorPDS, session, "")
}
parents = append(parents, *note)
find me elsewhere · generated with cgit