added slingshot fetching + refactored fetching in general

6 files changed, 180 insertions, 125 deletions

diff --git a/backend/fetchers/app.bsky.actor.go b/backend/fetchers/app.bsky.actor.go
index 3c0154a..45ced77 100644
--- a/backend/fetchers/app.bsky.actor.go
+++ b/backend/fetchers/app.bsky.actor.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"llostre/backend/general"
+	"log/slog"
 	"net"
 	"net/http"
 	"strings"
@@ -128,57 +129,67 @@ func FetchHandleAndPDS(did string) (string, string, error) {
 	return handle, pds, nil
 }
 
-func FetchProfile(did, pdsURL string, cache *Cache) (*general.Profile, error) {
-	recordURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=app.bsky.actor.profile&rkey=self", pdsURL, did)
+func FetchProfile(logger *slog.Logger, did, pdsURL, slingshotURL string, cache *Cache) (*general.Profile, error) {
+	// construct profile record URI
+	uri := fmt.Sprintf("at://%s/app.bsky.actor.profile/self", did)
+
+	// try slingshot first
+	if slingshotURL != "" {
+		rawRecord, err := general.FetchRawRecordByURI(logger, uri, slingshotURL)
+		if err != nil {
+			logger.Warn("slingshot profile fetch failed", "did", did, "error", err)
+		} else {
+			return parseProfile(did, rawRecord, cache)
+		}
+	}
 
-	resp, err := http.Get(recordURL)
+	// fallback to PDS
+	rawRecord, err := general.FetchRawRecordByURI(logger, uri, pdsURL)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
 
-	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("profile fetch failed: %d", resp.StatusCode)
-	}
+	return parseProfile(did, rawRecord, cache)
+}
 
-	handle, err := FetchHandle(did)
-	if err != nil {
-		handle = "handle.invalid"
+func FetchProfiles(logger *slog.Logger, dids []string, pdsURL, slingshotURL string, cache *Cache) map[string]general.Profile {
+	profiles := make(map[string]general.Profile)
+	for _, did := range dids {
+		profile, err := FetchProfile(logger, did, pdsURL, slingshotURL, cache)
+		if err == nil {
+			profiles[did] = *profile
+		}
 	}
+	return profiles
+}
 
-	var response general.ProfileRecordResponse
-	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+func parseProfile(did string, rawRecord *general.RawRecord, cache *Cache) (*general.Profile, error) {
+	var profileData general.ProfileRecord
+	if err := json.Unmarshal(rawRecord.Value, &profileData); err != nil {
 		return nil, err
 	}
 
 	var avatarURL, bannerURL string
-	if response.Value.Avatar != nil {
-		avatarURL = fmt.Sprintf("https://cdn.bsky.app/img/avatar/plain/%s/%s@jpeg", did, response.Value.Avatar.Ref.Link)
+	if profileData.Avatar != nil {
+		avatarURL = fmt.Sprintf("https://cdn.bsky.app/img/avatar/plain/%s/%s@jpeg", did, profileData.Avatar.Ref.Link)
+	}
+	if profileData.Banner != nil {
+		bannerURL = fmt.Sprintf("https://cdn.bsky.app/img/banner/plain/%s/%s@jpeg", did, profileData.Banner.Ref.Link)
 	}
-	if response.Value.Banner != nil {
-		bannerURL = fmt.Sprintf("https://cdn.bsky.app/img/banner/plain/%s/%s@jpeg", did, response.Value.Banner.Ref.Link)
+
+	handle, err := FetchHandle(did)
+	if err != nil {
+		handle = "handle.invalid"
 	}
 
 	profile := &general.Profile{
 		Handle:      handle,
-		DisplayName: response.Value.DisplayName,
+		DisplayName: profileData.DisplayName,
 		AvatarURL:   avatarURL,
 		BannerURL:   bannerURL,
-		Pronouns:    response.Value.Pronouns,
+		Pronouns:    profileData.Pronouns,
 	}
 
-	cache.SetProfile(handle, profile)
+	cache.SetProfile(did, profile)
 	return profile, nil
 }
-
-func FetchProfiles(dids []string, pdsURL string, cache *Cache) map[string]general.Profile {
-	profiles := make(map[string]general.Profile)
-	for _, did := range dids {
-		profile, err := FetchProfile(did, pdsURL, cache)
-		if err == nil {
-			cache.SetProfile(did, profile)
-			profiles[did] = *profile
-		}
-	}
-	return profiles
-}
diff --git a/backend/fetchers/app.bsky.feed.go b/backend/fetchers/app.bsky.feed.go
index 92303fc..abc3b46 100644
--- a/backend/fetchers/app.bsky.feed.go
+++ b/backend/fetchers/app.bsky.feed.go
@@ -3,130 +3,91 @@ package fetchers
 import (
 	"encoding/json"
 	"fmt"
-	"io"
-	"llostre/backend/auth"
 	"llostre/backend/general"
 	"llostre/backend/notes"
 	"log/slog"
-	"net/http"
 )
 
-type ListRecordsResponse struct {
-	Records []notes.AppBskyFeedPostRecord `json:"records"`
-}
-
-func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, accessToken string, keypair *auth.DPoPKeyPair, notesCache *Cache) ([]notes.Note, error) {
-	url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=app.bsky.feed.post", pdsURL, did)
-
-	logger.Debug("generating DPoP proof", "url", url)
-	dpopProof, err := keypair.GenerateDPoPProof("GET", url, "")
-	if err != nil {
-		logger.Error("DPoP generation failed", "error", err)
-		return nil, err
-	}
-
-	logger.Debug("making request to PDS")
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
+func FetchAppBskyFeedPosts(logger *slog.Logger, did, pdsURL, slingshotURL string, cache *Cache) ([]notes.Note, error) {
+	// get author of the note
+	profile, found := cache.GetProfile(did)
+	var err error
+	if !found {
+		profile, err = FetchProfile(logger, did, pdsURL, slingshotURL, cache)
 	}
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-	req.Header.Set("DPoP", dpopProof)
-
-	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
 
-	var response ListRecordsResponse
-	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
-		return nil, err
+	// try slingshot first if given
+	if slingshotURL != "" {
+		values, err := general.FetchRawRecordCollection(logger, did, "app.bsky.feed.post", slingshotURL)
+		if err != nil {
+			logger.Error("Fetching from slingshot failed")
+		} else {
+			return parseAppBskyFeedPostCollection(logger, values, profile, cache)
+		}
 	}
 
-	profile, err := FetchProfile(did, pdsURL, notesCache)
+	values, err := general.FetchRawRecordCollection(logger, did, "app.bsky.feed.post", pdsURL)
 	if err != nil {
-		logger.Error("profile fetch failed", "did", did, "error", err)
-	} else {
-		logger.Debug("fetched profile")
-	}
-
-	posts := make([]notes.Note, 0, len(response.Records))
-	for _, record := range response.Records {
-		bp, err := notes.NoteFromAppBskyFeedPostRecord(record, profile)
-		if err != nil {
-			logger.Error("failed to convert record", "error", err, "uri", record.URI)
-			continue
-		}
-		notesCache.Set(bp.URI, bp)
-		posts = append(posts, *bp)
+		return nil, err
 	}
 
-	return posts, nil
+	return parseAppBskyFeedPostCollection(logger, values, profile, cache)
 }
 
-func FetchPostByURI(uri, accessToken string, keypair *auth.DPoPKeyPair, cache *Cache) (*notes.Note, error) {
+func FetchAppBskyFeedPostByURI(logger *slog.Logger, uri, slingshotURL string, cache *Cache) (*notes.Note, error) {
 	// check cache
 	if cached, found := cache.Get(uri); found {
 		return cached, nil
 	}
 
-	repo, collection, rkey, err := general.ParseURI(uri)
+	repo, _, _, err := general.ParseURI(uri)
 
 	// need to resolve DID to PDS
-	pds, err := FetchPDS(repo)
+	pdsURL, err := FetchPDS(repo)
 	if err != nil {
 		return nil, err
 	}
 
-	url := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
-		pds, repo, collection, rkey)
-
-	// fetch with DPoP
-	dpopProof, err := keypair.GenerateDPoPProof("GET", url, "")
+	// get author of the note
+	profile, found := cache.GetProfile(repo)
+	if !found {
+		profile, err = FetchProfile(logger, repo, pdsURL, slingshotURL, cache)
+	}
 	if err != nil {
 		return nil, err
 	}
 
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
+	// try slingshot first if given
+	if slingshotURL != "" {
+		rawRecord, err := general.FetchRawRecordByURI(logger, uri, slingshotURL)
+		if err != nil {
+			logger.Error("fetching from slingshot failed", "error", err)
+		} else {
+			return parseAppBskyFeedPostRecord(rawRecord, profile, cache)
+		}
 	}
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-	req.Header.Set("DPoP", dpopProof)
 
-	resp, err := http.DefaultClient.Do(req)
+	rawRecord, err := general.FetchRawRecordByURI(logger, uri, pdsURL)
 	if err != nil {
+		logger.Error("Fetching from PDS failed")
 		return nil, err
 	}
-	defer resp.Body.Close()
 
-	if resp.StatusCode != 200 {
-		body, _ := io.ReadAll(resp.Body)
-		return nil, fmt.Errorf("PDS returned status %d: %s", resp.StatusCode, string(body))
-	}
+	return parseAppBskyFeedPostRecord(rawRecord, profile, cache)
+}
 
-	var result struct {
-		URI   string          `json:"uri"`
-		Value json.RawMessage `json:"value"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		return nil, err
+func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, profile *general.Profile, cache *Cache) (*notes.Note, error) {
+	if rawRecord == nil {
+		return nil, fmt.Errorf("unable to parse nil record")
 	}
 
-	// convert to AppBskyFeedPostRecord
 	record := notes.AppBskyFeedPostRecord{
-		URI: result.URI,
-	}
-	if err := json.Unmarshal(result.Value, &record.Value); err != nil {
-		return nil, err
-	}
-
-	// get author of the note
-	profile, found := cache.GetProfile(repo)
-	if !found {
-		profile, err = FetchProfile(repo, pds, cache)
+		URI: rawRecord.URI,
 	}
+	err := json.Unmarshal(rawRecord.Value, &record.Value)
 	if err != nil {
 		return nil, err
 	}
@@ -141,3 +102,18 @@ func FetchPostByURI(uri, accessToken string, keypair *auth.DPoPKeyPair, cache *C
 
 	return note, nil
 }
+
+func parseAppBskyFeedPostCollection(logger *slog.Logger, records []general.RawRecord, profile *general.Profile, cache *Cache) ([]notes.Note, error) {
+	posts := make([]notes.Note, 0, len(records))
+	for _, record := range records {
+		note, err := parseAppBskyFeedPostRecord(&record, profile, cache)
+		if err != nil {
+			logger.Error("failed to parse post record", "error", err)
+			continue
+		}
+		cache.Set(note.URI, note)
+		posts = append(posts, *note)
+	}
+
+	return posts, nil
+}
diff --git a/backend/general/profiles.go b/backend/general/profiles.go
index be74364..b545e60 100644
--- a/backend/general/profiles.go
+++ b/backend/general/profiles.go
@@ -22,8 +22,3 @@ type ProfileRecord struct {
 	} `json:"banner,omitempty"`
 	Pronouns string `json:"pronouns"`
 }
-
-type ProfileRecordResponse struct {
-	URI   string        `json:"uri"`
-	Value ProfileRecord `json:"value"`
-}
diff --git a/backend/general/records.go b/backend/general/records.go
new file mode 100644
index 0000000..d623862
--- /dev/null
+++ b/backend/general/records.go
@@ -0,0 +1,70 @@
+package general
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+)
+
+type RawRecord struct {
+	URI   string          `json:"uri"`
+	Value json.RawMessage `json:"value"`
+}
+
+func FetchRawRecordCollection(logger *slog.Logger, did, collection, baseURL string) ([]RawRecord, error) {
+	url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s", baseURL, did, collection)
+
+	logger.Debug("fetching raw record collection", "did", did, "endpoint", baseURL, "collection", collection)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("server returned %d: %s", resp.StatusCode, string(body))
+	}
+
+	var result struct {
+		Records []RawRecord `json:"records"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, err
+	}
+
+	return result.Records, nil
+}
+
+func FetchRawRecordByURI(logger *slog.Logger, uri, baseURL string) (*RawRecord, error) {
+	repo, collection, rkey, err := ParseURI(uri)
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
+		baseURL, repo, collection, rkey)
+
+	logger.Debug("fetching raw record by URI", "uri", uri, "endpoint", baseURL)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("endpoint returned %d: %s", resp.StatusCode, string(body))
+	}
+
+	var result RawRecord
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, err
+	}
+
+	return &result, nil
+}
diff --git a/backend/main.go b/backend/main.go
index f6851da..3c10887 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -16,12 +16,13 @@ import (
 )
 
 var (
-	logger		  *slog.Logger
+	logger        *slog.Logger
 	backendURL    string
 	frontendURL   string
+	slingshotURL  string
 	sessionsMutex sync.RWMutex
 	sessions      = make(map[string]*auth.OAuthState)
-	notesCache 	  *fetchers.Cache
+	cache         *fetchers.Cache
 )
 
 func init() {
@@ -53,7 +54,9 @@ func init() {
 		Level: logLevel,
 	}))
 
-	notesCache = fetchers.NewCache()
+	slingshotURL = os.Getenv("SLINGSHOT_URL")
+
+	cache = fetchers.NewCache()
 }
 
 func loginHandler(w http.ResponseWriter, r *http.Request) {
@@ -166,7 +169,7 @@ func callbackHandler(w http.ResponseWriter, r *http.Request) {
 		oauthState.PDS,
 		code,
 		oauthState.CodeVerifier,
-		backendURL + "/callback",
+		backendURL+"/callback",
 		oauthState.KeyPair,
 	)
 	if err != nil {
@@ -277,7 +280,7 @@ func postsHandler(w http.ResponseWriter, r *http.Request) {
 	sessionsMutex.RUnlock()
 
 	if needsRefresh {
-		newTokens, err := auth.RefreshAccessToken(pds, refreshToken, backendURL + "/callback", keypair)
+		newTokens, err := auth.RefreshAccessToken(pds, refreshToken, backendURL+"/callback", keypair)
 		if err != nil {
 			sessionsMutex.Lock()
 			delete(sessions, cookie.Value)
@@ -294,7 +297,7 @@ func postsHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Fetch app.bsky.feed.post records
-	posts, err := fetchers.FetchAppBskyFeedPosts(logger, session.DID, session.PDS, session.AccessToken, session.KeyPair, notesCache)
+	posts, err := fetchers.FetchAppBskyFeedPosts(logger, session.DID, session.PDS, slingshotURL, cache)
 	if err != nil {
 		logger.Error("fetching app.bsky.feed.posts failed", "error", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
diff --git a/backend/notes/app.bsky.feed.post.go b/backend/notes/app.bsky.feed.post.go
index 6b74257..451633e 100644
--- a/backend/notes/app.bsky.feed.post.go
+++ b/backend/notes/app.bsky.feed.post.go
@@ -46,8 +46,6 @@ func NoteFromAppBskyFeedPostRecord(record AppBskyFeedPostRecord, profile *genera
 		replyTo = &record.Value.Reply.Parent.URI
 	}
 
-	did, _, _, err := general.ParseURI(record.URI)
-
 	var facets []Facet
 	for _, f := range record.Value.Facets {
 		for _, feature := range f.Features {
@@ -88,6 +86,8 @@ func NoteFromAppBskyFeedPostRecord(record AppBskyFeedPostRecord, profile *genera
 		avatarURL = ""
 	}
 
+	did, _, _, err := general.ParseURI(record.URI)
+
 	return &Note{
 		URI:          record.URI,
 		AuthorDID:    did,