From 9f0a9a6779f00786e4728ad5c3fad36814a293b0 Mon Sep 17 00:00:00 2001 From: Guillermo Serrahima Date: Sat, 11 Apr 2026 07:40:30 +0200 Subject: project restructure --- auth/dpop.go | 83 +++++++ auth/oauth.go | 221 +++++++++++++++++++ backend/auth/dpop.go | 83 ------- backend/auth/oauth.go | 221 ------------------- backend/fetchers/app.bsky.actor.go | 184 ---------------- backend/fetchers/app.bsky.feed.go | 217 ------------------ backend/fetchers/cache.go | 166 -------------- backend/fetchers/config.go | 10 - backend/general/records.go | 148 ------------- backend/general/utils.go | 35 --- backend/go.mod | 15 -- backend/go.sum | 12 - backend/main.go | 428 ----------------------------------- backend/notes/app.bsky.actor.go | 58 ----- backend/notes/app.bsky.feed.go | 150 ------------- backend/notes/author.go | 11 - backend/notes/note.go | 80 ------- fetchers/app.bsky.actor.go | 184 ++++++++++++++++ fetchers/app.bsky.feed.go | 217 ++++++++++++++++++ fetchers/cache.go | 166 ++++++++++++++ fetchers/config.go | 10 + general/records.go | 148 +++++++++++++ general/utils.go | 35 +++ go.mod | 15 ++ go.sum | 12 + main.go | 441 +++++++++++++++++++++++++++++++++++++ notes/app.bsky.actor.go | 58 +++++ notes/app.bsky.feed.go | 150 +++++++++++++ notes/author.go | 11 + notes/note.go | 80 +++++++ 30 files changed, 1831 insertions(+), 1818 deletions(-) create mode 100644 auth/dpop.go create mode 100644 auth/oauth.go delete mode 100644 backend/auth/dpop.go delete mode 100644 backend/auth/oauth.go delete mode 100644 backend/fetchers/app.bsky.actor.go delete mode 100644 backend/fetchers/app.bsky.feed.go delete mode 100644 backend/fetchers/cache.go delete mode 100644 backend/fetchers/config.go delete mode 100644 backend/general/records.go delete mode 100644 backend/general/utils.go delete mode 100644 backend/go.mod delete mode 100644 backend/go.sum delete mode 100644 backend/main.go delete mode 100644 backend/notes/app.bsky.actor.go delete mode 100644 backend/notes/app.bsky.feed.go delete mode 100644 backend/notes/author.go delete mode 100644 backend/notes/note.go create mode 100644 fetchers/app.bsky.actor.go create mode 100644 fetchers/app.bsky.feed.go create mode 100644 fetchers/cache.go create mode 100644 fetchers/config.go create mode 100644 general/records.go create mode 100644 general/utils.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 notes/app.bsky.actor.go create mode 100644 notes/app.bsky.feed.go create mode 100644 notes/author.go create mode 100644 notes/note.go diff --git a/auth/dpop.go b/auth/dpop.go new file mode 100644 index 0000000..2a8f6ed --- /dev/null +++ b/auth/dpop.go @@ -0,0 +1,83 @@ +package auth + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "encoding/base64" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +type DPoPKeyPair struct { + PrivateKey *ecdsa.PrivateKey + PublicKey *ecdsa.PublicKey + JWK map[string]string +} + +func GenerateDPoPKeyPair() (*DPoPKeyPair, error) { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + + jwk := map[string]string{ + "kty": "EC", + "crv": "P-256", + "x": base64.RawURLEncoding.EncodeToString(privateKey.PublicKey.X.Bytes()), + "y": base64.RawURLEncoding.EncodeToString(privateKey.PublicKey.Y.Bytes()), + } + + return &DPoPKeyPair{ + PrivateKey: privateKey, + PublicKey: &privateKey.PublicKey, + JWK: jwk, + }, nil +} + +func (kp *DPoPKeyPair) GenerateDPoPProof(method, url, nonce string) (string, error) { + // generate unique ID for this proof + jti, err := GenerateRandomToken() + if err != nil { + return "", err + } + + // create JWT header with JWK + header := map[string]interface{}{ + "typ": "dpop+jwt", + "alg": "ES256", + "jwk": kp.JWK, + } + + // create JWT claims + claims := jwt.MapClaims{ + "jti": jti, + "htm": method, // HTTP method (GET, POST, etc) + "htu": url, // HTTP URL (the endpoint you're calling) + "iat": time.Now().Unix(), + } + + // add nonce if provided + if nonce != "" { + claims["nonce"] = nonce + } + + // create token with custom header + token := jwt.NewWithClaims(&jwt.SigningMethodECDSA{ + Name: "ES256", + Hash: crypto.SHA256, + KeySize: 32, + CurveBits: 256, + }, claims) + token.Header = header + + // sign with private key + tokenString, err := token.SignedString(kp.PrivateKey) + if err != nil { + return "", err + } + + return tokenString, nil +} diff --git a/auth/oauth.go b/auth/oauth.go new file mode 100644 index 0000000..a1b0419 --- /dev/null +++ b/auth/oauth.go @@ -0,0 +1,221 @@ +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type OAuthConfig struct { + ClientID string + RedirectURI string + Scopes []string +} + +type OAuthState struct { + State string + CodeVerifier string + DID string + PDS string + AccessToken string + RefreshToken string + ExpiresAt time.Time + KeyPair *DPoPKeyPair +} + +type OAuthMetadata struct { + Issuer string `json:"issuer"` + AuthorizationEndpoint string `json:"authorization_endpoint"` + TokenEndpoint string `json:"token_endpoint"` + Scopes []string `json:"scopes_supported"` +} + +type TokenResponse struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` +} + +func (s *OAuthState) Copy() *OAuthState { + return &OAuthState{ + State: s.State, + DID: s.DID, + PDS: s.PDS, + AccessToken: s.AccessToken, + RefreshToken: s.RefreshToken, + ExpiresAt: s.ExpiresAt, + KeyPair: s.KeyPair, + } +} + +func GenerateRandomToken() (string, error) { + b := make([]byte, 32) + _, err := rand.Read(b) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +func GenerateCodeChallenge(verifier string) string { + h := sha256.Sum256([]byte(verifier)) + return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(h[:]) +} + +func FetchOAuthMetadata(pdsURL string) (*OAuthMetadata, error) { + // first get the resource metadata to find the auth server + resourceURL := fmt.Sprintf("%s/.well-known/oauth-protected-resource", pdsURL) + fmt.Printf("Fetching oauth metadata from: %s\n", resourceURL) + resp, err := http.Get(resourceURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var resource struct { + AuthorizationServers []string `json:"authorization_servers"` + } + if err := json.NewDecoder(resp.Body).Decode(&resource); err != nil { + return nil, err + } + + if len(resource.AuthorizationServers) == 0 { + return nil, fmt.Errorf("no authorization servers found") + } + + // now fetch the actual oauth metadata from the auth server + authServer := resource.AuthorizationServers[0] + metadataURL := fmt.Sprintf("%s/.well-known/oauth-authorization-server", authServer) + + resp, err = http.Get(metadataURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var metadata OAuthMetadata + if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + return nil, err + } + + return &metadata, nil +} + +func BuildAuthorizationURL(metadata *OAuthMetadata, config *OAuthConfig, state, codeChallenge, handle string) string { + params := url.Values{ + "response_type": {"code"}, + "client_id": {config.ClientID}, + "redirect_uri": {config.RedirectURI}, + "state": {state}, + "code_challenge": {codeChallenge}, + "code_challenge_method": {"S256"}, + "scope": {"atproto"}, + "login_hint": {handle}, + } + + return metadata.AuthorizationEndpoint + "?" + params.Encode() +} + +func postWithDPoP(url string, data url.Values, keypair *DPoPKeyPair) (*TokenResponse, error) { + dpopProof, err := keypair.GenerateDPoPProof("POST", url, "") + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", url, strings.NewReader(data.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("DPoP", dpopProof) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == 400 { + body, _ := io.ReadAll(resp.Body) + bodyStr := string(body) + + if strings.Contains(bodyStr, "use_dpop_nonce") { + nonce := resp.Header.Get("DPoP-Nonce") + if nonce == "" { + return nil, fmt.Errorf("server requires nonce but didn't provide one") + } + + dpopProof, err = keypair.GenerateDPoPProof("POST", url, nonce) + if err != nil { + return nil, err + } + + retryReq, err := http.NewRequest("POST", url, strings.NewReader(data.Encode())) + if err != nil { + return nil, err + } + retryReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") + retryReq.Header.Set("DPoP", dpopProof) + + resp, err = http.DefaultClient.Do(retryReq) + if err != nil { + return nil, err + } + defer resp.Body.Close() + } else { + return nil, fmt.Errorf("request failed: 400 - %s", bodyStr) + } + } + + if resp.StatusCode != 200 { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed: %d - %s", resp.StatusCode, string(body)) + } + + var tokens TokenResponse + if err := json.NewDecoder(resp.Body).Decode(&tokens); err != nil { + return nil, err + } + + return &tokens, nil +} + +func ExchangeCodeForTokens(pdsURL, code, verifier, callbackURL string, keypair *DPoPKeyPair) (*TokenResponse, error) { + metadata, err := FetchOAuthMetadata(pdsURL) + if err != nil { + return nil, err + } + + data := url.Values{ + "grant_type": {"authorization_code"}, + "code": {code}, + "redirect_uri": {callbackURL}, + "code_verifier": {verifier}, + "client_id": {callbackURL}, + } + + return postWithDPoP(metadata.TokenEndpoint, data, keypair) +} + +func RefreshAccessToken(pdsURL, refreshToken, callbackURL string, keypair *DPoPKeyPair) (*TokenResponse, error) { + metadata, err := FetchOAuthMetadata(pdsURL) + if err != nil { + return nil, err + } + + data := url.Values{ + "grant_type": {"refresh_token"}, + "refresh_token": {refreshToken}, + "client_id": {callbackURL}, + } + + return postWithDPoP(metadata.TokenEndpoint, data, keypair) +} diff --git a/backend/auth/dpop.go b/backend/auth/dpop.go deleted file mode 100644 index 2a8f6ed..0000000 --- a/backend/auth/dpop.go +++ /dev/null @@ -1,83 +0,0 @@ -package auth - -import ( - "crypto" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "encoding/base64" - "time" - - "github.com/golang-jwt/jwt/v5" -) - -type DPoPKeyPair struct { - PrivateKey *ecdsa.PrivateKey - PublicKey *ecdsa.PublicKey - JWK map[string]string -} - -func GenerateDPoPKeyPair() (*DPoPKeyPair, error) { - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return nil, err - } - - jwk := map[string]string{ - "kty": "EC", - "crv": "P-256", - "x": base64.RawURLEncoding.EncodeToString(privateKey.PublicKey.X.Bytes()), - "y": base64.RawURLEncoding.EncodeToString(privateKey.PublicKey.Y.Bytes()), - } - - return &DPoPKeyPair{ - PrivateKey: privateKey, - PublicKey: &privateKey.PublicKey, - JWK: jwk, - }, nil -} - -func (kp *DPoPKeyPair) GenerateDPoPProof(method, url, nonce string) (string, error) { - // generate unique ID for this proof - jti, err := GenerateRandomToken() - if err != nil { - return "", err - } - - // create JWT header with JWK - header := map[string]interface{}{ - "typ": "dpop+jwt", - "alg": "ES256", - "jwk": kp.JWK, - } - - // create JWT claims - claims := jwt.MapClaims{ - "jti": jti, - "htm": method, // HTTP method (GET, POST, etc) - "htu": url, // HTTP URL (the endpoint you're calling) - "iat": time.Now().Unix(), - } - - // add nonce if provided - if nonce != "" { - claims["nonce"] = nonce - } - - // create token with custom header - token := jwt.NewWithClaims(&jwt.SigningMethodECDSA{ - Name: "ES256", - Hash: crypto.SHA256, - KeySize: 32, - CurveBits: 256, - }, claims) - token.Header = header - - // sign with private key - tokenString, err := token.SignedString(kp.PrivateKey) - if err != nil { - return "", err - } - - return tokenString, nil -} diff --git a/backend/auth/oauth.go b/backend/auth/oauth.go deleted file mode 100644 index a1b0419..0000000 --- a/backend/auth/oauth.go +++ /dev/null @@ -1,221 +0,0 @@ -package auth - -import ( - "crypto/rand" - "crypto/sha256" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" -) - -type OAuthConfig struct { - ClientID string - RedirectURI string - Scopes []string -} - -type OAuthState struct { - State string - CodeVerifier string - DID string - PDS string - AccessToken string - RefreshToken string - ExpiresAt time.Time - KeyPair *DPoPKeyPair -} - -type OAuthMetadata struct { - Issuer string `json:"issuer"` - AuthorizationEndpoint string `json:"authorization_endpoint"` - TokenEndpoint string `json:"token_endpoint"` - Scopes []string `json:"scopes_supported"` -} - -type TokenResponse struct { - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - TokenType string `json:"token_type"` - ExpiresIn int `json:"expires_in"` -} - -func (s *OAuthState) Copy() *OAuthState { - return &OAuthState{ - State: s.State, - DID: s.DID, - PDS: s.PDS, - AccessToken: s.AccessToken, - RefreshToken: s.RefreshToken, - ExpiresAt: s.ExpiresAt, - KeyPair: s.KeyPair, - } -} - -func GenerateRandomToken() (string, error) { - b := make([]byte, 32) - _, err := rand.Read(b) - if err != nil { - return "", err - } - return base64.RawURLEncoding.EncodeToString(b), nil -} - -func GenerateCodeChallenge(verifier string) string { - h := sha256.Sum256([]byte(verifier)) - return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(h[:]) -} - -func FetchOAuthMetadata(pdsURL string) (*OAuthMetadata, error) { - // first get the resource metadata to find the auth server - resourceURL := fmt.Sprintf("%s/.well-known/oauth-protected-resource", pdsURL) - fmt.Printf("Fetching oauth metadata from: %s\n", resourceURL) - resp, err := http.Get(resourceURL) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - var resource struct { - AuthorizationServers []string `json:"authorization_servers"` - } - if err := json.NewDecoder(resp.Body).Decode(&resource); err != nil { - return nil, err - } - - if len(resource.AuthorizationServers) == 0 { - return nil, fmt.Errorf("no authorization servers found") - } - - // now fetch the actual oauth metadata from the auth server - authServer := resource.AuthorizationServers[0] - metadataURL := fmt.Sprintf("%s/.well-known/oauth-authorization-server", authServer) - - resp, err = http.Get(metadataURL) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - var metadata OAuthMetadata - if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { - return nil, err - } - - return &metadata, nil -} - -func BuildAuthorizationURL(metadata *OAuthMetadata, config *OAuthConfig, state, codeChallenge, handle string) string { - params := url.Values{ - "response_type": {"code"}, - "client_id": {config.ClientID}, - "redirect_uri": {config.RedirectURI}, - "state": {state}, - "code_challenge": {codeChallenge}, - "code_challenge_method": {"S256"}, - "scope": {"atproto"}, - "login_hint": {handle}, - } - - return metadata.AuthorizationEndpoint + "?" + params.Encode() -} - -func postWithDPoP(url string, data url.Values, keypair *DPoPKeyPair) (*TokenResponse, error) { - dpopProof, err := keypair.GenerateDPoPProof("POST", url, "") - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", url, strings.NewReader(data.Encode())) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("DPoP", dpopProof) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode == 400 { - body, _ := io.ReadAll(resp.Body) - bodyStr := string(body) - - if strings.Contains(bodyStr, "use_dpop_nonce") { - nonce := resp.Header.Get("DPoP-Nonce") - if nonce == "" { - return nil, fmt.Errorf("server requires nonce but didn't provide one") - } - - dpopProof, err = keypair.GenerateDPoPProof("POST", url, nonce) - if err != nil { - return nil, err - } - - retryReq, err := http.NewRequest("POST", url, strings.NewReader(data.Encode())) - if err != nil { - return nil, err - } - retryReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") - retryReq.Header.Set("DPoP", dpopProof) - - resp, err = http.DefaultClient.Do(retryReq) - if err != nil { - return nil, err - } - defer resp.Body.Close() - } else { - return nil, fmt.Errorf("request failed: 400 - %s", bodyStr) - } - } - - if resp.StatusCode != 200 { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("request failed: %d - %s", resp.StatusCode, string(body)) - } - - var tokens TokenResponse - if err := json.NewDecoder(resp.Body).Decode(&tokens); err != nil { - return nil, err - } - - return &tokens, nil -} - -func ExchangeCodeForTokens(pdsURL, code, verifier, callbackURL string, keypair *DPoPKeyPair) (*TokenResponse, error) { - metadata, err := FetchOAuthMetadata(pdsURL) - if err != nil { - return nil, err - } - - data := url.Values{ - "grant_type": {"authorization_code"}, - "code": {code}, - "redirect_uri": {callbackURL}, - "code_verifier": {verifier}, - "client_id": {callbackURL}, - } - - return postWithDPoP(metadata.TokenEndpoint, data, keypair) -} - -func RefreshAccessToken(pdsURL, refreshToken, callbackURL string, keypair *DPoPKeyPair) (*TokenResponse, error) { - metadata, err := FetchOAuthMetadata(pdsURL) - if err != nil { - return nil, err - } - - data := url.Values{ - "grant_type": {"refresh_token"}, - "refresh_token": {refreshToken}, - "client_id": {callbackURL}, - } - - return postWithDPoP(metadata.TokenEndpoint, data, keypair) -} diff --git a/backend/fetchers/app.bsky.actor.go b/backend/fetchers/app.bsky.actor.go deleted file mode 100644 index 41a9c17..0000000 --- a/backend/fetchers/app.bsky.actor.go +++ /dev/null @@ -1,184 +0,0 @@ -package fetchers - -import ( - "encoding/json" - "fmt" - "io" - "llostre/backend/auth" - "llostre/backend/general" - "llostre/backend/notes" - "net" - "net/http" - "strings" - "time" -) - -type DIDDocument struct { - AlsoKnownAs []string `json:"alsoKnownAs"` - Service []struct { - ID string `json:"id"` - Type string `json:"type"` - ServiceEndpoint string `json:"serviceEndpoint"` - } `json:"service"` -} - -func FetchDID(handle string) (string, error) { - // return if it's a DID already - if strings.HasPrefix(handle, "did:") { - return handle, nil - } - - // resolve DNS record first - txtRecords, err := net.LookupTXT("_atproto." + handle) - if err == nil { - for _, txt := range txtRecords { - if after, ok := strings.CutPrefix(txt, "did="); ok { - return after, nil - } - } - } - - // fallback to HTTPS well-known - resp, err := http.Get(fmt.Sprintf("https://%s/.well-known/atproto-did", handle)) - if err != nil { - return "", err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - did := strings.TrimSpace(string(body)) - if !strings.HasPrefix(did, "did:") { - return "", fmt.Errorf("invalid DID returned: %s", did) - } - - return did, nil -} - -func FetchDidDoc(did string) (*DIDDocument, error) { - didDocURL := fmt.Sprintf("https://plc.directory/%s", did) - - resp, err := http.Get(didDocURL) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("DID resolution failed: %d", resp.StatusCode) - } - - var didDoc DIDDocument - if err := json.NewDecoder(resp.Body).Decode(&didDoc); err != nil { - return nil, err - } - - return &didDoc, nil -} - -func FetchHandle(did string) (string, error) { - var didDoc, err = FetchDidDoc(did) - if err != nil { - return "", err - } - - if len(didDoc.AlsoKnownAs) == 0 { - return "", fmt.Errorf("no handle found") - } - - handle := strings.TrimPrefix(didDoc.AlsoKnownAs[0], "at://") - return handle, nil -} - -func FetchPDS(did string) (string, error) { - var didDoc, err = FetchDidDoc(did) - if err != nil { - return "", err - } - - for _, svc := range didDoc.Service { - if svc.Type == "AtprotoPersonalDataServer" { - return svc.ServiceEndpoint, nil - } - } - - return "", fmt.Errorf("no PDS found for DID %s", did) -} - -func FetchHandleAndPDS(did string) (string, string, error) { - var didDoc, err = FetchDidDoc(did) - if err != nil { - return "", "", err - } - - if len(didDoc.AlsoKnownAs) == 0 { - return "", "", fmt.Errorf("no handle found") - } - - var pds string - for _, svc := range didDoc.Service { - if svc.Type == "AtprotoPersonalDataServer" { - pds = svc.ServiceEndpoint - break - } - } - - handle := strings.TrimPrefix(didDoc.AlsoKnownAs[0], "at://") - - return handle, pds, nil -} - -func FetchAppBskyActorProfile(conf *Config, did, pdsURL string, session *auth.OAuthState) (*notes.Author, error) { - // construct profile record URI - uri := fmt.Sprintf("at://%s/app.bsky.actor.profile/self", did) - - // try slingshot first - if conf.SlingshotURL != "" && general.SlingshotAvailable() { - rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, conf.SlingshotURL, session) - if err != nil { - conf.Logger.Warn("slingshot profile fetch failed", "did", did, "error", err) - general.SetSlingshotBackoff(5 * time.Minute) - } else { - return parseAppBskyActorProfile(did, pdsURL, rawRecord, conf.Cache) - } - } - - // fallback to PDS - rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, pdsURL, session) - if err != nil { - return nil, err - } - - return parseAppBskyActorProfile(did, pdsURL, rawRecord, conf.Cache) -} - -func FetchAppBskyActorProfiles(conf *Config, dids []string, pdsURL string, session *auth.OAuthState) map[string]notes.Author { - authors := make(map[string]notes.Author) - for _, did := range dids { - author, err := FetchAppBskyActorProfile(conf, did, pdsURL, session) - if err == nil { - authors[did] = *author - } - } - return authors -} - -func parseAppBskyActorProfile(did, pds string, rawRecord *general.RawRecord, cache *Cache) (*notes.Author, error) { - var profile notes.AppBskyActorProfileRecord - if err := json.Unmarshal(rawRecord.Value, &profile); err != nil { - return nil, err - } - - handle, err := FetchHandle(did) - if err != nil { - handle = "handle.invalid" - } - - author, err := notes.AuthorFromAppBskyActorProfileRecord(profile, did, handle, pds) - - cache.SetAuthor(did, author) - return author, nil -} diff --git a/backend/fetchers/app.bsky.feed.go b/backend/fetchers/app.bsky.feed.go deleted file mode 100644 index 8562a9c..0000000 --- a/backend/fetchers/app.bsky.feed.go +++ /dev/null @@ -1,217 +0,0 @@ -package fetchers - -import ( - "encoding/json" - "fmt" - "llostre/backend/auth" - "llostre/backend/general" - "llostre/backend/notes" - "time" -) - -func FetchAppBskyFeedPosts(conf *Config, did, pdsURL string, session *auth.OAuthState) ([]notes.Note, error) { - // get author of the note - author, found := conf.Cache.GetAuthor(did) - var err error - if !found { - author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session) - } - if err != nil { - return nil, err - } - - // try slingshot first if given - if conf.SlingshotURL != "" && general.SlingshotAvailable() { - records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session) - if err != nil { - conf.Logger.Error("fetching from slingshot failed, falling back to PDS") - general.SetSlingshotBackoff(5 * time.Minute) - } else { - return parseAppBskyFeedPostCollection(conf, session, records, author) - } - } - - records, err := general.FetchRawRecordCollection(conf.Logger, did, "app.bsky.feed.post", pdsURL, session) - if err != nil { - return nil, err - } - - return parseAppBskyFeedPostCollection(conf, session, records, author) -} - -func FetchAppBskyFeedPostsWithCursor(conf *Config, did, pdsURL string, session *auth.OAuthState, cursor string) ([]notes.Note, error) { - author, found := conf.Cache.GetAuthor(did) - var err error - if !found { - author, err = FetchAppBskyActorProfile(conf, did, pdsURL, session) - } - if err != nil { - return nil, err - } - - if conf.SlingshotURL != "" && general.SlingshotAvailable() { - records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", conf.SlingshotURL, session, cursor, conf.PageSize) - if err != nil { - conf.Logger.Error("fetching from slingshot failed, falling back to PDS") - general.SetSlingshotBackoff(5 * time.Minute) - } else { - notes, err := parseAppBskyFeedPostCollection(conf, session, records, author) - conf.Cache.SetCursor(did, nextCursor) - return notes, err - } - } - - records, nextCursor, err := general.FetchRawRecordCollectionWithCursor(conf.Logger, did, "app.bsky.feed.post", pdsURL, session, cursor, conf.PageSize) - if err != nil { - return nil, err - } - - notes, err := parseAppBskyFeedPostCollection(conf, session, records, author) - - conf.Cache.SetCursor(did, nextCursor) - return notes, err -} - -func GetAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) { - // check cache, return if it was cached without auth or if it was cached with auth and we have auth - conf.Logger.Debug("Getting app.bsky.feed.post", "uri", uri) - if cached, found := conf.Cache.Get(uri); found { - conf.Logger.Debug("Found in the cache", "public", !cached.AuthorNoUnauth) - if !cached.AuthorNoUnauth { - return cached, nil - } - - // author requires auth - if we have a session, serve it - // if the token was invalid, trust that it would have been caught by the "refresh token" call in main - if session != nil { - return cached, nil - } - } - - return FetchAppBskyFeedPostByURI(conf, uri, session) -} - -func FetchAppBskyFeedPostByURI(conf *Config, uri string, session *auth.OAuthState) (*notes.Note, error) { - repo, _, _, err := general.ParseURI(uri) - - // need to resolve DID to PDS - pdsURL, err := FetchPDS(repo) - if err != nil { - return nil, err - } - conf.Logger.Debug("fetched origin PDS", "did", repo, "pdsURL", pdsURL) - - // get author of the note - author, found := conf.Cache.GetAuthor(repo) - if !found { - author, err = FetchAppBskyActorProfile(conf, repo, pdsURL, session) - } - if err != nil { - return nil, err - } - - // try slingshot first if given - if conf.SlingshotURL != "" && general.SlingshotAvailable() { - rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, conf.SlingshotURL, session) - if err != nil { - conf.Logger.Error("fetching from slingshot failed", "error", err) - general.SetSlingshotBackoff(5 * time.Minute) - } else { - return parseAppBskyFeedPostRecord(rawRecord, author, conf) - } - } - - rawRecord, err := general.FetchRawRecordByURI(conf.Logger, uri, pdsURL, session) - if err != nil { - conf.Logger.Error("Fetching from PDS failed") - return nil, err - } - - return parseAppBskyFeedPostRecord(rawRecord, author, conf) -} - -func parseAppBskyFeedPostRecord(rawRecord *general.RawRecord, author *notes.Author, conf *Config) (*notes.Note, error) { - if rawRecord == nil { - return nil, fmt.Errorf("unable to parse nil record") - } - - record := notes.AppBskyFeedPostRecord{ - URI: rawRecord.URI, - } - err := json.Unmarshal(rawRecord.Value, &record.Value) - if err != nil { - return nil, err - } - - note, err := notes.NoteFromAppBskyFeedPostRecord(record, author) - if err != nil { - return nil, err - } - - // cache it - conf.Cache.Set(note.URI, note) - - conf.Logger.Debug("parsed note", "uri", note.URI, "media_count", len(note.Media), "embed", record.Value.Embed) - - return note, nil -} - -func parseAppBskyFeedPostCollection(conf *Config, session *auth.OAuthState, records []general.RawRecord, author *notes.Author) ([]notes.Note, error) { - posts := make([]notes.Note, 0, len(records)) - for _, record := range records { - note, err := parseAppBskyFeedPostRecord(&record, author, conf) - if err != nil { - conf.Logger.Error("failed to parse post record", "error", err) - continue - } - if note.ReplyTo != nil { - parents, err := FetchParents(conf, *note.ReplyTo, session, 10) - if err != nil { - conf.Logger.Warn("failed to fetch parent chain", "uri", note.URI, "error", err) - } - note.Parents = parents - } - conf.Cache.Set(note.URI, note) - posts = append(posts, *note) - } - - return notes.DeduplicateThreads(posts), nil -} - -func FetchParents(conf *Config, replyToURI string, session *auth.OAuthState, maxDepth int) ([]notes.Note, error) { - parents := []notes.Note{} - currentURI := replyToURI - - for currentURI != "" { - note, err := GetAppBskyFeedPostByURI(conf, currentURI, session) - if err != nil { - conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err) - note = notes.MissingNote(currentURI) - - conf.Cache.Set(currentURI, note) - parents = append(parents, *note) - break - } - - // preemptively fetch posts from author to avoid hammering servers if there's a back and forth - if _, found := conf.Cache.GetAuthor(note.AuthorDID); !found { - conf.Logger.Debug("caching posts from thread participant", "did", note.AuthorDID) - authorPDS, err := FetchPDS(note.AuthorDID) - if err != nil { - conf.Logger.Warn("failed to fetch parent", "uri", currentURI, "error", err) - break - } - go FetchAppBskyFeedPostsWithCursor(conf, note.AuthorDID, authorPDS, session, "") - } - - parents = append(parents, *note) - - if note.ReplyTo == nil { - break - } - - currentURI = *note.ReplyTo - } - - return parents, nil -} diff --git a/backend/fetchers/cache.go b/backend/fetchers/cache.go deleted file mode 100644 index ac91259..0000000 --- a/backend/fetchers/cache.go +++ /dev/null @@ -1,166 +0,0 @@ -package fetchers - -import ( - "context" - "encoding/json" - "fmt" - "llostre/backend/notes" - "sort" - "time" - - "github.com/redis/go-redis/v9" -) - -type Cache struct { - client *redis.Client - ctx context.Context - expiryTime time.Duration -} - -func NewCache(addr, password string, expiryMinutes int) (*Cache, error) { - client := redis.NewClient(&redis.Options{ - Addr: addr, - Password: password, - DB: 0, - }) - - ctx := context.Background() - if err := client.Ping(ctx).Err(); err != nil { - return nil, fmt.Errorf("redis connection failed: %w", err) - } - - return &Cache{ - client: client, - ctx: ctx, - expiryTime: time.Duration(expiryMinutes) * time.Minute, - }, nil -} - -func (c *Cache) Get(uri string) (*notes.Note, bool) { - key := fmt.Sprintf("post:%s", uri) - data, err := c.client.Get(c.ctx, key).Bytes() - if err != nil { - return nil, false - } - - var note notes.Note - if err := json.Unmarshal(data, ¬e); err != nil { - return nil, false - } - - return ¬e, true -} - -func (c *Cache) Set(uri string, note *notes.Note) { - key := fmt.Sprintf("post:%s", uri) - data, err := json.Marshal(note) - if err != nil { - return - } - - c.client.Set(c.ctx, key, data, c.expiryTime) -} - -func (c *Cache) GetAuthor(did string) (*notes.Author, bool) { - key := fmt.Sprintf("profile:%s", did) - data, err := c.client.Get(c.ctx, key).Bytes() - if err != nil { - return nil, false - } - - var author notes.Author - if err := json.Unmarshal(data, &author); err != nil { - return nil, false - } - - return &author, true -} - -func (c *Cache) SetAuthor(did string, author *notes.Author) { - key := fmt.Sprintf("profile:%s", did) - data, err := json.Marshal(author) - if err != nil { - return - } - - c.client.Set(c.ctx, key, data, c.expiryTime) -} - -func (c *Cache) GetCursor(did string) (string, bool) { - key := fmt.Sprintf("cursor:%s", did) - cursor, err := c.client.Get(c.ctx, key).Result() - if err != nil { - return "", false - } - return cursor, true -} - -func (c *Cache) SetCursor(did, cursor string) { - key := fmt.Sprintf("cursor:%s", did) - if cursor == "" { - c.client.Del(c.ctx, key) - return - } - c.client.Set(c.ctx, key, cursor, 24*time.Hour) -} - -func (c *Cache) GetPostsByDID(did string) ([]notes.Note, error) { - pattern := fmt.Sprintf("post:at://%s/*", did) - var posts []notes.Note - - iter := c.client.Scan(c.ctx, 0, pattern, 0).Iterator() - for iter.Next(c.ctx) { - data, err := c.client.Get(c.ctx, iter.Val()).Bytes() - if err != nil { - continue - } - var note notes.Note - if err := json.Unmarshal(data, ¬e); err != nil { - continue - } - posts = append(posts, note) - } - - if err := iter.Err(); err != nil { - return nil, err - } - - sort.Slice(posts, func(i, j int) bool { - return posts[i].CreatedAt.After(posts[j].CreatedAt) - }) - - return posts, nil -} - -// track oldest sent post per profile -func (c *Cache) GetOldestSent(did string) (time.Time, bool) { - key := fmt.Sprintf("oldest_sent:%s", did) - ts, err := c.client.Get(c.ctx, key).Int64() - if err != nil { - return time.Time{}, false - } - return time.Unix(ts, 0), true -} - -func (c *Cache) SetOldestSent(did string, timestamp time.Time) { - key := fmt.Sprintf("oldest_sent:%s", did) - c.client.Set(c.ctx, key, timestamp.Unix(), 24*time.Hour) -} - -// count posts older than timestamp for a profile -func (c *Cache) CountPostsOlderThan(did string, timestamp time.Time) (int, error) { - posts, err := c.GetPostsByDID(did) - if err != nil { - return 0, err - } - for i, p := range posts { - if p.CreatedAt.Before(timestamp) { - return len(posts) - i, nil - } - } - return 0, nil -} - -func (c *Cache) Close() error { - return c.client.Close() -} diff --git a/backend/fetchers/config.go b/backend/fetchers/config.go deleted file mode 100644 index 090eb09..0000000 --- a/backend/fetchers/config.go +++ /dev/null @@ -1,10 +0,0 @@ -package fetchers - -import "log/slog" - -type Config struct { - SlingshotURL string - PageSize int - Logger *slog.Logger - Cache *Cache -} diff --git a/backend/general/records.go b/backend/general/records.go deleted file mode 100644 index d82b289..0000000 --- a/backend/general/records.go +++ /dev/null @@ -1,148 +0,0 @@ -package general - -import ( - "encoding/json" - "fmt" - "io" - "llostre/backend/auth" - "log/slog" - "net/http" -) - -type RawRecord struct { - URI string `json:"uri"` - Value json.RawMessage `json:"value"` -} - -func FetchRawRecordCollection(logger *slog.Logger, did, collection, baseURL string, session *auth.OAuthState) ([]RawRecord, error) { - url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s", baseURL, did, collection) - - logger.Debug("fetching raw record collection", "did", did, "endpoint", baseURL, "collection", collection) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - - // only add auth if provided - if session != nil { - dpopProof, err := session.KeyPair.GenerateDPoPProof("GET", url, "") - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+session.AccessToken) - req.Header.Set("DPoP", dpopProof) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("server returned %d: %s", resp.StatusCode, string(body)) - } - - var result struct { - Records []RawRecord `json:"records"` - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, err - } - - return result.Records, nil -} - -func FetchRawRecordCollectionWithCursor(logger *slog.Logger, did, collection, baseURL string, session *auth.OAuthState, cursor string, limit int) ([]RawRecord, string, error) { - url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s", baseURL, did, collection) - if cursor != "" { - url += "&cursor=" + cursor - } - if limit > 0 { - url += fmt.Sprintf("&limit=%d", limit) - } - - logger.Debug("fetching raw record collection", "did", did, "endpoint", baseURL, "collection", collection, "cursor", cursor) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, "", err - } - - if session != nil { - dpopProof, err := session.KeyPair.GenerateDPoPProof("GET", url, "") - if err != nil { - return nil, "", err - } - req.Header.Set("Authorization", "Bearer "+session.AccessToken) - req.Header.Set("DPoP", dpopProof) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, "", err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - body, _ := io.ReadAll(resp.Body) - return nil, "", fmt.Errorf("server returned %d: %s", resp.StatusCode, string(body)) - } - - var result struct { - Records []RawRecord `json:"records"` - Cursor string `json:"cursor"` - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, "", err - } - - return result.Records, result.Cursor, nil -} - -func FetchRawRecordByURI(logger *slog.Logger, uri, baseURL string, session *auth.OAuthState) (*RawRecord, error) { - repo, collection, rkey, err := ParseURI(uri) - if err != nil { - return nil, err - } - - url := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s", - baseURL, repo, collection, rkey) - - logger.Debug("fetching raw record by URI", "uri", uri, "endpoint", baseURL) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - - // only add auth if provided - if session != nil { - dpopProof, err := session.KeyPair.GenerateDPoPProof("GET", url, "") - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+session.AccessToken) - req.Header.Set("DPoP", dpopProof) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("endpoint returned %d: %s", resp.StatusCode, string(body)) - } - - var result RawRecord - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, err - } - - return &result, nil -} diff --git a/backend/general/utils.go b/backend/general/utils.go deleted file mode 100644 index ac27443..0000000 --- a/backend/general/utils.go +++ /dev/null @@ -1,35 +0,0 @@ -package general - -import ( - "fmt" - "strings" - "sync" - "time" -) - -var ( - slingshotBackoff time.Time - slingshotBackoffMu sync.Mutex -) - -func ParseURI(uri string) (repo, collection, rkey string, err error) { - // URI format: 'at://DID/collection/recordkey' - parts := strings.Split(strings.TrimPrefix(uri, "at://"), "/") - if len(parts) != 3 { - return "", "", "", fmt.Errorf("invalid URI format: %s", uri) - } - - return parts[0], parts[1], parts[2], nil -} - -func SlingshotAvailable() bool { - slingshotBackoffMu.Lock() - defer slingshotBackoffMu.Unlock() - return time.Now().After(slingshotBackoff) -} - -func SetSlingshotBackoff(d time.Duration) { - slingshotBackoffMu.Lock() - defer slingshotBackoffMu.Unlock() - slingshotBackoff = time.Now().Add(d) -} diff --git a/backend/go.mod b/backend/go.mod deleted file mode 100644 index 7f2db06..0000000 --- a/backend/go.mod +++ /dev/null @@ -1,15 +0,0 @@ -module llostre/backend - -go 1.25.7 - -require ( - github.com/golang-jwt/jwt/v5 v5.3.1 - github.com/joho/godotenv v1.5.1 -) - -require ( - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/redis/go-redis/v9 v9.18.0 // indirect - go.uber.org/atomic v1.11.0 // indirect -) diff --git a/backend/go.sum b/backend/go.sum deleted file mode 100644 index 1d779d1..0000000 --- a/backend/go.sum +++ /dev/null @@ -1,12 +0,0 @@ -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= -github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= -github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= diff --git a/backend/main.go b/backend/main.go deleted file mode 100644 index 6585e56..0000000 --- a/backend/main.go +++ /dev/null @@ -1,428 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "llostre/backend/auth" - "llostre/backend/fetchers" - "llostre/backend/notes" - "log/slog" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "sync" - "time" - - "github.com/joho/godotenv" -) - -var ( - backendURL string - frontendURL string - sessionsMutex sync.RWMutex - sessions = make(map[string]*auth.OAuthState) - config *fetchers.Config -) - -func init() { - godotenv.Load("../.env") - - // setup URLs - appDomain := os.Getenv("APP_DOMAIN") - if appDomain == "" { - appDomain = "http://127.0.0.1:8080" - } - - backendURL = appDomain + "/api" - frontendURL = appDomain - - // setup logger - logLevel := slog.LevelInfo - switch strings.ToLower(os.Getenv("LOG_LEVEL")) { - case "debug": - logLevel = slog.LevelDebug - case "info": - logLevel = slog.LevelInfo - case "warn": - logLevel = slog.LevelWarn - case "error": - logLevel = slog.LevelError - } - - logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ - Level: logLevel, - })) - - slingshotURL := os.Getenv("SLINGSHOT_URL") - ttlMin := os.Getenv("TTL_MIN") - - var expiryMinutes int - if ttlMin != "" { - var err error - expiryMinutes, err = strconv.Atoi(ttlMin) - if err != nil { - panic(err) - } - } else { - expiryMinutes = 15 - } - redisAddr := os.Getenv("REDIS_ADDR") - redisPassword := os.Getenv("REDIS_PASSWORD") - postCache, err := fetchers.NewCache(redisAddr, redisPassword, expiryMinutes) - if err != nil { - config.Logger.Error("failed to connect to redis", "error", err) - panic(err) - } - - config = &fetchers.Config{ - SlingshotURL: slingshotURL, - PageSize: 50, - Logger: logger, - Cache: postCache, - } -} - -func loginHandler(w http.ResponseWriter, r *http.Request) { - handle := r.URL.Query().Get("handle") - if handle == "" { - http.Error(w, "handle parameter required", http.StatusBadRequest) - return - } - - did, err := fetchers.FetchDID(handle) - if err != nil { - http.Error(w, fmt.Sprintf("failed to resolve handle: %v", err), http.StatusInternalServerError) - return - } - - pds, err := fetchers.FetchPDS(did) - if err != nil { - http.Error(w, fmt.Sprintf("failed to fetch PDS: %v", err), http.StatusInternalServerError) - return - } - config.Logger.Debug("Resolved DID and PDS", "did", did, "pds", pds) - - // fetch oauth metadata from PDS - metadata, err := auth.FetchOAuthMetadata(pds) - if err != nil { - http.Error(w, fmt.Sprintf("failed to fetch oauth metadata: %v", err), http.StatusInternalServerError) - return - } - - state, err := auth.GenerateRandomToken() - if err != nil { - http.Error(w, "failed to generate state", http.StatusInternalServerError) - return - } - - verifier, err := auth.GenerateRandomToken() - if err != nil { - http.Error(w, "failed to generate verifier", http.StatusInternalServerError) - return - } - - // generate DPoP keypair - keypair, err := auth.GenerateDPoPKeyPair() - if err != nil { - http.Error(w, "failed to generate keypair", http.StatusInternalServerError) - return - } - - sessionsMutex.Lock() - sessions[state] = &auth.OAuthState{ - State: state, - CodeVerifier: verifier, - DID: did, - PDS: pds, - KeyPair: keypair, - } - sessionsMutex.Unlock() - - config := auth.OAuthConfig{ - ClientID: backendURL + "/callback", - RedirectURI: backendURL + "/callback", - Scopes: []string{"atproto"}, - } - - challenge := auth.GenerateCodeChallenge(verifier) - authURL := auth.BuildAuthorizationURL(metadata, &config, state, challenge, handle) - - // redirect user to PDS - http.Redirect(w, r, authURL, http.StatusFound) -} - -func callbackHandler(w http.ResponseWriter, r *http.Request) { - // serve client metadata if no oauth params present - if r.URL.Query().Get("code") == "" && r.URL.Query().Get("state") == "" && r.URL.Query().Get("error") == "" { - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]interface{}{ - "client_id": backendURL + "/callback", - "client_name": "llostre", - "redirect_uris": []string{backendURL + "/callback"}, - "scope": "atproto", - "grant_types": []string{"authorization_code", "refresh_token"}, - "response_types": []string{"code"}, - "token_endpoint_auth_method": "none", - "application_type": "web", - "dpop_bound_access_tokens": true, - }) - return - } - - code := r.URL.Query().Get("code") - state := r.URL.Query().Get("state") - - if code == "" || state == "" { - http.Error(w, "missing code or state", http.StatusBadRequest) - return - } - - // verify state matches stored session - sessionsMutex.RLock() - oauthState, exists := sessions[state] - sessionsMutex.RUnlock() - - if !exists { - http.Error(w, "invalid state", http.StatusBadRequest) - return - } - - // exchange code for tokens - tokens, err := auth.ExchangeCodeForTokens( - oauthState.PDS, - code, - oauthState.CodeVerifier, - backendURL+"/callback", - oauthState.KeyPair, - ) - if err != nil { - http.Error(w, fmt.Sprintf("token exchange failed: %v", err), http.StatusInternalServerError) - return - } - - // create session with tokens - sessionToken, err := auth.GenerateRandomToken() - if err != nil { - http.Error(w, "failed to generate session token", http.StatusInternalServerError) - return - } - - // store session with tokens from the callback - sessionsMutex.Lock() - sessions[sessionToken] = &auth.OAuthState{ - State: sessionToken, - DID: oauthState.DID, - PDS: oauthState.PDS, - AccessToken: tokens.AccessToken, - RefreshToken: tokens.RefreshToken, - ExpiresAt: time.Now().Add(time.Duration(tokens.ExpiresIn) * time.Second), - KeyPair: oauthState.KeyPair, - } - delete(sessions, state) // remove the partial oauth state from login - sessionsMutex.Unlock() - - config.Logger.Debug("About to set cookie", "token", sessionToken) - - // set session cookie - http.SetCookie(w, &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Path: "/", - HttpOnly: true, - Secure: strings.HasPrefix(backendURL, "https"), - SameSite: http.SameSiteLaxMode, - MaxAge: 86400 * 30, - }) - - config.Logger.Debug("Cookie set, redirecting to frontend", "destination", frontendURL+"/") - - // redirect to main app - http.Redirect(w, r, frontendURL, http.StatusFound) -} - -func sessionHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", frontendURL) - w.Header().Set("Access-Control-Allow-Credentials", "true") - w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type") - - if r.Method == "OPTIONS" { - w.WriteHeader(http.StatusOK) - return - } - - // get session token from cookie - cookie, err := r.Cookie("session_token") - if err != nil { - w.WriteHeader(http.StatusUnauthorized) - return - } - - // check if session exists - sessionsMutex.RLock() - _, exists := sessions[cookie.Value] - sessionsMutex.RUnlock() - - if !exists { - w.WriteHeader(http.StatusUnauthorized) - return - } - w.WriteHeader(http.StatusOK) -} - -func getSession(cookieValue string) (*auth.OAuthState, error) { - // Check for access token refresh - sessionsMutex.RLock() - session, exists := sessions[cookieValue] - if !exists { - return nil, fmt.Errorf("no session found") - } - sessionRead := session.Copy() - sessionsMutex.RUnlock() - - config.Logger.Debug("session found", "did", session.DID, "pds", session.PDS, "access token length", len(session.AccessToken), "session keypair", session.KeyPair) - if session.KeyPair != nil { - config.Logger.Debug("session public key", "key", session.KeyPair.PublicKey) - } - - needsRefresh := time.Now().After(sessionRead.ExpiresAt) - - if needsRefresh { - newTokens, err := auth.RefreshAccessToken(sessionRead.PDS, sessionRead.RefreshToken, backendURL+"/callback", sessionRead.KeyPair) - if err != nil { - sessionsMutex.Lock() - delete(sessions, cookieValue) - sessionsMutex.Unlock() - config.Logger.Error("no authorization servers found for token refresh") - return nil, fmt.Errorf("no authorization servers found") - } - - sessionsMutex.Lock() - session, _ = sessions[cookieValue] - session.AccessToken = newTokens.AccessToken - session.RefreshToken = newTokens.RefreshToken - session.ExpiresAt = time.Now().Add(time.Duration(newTokens.ExpiresIn) * time.Second) - sessionsMutex.Unlock() - } - - return session, nil -} - -func postsHandler(w http.ResponseWriter, r *http.Request) { - config.Logger.Debug("posts handler start") - - cookie, err := r.Cookie("session_token") - if err != nil { - config.Logger.Error("cookie not received", "error", err) - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - - config.Logger.Debug("cookie received", "value", cookie.Value) - - session, err := getSession(cookie.Value) - if err != nil { - http.Error(w, "invalid session", http.StatusUnauthorized) - return - } - - // Fetch app.bsky.feed.post records - var posts []notes.Note - - cached, err := config.Cache.GetPostsByDID(session.DID) - config.Logger.Debug("cache read", "count", len(cached), "err", err) - if err != nil { - config.Logger.Warn("cache read failed, falling back to fetch", "error", err) - } - - if len(cached) > 0 { - posts = notes.DeduplicateThreads(cached) - } else { - posts, err = fetchers.FetchAppBskyFeedPostsWithCursor(config, session.DID, session.PDS, session, "") - if err != nil { - config.Logger.Error("fetching app.bsky.feed.posts failed", "error", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - config.Logger.Debug("posts ready", "count", len(posts)) - - // background refetch if cache is running low - oldestSent, hasOldest := config.Cache.GetOldestSent(session.DID) - if !hasOldest || len(posts) > 0 { - oldest := posts[len(posts)-1].CreatedAt - count, _ := config.Cache.CountPostsOlderThan(session.DID, oldest) - cursor, hasCursor := config.Cache.GetCursor(session.DID) - config.Logger.Debug("refetch check", "count", count, "pageSize", config.PageSize, "hasCursor", hasCursor, "oldest", oldest) - if count < config.PageSize && hasCursor { - sessionCopy := session.Copy() - go fetchers.FetchAppBskyFeedPostsWithCursor(config, sessionCopy.DID, sessionCopy.PDS, sessionCopy, cursor) - } - if !hasOldest || oldest.Before(oldestSent) { - config.Cache.SetOldestSent(session.DID, oldest) - } - } - - // return posts - config.Logger.Debug("successfully fetched posts", "number", len(posts)) - - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(posts) -} - -func postHandler(w http.ResponseWriter, r *http.Request) { - uri := strings.TrimPrefix(r.URL.Path, "/api/post/") - if uri == "" { - http.Error(w, "uri required", http.StatusBadRequest) - return - } - - // decode if URL-encoded - uri, err := url.QueryUnescape(uri) - if err != nil { - http.Error(w, "invalid uri", http.StatusBadRequest) - return - } - - var currentSession *auth.OAuthState - cookie, err := r.Cookie("session_token") - if err != nil { - config.Logger.Warn("unauthorized") - } else { - session, err := getSession(cookie.Value) - - if err != nil { - config.Logger.Warn("invalid session", "error", err) - } else { - currentSession = session - } - } - - note, err := fetchers.GetAppBskyFeedPostByURI(config, uri, currentSession) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(note) -} - -func main() { - defer config.Cache.Close() - - // serve static files - fs := http.FileServer(http.Dir("../frontend/dist")) - http.Handle("/", fs) - - // api routes - http.HandleFunc("/api/posts", postsHandler) - http.HandleFunc("/api/session", sessionHandler) - http.HandleFunc("/api/login", loginHandler) - http.HandleFunc("/api/callback", callbackHandler) - http.HandleFunc("/api/post/", postHandler) // /api/post/{uri} - http.ListenAndServe("0.0.0.0:8080", nil) -} diff --git a/backend/notes/app.bsky.actor.go b/backend/notes/app.bsky.actor.go deleted file mode 100644 index 00a1ff1..0000000 --- a/backend/notes/app.bsky.actor.go +++ /dev/null @@ -1,58 +0,0 @@ -package notes - -import ( - "fmt" -) - -type AppBskyActorProfileRecord struct { - DisplayName string `json:"displayName"` - Avatar *struct { - Ref struct { - Link string `json:"$link"` - } `json:"ref"` - } `json:"avatar,omitempty"` - Banner *struct { - Ref struct { - Link string `json:"$link"` - } `json:"ref"` - } `json:"banner,omitempty"` - Pronouns string `json:"pronouns"` - Labels *struct { - Type string `json:"$type"` - Values []struct { - Val string `json:"val"` - } `json:"values"` - } `json:"labels,omitempty"` -} - -func AuthorFromAppBskyActorProfileRecord(record AppBskyActorProfileRecord, did, handle, pds string) (*Author, error) { - var avatarURL, bannerURL string - if record.Avatar != nil { - avatarURL = fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pds, did, record.Avatar.Ref.Link) - } - if record.Banner != nil { - bannerURL = fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pds, did, record.Banner.Ref.Link) - } - - // check for !no-unauthenticated label - noUnauth := false - if record.Labels != nil && record.Labels.Type == "com.atproto.label.defs#selfLabels" { - for _, label := range record.Labels.Values { - if label.Val == "!no-unauthenticated" { - noUnauth = true - } - } - } - - author := &Author{ - PDS: pds, - Handle: handle, - DisplayName: record.DisplayName, - AvatarURL: avatarURL, - BannerURL: bannerURL, - Pronouns: record.Pronouns, - NoUnauth: noUnauth, - } - - return author, nil -} diff --git a/backend/notes/app.bsky.feed.go b/backend/notes/app.bsky.feed.go deleted file mode 100644 index 3dd7b95..0000000 --- a/backend/notes/app.bsky.feed.go +++ /dev/null @@ -1,150 +0,0 @@ -package notes - -import ( - "fmt" - "llostre/backend/general" - "time" -) - -type EmbedImage struct { - Alt string `json:"alt"` - Image struct { - CID string `json:"$link"` - } `json:"image"` - AspectRatio *struct { - Width int `json:"width"` - Height int `json:"height"` - } `json:"aspectRatio,omitempty"` -} - -type EmbedImages struct { - Type string `json:"$type"` - Images []EmbedImage `json:"images,omitempty"` -} - -type AppBskyFeedPostEmbed struct { - Type string `json:"$type"` - Images []EmbedImage `json:"images,omitempty"` - Media *EmbedImages `json:"media,omitempty"` -} - -type AppBskyFeedPostRecord struct { - URI string `json:"uri"` - Value struct { - Text string `json:"text"` - CreatedAt string `json:"createdAt"` - Reply *struct { - Parent struct { - URI string `json:"uri"` - } `json:"parent"` - } `json:"reply,omitempty"` - Facets []struct { - Index struct { - ByteStart int `json:"byteStart"` - ByteEnd int `json:"byteEnd"` - } `json:"index"` - Features []struct { - Type string `json:"$type"` - URI string `json:"uri,omitempty"` - DID string `json:"did,omitempty"` - Tag string `json:"tag,omitempty"` - } `json:"features"` - } `json:"facets,omitempty"` - Embed *AppBskyFeedPostEmbed `json:"embed,omitempty"` - } `json:"value"` -} - -func extractImages(embed *AppBskyFeedPostEmbed, did, pdsURL string) []Media { - if embed == nil { - return nil - } - images := embed.Images - if embed.Media != nil { - images = embed.Media.Images - } - var media []Media - for _, img := range images { - m := Media{ - URL: fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pdsURL, did, img.Image.CID), - Alt: img.Alt, - } - if img.AspectRatio != nil { - m.Width = img.AspectRatio.Width - m.Height = img.AspectRatio.Height - } - media = append(media, m) - } - return media -} - -func NoteFromAppBskyFeedPostRecord(record AppBskyFeedPostRecord, author *Author) (*Note, error) { - if record.URI == "" { - return nil, fmt.Errorf("missing required field: uri") - } - - createdAt, err := time.Parse(time.RFC3339, record.Value.CreatedAt) - if err != nil { - return nil, fmt.Errorf("failed to parse timestamp: %w", err) - } - - var replyTo *string - if record.Value.Reply != nil { - replyTo = &record.Value.Reply.Parent.URI - } - - var facets []Facet - for _, f := range record.Value.Facets { - for _, feature := range f.Features { - var value string - var facetType string - - switch feature.Type { - case "app.bsky.richtext.facet#link": - facetType = "link" - value = feature.URI - case "app.bsky.richtext.facet#mention": - facetType = "mention" - value = feature.DID - case "app.bsky.richtext.facet#tag": - facetType = "tag" - value = feature.Tag - default: - continue - } - - facets = append(facets, Facet{ - ByteStart: f.Index.ByteStart, - ByteEnd: f.Index.ByteEnd, - Type: facetType, - Value: value, - }) - } - } - - var handle, displayName, avatarURL string - if author != nil { - handle = author.Handle - displayName = author.DisplayName - avatarURL = author.AvatarURL - } else { - handle = "handle.invalid" - displayName = "" - avatarURL = "" - } - - did, _, _, err := general.ParseURI(record.URI) - - return &Note{ - URI: record.URI, - AuthorDID: did, - AuthorHandle: handle, - AuthorName: displayName, - AuthorNoUnauth: author.NoUnauth, - AvatarURL: avatarURL, - CreatedAt: createdAt, - Text: record.Value.Text, - ReplyTo: replyTo, - Facets: facets, - Media: extractImages(record.Value.Embed, did, author.PDS), - }, nil -} diff --git a/backend/notes/author.go b/backend/notes/author.go deleted file mode 100644 index faf4f7b..0000000 --- a/backend/notes/author.go +++ /dev/null @@ -1,11 +0,0 @@ -package notes - -type Author struct { - PDS string - Handle string - DisplayName string - AvatarURL string - BannerURL string - Pronouns string - NoUnauth bool -} diff --git a/backend/notes/note.go b/backend/notes/note.go deleted file mode 100644 index 15f6f67..0000000 --- a/backend/notes/note.go +++ /dev/null @@ -1,80 +0,0 @@ -package notes - -import ( - "time" -) - -type Media struct { - URL string `json:"url"` - Alt string `json:"alt"` - Width int `json:"width,omitempty"` - Height int `json:"height,omitempty"` -} - -type Note struct { - URI string `json:"uri"` - AuthorDID string `json:"authorDID"` - AuthorHandle string `json:"authorHandle"` - AuthorName string `json:"authorName"` - AuthorNoUnauth bool `json:"authorNoUnauth"` // author has !no-unauthenticated label - AvatarURL string `json:"avatarURL"` - Text string `json:"text"` - CreatedAt time.Time `json:"createdAt"` - ReplyTo *string `json:"replyTo,omitempty"` - Facets []Facet `json:"facets,omitempty"` - Parents []Note `json:"parents,omitempty"` - Media []Media `json:"media,omitempty"` -} - -type Facet struct { - ByteStart int `json:"byteStart"` - ByteEnd int `json:"byteEnd"` - Type string `json:"type"` - Value string `json:"value"` // the link destination, whether url, profile or tag -} - -func MissingNote(uri string) *Note { - return &Note{ - URI: uri, - AuthorDID: "MISSING", - Text: "This post could not be fetched. It may have been deleted.", - CreatedAt: time.Time{}, - } -} - -func isChainContained(smallChainPost, bigChainPost Note) bool { - if len(smallChainPost.Parents) > len(bigChainPost.Parents) { - return false - } - - // if the thread is going to differ at any post, it's going to be at the end - // thus if the posts match their parents MUST match too - // (not that I expect to ever compare a post to itself but this way it won't break if I do. And technically a thread contains itself) - if len(smallChainPost.Parents) == len(bigChainPost.Parents) { - return smallChainPost.URI == bigChainPost.URI - } - - // check if the last post of the smaller chain is the (n+1)-th post of the bigger chain - // where n is the number of parents in the smaller reply chain - offset := len(bigChainPost.Parents) - len(smallChainPost.Parents) - return smallChainPost.URI == bigChainPost.Parents[offset-1].URI -} - -func DeduplicateThreads(posts []Note) []Note { - filtered := []Note{} - for _, post := range posts { - isRedundant := false - for _, kept := range filtered { - // check if this post's chain is contained in any kept post - if isChainContained(post, kept) { - isRedundant = true - break - } - } - if !isRedundant { - filtered = append(filtered, post) - } - } - - return filtered -} diff --git a/fetchers/app.bsky.actor.go b/fetchers/app.bsky.actor.go new file mode 100644 index 0000000..ebac250 --- /dev/null +++ b/fetchers/app.bsky.actor.go @@ -0,0 +1,184 @@ +package fetchers + +import ( + "encoding/json" + "fmt" + "io" + "llostre/auth" + "llostre/general" + "llostre/notes" + "net" + "net/http" + "strings" + "time" +) + +type DIDDocument struct { + AlsoKnownAs []string `json:"alsoKnownAs"` + Service []struct { + ID string `json:"id"` + Type string `json:"type"` + ServiceEndpoint string `json:"serviceEndpoint"` + } `json:"service"` +} + +func FetchDID(handle string) (string, error) { + // return if it's a DID already + if strings.HasPrefix(handle, "did:") { + return handle, nil + } + + // resolve DNS record first + txtRecords, err := net.LookupTXT("_atproto." + handle) + if err == nil { + for _, txt := range txtRecords { + if after, ok := strings.CutPrefix(txt, "did="); ok { + return after, nil + } + } + } + + // fallback to HTTPS well-known + resp, err := http.Get(fmt.Sprintf("https://%s/.well-known/atproto-did", handle)) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + did := strings.TrimSpace(string(body)) + if !strings.HasPrefix(did, "did:") { + return "", fmt.Errorf("invalid DID returned: %s", did) + } + + return did, nil +} + +func FetchDidDoc(did string) (*DIDDocument, error) { + didDocURL := fmt.Sprintf("https://plc.directory/%s", did) + + resp, err := http.Get(didDocURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("DID resolution failed: %d", resp.StatusCode) + } + + var didDoc DIDDocument + if err := json.NewDecoder(resp.Body).Decode(&didDoc); err != nil { + return nil, err + } + + return &didDoc, nil +} + +func FetchHandle(did string) (string, error) { + var didDoc, err = FetchDidDoc(did) + if err != nil { + return "", err + } + + if len(didDoc.AlsoKnownAs) == 0 { + return "", fmt.Errorf("no handle found") + } + + handle := strings.TrimPrefix(didDoc.AlsoKnownAs[0], "at://") + return handle, nil +} + +func FetchPDS(did string) (string, error) { + var didDoc, e