[WIP] music platform user data scraper
teal-fm atproto
31
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge pull request #57 from charlesharries/repeated-uploaded-tracks

Don't repeatedly scrobble uploaded tracks

authored by matt.evil.gay and committed by

GitHub 6ac5d464 10f44dfc

+178 -4
+1 -1
models/constants.go
··· 1 1 package models 2 2 3 - const SubmissionAgent = "piper/v0.0.3" 3 + const SubmissionAgent = "piper/v0.0.4"
+9 -3
service/applemusic/applemusic.go
··· 439 439 s.logger.Printf("failed to get last tracks for user %d: %v", user.ID, err) 440 440 } 441 441 442 - // Check if this is a new track (by PlayParams.id) 442 + // Pre-compute the hash for uploaded tracks so comparisons against stored 443 + // latest tracks will work 444 + currentURL := currentAppleTrack.Attributes.URL 445 + if currentURL == "" { 446 + currentURL = generateUploadHash(currentAppleTrack) 447 + } 448 + 449 + // Check if this is a new track (by URL / upload hash) 443 450 if len(lastTracks) > 0 { 444 451 lastTrack := lastTracks[0] 445 - // If the URL matches, it's the same track 446 - if lastTrack.URL == currentAppleTrack.Attributes.URL { 452 + if lastTrack.URL == currentURL { 447 453 s.logger.Printf("track unchanged for user %d: %s by %s", user.ID, currentAppleTrack.Attributes.Name, currentAppleTrack.Attributes.ArtistName) 448 454 return nil 449 455 }
+168
service/applemusic/applemusic_test.go
··· 1 1 package applemusic 2 2 3 3 import ( 4 + "context" 5 + "encoding/base64" 6 + "encoding/json" 7 + "io" 8 + "log" 9 + "net/http" 4 10 "strings" 5 11 "testing" 12 + "time" 13 + 14 + "github.com/teal-fm/piper/db" 15 + "github.com/teal-fm/piper/models" 6 16 ) 7 17 18 + // createTestJWT creates a minimal JWT for testing that will pass structural validation 19 + func createTestJWT(teamID string, expiry time.Time) string { 20 + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"ES256","typ":"JWT"}`)) 21 + 22 + claims := map[string]any{ 23 + "iss": teamID, 24 + "iat": time.Now().Unix(), 25 + "exp": expiry.Unix(), 26 + } 27 + claimsJSON, _ := json.Marshal(claims) 28 + payload := base64.RawURLEncoding.EncodeToString(claimsJSON) 29 + 30 + // Signature doesn't need to be valid for structural validation 31 + signature := base64.RawURLEncoding.EncodeToString([]byte("fake-signature")) 32 + 33 + return header + "." + payload + "." + signature 34 + } 35 + 8 36 // Helper to create AppleRecentTrack for testing 9 37 func makeTestTrack(name, album, artist string) *AppleRecentTrack { 10 38 track := &AppleRecentTrack{} ··· 12 40 track.Attributes.AlbumName = album 13 41 track.Attributes.ArtistName = artist 14 42 return track 43 + } 44 + 45 + // trackResponseTransport returns a fixed JSON response simulating the Apple Music API. 46 + type trackResponseTransport struct { 47 + response string 48 + } 49 + 50 + func (t *trackResponseTransport) RoundTrip(req *http.Request) (*http.Response, error) { 51 + return &http.Response{ 52 + StatusCode: http.StatusOK, 53 + Body: io.NopCloser(strings.NewReader(t.response)), 54 + Header: make(http.Header), 55 + }, nil 56 + } 57 + 58 + // newTestDB creates an in-memory SQLite database for testing. 59 + func newTestDB(t *testing.T) *db.DB { 60 + t.Helper() 61 + testDB, err := db.New(":memory:") 62 + if err != nil { 63 + t.Fatalf("failed to create test db: %v", err) 64 + } 65 + if err := testDB.Initialize(); err != nil { 66 + t.Fatalf("failed to initialize test db: %v", err) 67 + } 68 + t.Cleanup(func() { testDB.Close() }) 69 + return testDB 70 + } 71 + 72 + // createTestUser creates a user in the DB with an Apple Music token set. 73 + func createTestUser(t *testing.T, testDB *db.DB) *models.User { 74 + t.Helper() 75 + userID, err := testDB.CreateUser(&models.User{}) 76 + if err != nil { 77 + t.Fatalf("failed to create user: %v", err) 78 + } 79 + if err := testDB.UpdateAppleMusicUserToken(userID, "fake-token"); err != nil { 80 + t.Fatalf("failed to set apple music token: %v", err) 81 + } 82 + user, err := testDB.GetUserByID(userID) 83 + if err != nil { 84 + t.Fatalf("failed to get user: %v", err) 85 + } 86 + return user 87 + } 88 + 89 + // newTestService creates a Service backed by an in-memory DB and the given transport. 90 + func newTestService(t *testing.T, testDB *db.DB, transport http.RoundTripper) *Service { 91 + t.Helper() 92 + tokenExpiry := time.Now().Add(1 * time.Hour) 93 + return &Service{ 94 + DB: testDB, 95 + httpClient: &http.Client{Transport: transport}, 96 + logger: log.New(io.Discard, "", 0), 97 + teamID: "test-team", 98 + keyID: "test-key", 99 + cachedToken: createTestJWT("test-team", tokenExpiry), 100 + cachedExpiry: tokenExpiry, 101 + } 102 + } 103 + 104 + // uploadedTrackJSON builds an Apple Music API response for an uploaded track (no URL). 105 + func uploadedTrackJSON(name, artist, album string) string { 106 + track := map[string]any{ 107 + "id": "1", 108 + "attributes": map[string]string{ 109 + "name": name, 110 + "artistName": artist, 111 + "albumName": album, 112 + }, 113 + } 114 + data, _ := json.Marshal(map[string]any{"data": []any{track}}) 115 + return string(data) 116 + } 117 + 118 + // processUserTestEnv sets up a DB, user, and service wired to the given API response. 119 + type processUserTestEnv struct { 120 + testDB *db.DB 121 + user *models.User 122 + svc *Service 123 + } 124 + 125 + func newProcessUserTestEnv(t *testing.T, apiResponse string) *processUserTestEnv { 126 + t.Helper() 127 + testDB := newTestDB(t) 128 + user := createTestUser(t, testDB) 129 + transport := &trackResponseTransport{response: apiResponse} 130 + svc := newTestService(t, testDB, transport) 131 + return &processUserTestEnv{testDB: testDB, user: user, svc: svc} 132 + } 133 + 134 + // seedUploadedTrack saves an uploaded track to the DB, using its upload hash as the URL. 135 + func (env *processUserTestEnv) seedUploadedTrack(t *testing.T, name, artist, album string) { 136 + t.Helper() 137 + hash := generateUploadHash(makeTestTrack(name, album, artist)) 138 + _, err := env.testDB.SaveTrack(env.user.ID, &models.Track{ 139 + Name: name, 140 + Artist: []models.Artist{{Name: artist}}, 141 + Album: album, 142 + URL: hash, 143 + }) 144 + if err != nil { 145 + t.Fatalf("failed to seed track: %v", err) 146 + } 147 + } 148 + 149 + // trackCount returns the number of tracks stored for the test user. 150 + func (env *processUserTestEnv) trackCount(t *testing.T) int { 151 + t.Helper() 152 + tracks, err := env.testDB.GetRecentTracks(env.user.ID, 100) 153 + if err != nil { 154 + t.Fatalf("failed to get recent tracks: %v", err) 155 + } 156 + return len(tracks) 157 + } 158 + 159 + func TestProcessUserSkipsDuplicateUploadedTrack(t *testing.T) { 160 + env := newProcessUserTestEnv(t, uploadedTrackJSON("My Upload", "Local Artist", "Local Album")) 161 + env.seedUploadedTrack(t, "My Upload", "Local Artist", "Local Album") 162 + 163 + if err := env.svc.ProcessUser(context.Background(), env.user); err != nil { 164 + t.Fatalf("ProcessUser returned error: %v", err) 165 + } 166 + 167 + if got := env.trackCount(t); got != 1 { 168 + t.Errorf("expected 1 track (no duplicate save), got %d", got) 169 + } 170 + } 171 + 172 + func TestProcessUserSavesDifferentUploadedTrack(t *testing.T) { 173 + env := newProcessUserTestEnv(t, uploadedTrackJSON("New Upload", "New Artist", "New Album")) 174 + env.seedUploadedTrack(t, "Old Upload", "Old Artist", "Old Album") 175 + 176 + if err := env.svc.ProcessUser(context.Background(), env.user); err != nil { 177 + t.Fatalf("ProcessUser returned error: %v", err) 178 + } 179 + 180 + if got := env.trackCount(t); got != 2 { 181 + t.Errorf("expected 2 tracks (new upload saved), got %d", got) 182 + } 15 183 } 16 184 17 185 func TestGenerateUploadHash(t *testing.T) {