[WIP] music platform user data scraper
teal-fm atproto

Compare changes

Choose any two refs to compare.

+312 -65
-46
.github/workflows/test.yaml
··· 1 - name: Go Tests 2 - 3 - on: 4 - push: 5 - branches: ["main"] 6 - pull_request: 7 - branches: ["main"] 8 - 9 - jobs: 10 - test: 11 - name: Run Tests 12 - runs-on: ubuntu-latest 13 - 14 - steps: 15 - - name: Checkout code 16 - uses: actions/checkout@v4 17 - 18 - - name: Set up Go 19 - uses: actions/setup-go@v5 20 - with: 21 - # Automatically detects go.mod version 22 - go-version-file: "go.mod" 23 - cache: true 24 - 25 - - name: Install dependencies 26 - run: go mod download 27 - 28 - # Optional: Runs "make" setup if your project requires generated files 29 - # Based on your README, you use lexicons and tailwind. 30 - - name: Run project setup 31 - run: make 32 - 33 - - name: Run Tests 34 - # Runs all tests in the repository recursively 35 - # -v provides verbose output 36 - # -race enables the race detector (recommended for Go) 37 - run: go test -v -race ./... 38 - 39 - - name: Check Formatting 40 - # Ensures all code is properly formatted 41 - run: | 42 - if [ "$(gofmt -l . | wc -l)" -gt 0 ]; then 43 - echo "The following files are not formatted:" 44 - gofmt -l . 45 - exit 1 46 - fi
+1 -1
models/constants.go
··· 1 1 package models 2 2 3 - const SubmissionAgent = "piper/v0.0.3" 3 + const SubmissionAgent = "piper/v0.0.5"
+54 -10
pages/templates/applemusic_link.gohtml
··· 16 16 ></script> 17 17 </head> 18 18 <body> 19 - <main style="max-width: 720px; margin: 4rem auto; padding: 0 1.5rem;"> 20 - <h1 style="font-size: 2.5rem; font-weight: 700; letter-spacing: -0.03em; margin-bottom: 0.75rem; background: linear-gradient(135deg, #fa2d48, #fc5c7d); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text;">Link Apple Music</h1> 21 - <p style="font-size: 1.125rem; color: #8e8e93; max-width: 480px;">Authorize with Apple Music to enable MusicKit features and sync your library.</p> 19 + <main style="max-width: 720px; margin: 4rem auto; padding: 0 1.5rem"> 20 + <h1 21 + style=" 22 + font-size: 2.5rem; 23 + font-weight: 700; 24 + letter-spacing: -0.03em; 25 + margin-bottom: 0.75rem; 26 + background: linear-gradient(135deg, #fa2d48, #fc5c7d); 27 + -webkit-background-clip: text; 28 + -webkit-text-fill-color: transparent; 29 + background-clip: text; 30 + " 31 + > 32 + Link Apple Music 33 + </h1> 34 + <p style="font-size: 1.125rem; color: #8e8e93; max-width: 480px"> 35 + Authorize with Apple Music to enable MusicKit features and sync your 36 + library. 37 + </p> 22 38 <div 23 - style="display: flex; gap: 1.5rem; align-items: center; flex-wrap: wrap; margin-top: 2rem;" 39 + style=" 40 + display: flex; 41 + gap: 1.5rem; 42 + align-items: center; 43 + flex-wrap: wrap; 44 + margin-top: 2rem; 45 + " 24 46 > 25 47 <button id="authorizeBtn" class="am-btn am-btn-primary"> 26 48 ๐ŸŽต Authorize Apple Music ··· 29 51 id="unlinkForm" 30 52 method="post" 31 53 action="/api/v1/applemusic/unlink" 32 - onsubmit="handleUnlink(event)" 54 + onsubmit=" 55 + (async () => { 56 + const music = window.MusicKit.getInstance(); 57 + await music.unauthorize(); 58 + handleUnlink(event); 59 + })() 60 + " 33 61 > 34 62 <button type="submit" class="am-btn am-btn-danger"> 35 - <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="flex-shrink: 0;"> 36 - <path d="M18 6L6 18M6 6l12 12"/> 63 + <svg 64 + xmlns="http://www.w3.org/2000/svg" 65 + width="24" 66 + height="24" 67 + viewBox="0 0 24 24" 68 + fill="none" 69 + stroke="currentColor" 70 + stroke-width="2.5" 71 + stroke-linecap="round" 72 + stroke-linejoin="round" 73 + style="flex-shrink: 0" 74 + > 75 + <path d="M18 6L6 18M6 6l12 12" /> 37 76 </svg> 38 77 Unlink Apple Music 39 78 </button> ··· 56 95 letter-spacing: -0.01em; 57 96 } 58 97 .am-btn-primary { 59 - background: linear-gradient(135deg, #fa2d48 0%, #fc5c7d 50%, #fa2d48 100%); 98 + background: linear-gradient( 99 + 135deg, 100 + #fa2d48 0%, 101 + #fc5c7d 50%, 102 + #fa2d48 100% 103 + ); 60 104 background-size: 200% 200%; 61 105 color: white; 62 106 } ··· 146 190 music.addEventListener("authorizationStatusDidChange", (e) => { 147 191 console.debug( 148 192 "authorizationStatusDidChange", 149 - e && e.authorizationStatus 193 + e && e.authorizationStatus, 150 194 ); 151 195 }); 152 196 music.addEventListener("userTokenDidChange", (e) => { ··· 161 205 } catch (evtErr) { 162 206 console.warn( 163 207 "Failed to attach some MusicKit event listeners", 164 - evtErr 208 + evtErr, 165 209 ); 166 210 } 167 211 document
+9 -3
service/applemusic/applemusic.go
··· 439 439 s.logger.Printf("failed to get last tracks for user %d: %v", user.ID, err) 440 440 } 441 441 442 - // Check if this is a new track (by PlayParams.id) 442 + // Pre-compute the hash for uploaded tracks so comparisons against stored 443 + // latest tracks will work 444 + currentURL := currentAppleTrack.Attributes.URL 445 + if currentURL == "" { 446 + currentURL = generateUploadHash(currentAppleTrack) 447 + } 448 + 449 + // Check if this is a new track (by URL / upload hash) 443 450 if len(lastTracks) > 0 { 444 451 lastTrack := lastTracks[0] 445 - // If the URL matches, it's the same track 446 - if lastTrack.URL == currentAppleTrack.Attributes.URL { 452 + if lastTrack.URL == currentURL { 447 453 s.logger.Printf("track unchanged for user %d: %s by %s", user.ID, currentAppleTrack.Attributes.Name, currentAppleTrack.Attributes.ArtistName) 448 454 return nil 449 455 }
+168
service/applemusic/applemusic_test.go
··· 1 1 package applemusic 2 2 3 3 import ( 4 + "context" 5 + "encoding/base64" 6 + "encoding/json" 7 + "io" 8 + "log" 9 + "net/http" 4 10 "strings" 5 11 "testing" 12 + "time" 13 + 14 + "github.com/teal-fm/piper/db" 15 + "github.com/teal-fm/piper/models" 6 16 ) 7 17 18 + // createTestJWT creates a minimal JWT for testing that will pass structural validation 19 + func createTestJWT(teamID string, expiry time.Time) string { 20 + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"ES256","typ":"JWT"}`)) 21 + 22 + claims := map[string]any{ 23 + "iss": teamID, 24 + "iat": time.Now().Unix(), 25 + "exp": expiry.Unix(), 26 + } 27 + claimsJSON, _ := json.Marshal(claims) 28 + payload := base64.RawURLEncoding.EncodeToString(claimsJSON) 29 + 30 + // Signature doesn't need to be valid for structural validation 31 + signature := base64.RawURLEncoding.EncodeToString([]byte("fake-signature")) 32 + 33 + return header + "." + payload + "." + signature 34 + } 35 + 8 36 // Helper to create AppleRecentTrack for testing 9 37 func makeTestTrack(name, album, artist string) *AppleRecentTrack { 10 38 track := &AppleRecentTrack{} ··· 12 40 track.Attributes.AlbumName = album 13 41 track.Attributes.ArtistName = artist 14 42 return track 43 + } 44 + 45 + // trackResponseTransport returns a fixed JSON response simulating the Apple Music API. 46 + type trackResponseTransport struct { 47 + response string 48 + } 49 + 50 + func (t *trackResponseTransport) RoundTrip(req *http.Request) (*http.Response, error) { 51 + return &http.Response{ 52 + StatusCode: http.StatusOK, 53 + Body: io.NopCloser(strings.NewReader(t.response)), 54 + Header: make(http.Header), 55 + }, nil 56 + } 57 + 58 + // newTestDB creates an in-memory SQLite database for testing. 59 + func newTestDB(t *testing.T) *db.DB { 60 + t.Helper() 61 + testDB, err := db.New(":memory:") 62 + if err != nil { 63 + t.Fatalf("failed to create test db: %v", err) 64 + } 65 + if err := testDB.Initialize(); err != nil { 66 + t.Fatalf("failed to initialize test db: %v", err) 67 + } 68 + t.Cleanup(func() { testDB.Close() }) 69 + return testDB 70 + } 71 + 72 + // createTestUser creates a user in the DB with an Apple Music token set. 73 + func createTestUser(t *testing.T, testDB *db.DB) *models.User { 74 + t.Helper() 75 + userID, err := testDB.CreateUser(&models.User{}) 76 + if err != nil { 77 + t.Fatalf("failed to create user: %v", err) 78 + } 79 + if err := testDB.UpdateAppleMusicUserToken(userID, "fake-token"); err != nil { 80 + t.Fatalf("failed to set apple music token: %v", err) 81 + } 82 + user, err := testDB.GetUserByID(userID) 83 + if err != nil { 84 + t.Fatalf("failed to get user: %v", err) 85 + } 86 + return user 87 + } 88 + 89 + // newTestService creates a Service backed by an in-memory DB and the given transport. 90 + func newTestService(t *testing.T, testDB *db.DB, transport http.RoundTripper) *Service { 91 + t.Helper() 92 + tokenExpiry := time.Now().Add(1 * time.Hour) 93 + return &Service{ 94 + DB: testDB, 95 + httpClient: &http.Client{Transport: transport}, 96 + logger: log.New(io.Discard, "", 0), 97 + teamID: "test-team", 98 + keyID: "test-key", 99 + cachedToken: createTestJWT("test-team", tokenExpiry), 100 + cachedExpiry: tokenExpiry, 101 + } 102 + } 103 + 104 + // uploadedTrackJSON builds an Apple Music API response for an uploaded track (no URL). 105 + func uploadedTrackJSON(name, artist, album string) string { 106 + track := map[string]any{ 107 + "id": "1", 108 + "attributes": map[string]string{ 109 + "name": name, 110 + "artistName": artist, 111 + "albumName": album, 112 + }, 113 + } 114 + data, _ := json.Marshal(map[string]any{"data": []any{track}}) 115 + return string(data) 116 + } 117 + 118 + // processUserTestEnv sets up a DB, user, and service wired to the given API response. 119 + type processUserTestEnv struct { 120 + testDB *db.DB 121 + user *models.User 122 + svc *Service 123 + } 124 + 125 + func newProcessUserTestEnv(t *testing.T, apiResponse string) *processUserTestEnv { 126 + t.Helper() 127 + testDB := newTestDB(t) 128 + user := createTestUser(t, testDB) 129 + transport := &trackResponseTransport{response: apiResponse} 130 + svc := newTestService(t, testDB, transport) 131 + return &processUserTestEnv{testDB: testDB, user: user, svc: svc} 132 + } 133 + 134 + // seedUploadedTrack saves an uploaded track to the DB, using its upload hash as the URL. 135 + func (env *processUserTestEnv) seedUploadedTrack(t *testing.T, name, artist, album string) { 136 + t.Helper() 137 + hash := generateUploadHash(makeTestTrack(name, album, artist)) 138 + _, err := env.testDB.SaveTrack(env.user.ID, &models.Track{ 139 + Name: name, 140 + Artist: []models.Artist{{Name: artist}}, 141 + Album: album, 142 + URL: hash, 143 + }) 144 + if err != nil { 145 + t.Fatalf("failed to seed track: %v", err) 146 + } 147 + } 148 + 149 + // trackCount returns the number of tracks stored for the test user. 150 + func (env *processUserTestEnv) trackCount(t *testing.T) int { 151 + t.Helper() 152 + tracks, err := env.testDB.GetRecentTracks(env.user.ID, 100) 153 + if err != nil { 154 + t.Fatalf("failed to get recent tracks: %v", err) 155 + } 156 + return len(tracks) 157 + } 158 + 159 + func TestProcessUserSkipsDuplicateUploadedTrack(t *testing.T) { 160 + env := newProcessUserTestEnv(t, uploadedTrackJSON("My Upload", "Local Artist", "Local Album")) 161 + env.seedUploadedTrack(t, "My Upload", "Local Artist", "Local Album") 162 + 163 + if err := env.svc.ProcessUser(context.Background(), env.user); err != nil { 164 + t.Fatalf("ProcessUser returned error: %v", err) 165 + } 166 + 167 + if got := env.trackCount(t); got != 1 { 168 + t.Errorf("expected 1 track (no duplicate save), got %d", got) 169 + } 170 + } 171 + 172 + func TestProcessUserSavesDifferentUploadedTrack(t *testing.T) { 173 + env := newProcessUserTestEnv(t, uploadedTrackJSON("New Upload", "New Artist", "New Album")) 174 + env.seedUploadedTrack(t, "Old Upload", "Old Artist", "Old Album") 175 + 176 + if err := env.svc.ProcessUser(context.Background(), env.user); err != nil { 177 + t.Fatalf("ProcessUser returned error: %v", err) 178 + } 179 + 180 + if got := env.trackCount(t); got != 2 { 181 + t.Errorf("expected 2 tracks (new upload saved), got %d", got) 182 + } 15 183 } 16 184 17 185 func TestGenerateUploadHash(t *testing.T) {
+4 -5
service/playingnow/playingnow.go
··· 6 6 "fmt" 7 7 "log" 8 8 "os" 9 - "strconv" 10 9 "sync" 11 10 "time" 12 11 ··· 78 77 79 78 status := &teal.AlphaActorStatus{ 80 79 LexiconTypeID: "fm.teal.alpha.actor.status", 81 - Time: strconv.FormatInt(now.Unix(), 10), 82 - Expiry: func() *string { s := strconv.FormatInt(expiry.Unix(), 10); return &s }(), 80 + Time: now.Format(time.RFC3339), 81 + Expiry: func() *string { s := expiry.Format(time.RFC3339); return &s }(), 83 82 Item: playView, 84 83 } 85 84 ··· 166 165 167 166 status := &teal.AlphaActorStatus{ 168 167 LexiconTypeID: "fm.teal.alpha.actor.status", 169 - Time: strconv.FormatInt(now.Unix(), 10), 170 - Expiry: func() *string { s := strconv.FormatInt(expiredTime.Unix(), 10); return &s }(), 168 + Time: now.Format(time.RFC3339), 169 + Expiry: func() *string { s := expiredTime.Format(time.RFC3339); return &s }(), 171 170 Item: emptyPlayView, 172 171 } 173 172
+13
service/spotify/spotify.go
··· 1 1 package spotify 2 2 3 3 import ( 4 + "crypto/sha256" 4 5 "encoding/base64" 5 6 "encoding/json" 6 7 "errors" ··· 433 434 } 434 435 } 435 436 437 + func generateLocalHash(track *models.Track) string { 438 + input := track.Name + track.Album + getFirstArtist(track) 439 + hash := sha256.Sum256([]byte(input)) 440 + return fmt.Sprintf("sp_local_%x", hash) 441 + } 442 + 436 443 func (s *Service) FetchCurrentTrack(userID int64) (*SpotifyTrackResponse, error) { 437 444 s.mu.RLock() 438 445 token, exists := s.userTokens[userID] ··· 567 574 ISRC: response.Item.ExternalIDs.ISRC, 568 575 HasStamped: false, 569 576 Timestamp: time.Now().UTC(), 577 + } 578 + 579 + // Local files have no URL. We hash the song name, album name, and 580 + // artist name to form a consistent URL. 581 + if track.URL == "" { 582 + track.URL = generateLocalHash(track) 570 583 } 571 584 572 585 return &SpotifyTrackResponse{Track: track, IsPlaying: response.IsPlaying}, nil
+63
service/spotify/spotify_test.go
··· 7 7 "log" 8 8 "net/http" 9 9 "net/http/httptest" 10 + "strings" 10 11 "testing" 11 12 "time" 12 13 ··· 1350 1351 } 1351 1352 }) 1352 1353 } 1354 + 1355 + // ===== generateLocalHash Tests ===== 1356 + 1357 + func TestGenerateLocalHash(t *testing.T) { 1358 + t.Run("deterministic output", func(t *testing.T) { 1359 + track := createTestTrack("My Song", "My Artist", "", 240000, 0) 1360 + hash1 := generateLocalHash(track) 1361 + hash2 := generateLocalHash(track) 1362 + if hash1 != hash2 { 1363 + t.Errorf("Expected deterministic output, got %s and %s", hash1, hash2) 1364 + } 1365 + }) 1366 + 1367 + t.Run("has correct prefix", func(t *testing.T) { 1368 + track := createTestTrack("My Song", "My Artist", "", 240000, 0) 1369 + hash := generateLocalHash(track) 1370 + if !strings.HasPrefix(hash, "sp_local_") { 1371 + t.Errorf("Expected hash to start with 'sp_local_', got '%s'", hash) 1372 + } 1373 + }) 1374 + 1375 + t.Run("different tracks produce different hashes", func(t *testing.T) { 1376 + trackA := createTestTrack("Song A", "Artist A", "", 240000, 0) 1377 + trackB := createTestTrack("Song B", "Artist B", "", 240000, 0) 1378 + hashA := generateLocalHash(trackA) 1379 + hashB := generateLocalHash(trackB) 1380 + if hashA == hashB { 1381 + t.Errorf("Expected different hashes for different tracks, both got %s", hashA) 1382 + } 1383 + }) 1384 + 1385 + t.Run("uses Unknown Artist for empty artist list", func(t *testing.T) { 1386 + trackWithArtist := &models.Track{ 1387 + Name: "Local File", 1388 + Album: "My Album", 1389 + // Hardcoding the other song to Unknown Artist to show that they result in the same hash. 1390 + // This would (probably) never happen, but just for testing sake 1391 + Artist: []models.Artist{{Name: "Unknown Artist", ID: ""}}, 1392 + } 1393 + trackNoArtist := &models.Track{ 1394 + Name: "Local File", 1395 + Album: "My Album", 1396 + Artist: []models.Artist{}, 1397 + } 1398 + hashWith := generateLocalHash(trackWithArtist) 1399 + hashWithout := generateLocalHash(trackNoArtist) 1400 + if hashWith != hashWithout { 1401 + t.Errorf("Expected same hash when artist is 'Unknown Artist' vs empty list, got %s and %s", hashWith, hashWithout) 1402 + } 1403 + }) 1404 + 1405 + t.Run("album name affects hash", func(t *testing.T) { 1406 + trackA := createTestTrack("Same Song", "Same Artist", "", 240000, 0) 1407 + trackB := createTestTrack("Same Song", "Same Artist", "", 240000, 0) 1408 + trackB.Album = "Different Album" 1409 + hashA := generateLocalHash(trackA) 1410 + hashB := generateLocalHash(trackB) 1411 + if hashA == hashB { 1412 + t.Errorf("Expected different hashes for different albums, both got %s", hashA) 1413 + } 1414 + }) 1415 + }