A decentralized music tracking and discovery platform built on AT Protocol 🎵
listenbrainz spotify atproto lastfm musicbrainz scrobbling

feat: add genres script and update turbo configuration for genres task

Changed files
+121
apps
api
src
scripts
+1
apps/api/package.json
··· 13 13 "meili:sync": "tsx ./src/scripts/meili.ts", 14 14 "sync:library": "tsx ./src/scripts/sync-library.ts", 15 15 "avatar": "tsx ./src/scripts/avatar.ts", 16 + "genres": "tsx ./src/scripts/genres.ts", 16 17 "pkl:eval": "pkl eval -f json", 17 18 "pkl:gen": "tsx ./scripts/pkl.ts", 18 19 "dev:xrpc": "tsx --watch ./src/server.ts",
+116
apps/api/src/scripts/genres.ts
··· 1 + import { ctx } from "context"; 2 + import { eq, isNull } from "drizzle-orm"; 3 + import { decrypt } from "lib/crypto"; 4 + import { env } from "lib/env"; 5 + import _ from "lodash"; 6 + import tables from "schema"; 7 + 8 + async function getSpotifyToken(): Promise<string> { 9 + const spotifyTokens = await ctx.db 10 + .select() 11 + .from(tables.spotifyTokens) 12 + .leftJoin( 13 + tables.spotifyAccounts, 14 + eq(tables.spotifyAccounts.userId, tables.spotifyTokens.userId) 15 + ) 16 + .where(eq(tables.spotifyAccounts.isBetaUser, true)) 17 + .execute() 18 + .then((res) => res.map(({ spotify_tokens }) => spotify_tokens)); 19 + 20 + const record = 21 + spotifyTokens[Math.floor(Math.random() * spotifyTokens.length)]; 22 + const refreshToken = decrypt(record.refreshToken, env.SPOTIFY_ENCRYPTION_KEY); 23 + 24 + const accessToken = await fetch("https://accounts.spotify.com/api/token", { 25 + method: "POST", 26 + headers: { 27 + "Content-Type": "application/x-www-form-urlencoded", 28 + }, 29 + body: new URLSearchParams({ 30 + grant_type: "refresh_token", 31 + refresh_token: refreshToken, 32 + client_id: env.SPOTIFY_CLIENT_ID, 33 + client_secret: env.SPOTIFY_CLIENT_SECRET, 34 + }), 35 + }) 36 + .then((res) => res.json() as Promise<{ access_token: string }>) 37 + .then((data) => data.access_token); 38 + 39 + return accessToken; 40 + } 41 + 42 + async function getGenresAndPicture(artists) { 43 + for (const artist of artists) { 44 + const token = await getSpotifyToken(); 45 + // search artist by name on spotify 46 + const result = await fetch( 47 + `https://api.spotify.com/v1/search?q=${encodeURIComponent(artist.name)}&type=artist&limit=1`, 48 + { 49 + headers: { 50 + Authorization: `Bearer ${token}`, 51 + }, 52 + } 53 + ) 54 + .then( 55 + (res) => 56 + res.json() as Promise<{ 57 + artists: { 58 + items: Array<{ 59 + id: string; 60 + name: string; 61 + genres: string[]; 62 + images: Array<{ url: string }>; 63 + }>; 64 + }; 65 + }> 66 + ) 67 + .then(async (data) => _.get(data, "artists.items.0")); 68 + 69 + if (result) { 70 + console.log(JSON.stringify(result, null, 2), "\n"); 71 + if (result.genres && result.genres.length > 0) { 72 + await ctx.db 73 + .update(tables.artists) 74 + .set({ genres: result.genres }) 75 + .where(eq(tables.artists.id, artist.id)) 76 + .execute(); 77 + } 78 + // update artist picture if not set 79 + if (!artist.picture && result.images && result.images.length > 0) { 80 + await ctx.db 81 + .update(tables.artists) 82 + .set({ picture: result.images[0].url }) 83 + .where(eq(tables.artists.id, artist.id)) 84 + .execute(); 85 + } 86 + } 87 + 88 + // sleep for a while to avoid rate limiting 89 + await new Promise((resolve) => setTimeout(resolve, 1000)); 90 + } 91 + } 92 + 93 + const PAGE_SIZE = 1000; 94 + 95 + const count = await ctx.db 96 + .select() 97 + .from(tables.artists) 98 + .where(isNull(tables.artists.genres)) 99 + .execute() 100 + .then((res) => res.length); 101 + 102 + for (let offset = 0; offset < count; offset += PAGE_SIZE) { 103 + const artists = await ctx.db 104 + .select() 105 + .from(tables.artists) 106 + .where(isNull(tables.artists.genres)) 107 + .offset(offset) 108 + .limit(PAGE_SIZE) 109 + .execute(); 110 + 111 + await getGenresAndPicture(artists); 112 + } 113 + 114 + console.log(`Artists without genres: ${count}`); 115 + 116 + process.exit(0);
+4
turbo.json
··· 34 34 "persistent": true, 35 35 "cache": false 36 36 }, 37 + "genres": { 38 + "persistent": true, 39 + "cache": false 40 + }, 37 41 "db:migrate": { 38 42 "cache": false 39 43 },