From b120a7cf6deb9d92a72c092c6fffbe74956ce7a8 Mon Sep 17 00:00:00 2001 From: Geert Rademakes Date: Wed, 6 Aug 2025 14:07:24 +0200 Subject: [PATCH] feat: Preserve original file paths and add S3 information alongside - Update Song model to preserve original location field from Rekordbox XML - Add s3File object to Song model with S3 information alongside original location - Update SongMatchingService to link S3 files while preserving original paths - Add location-based matching for better accuracy - Update XML export to include S3 information while preserving original location - Update frontend components to show both original paths and S3 information - Add new section in SongMatching to show songs with music files - Enhance SongList to display original file paths with folder icon This ensures that: - Original file paths from Rekordbox XML are preserved - S3 information is added alongside, not replacing original data - XML export maintains compatibility while adding S3 data - Users can see both original paths and S3 streaming URLs - Matching algorithms consider original file paths for better accuracy --- packages/backend/src/models/Song.ts | 14 ++- packages/backend/src/routes/matching.ts | 59 +++++++-- packages/backend/src/routes/songs.ts | 35 +----- .../src/services/songMatchingService.ts | 92 +++++++++++++- packages/backend/src/services/xmlService.ts | 17 ++- packages/frontend/src/components/SongList.tsx | 22 +++- .../frontend/src/components/SongMatching.tsx | 113 ++++++++++++++++-- packages/frontend/src/types/interfaces.ts | 23 +++- 8 files changed, 314 insertions(+), 61 deletions(-) diff --git a/packages/backend/src/models/Song.ts b/packages/backend/src/models/Song.ts index 19c9f64..aa25f34 100644 --- a/packages/backend/src/models/Song.ts +++ b/packages/backend/src/models/Song.ts @@ -28,12 +28,20 @@ const songSchema = new mongoose.Schema({ comments: String, playCount: String, rating: String, - location: String, + location: String, // Original file path from Rekordbox XML remixer: String, tonality: String, label: String, mix: String, tempo: tempoSchema, + // S3 file integration (preserves original location) + s3File: { + musicFileId: { type: mongoose.Schema.Types.ObjectId, ref: 'MusicFile' }, + s3Key: String, + s3Url: String, + streamingUrl: String, + hasS3File: { type: Boolean, default: false } + } }, { timestamps: true, versionKey: false, @@ -47,4 +55,8 @@ const songSchema = new mongoose.Schema({ } }); +// Create indexes for performance +songSchema.index({ 's3File.hasS3File': 1 }); +songSchema.index({ location: 1 }); + export const Song = mongoose.model('Song', songSchema); \ No newline at end of file diff --git a/packages/backend/src/routes/matching.ts b/packages/backend/src/routes/matching.ts index dff26ef..c2b5563 100644 --- a/packages/backend/src/routes/matching.ts +++ b/packages/backend/src/routes/matching.ts @@ -104,12 +104,11 @@ router.post('/link/:musicFileId/:songId', async (req, res) => { return res.status(404).json({ error: 'Song not found' }); } - musicFile.songId = song._id; - await musicFile.save(); + await matchingService.linkMusicFileToSong(musicFile, song); res.json({ message: 'Music file linked to song successfully', - musicFile: await musicFile.populate('songId') + song: await song.populate('s3File.musicFileId') }); } catch (error) { console.error('Error linking music file to song:', error); @@ -120,19 +119,18 @@ router.post('/link/:musicFileId/:songId', async (req, res) => { /** * Unlink a music file from a song */ -router.delete('/unlink/:musicFileId', async (req, res) => { +router.delete('/unlink/:songId', async (req, res) => { try { - const musicFile = await MusicFile.findById(req.params.musicFileId); - if (!musicFile) { - return res.status(404).json({ error: 'Music file not found' }); + const song = await Song.findById(req.params.songId); + if (!song) { + return res.status(404).json({ error: 'Song not found' }); } - musicFile.songId = undefined; - await musicFile.save(); + await matchingService.unlinkMusicFileFromSong(song); res.json({ message: 'Music file unlinked from song successfully', - musicFile + song }); } catch (error) { console.error('Error unlinking music file from song:', error); @@ -149,12 +147,14 @@ router.get('/stats', async (req, res) => { unmatchedMusicFiles, matchedMusicFiles, songsWithoutMusicFiles, + songsWithMusicFiles, totalSongs, totalMusicFiles ] = await Promise.all([ matchingService.getUnmatchedMusicFiles(), matchingService.getMatchedMusicFiles(), matchingService.getSongsWithoutMusicFiles(), + matchingService.getSongsWithMusicFiles(), Song.countDocuments(), MusicFile.countDocuments() ]); @@ -166,6 +166,7 @@ router.get('/stats', async (req, res) => { matchedMusicFiles: matchedMusicFiles.length, unmatchedMusicFiles: unmatchedMusicFiles.length, songsWithoutMusicFiles: songsWithoutMusicFiles.length, + songsWithMusicFiles: songsWithMusicFiles.length, matchRate: totalMusicFiles > 0 ? (matchedMusicFiles.length / totalMusicFiles * 100).toFixed(1) : '0' } }); @@ -249,13 +250,12 @@ router.get('/songs-without-music-files', async (req, res) => { const limit = parseInt(req.query.limit as string) || 20; const skip = (page - 1) * limit; - const songsWithFiles = await MusicFile.distinct('songId'); const [songs, total] = await Promise.all([ - Song.find({ _id: { $nin: songsWithFiles } }) + Song.find({ 's3File.hasS3File': { $ne: true } }) .sort({ title: 1 }) .skip(skip) .limit(limit), - Song.countDocuments({ _id: { $nin: songsWithFiles } }) + Song.countDocuments({ 's3File.hasS3File': { $ne: true } }) ]); res.json({ @@ -273,4 +273,37 @@ router.get('/songs-without-music-files', async (req, res) => { } }); +/** + * Get songs with music files + */ +router.get('/songs-with-music-files', async (req, res) => { + try { + const page = parseInt(req.query.page as string) || 1; + const limit = parseInt(req.query.limit as string) || 20; + const skip = (page - 1) * limit; + + const [songs, total] = await Promise.all([ + Song.find({ 's3File.hasS3File': true }) + .populate('s3File.musicFileId') + .sort({ title: 1 }) + .skip(skip) + .limit(limit), + Song.countDocuments({ 's3File.hasS3File': true }) + ]); + + res.json({ + songs, + pagination: { + page, + limit, + total, + pages: Math.ceil(total / limit) + } + }); + } catch (error) { + console.error('Error getting songs with music files:', error); + res.status(500).json({ error: 'Failed to get songs with music files' }); + } +}); + export { router as matchingRouter }; \ No newline at end of file diff --git a/packages/backend/src/routes/songs.ts b/packages/backend/src/routes/songs.ts index 8ec34b1..07d35e6 100644 --- a/packages/backend/src/routes/songs.ts +++ b/packages/backend/src/routes/songs.ts @@ -1,7 +1,6 @@ import express, { Request, Response } from 'express'; import { Song } from '../models/Song.js'; import { Playlist } from '../models/Playlist.js'; -import { MusicFile } from '../models/MusicFile.js'; const router = express.Router(); @@ -37,24 +36,13 @@ router.get('/', async (req: Request, res: Response) => { .sort({ title: 1 }) .skip(skip) .limit(limit) + .populate('s3File.musicFileId') .lean(); - // Get music file information for these songs - const songIds = songs.map(song => song._id); - const musicFiles = await MusicFile.find({ songId: { $in: songIds } }).lean(); - const musicFileMap = new Map(musicFiles.map(mf => [mf.songId?.toString() || '', mf])); - - // Add music file information to songs - const songsWithMusicFiles = songs.map(song => ({ - ...song, - hasMusicFile: musicFileMap.has(song._id.toString()), - musicFile: musicFileMap.get(song._id.toString()) || null - })); - - console.log(`Found ${songs.length} songs (${totalSongs} total), ${musicFiles.length} with music files`); + console.log(`Found ${songs.length} songs (${totalSongs} total), ${songs.filter((s: any) => s.s3File?.hasS3File).length} with S3 files`); res.json({ - songs: songsWithMusicFiles, + songs, pagination: { page, limit, @@ -175,24 +163,13 @@ router.get('/playlist/*', async (req: Request, res: Response) => { .sort({ title: 1 }) .skip(skip) .limit(limit) + .populate('s3File.musicFileId') .lean(); - // Get music file information for these songs - const songIds = songs.map(song => song._id); - const musicFiles = await MusicFile.find({ songId: { $in: songIds } }).lean(); - const musicFileMap = new Map(musicFiles.map(mf => [mf.songId?.toString() || '', mf])); - - // Add music file information to songs - const songsWithMusicFiles = songs.map(song => ({ - ...song, - hasMusicFile: musicFileMap.has(song._id.toString()), - musicFile: musicFileMap.get(song._id.toString()) || null - })); - - console.log(`Found ${songs.length} songs for playlist "${playlistName}" (${totalSongs} total), ${musicFiles.length} with music files`); + console.log(`Found ${songs.length} songs for playlist "${playlistName}" (${totalSongs} total), ${songs.filter((s: any) => s.s3File?.hasS3File).length} with S3 files`); res.json({ - songs: songsWithMusicFiles, + songs, pagination: { page, limit, diff --git a/packages/backend/src/services/songMatchingService.ts b/packages/backend/src/services/songMatchingService.ts index 8e1ae59..e403de9 100644 --- a/packages/backend/src/services/songMatchingService.ts +++ b/packages/backend/src/services/songMatchingService.ts @@ -103,8 +103,7 @@ export class SongMatchingService { if (matches.length > 0 && matches[0].confidence >= minConfidence) { // Link the music file to the best match - musicFile.songId = matches[0].song._id; - await musicFile.save(); + await this.linkMusicFileToSong(musicFile, matches[0].song); linked++; } else { unmatched++; @@ -114,6 +113,51 @@ export class SongMatchingService { return { linked, unmatched }; } + /** + * Link a music file to a song (preserves original location) + */ + async linkMusicFileToSong(musicFile: any, song: any): Promise { + // Update the song with S3 file information + song.s3File = { + musicFileId: musicFile._id, + s3Key: musicFile.s3Key, + s3Url: musicFile.s3Url, + streamingUrl: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${musicFile.s3Key}`, + hasS3File: true + }; + + await song.save(); + + // Also update the music file to reference the song + musicFile.songId = song._id; + await musicFile.save(); + } + + /** + * Unlink a music file from a song + */ + async unlinkMusicFileFromSong(song: any): Promise { + // Remove S3 file information from song + song.s3File = { + musicFileId: null, + s3Key: null, + s3Url: null, + streamingUrl: null, + hasS3File: false + }; + + await song.save(); + + // Remove song reference from music file + if (song.s3File?.musicFileId) { + const musicFile = await MusicFile.findById(song.s3File.musicFileId); + if (musicFile) { + musicFile.songId = undefined; + await musicFile.save(); + } + } + } + /** * Calculate match confidence between a music file and a song */ @@ -156,6 +200,14 @@ export class SongMatchingService { } } + // 6. Original location match (if available) + if (song.location) { + const locationScore = this.matchLocation(musicFile.originalName, song.location); + if (locationScore.score > 0) { + scores.push(locationScore); + } + } + // Calculate weighted average score const totalScore = scores.reduce((sum, s) => sum + s.score, 0); const averageScore = scores.length > 0 ? totalScore / scores.length : 0; @@ -221,6 +273,32 @@ export class SongMatchingService { return { score: 0, reason: '' }; } + /** + * Match original location to filename + */ + private matchLocation(filename: string, location: string): { score: number; reason: string } { + if (!filename || !location) return { score: 0, reason: '' }; + + const cleanFilename = this.cleanString(filename); + const cleanLocation = this.cleanString(location); + + // Extract filename from location path + const locationFilename = cleanLocation.split('/').pop() || cleanLocation; + const locationFilenameNoExt = locationFilename.replace(/\.[^/.]+$/, ''); + + // Exact filename match + if (cleanFilename.includes(locationFilenameNoExt) || locationFilenameNoExt.includes(cleanFilename.replace(/\.[^/.]+$/, ''))) { + return { score: 0.9, reason: 'Original location filename match' }; + } + + // Path contains filename + if (cleanLocation.includes(cleanFilename.replace(/\.[^/.]+$/, ''))) { + return { score: 0.7, reason: 'Original location contains filename' }; + } + + return { score: 0, reason: '' }; + } + /** * Match title */ @@ -390,7 +468,13 @@ export class SongMatchingService { * Get songs without music files */ async getSongsWithoutMusicFiles(): Promise { - const songsWithFiles = await MusicFile.distinct('songId'); - return await Song.find({ _id: { $nin: songsWithFiles } }); + return await Song.find({ 's3File.hasS3File': { $ne: true } }); + } + + /** + * Get songs with music files + */ + async getSongsWithMusicFiles(): Promise { + return await Song.find({ 's3File.hasS3File': true }).populate('s3File.musicFileId'); } } \ No newline at end of file diff --git a/packages/backend/src/services/xmlService.ts b/packages/backend/src/services/xmlService.ts index 06608ee..3d772c7 100644 --- a/packages/backend/src/services/xmlService.ts +++ b/packages/backend/src/services/xmlService.ts @@ -53,12 +53,18 @@ export const streamToXml = async (res: any) => { const songs = await Song.find({}) .skip(processedSongs) .limit(batchSize) + .populate('s3File.musicFileId') .lean(); for (const song of songs) { // Include ALL track attributes like master collection res.write(`\n `); + // Add S3 file information if available (preserves original location) + if (song.s3File?.hasS3File) { + res.write(`\n `); + } + // Add TEMPO entries if they exist if (song.tempo) { res.write(`\n `); @@ -217,11 +223,20 @@ export const exportToXml = (songs: any[], playlists: any[]): string => { '@Comments': song.comments, '@PlayCount': song.playCount, '@Rating': song.rating, - '@Location': song.location, + '@Location': song.location, // Preserve original location '@Remixer': song.remixer, '@Tonality': song.tonality, '@Label': song.label, '@Mix': song.mix, + // Add S3 file information if available + ...(song.s3File?.hasS3File ? { + S3_FILE: { + '@S3Key': song.s3File.s3Key, + '@S3Url': song.s3File.s3Url, + '@StreamingUrl': song.s3File.streamingUrl, + '@MusicFileId': song.s3File.musicFileId?._id || song.s3File.musicFileId + } + } : {}), ...(song.tempo ? { TEMPO: { '@Inizio': song.tempo.inizio, diff --git a/packages/frontend/src/components/SongList.tsx b/packages/frontend/src/components/SongList.tsx index 610679f..0b8171f 100644 --- a/packages/frontend/src/components/SongList.tsx +++ b/packages/frontend/src/components/SongList.tsx @@ -102,11 +102,16 @@ export const SongList: React.FC = ({ const handlePlaySong = (song: Song, e: React.MouseEvent) => { e.stopPropagation(); - if (onPlaySong && song.hasMusicFile) { + if (onPlaySong && (song.s3File?.hasS3File || song.hasMusicFile)) { onPlaySong(song); } }; + // Helper function to check if song has music file + const hasMusicFile = (song: Song): boolean => { + return song.s3File?.hasS3File || song.hasMusicFile || false; + }; + // Calculate total duration const totalDuration = useMemo(() => { return filteredSongs.reduce((total, song) => { @@ -119,7 +124,7 @@ export const SongList: React.FC = ({ // Count songs with music files const songsWithMusicFiles = useMemo(() => { - return filteredSongs.filter(song => song.hasMusicFile).length; + return filteredSongs.filter(song => hasMusicFile(song)).length; }, [filteredSongs]); return ( @@ -253,7 +258,7 @@ export const SongList: React.FC = ({ > {song.title} - {song.hasMusicFile && ( + {hasMusicFile(song) && ( @@ -267,10 +272,19 @@ export const SongList: React.FC = ({ > {song.artist} • {formatDuration(song.totalTime)} + {song.location && ( + + 📁 {song.location} + + )} {/* Play Button */} - {song.hasMusicFile && onPlaySong && ( + {hasMusicFile(song) && onPlaySong && ( { const [unmatchedMusicFiles, setUnmatchedMusicFiles] = useState([]); const [matchedMusicFiles, setMatchedMusicFiles] = useState([]); const [songsWithoutMusicFiles, setSongsWithoutMusicFiles] = useState([]); + const [songsWithMusicFiles, setSongsWithMusicFiles] = useState([]); const [isLoading, setIsLoading] = useState(false); const [autoLinking, setAutoLinking] = useState(false); const [selectedMusicFile, setSelectedMusicFile] = useState(null); @@ -72,11 +74,12 @@ export const SongMatching: React.FC = () => { const loadData = async () => { setIsLoading(true); try { - const [statsRes, unmatchedRes, matchedRes, songsWithoutRes] = await Promise.all([ + const [statsRes, unmatchedRes, matchedRes, songsWithoutRes, songsWithRes] = await Promise.all([ fetch('/api/matching/stats'), fetch('/api/matching/unmatched-music-files'), fetch('/api/matching/matched-music-files'), - fetch('/api/matching/songs-without-music-files') + fetch('/api/matching/songs-without-music-files'), + fetch('/api/matching/songs-with-music-files') ]); if (statsRes.ok) { @@ -98,6 +101,11 @@ export const SongMatching: React.FC = () => { const songsData = await songsWithoutRes.json(); setSongsWithoutMusicFiles(songsData.songs); } + + if (songsWithRes.ok) { + const songsData = await songsWithRes.json(); + setSongsWithMusicFiles(songsData.songs); + } } catch (error) { console.error('Error loading data:', error); toast({ @@ -211,9 +219,9 @@ export const SongMatching: React.FC = () => { } }; - const handleUnlinkMusicFile = async (musicFileId: string) => { + const handleUnlinkMusicFile = async (songId: string) => { try { - const response = await fetch(`/api/matching/unlink/${musicFileId}`, { + const response = await fetch(`/api/matching/unlink/${songId}`, { method: 'DELETE', }); @@ -312,6 +320,7 @@ export const SongMatching: React.FC = () => { Automatically match and link music files to songs in your Rekordbox library. This will attempt to find matches based on filename, title, artist, and other metadata. + Original file paths are preserved and S3 information is added alongside. @@ -432,16 +441,21 @@ export const SongMatching: React.FC = () => { → {musicFile.songId.title} by {musicFile.songId.artist} )} + {musicFile.songId?.location && ( + + 📁 {musicFile.songId.location} + + )} } + icon={} size="sm" variant="ghost" colorScheme="red" - onClick={() => handleUnlinkMusicFile(musicFile._id)} + onClick={() => handleUnlinkMusicFile(musicFile.songId._id)} /> @@ -467,6 +481,86 @@ export const SongMatching: React.FC = () => { + {/* Songs with Music Files */} + + + Songs with Music Files ({songsWithMusicFiles.length}) + + + {songsWithMusicFiles.length === 0 ? ( + + No songs have music files linked yet. + + ) : ( + + {songsWithMusicFiles.slice(0, 10).map((song) => ( + + + + + + {song.title} + + + + Has S3 File + + + + {song.artist} + + {song.location && ( + + 📁 {song.location} + + )} + {song.s3File?.streamingUrl && ( + + 🎵 S3: {song.s3File.s3Key} + + )} + + + + } + size="sm" + variant="ghost" + colorScheme="red" + onClick={() => handleUnlinkMusicFile(song._id)} + /> + + + } + size="sm" + variant="ghost" + colorScheme="blue" + /> + + + + + ))} + {songsWithMusicFiles.length > 10 && ( + + Showing first 10 of {songsWithMusicFiles.length} songs with music files + + )} + + )} + + + {/* Suggestions Modal */} @@ -510,6 +604,11 @@ export const SongMatching: React.FC = () => { {match.song.album} )} + {match.song.location && ( + + 📁 {match.song.location} + + )} {match.matchReason} diff --git a/packages/frontend/src/types/interfaces.ts b/packages/frontend/src/types/interfaces.ts index c748198..391976f 100644 --- a/packages/frontend/src/types/interfaces.ts +++ b/packages/frontend/src/types/interfaces.ts @@ -19,7 +19,7 @@ export interface Song { comments?: string; playCount?: string; rating?: string; - location?: string; + location?: string; // Original file path from Rekordbox XML remixer?: string; tonality?: string; label?: string; @@ -30,7 +30,26 @@ export interface Song { metro?: string; battito?: string; }; - // Music file integration + // S3 file integration (preserves original location) + s3File?: { + musicFileId?: string | { + _id: string; + originalName: string; + title?: string; + artist?: string; + album?: string; + duration?: number; + size: number; + format?: string; + s3Key: string; + s3Url: string; + }; + s3Key?: string; + s3Url?: string; + streamingUrl?: string; + hasS3File: boolean; + }; + // Legacy support for backward compatibility hasMusicFile?: boolean; musicFile?: { _id: string;