feat: Preserve original file paths and add S3 information alongside

- Update Song model to preserve original location field from Rekordbox XML
- Add s3File object to Song model with S3 information alongside original location
- Update SongMatchingService to link S3 files while preserving original paths
- Add location-based matching for better accuracy
- Update XML export to include S3 information while preserving original location
- Update frontend components to show both original paths and S3 information
- Add new section in SongMatching to show songs with music files
- Enhance SongList to display original file paths with folder icon

This ensures that:
- Original file paths from Rekordbox XML are preserved
- S3 information is added alongside, not replacing original data
- XML export maintains compatibility while adding S3 data
- Users can see both original paths and S3 streaming URLs
- Matching algorithms consider original file paths for better accuracy
This commit is contained in:
Geert Rademakes 2025-08-06 14:07:24 +02:00
parent 8684f2e59d
commit b120a7cf6d
8 changed files with 314 additions and 61 deletions

View File

@ -28,12 +28,20 @@ const songSchema = new mongoose.Schema({
comments: String,
playCount: String,
rating: String,
location: String,
location: String, // Original file path from Rekordbox XML
remixer: String,
tonality: String,
label: String,
mix: String,
tempo: tempoSchema,
// S3 file integration (preserves original location)
s3File: {
musicFileId: { type: mongoose.Schema.Types.ObjectId, ref: 'MusicFile' },
s3Key: String,
s3Url: String,
streamingUrl: String,
hasS3File: { type: Boolean, default: false }
}
}, {
timestamps: true,
versionKey: false,
@ -47,4 +55,8 @@ const songSchema = new mongoose.Schema({
}
});
// Create indexes for performance
songSchema.index({ 's3File.hasS3File': 1 });
songSchema.index({ location: 1 });
export const Song = mongoose.model('Song', songSchema);

View File

@ -104,12 +104,11 @@ router.post('/link/:musicFileId/:songId', async (req, res) => {
return res.status(404).json({ error: 'Song not found' });
}
musicFile.songId = song._id;
await musicFile.save();
await matchingService.linkMusicFileToSong(musicFile, song);
res.json({
message: 'Music file linked to song successfully',
musicFile: await musicFile.populate('songId')
song: await song.populate('s3File.musicFileId')
});
} catch (error) {
console.error('Error linking music file to song:', error);
@ -120,19 +119,18 @@ router.post('/link/:musicFileId/:songId', async (req, res) => {
/**
* Unlink a music file from a song
*/
router.delete('/unlink/:musicFileId', async (req, res) => {
router.delete('/unlink/:songId', async (req, res) => {
try {
const musicFile = await MusicFile.findById(req.params.musicFileId);
if (!musicFile) {
return res.status(404).json({ error: 'Music file not found' });
const song = await Song.findById(req.params.songId);
if (!song) {
return res.status(404).json({ error: 'Song not found' });
}
musicFile.songId = undefined;
await musicFile.save();
await matchingService.unlinkMusicFileFromSong(song);
res.json({
message: 'Music file unlinked from song successfully',
musicFile
song
});
} catch (error) {
console.error('Error unlinking music file from song:', error);
@ -149,12 +147,14 @@ router.get('/stats', async (req, res) => {
unmatchedMusicFiles,
matchedMusicFiles,
songsWithoutMusicFiles,
songsWithMusicFiles,
totalSongs,
totalMusicFiles
] = await Promise.all([
matchingService.getUnmatchedMusicFiles(),
matchingService.getMatchedMusicFiles(),
matchingService.getSongsWithoutMusicFiles(),
matchingService.getSongsWithMusicFiles(),
Song.countDocuments(),
MusicFile.countDocuments()
]);
@ -166,6 +166,7 @@ router.get('/stats', async (req, res) => {
matchedMusicFiles: matchedMusicFiles.length,
unmatchedMusicFiles: unmatchedMusicFiles.length,
songsWithoutMusicFiles: songsWithoutMusicFiles.length,
songsWithMusicFiles: songsWithMusicFiles.length,
matchRate: totalMusicFiles > 0 ? (matchedMusicFiles.length / totalMusicFiles * 100).toFixed(1) : '0'
}
});
@ -249,13 +250,12 @@ router.get('/songs-without-music-files', async (req, res) => {
const limit = parseInt(req.query.limit as string) || 20;
const skip = (page - 1) * limit;
const songsWithFiles = await MusicFile.distinct('songId');
const [songs, total] = await Promise.all([
Song.find({ _id: { $nin: songsWithFiles } })
Song.find({ 's3File.hasS3File': { $ne: true } })
.sort({ title: 1 })
.skip(skip)
.limit(limit),
Song.countDocuments({ _id: { $nin: songsWithFiles } })
Song.countDocuments({ 's3File.hasS3File': { $ne: true } })
]);
res.json({
@ -273,4 +273,37 @@ router.get('/songs-without-music-files', async (req, res) => {
}
});
/**
* Get songs with music files
*/
router.get('/songs-with-music-files', async (req, res) => {
try {
const page = parseInt(req.query.page as string) || 1;
const limit = parseInt(req.query.limit as string) || 20;
const skip = (page - 1) * limit;
const [songs, total] = await Promise.all([
Song.find({ 's3File.hasS3File': true })
.populate('s3File.musicFileId')
.sort({ title: 1 })
.skip(skip)
.limit(limit),
Song.countDocuments({ 's3File.hasS3File': true })
]);
res.json({
songs,
pagination: {
page,
limit,
total,
pages: Math.ceil(total / limit)
}
});
} catch (error) {
console.error('Error getting songs with music files:', error);
res.status(500).json({ error: 'Failed to get songs with music files' });
}
});
export { router as matchingRouter };

View File

@ -1,7 +1,6 @@
import express, { Request, Response } from 'express';
import { Song } from '../models/Song.js';
import { Playlist } from '../models/Playlist.js';
import { MusicFile } from '../models/MusicFile.js';
const router = express.Router();
@ -37,24 +36,13 @@ router.get('/', async (req: Request, res: Response) => {
.sort({ title: 1 })
.skip(skip)
.limit(limit)
.populate('s3File.musicFileId')
.lean();
// Get music file information for these songs
const songIds = songs.map(song => song._id);
const musicFiles = await MusicFile.find({ songId: { $in: songIds } }).lean();
const musicFileMap = new Map(musicFiles.map(mf => [mf.songId?.toString() || '', mf]));
// Add music file information to songs
const songsWithMusicFiles = songs.map(song => ({
...song,
hasMusicFile: musicFileMap.has(song._id.toString()),
musicFile: musicFileMap.get(song._id.toString()) || null
}));
console.log(`Found ${songs.length} songs (${totalSongs} total), ${musicFiles.length} with music files`);
console.log(`Found ${songs.length} songs (${totalSongs} total), ${songs.filter((s: any) => s.s3File?.hasS3File).length} with S3 files`);
res.json({
songs: songsWithMusicFiles,
songs,
pagination: {
page,
limit,
@ -175,24 +163,13 @@ router.get('/playlist/*', async (req: Request, res: Response) => {
.sort({ title: 1 })
.skip(skip)
.limit(limit)
.populate('s3File.musicFileId')
.lean();
// Get music file information for these songs
const songIds = songs.map(song => song._id);
const musicFiles = await MusicFile.find({ songId: { $in: songIds } }).lean();
const musicFileMap = new Map(musicFiles.map(mf => [mf.songId?.toString() || '', mf]));
// Add music file information to songs
const songsWithMusicFiles = songs.map(song => ({
...song,
hasMusicFile: musicFileMap.has(song._id.toString()),
musicFile: musicFileMap.get(song._id.toString()) || null
}));
console.log(`Found ${songs.length} songs for playlist "${playlistName}" (${totalSongs} total), ${musicFiles.length} with music files`);
console.log(`Found ${songs.length} songs for playlist "${playlistName}" (${totalSongs} total), ${songs.filter((s: any) => s.s3File?.hasS3File).length} with S3 files`);
res.json({
songs: songsWithMusicFiles,
songs,
pagination: {
page,
limit,

View File

@ -103,8 +103,7 @@ export class SongMatchingService {
if (matches.length > 0 && matches[0].confidence >= minConfidence) {
// Link the music file to the best match
musicFile.songId = matches[0].song._id;
await musicFile.save();
await this.linkMusicFileToSong(musicFile, matches[0].song);
linked++;
} else {
unmatched++;
@ -114,6 +113,51 @@ export class SongMatchingService {
return { linked, unmatched };
}
/**
* Link a music file to a song (preserves original location)
*/
async linkMusicFileToSong(musicFile: any, song: any): Promise<void> {
// Update the song with S3 file information
song.s3File = {
musicFileId: musicFile._id,
s3Key: musicFile.s3Key,
s3Url: musicFile.s3Url,
streamingUrl: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${musicFile.s3Key}`,
hasS3File: true
};
await song.save();
// Also update the music file to reference the song
musicFile.songId = song._id;
await musicFile.save();
}
/**
* Unlink a music file from a song
*/
async unlinkMusicFileFromSong(song: any): Promise<void> {
// Remove S3 file information from song
song.s3File = {
musicFileId: null,
s3Key: null,
s3Url: null,
streamingUrl: null,
hasS3File: false
};
await song.save();
// Remove song reference from music file
if (song.s3File?.musicFileId) {
const musicFile = await MusicFile.findById(song.s3File.musicFileId);
if (musicFile) {
musicFile.songId = undefined;
await musicFile.save();
}
}
}
/**
* Calculate match confidence between a music file and a song
*/
@ -156,6 +200,14 @@ export class SongMatchingService {
}
}
// 6. Original location match (if available)
if (song.location) {
const locationScore = this.matchLocation(musicFile.originalName, song.location);
if (locationScore.score > 0) {
scores.push(locationScore);
}
}
// Calculate weighted average score
const totalScore = scores.reduce((sum, s) => sum + s.score, 0);
const averageScore = scores.length > 0 ? totalScore / scores.length : 0;
@ -221,6 +273,32 @@ export class SongMatchingService {
return { score: 0, reason: '' };
}
/**
* Match original location to filename
*/
private matchLocation(filename: string, location: string): { score: number; reason: string } {
if (!filename || !location) return { score: 0, reason: '' };
const cleanFilename = this.cleanString(filename);
const cleanLocation = this.cleanString(location);
// Extract filename from location path
const locationFilename = cleanLocation.split('/').pop() || cleanLocation;
const locationFilenameNoExt = locationFilename.replace(/\.[^/.]+$/, '');
// Exact filename match
if (cleanFilename.includes(locationFilenameNoExt) || locationFilenameNoExt.includes(cleanFilename.replace(/\.[^/.]+$/, ''))) {
return { score: 0.9, reason: 'Original location filename match' };
}
// Path contains filename
if (cleanLocation.includes(cleanFilename.replace(/\.[^/.]+$/, ''))) {
return { score: 0.7, reason: 'Original location contains filename' };
}
return { score: 0, reason: '' };
}
/**
* Match title
*/
@ -390,7 +468,13 @@ export class SongMatchingService {
* Get songs without music files
*/
async getSongsWithoutMusicFiles(): Promise<any[]> {
const songsWithFiles = await MusicFile.distinct('songId');
return await Song.find({ _id: { $nin: songsWithFiles } });
return await Song.find({ 's3File.hasS3File': { $ne: true } });
}
/**
* Get songs with music files
*/
async getSongsWithMusicFiles(): Promise<any[]> {
return await Song.find({ 's3File.hasS3File': true }).populate('s3File.musicFileId');
}
}

View File

@ -53,12 +53,18 @@ export const streamToXml = async (res: any) => {
const songs = await Song.find({})
.skip(processedSongs)
.limit(batchSize)
.populate('s3File.musicFileId')
.lean();
for (const song of songs) {
// Include ALL track attributes like master collection
res.write(`\n <TRACK TrackID="${song.id}" Name="${escapeXml(song.title || '')}" Artist="${escapeXml(song.artist || '')}" Composer="${escapeXml(song.composer || '')}" Album="${escapeXml(song.album || '')}" Grouping="${escapeXml(song.grouping || '')}" Genre="${escapeXml(song.genre || '')}" Kind="${escapeXml(song.kind || '')}" Size="${song.size || ''}" TotalTime="${song.totalTime || ''}" DiscNumber="${song.discNumber || ''}" TrackNumber="${song.trackNumber || ''}" Year="${song.year || ''}" AverageBpm="${song.averageBpm || ''}" DateAdded="${song.dateAdded || ''}" BitRate="${song.bitRate || ''}" SampleRate="${song.sampleRate || ''}" Comments="${escapeXml(song.comments || '')}" PlayCount="${song.playCount || ''}" Rating="${song.rating || ''}" Location="${escapeXml(song.location || '')}" Remixer="${escapeXml(song.remixer || '')}" Tonality="${escapeXml(song.tonality || '')}" Label="${escapeXml(song.label || '')}" Mix="${escapeXml(song.mix || '')}">`);
// Add S3 file information if available (preserves original location)
if (song.s3File?.hasS3File) {
res.write(`\n <S3_FILE S3Key="${escapeXml(song.s3File.s3Key || '')}" S3Url="${escapeXml(song.s3File.s3Url || '')}" StreamingUrl="${escapeXml(song.s3File.streamingUrl || '')}" MusicFileId="${song.s3File.musicFileId?._id || ''}"/>`);
}
// Add TEMPO entries if they exist
if (song.tempo) {
res.write(`\n <TEMPO Inizio="${song.tempo.inizio}" Bpm="${song.tempo.bpm}" Metro="${song.tempo.metro}" Battito="${song.tempo.battito}"/>`);
@ -217,11 +223,20 @@ export const exportToXml = (songs: any[], playlists: any[]): string => {
'@Comments': song.comments,
'@PlayCount': song.playCount,
'@Rating': song.rating,
'@Location': song.location,
'@Location': song.location, // Preserve original location
'@Remixer': song.remixer,
'@Tonality': song.tonality,
'@Label': song.label,
'@Mix': song.mix,
// Add S3 file information if available
...(song.s3File?.hasS3File ? {
S3_FILE: {
'@S3Key': song.s3File.s3Key,
'@S3Url': song.s3File.s3Url,
'@StreamingUrl': song.s3File.streamingUrl,
'@MusicFileId': song.s3File.musicFileId?._id || song.s3File.musicFileId
}
} : {}),
...(song.tempo ? {
TEMPO: {
'@Inizio': song.tempo.inizio,

View File

@ -102,11 +102,16 @@ export const SongList: React.FC<SongListProps> = ({
const handlePlaySong = (song: Song, e: React.MouseEvent) => {
e.stopPropagation();
if (onPlaySong && song.hasMusicFile) {
if (onPlaySong && (song.s3File?.hasS3File || song.hasMusicFile)) {
onPlaySong(song);
}
};
// Helper function to check if song has music file
const hasMusicFile = (song: Song): boolean => {
return song.s3File?.hasS3File || song.hasMusicFile || false;
};
// Calculate total duration
const totalDuration = useMemo(() => {
return filteredSongs.reduce((total, song) => {
@ -119,7 +124,7 @@ export const SongList: React.FC<SongListProps> = ({
// Count songs with music files
const songsWithMusicFiles = useMemo(() => {
return filteredSongs.filter(song => song.hasMusicFile).length;
return filteredSongs.filter(song => hasMusicFile(song)).length;
}, [filteredSongs]);
return (
@ -253,7 +258,7 @@ export const SongList: React.FC<SongListProps> = ({
>
{song.title}
</Text>
{song.hasMusicFile && (
{hasMusicFile(song) && (
<Tooltip label="Has music file available for playback">
<Badge colorScheme="green" size="sm" variant="subtle">
<FiMusic size={10} />
@ -267,10 +272,19 @@ export const SongList: React.FC<SongListProps> = ({
>
{song.artist} {formatDuration(song.totalTime)}
</Text>
{song.location && (
<Text
fontSize="xs"
color="gray.600"
noOfLines={1}
>
📁 {song.location}
</Text>
)}
</Box>
{/* Play Button */}
{song.hasMusicFile && onPlaySong && (
{hasMusicFile(song) && onPlaySong && (
<Tooltip label="Play music file">
<IconButton
aria-label="Play song"

View File

@ -32,7 +32,7 @@ import {
IconButton,
Tooltip,
} from '@chakra-ui/react';
import { FiPlay, FiLink, FiUnlink, FiSearch, FiZap, FiMusic, FiCheck } from 'react-icons/fi';
import { FiPlay, FiLink, FiSearch, FiZap, FiMusic, FiCheck, FiX } from 'react-icons/fi';
interface MatchResult {
song: any;
@ -48,6 +48,7 @@ interface MatchingStats {
matchedMusicFiles: number;
unmatchedMusicFiles: number;
songsWithoutMusicFiles: number;
songsWithMusicFiles: number;
matchRate: string;
}
@ -56,6 +57,7 @@ export const SongMatching: React.FC = () => {
const [unmatchedMusicFiles, setUnmatchedMusicFiles] = useState<any[]>([]);
const [matchedMusicFiles, setMatchedMusicFiles] = useState<any[]>([]);
const [songsWithoutMusicFiles, setSongsWithoutMusicFiles] = useState<any[]>([]);
const [songsWithMusicFiles, setSongsWithMusicFiles] = useState<any[]>([]);
const [isLoading, setIsLoading] = useState(false);
const [autoLinking, setAutoLinking] = useState(false);
const [selectedMusicFile, setSelectedMusicFile] = useState<any>(null);
@ -72,11 +74,12 @@ export const SongMatching: React.FC = () => {
const loadData = async () => {
setIsLoading(true);
try {
const [statsRes, unmatchedRes, matchedRes, songsWithoutRes] = await Promise.all([
const [statsRes, unmatchedRes, matchedRes, songsWithoutRes, songsWithRes] = await Promise.all([
fetch('/api/matching/stats'),
fetch('/api/matching/unmatched-music-files'),
fetch('/api/matching/matched-music-files'),
fetch('/api/matching/songs-without-music-files')
fetch('/api/matching/songs-without-music-files'),
fetch('/api/matching/songs-with-music-files')
]);
if (statsRes.ok) {
@ -98,6 +101,11 @@ export const SongMatching: React.FC = () => {
const songsData = await songsWithoutRes.json();
setSongsWithoutMusicFiles(songsData.songs);
}
if (songsWithRes.ok) {
const songsData = await songsWithRes.json();
setSongsWithMusicFiles(songsData.songs);
}
} catch (error) {
console.error('Error loading data:', error);
toast({
@ -211,9 +219,9 @@ export const SongMatching: React.FC = () => {
}
};
const handleUnlinkMusicFile = async (musicFileId: string) => {
const handleUnlinkMusicFile = async (songId: string) => {
try {
const response = await fetch(`/api/matching/unlink/${musicFileId}`, {
const response = await fetch(`/api/matching/unlink/${songId}`, {
method: 'DELETE',
});
@ -312,6 +320,7 @@ export const SongMatching: React.FC = () => {
<Text color="gray.600">
Automatically match and link music files to songs in your Rekordbox library.
This will attempt to find matches based on filename, title, artist, and other metadata.
Original file paths are preserved and S3 information is added alongside.
</Text>
</CardBody>
</Card>
@ -432,16 +441,21 @@ export const SongMatching: React.FC = () => {
{musicFile.songId.title} by {musicFile.songId.artist}
</Text>
)}
{musicFile.songId?.location && (
<Text fontSize="xs" color="gray.500">
📁 {musicFile.songId.location}
</Text>
)}
</VStack>
<HStack spacing={2}>
<Tooltip label="Unlink from song">
<IconButton
aria-label="Unlink"
icon={<FiUnlink />}
icon={<FiX />}
size="sm"
variant="ghost"
colorScheme="red"
onClick={() => handleUnlinkMusicFile(musicFile._id)}
onClick={() => handleUnlinkMusicFile(musicFile.songId._id)}
/>
</Tooltip>
<Tooltip label="Play music file">
@ -467,6 +481,86 @@ export const SongMatching: React.FC = () => {
</CardBody>
</Card>
{/* Songs with Music Files */}
<Card>
<CardHeader>
<Heading size="md">Songs with Music Files ({songsWithMusicFiles.length})</Heading>
</CardHeader>
<CardBody>
{songsWithMusicFiles.length === 0 ? (
<Text color="gray.500" textAlign="center">
No songs have music files linked yet.
</Text>
) : (
<VStack spacing={3} align="stretch">
{songsWithMusicFiles.slice(0, 10).map((song) => (
<Box
key={song._id}
p={3}
border="1px"
borderColor="blue.200"
borderRadius="md"
bg="blue.50"
>
<HStack justify="space-between">
<VStack align="start" spacing={1} flex={1}>
<HStack spacing={2}>
<Text fontWeight="bold" fontSize="sm">
{song.title}
</Text>
<Badge colorScheme="blue" size="sm">
<FiMusic style={{ marginRight: '4px' }} />
Has S3 File
</Badge>
</HStack>
<Text fontSize="xs" color="gray.600">
{song.artist}
</Text>
{song.location && (
<Text fontSize="xs" color="gray.500">
📁 {song.location}
</Text>
)}
{song.s3File?.streamingUrl && (
<Text fontSize="xs" color="green.600">
🎵 S3: {song.s3File.s3Key}
</Text>
)}
</VStack>
<HStack spacing={2}>
<Tooltip label="Unlink music file">
<IconButton
aria-label="Unlink"
icon={<FiX />}
size="sm"
variant="ghost"
colorScheme="red"
onClick={() => handleUnlinkMusicFile(song._id)}
/>
</Tooltip>
<Tooltip label="Play music file">
<IconButton
aria-label="Play"
icon={<FiPlay />}
size="sm"
variant="ghost"
colorScheme="blue"
/>
</Tooltip>
</HStack>
</HStack>
</Box>
))}
{songsWithMusicFiles.length > 10 && (
<Text fontSize="sm" color="gray.500" textAlign="center">
Showing first 10 of {songsWithMusicFiles.length} songs with music files
</Text>
)}
</VStack>
)}
</CardBody>
</Card>
{/* Suggestions Modal */}
<Modal isOpen={isOpen} onClose={onClose} size="xl">
<ModalOverlay />
@ -510,6 +604,11 @@ export const SongMatching: React.FC = () => {
{match.song.album}
</Text>
)}
{match.song.location && (
<Text fontSize="xs" color="gray.500">
📁 {match.song.location}
</Text>
)}
<Text fontSize="xs" color="gray.500">
{match.matchReason}
</Text>

View File

@ -19,7 +19,7 @@ export interface Song {
comments?: string;
playCount?: string;
rating?: string;
location?: string;
location?: string; // Original file path from Rekordbox XML
remixer?: string;
tonality?: string;
label?: string;
@ -30,7 +30,26 @@ export interface Song {
metro?: string;
battito?: string;
};
// Music file integration
// S3 file integration (preserves original location)
s3File?: {
musicFileId?: string | {
_id: string;
originalName: string;
title?: string;
artist?: string;
album?: string;
duration?: number;
size: number;
format?: string;
s3Key: string;
s3Url: string;
};
s3Key?: string;
s3Url?: string;
streamingUrl?: string;
hasS3File: boolean;
};
// Legacy support for backward compatibility
hasMusicFile?: boolean;
musicFile?: {
_id: string;