master/rekordbox-viewer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: master:3cd83ff2b52ebb6c6428538e5bcb74d1b60f25a0
Branches Tags
master:main
master:frontend-ux-improvements
master:feature/desktop-sync
...
compare: master:b436d1dabc9ed3ccf4f54bfeba07a5110b3beaa6
Branches Tags
master:frontend-ux-improvements
master:main
master:feature/desktop-sync

2 Commits
3cd83ff2b5 ... b436d1dabc

Author SHA1 Message Date
Geert Rademakes
b436d1dabc feat: implement immediate song document updates for perfect sync resilience 
- Remove end-of-job cleanup phases from S3 sync and song matching jobs
- Update Song documents immediately after each successful match in both Phase 1 and Phase 2
- Ensure hasS3File flag is set to true immediately for each matched song
- Enable play buttons to appear instantly as songs are processed
- Make system perfectly resilient to interruptions - no orphaned files
- Allow seamless resume capability for long-running sync jobs
- Provide real-time availability of matched songs without waiting for job completion
- Maintain system consistency regardless of when sync gets interrupted
 2025-08-07 23:36:30 +02:00
Geert Rademakes
96c43dbcff feat: implement infinite scroll with 100 songs per page 
- Update pagination from 50 to 100 songs per page for better UX
- Frontend: Update usePaginatedSongs hook default pageSize to 100
- Frontend: Update API service default limits to 100
- Frontend: Update App.tsx pageSize configuration to 100
- Backend: Update songs routes default limit to 100 for both main and playlist endpoints
- Maintain existing infinite scroll functionality with larger batch sizes
- Improve performance by reducing number of API calls needed
- Verified API endpoints return correct pagination info and song counts
 2025-08-07 20:23:05 +02:00
18 changed files with 1876 additions and 200 deletions
Show Stats Expand all files Collapse all files

2
packages/backend/src/index.ts
View File

@ -7,6 +7,7 @@ import { playlistsRouter } from './routes/playlists.js';
import { musicRouter } from './routes/music.js';
import { matchingRouter } from './routes/matching.js';
import { configRouter } from './routes/config.js';
import backgroundJobsRouter from './routes/backgroundJobs.js';
import { Song } from './models/Song.js';
import { Playlist } from './models/Playlist.js';
import { MusicFile } from './models/MusicFile.js';
@ -67,6 +68,7 @@ app.use('/api/playlists', playlistsRouter);
app.use('/api/music', musicRouter);
app.use('/api/matching', matchingRouter);
app.use('/api/config', configRouter);
app.use('/api/background-jobs', backgroundJobsRouter);
app.listen(port, () => {
console.log(`Server is running on port ${port}`);

78
packages/backend/src/routes/backgroundJobs.ts Normal file
View File

@ -0,0 +1,78 @@
import express from 'express';
import { backgroundJobService } from '../services/backgroundJobService.js';
const router = express.Router();
/**
* Start a new background job
*/
router.post('/start', async (req, res) => {
try {
const { type, options } = req.body;
if (!type || !['s3-sync', 'song-matching'].includes(type)) {
return res.status(400).json({ error: 'Invalid job type. Must be "s3-sync" or "song-matching"' });
}
console.log(`🚀 Starting background job: ${type}`);
const jobId = await backgroundJobService.startJob({ type, options });
res.json({
message: 'Background job started',
jobId,
type
});
} catch (error) {
console.error('Error starting background job:', error);
res.status(500).json({ error: 'Failed to start background job' });
}
});
/**
* Get job progress
*/
router.get('/progress/:jobId', async (req, res) => {
try {
const { jobId } = req.params;
const progress = backgroundJobService.getJobProgress(jobId);
if (!progress) {
return res.status(404).json({ error: 'Job not found' });
}
res.json(progress);
} catch (error) {
console.error('Error getting job progress:', error);
res.status(500).json({ error: 'Failed to get job progress' });
}
});
/**
* Get all jobs
*/
router.get('/jobs', async (req, res) => {
try {
const jobs = backgroundJobService.getAllJobs();
res.json({ jobs });
} catch (error) {
console.error('Error getting jobs:', error);
res.status(500).json({ error: 'Failed to get jobs' });
}
});
/**
* Clean up old jobs
*/
router.post('/cleanup', async (req, res) => {
try {
backgroundJobService.cleanupOldJobs();
res.json({ message: 'Old jobs cleaned up successfully' });
} catch (error) {
console.error('Error cleaning up jobs:', error);
res.status(500).json({ error: 'Failed to clean up jobs' });
}
});
export default router;

28
packages/backend/src/routes/matching.ts
View File

@ -74,12 +74,11 @@ router.get('/suggestions', async (req, res) => {
});
/**
* Auto-match and link music files to songs
* Auto-match and link music files to songs - now uses background job system
*/
router.post('/auto-link', async (req, res) => {
try {
console.log('🚀 Starting auto-match and link request...');
const startTime = Date.now();
console.log('🚀 Starting auto-match and link background job...');
const options = {
minConfidence: parseFloat(req.body.minConfidence as string) || 0.7,
@ -89,23 +88,24 @@ router.post('/auto-link', async (req, res) => {
console.log('⚙️ Auto-linking options:', options);
const result = await matchingService.autoMatchAndLink(options);
// Import background job service
const { backgroundJobService } = await import('../services/backgroundJobService.js');
const endTime = Date.now();
const duration = ((endTime - startTime) / 1000).toFixed(2);
console.log(`✅ Auto-linking completed in ${duration} seconds`);
console.log(`📊 Results: ${result.linked} linked, ${result.unmatched} unmatched`);
// Start the background job
const jobId = await backgroundJobService.startJob({
type: 'song-matching',
options
});
res.json({
message: 'Auto-linking completed',
result,
message: 'Auto-linking started as background job',
jobId,
options,
duration: `${duration}s`
status: 'started'
});
} catch (error) {
console.error('❌ Error during auto-linking:', error);
res.status(500).json({ error: 'Failed to auto-link music files' });
console.error('❌ Error starting auto-linking job:', error);
res.status(500).json({ error: 'Failed to start auto-linking job' });
}
});

176
packages/backend/src/routes/music.ts
View File

@ -166,130 +166,30 @@ router.get('/files', async (req, res) => {
});
/**
* Sync S3 files with database - recursively list all files in S3 and sync
* Sync S3 files with database - now uses background job system
*/
router.post('/sync-s3', async (req, res) => {
try {
console.log('🔄 Starting S3 sync process...');
const startTime = Date.now();
console.log('🔄 Starting S3 sync background job...');
// Get all files from S3 recursively
console.log('📁 Fetching files from S3 bucket...');
const s3Files = await s3Service.listAllFiles();
console.log(`✅ Found ${s3Files.length} files in S3 bucket`);
// Import background job service
const { backgroundJobService } = await import('../services/backgroundJobService.js');
const results = {
total: s3Files.length,
synced: 0,
skipped: 0,
errors: 0,
newFiles: 0,
audioFiles: 0,
nonAudioFiles: 0
};
console.log('🔍 Processing files...');
let processedCount = 0;
for (const s3File of s3Files) {
processedCount++;
const progress = ((processedCount / s3Files.length) * 100).toFixed(1);
try {
console.log(`📄 [${progress}%] Processing: ${s3File.key} (${audioMetadataService.formatFileSize(s3File.size)})`);
// Check if file already exists in database
const existingFile = await MusicFile.findOne({ s3Key: s3File.key });
if (existingFile) {
console.log(`⏭️ Skipping existing file: ${s3File.key}`);
results.skipped++;
continue;
}
// Extract filename from S3 key
const filename = s3File.key.split('/').pop() || s3File.key;
// Check if it's an audio file
if (!audioMetadataService.isAudioFile(filename)) {
console.log(`🚫 Skipping non-audio file: ${filename}`);
results.skipped++;
results.nonAudioFiles++;
continue;
}
results.audioFiles++;
console.log(`🎵 Processing audio file: ${filename}`);
// Get file content to extract metadata
try {
console.log(`⬇️ Downloading file content: ${s3File.key}`);
const fileBuffer = await s3Service.getFileContent(s3File.key);
console.log(`📊 Extracting metadata: ${filename}`);
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
// Save to database
console.log(`💾 Saving to database: ${filename}`);
const musicFile = new MusicFile({
originalName: filename,
s3Key: s3File.key,
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
contentType: 'audio/mpeg', // Default, will be updated by metadata
size: s3File.size,
...metadata,
// Start the background job
const jobId = await backgroundJobService.startJob({
type: 's3-sync',
options: req.body
});
await musicFile.save();
results.synced++;
results.newFiles++;
console.log(`✅ Successfully synced: ${filename}`);
} catch (metadataError) {
console.error(`❌ Error extracting metadata for ${s3File.key}:`, metadataError);
console.log(`🔄 Saving file without metadata: ${filename}`);
// Still save the file without metadata
const musicFile = new MusicFile({
originalName: filename,
s3Key: s3File.key,
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
contentType: 'audio/mpeg',
size: s3File.size,
});
await musicFile.save();
results.synced++;
results.newFiles++;
console.log(`✅ Saved without metadata: ${filename}`);
}
} catch (error) {
console.error(`❌ Error processing ${s3File.key}:`, error);
results.errors++;
}
}
const endTime = Date.now();
const duration = ((endTime - startTime) / 1000).toFixed(2);
console.log('🎉 S3 sync completed!');
console.log(`⏱️ Duration: ${duration} seconds`);
console.log(`📊 Results:`, results);
console.log(` Total files: ${results.total}`);
console.log(` Audio files: ${results.audioFiles}`);
console.log(` Non-audio files: ${results.nonAudioFiles}`);
console.log(` New files synced: ${results.newFiles}`);
console.log(` Files skipped: ${results.skipped}`);
console.log(` Errors: ${results.errors}`);
res.json({
message: 'S3 sync completed',
results,
duration: `${duration}s`
message: 'S3 sync started as background job',
jobId,
status: 'started'
});
} catch (error) {
console.error('❌ S3 sync error:', error);
res.status(500).json({ error: 'Failed to sync S3 files' });
console.error('❌ Error starting S3 sync job:', error);
res.status(500).json({ error: 'Failed to start S3 sync job' });
}
});
@ -465,4 +365,54 @@ router.post('/:id/link-song/:songId', async (req, res) => {
}
});
/**
* Fix orphaned music files (MusicFile exists but Song doesn't have hasS3File: true)
*/
router.post('/fix-orphaned', async (req, res) => {
try {
console.log('🔧 Starting orphaned music files fix...');
const orphanedMusicFiles = await MusicFile.find({
songId: { $exists: true },
s3Key: { $exists: true }
});
let fixedCount = 0;
const fixedFiles = [];
for (const musicFile of orphanedMusicFiles) {
// Check if the corresponding Song document needs to be updated
const song = await Song.findById(musicFile.songId);
if (song && !song.s3File?.hasS3File) {
await Song.updateOne(
{ _id: musicFile.songId },
{
$set: {
's3File.musicFileId': musicFile._id,
's3File.s3Key': musicFile.s3Key,
's3File.s3Url': musicFile.s3Url,
's3File.streamingUrl': musicFile.s3Url,
's3File.hasS3File': true
}
}
);
fixedCount++;
fixedFiles.push(musicFile.originalName);
console.log(`🔧 Fixed orphaned music file: ${musicFile.originalName}`);
}
}
console.log(`✅ Orphaned music files fix completed: Fixed ${fixedCount} files`);
res.json({
message: `Fixed ${fixedCount} orphaned music files`,
fixedCount,
fixedFiles
});
} catch (error) {
console.error('Error fixing orphaned music files:', error);
res.status(500).json({ message: 'Error fixing orphaned music files', error });
}
});
export { router as musicRouter };

19
packages/backend/src/routes/playlists.ts
View File

@ -20,18 +20,31 @@ router.get('/structure', async (req: Request, res: Response) => {
try {
const playlists = await Playlist.find({});
// Remove track data from playlists to reduce payload size
// Keep track counts but remove full track data to reduce payload size
const structureOnly = playlists.map(playlist => {
const cleanPlaylist = playlist.toObject() as any;
// Remove tracks from the main playlist
// Replace tracks array with track count for the main playlist
if (cleanPlaylist.tracks) {
cleanPlaylist.trackCount = cleanPlaylist.tracks.length;
delete cleanPlaylist.tracks;
} else {
cleanPlaylist.trackCount = 0;
}
// Recursively remove tracks from children
// Recursively process children
const cleanChildren = (children: any[]): any[] => {
return children.map(child => {
const cleanChild = { ...child } as any;
// Replace tracks array with track count
if (cleanChild.tracks) {
cleanChild.trackCount = cleanChild.tracks.length;
delete cleanChild.tracks;
} else {
cleanChild.trackCount = 0;
}
if (cleanChild.children && cleanChild.children.length > 0) {
cleanChild.children = cleanChildren(cleanChild.children);
}

4
packages/backend/src/routes/songs.ts
View File

@ -10,7 +10,7 @@ const router = express.Router();
router.get('/', async (req: Request, res: Response) => {
try {
const page = parseInt(req.query.page as string) || 1;
const limit = parseInt(req.query.limit as string) || 50;
const limit = parseInt(req.query.limit as string) || 100;
const search = req.query.search as string || '';
const skip = (page - 1) * limit;
@ -65,7 +65,7 @@ router.get('/playlist/*', async (req: Request, res: Response) => {
try {
const playlistName = decodeURIComponent(req.params[0]);
const page = parseInt(req.query.page as string) || 1;
const limit = parseInt(req.query.limit as string) || 50;
const limit = parseInt(req.query.limit as string) || 100;
const search = req.query.search as string || '';
const skip = (page - 1) * limit;

65
packages/backend/src/services/audioMetadataService.ts
View File

@ -54,6 +54,51 @@ export class AudioMetadataService {
return container.toUpperCase();
}
/**
* Extract basic metadata from filename (fallback method)
*/
extractBasicMetadataFromFilename(fileName: string): AudioMetadata {
console.log(`🔄 Using filename-based metadata extraction for: ${fileName}`);
// Remove file extension
const nameWithoutExt = fileName.replace(/\.[^/.]+$/, '');
// Try to extract artist and title from common patterns
let title = nameWithoutExt;
let artist: string | undefined;
// Common patterns: "Artist - Title", "Artist feat. Title", etc.
const patterns = [
/^(.+?)\s*[-–—]\s*(.+)$/, // Artist - Title
/^(.+?)\s+feat\.?\s+(.+)$/i, // Artist feat. Title
/^(.+?)\s+ft\.?\s+(.+)$/i, // Artist ft. Title
/^(.+?)\s+featuring\s+(.+)$/i, // Artist featuring Title
/^(.+?)\s*&\s*(.+)$/, // Artist & Title
/^(.+?)\s+vs\s+(.+)$/i, // Artist vs Title
/^(.+?)\s+x\s+(.+)$/i, // Artist x Title
];
for (const pattern of patterns) {
const match = nameWithoutExt.match(pattern);
if (match) {
artist = match[1].trim();
title = match[2].trim();
break;
}
}
// Determine format from extension
const extension = fileName.split('.').pop()?.toLowerCase();
const format = extension ? this.mapFormatToDisplayName(extension, fileName) : 'UNKNOWN';
return {
title,
artist,
format,
size: 0, // Will be set by caller
};
}
/**
* Validate and sanitize numeric values
*/
@ -99,7 +144,7 @@ export class AudioMetadataService {
}
/**
* Extract metadata from audio file buffer
* Extract metadata from audio file buffer with improved error handling
*/
async extractMetadata(fileBuffer: Buffer, fileName: string): Promise<AudioMetadata> {
console.log(`🎵 Extracting metadata for: ${fileName} (${this.formatFileSize(fileBuffer.length)})`);
@ -141,18 +186,16 @@ export class AudioMetadataService {
} catch (error) {
console.error(`❌ Error extracting metadata for ${fileName}:`, error);
// Try to determine format from file extension as fallback
const extension = fileName.split('.').pop()?.toLowerCase();
const fallbackFormat = extension ? this.mapFormatToDisplayName(extension, fileName) : 'UNKNOWN';
// Use filename-based extraction as fallback
const fallbackMetadata = this.extractBasicMetadataFromFilename(fileName);
fallbackMetadata.size = fileBuffer.length;
console.log(`🔄 Using fallback metadata for ${fileName}`);
console.log(`✅ Fallback metadata created for ${fileName}:`);
console.log(` Title: ${fallbackMetadata.title}`);
console.log(` Artist: ${fallbackMetadata.artist || 'Unknown'}`);
console.log(` Format: ${fallbackMetadata.format}`);
// Return basic metadata if extraction fails
return {
title: fileName.replace(/\.[^/.]+$/, ''), // Remove file extension
format: fallbackFormat,
size: fileBuffer.length,
};
return fallbackMetadata;
}
}

593
packages/backend/src/services/backgroundJobService.ts Normal file
View File

@ -0,0 +1,593 @@
export interface JobProgress {
jobId: string;
type: 's3-sync' | 'song-matching';
status: 'running' | 'completed' | 'failed';
progress: number; // 0-100
current: number;
total: number;
message: string;
startTime: Date;
endTime?: Date;
result?: any;
error?: string;
}
export interface JobOptions {
type: 's3-sync' | 'song-matching';
options?: any;
}
class BackgroundJobService {
private jobs: Map<string, JobProgress> = new Map();
private jobIdCounter = 0;
/**
* Start a new background job
*/
async startJob(jobOptions: JobOptions): Promise<string> {
const jobId = `job_${++this.jobIdCounter}_${Date.now()}`;
const job: JobProgress = {
jobId,
type: jobOptions.type,
status: 'running',
progress: 0,
current: 0,
total: 0,
message: `Starting ${jobOptions.type}...`,
startTime: new Date(),
};
this.jobs.set(jobId, job);
// Start the job in the background
this.runJob(jobId, jobOptions).catch(error => {
console.error(`Job ${jobId} failed:`, error);
const job = this.jobs.get(jobId);
if (job) {
job.status = 'failed';
job.error = error.message;
job.endTime = new Date();
}
});
return jobId;
}
/**
* Get job progress
*/
getJobProgress(jobId: string): JobProgress | null {
return this.jobs.get(jobId) || null;
}
/**
* Get all jobs
*/
getAllJobs(): JobProgress[] {
return Array.from(this.jobs.values());
}
/**
* Update job progress
*/
updateProgress(jobId: string, progress: Partial<JobProgress>): void {
const job = this.jobs.get(jobId);
if (job) {
Object.assign(job, progress);
// Calculate percentage if current and total are provided
if (progress.current !== undefined && progress.total !== undefined && progress.total > 0) {
job.progress = Math.round((progress.current / progress.total) * 100);
}
}
}
/**
* Complete a job
*/
completeJob(jobId: string, result?: any): void {
const job = this.jobs.get(jobId);
if (job) {
job.status = 'completed';
job.progress = 100;
job.endTime = new Date();
job.result = result;
job.message = 'Job completed successfully';
}
}
/**
* Fail a job
*/
failJob(jobId: string, error: string): void {
const job = this.jobs.get(jobId);
if (job) {
job.status = 'failed';
job.error = error;
job.endTime = new Date();
job.message = `Job failed: ${error}`;
}
}
/**
* Clean up old completed jobs (keep last 10)
*/
cleanupOldJobs(): void {
const allJobs = Array.from(this.jobs.values());
const completedJobs = allJobs.filter(job => job.status === 'completed' || job.status === 'failed');
if (completedJobs.length > 10) {
// Sort by end time and remove oldest
completedJobs.sort((a, b) => {
const aTime = a.endTime?.getTime() || 0;
const bTime = b.endTime?.getTime() || 0;
return aTime - bTime;
});
const toRemove = completedJobs.slice(0, completedJobs.length - 10);
toRemove.forEach(job => {
this.jobs.delete(job.jobId);
});
console.log(`🧹 Cleaned up ${toRemove.length} old jobs`);
}
}
/**
* Run the actual job
*/
private async runJob(jobId: string, jobOptions: JobOptions): Promise<void> {
try {
switch (jobOptions.type) {
case 's3-sync':
await this.runS3SyncJob(jobId, jobOptions.options);
break;
case 'song-matching':
await this.runSongMatchingJob(jobId, jobOptions.options);
break;
default:
throw new Error(`Unknown job type: ${jobOptions.type}`);
}
} catch (error) {
this.failJob(jobId, error instanceof Error ? error.message : 'Unknown error');
throw error;
}
}
/**
* Run S3 sync job
*/
private async runS3SyncJob(jobId: string, options?: any): Promise<void> {
try {
// Import here to avoid circular dependencies
const { S3Service } = await import('./s3Service.js');
const { AudioMetadataService } = await import('./audioMetadataService.js');
const { SongMatchingService } = await import('./songMatchingService.js');
const { MusicFile } = await import('../models/MusicFile.js');
const { Song } = await import('../models/Song.js');
const s3Service = await S3Service.createFromConfig();
const audioMetadataService = new AudioMetadataService();
const songMatchingService = new SongMatchingService();
// Phase 1: Quick filename matching
this.updateProgress(jobId, {
message: 'Phase 1: Fetching files from S3...',
current: 0,
total: 0
});
const s3Files = await s3Service.listAllFiles();
const audioFiles = s3Files.filter(s3File => {
const filename = s3File.key.split('/').pop() || s3File.key;
return audioMetadataService.isAudioFile(filename);
});
this.updateProgress(jobId, {
message: `Phase 1: Found ${audioFiles.length} audio files, checking database...`,
current: 0,
total: audioFiles.length
});
// Get existing files
const existingFiles = await MusicFile.find({}, { s3Key: 1 });
const existingS3Keys = new Set(existingFiles.map(f => f.s3Key));
const newAudioFiles = audioFiles.filter(s3File => !existingS3Keys.has(s3File.key));
this.updateProgress(jobId, {
message: `Phase 1: Processing ${newAudioFiles.length} new audio files...`,
current: 0,
total: newAudioFiles.length
});
// Get all songs for filename matching
const allSongs = await Song.find({}, { id: 1, title: 1, artist: 1, location: 1 });
const quickMatches: any[] = [];
const unmatchedFiles: any[] = [];
let processedCount = 0;
let phase1Errors = 0;
for (const s3File of newAudioFiles) {
processedCount++;
try {
const filename = s3File.key.split('/').pop() || s3File.key;
this.updateProgress(jobId, {
message: `Phase 1: Quick filename matching`,
current: processedCount,
total: newAudioFiles.length
});
// Quick filename matching logic
const normalizedS3Filename = filename.replace(/\.[^/.]+$/, '').toLowerCase();
let matchedSong = null;
for (const song of allSongs) {
if (song.location) {
const rekordboxFilename = song.location.split(/[/\\]/).pop() || song.location;
const normalizedRekordboxFilename = rekordboxFilename.replace(/\.[^/.]+$/, '').toLowerCase();
if (normalizedS3Filename === normalizedRekordboxFilename) {
matchedSong = song;
break;
}
}
}
if (matchedSong) {
const basicMetadata = audioMetadataService.extractBasicMetadataFromFilename(filename);
const musicFile = new MusicFile({
originalName: filename,
s3Key: s3File.key,
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
contentType: 'audio/mpeg',
size: s3File.size,
...basicMetadata,
songId: matchedSong._id,
});
// Save immediately for real-time availability
await musicFile.save();
quickMatches.push(musicFile);
// Update the Song document to indicate it has an S3 file
await Song.updateOne(
{ _id: matchedSong._id },
{
$set: {
's3File.musicFileId': musicFile._id,
's3File.s3Key': s3File.key,
's3File.s3Url': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
's3File.streamingUrl': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
's3File.hasS3File': true
}
}
);
console.log(`✅ Quick match saved immediately: ${filename}`);
} else {
unmatchedFiles.push(s3File);
}
} catch (error) {
console.error(`Error in quick matching ${s3File.key}:`, error);
unmatchedFiles.push(s3File);
phase1Errors++;
}
}
this.updateProgress(jobId, {
message: `Phase 1 completed: ${quickMatches.length} quick matches, ${unmatchedFiles.length} unmatched files`,
current: newAudioFiles.length,
total: newAudioFiles.length
});
// Phase 2: Complex matching for unmatched files
if (unmatchedFiles.length > 0) {
this.updateProgress(jobId, {
message: `Phase 2: Complex matching for ${unmatchedFiles.length} files...`,
current: 0,
total: unmatchedFiles.length
});
let complexMatches = 0;
let stillUnmatched = 0;
let phase2Errors = 0;
const processedFiles: any[] = [];
for (let i = 0; i < unmatchedFiles.length; i++) {
const s3File = unmatchedFiles[i];
try {
const filename = s3File.key.split('/').pop() || s3File.key;
this.updateProgress(jobId, {
message: `Phase 2: Complex matching`,
current: i + 1,
total: unmatchedFiles.length
});
// Download file and extract metadata
const fileBuffer = await s3Service.getFileContent(s3File.key);
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
const musicFile = new MusicFile({
originalName: filename,
s3Key: s3File.key,
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
contentType: 'audio/mpeg',
size: s3File.size,
...metadata,
});
processedFiles.push(musicFile);
// Try complex matching
const matchResult = await songMatchingService.matchMusicFileToSongs(musicFile, {
minConfidence: 0.7,
enableFuzzyMatching: true,
enablePartialMatching: true,
maxResults: 1
});
if (matchResult.length > 0 && matchResult[0].confidence >= 0.7) {
const bestMatch = matchResult[0];
musicFile.songId = bestMatch.song._id;
complexMatches++;
// Update the Song document to indicate it has an S3 file
await Song.updateOne(
{ _id: bestMatch.song._id },
{
$set: {
's3File.musicFileId': musicFile._id,
's3File.s3Key': s3File.key,
's3File.s3Url': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
's3File.streamingUrl': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
's3File.hasS3File': true
}
}
);
} else {
stillUnmatched++;
}
// Save immediately for real-time availability
await musicFile.save();
processedFiles.push(musicFile);
console.log(`✅ Complex match saved immediately: ${filename} (confidence: ${matchResult.length > 0 ? matchResult[0].confidence : 'N/A'})`);
} catch (error) {
console.error(`Error processing ${s3File.key}:`, error);
stillUnmatched++;
phase2Errors++;
}
}
this.updateProgress(jobId, {
message: `Phase 2 completed: ${complexMatches} complex matches, ${stillUnmatched} still unmatched`,
current: unmatchedFiles.length,
total: unmatchedFiles.length
});
// All files have been saved immediately during processing
console.log(`✅ All files saved immediately during processing`);
const result = {
phase1: {
totalFiles: newAudioFiles.length,
quickMatches: quickMatches.length,
unmatchedFiles: unmatchedFiles.length,
errors: 0
},
phase2: {
processedFiles: processedFiles.length,
complexMatches,
stillUnmatched,
errors: 0
},
total: {
processed: allMusicFiles.length,
matched: quickMatches.length + complexMatches,
unmatched: stillUnmatched,
errors: 0
}
};
this.completeJob(jobId, result);
} else {
// No unmatched files, all quick matches have been saved immediately
console.log(`✅ All quick matches saved immediately during processing`);
const result = {
phase1: {
totalFiles: newAudioFiles.length,
quickMatches: quickMatches.length,
unmatchedFiles: 0,
errors: 0
},
phase2: {
processedFiles: 0,
complexMatches: 0,
stillUnmatched: 0,
errors: 0
},
total: {
processed: quickMatches.length,
matched: quickMatches.length,
unmatched: 0,
errors: 0
}
};
this.completeJob(jobId, result);
}
} catch (error) {
this.failJob(jobId, error instanceof Error ? error.message : 'Unknown error');
throw error;
}
}
/**
* Run song matching job
*/
private async runSongMatchingJob(jobId: string, options?: any): Promise<void> {
try {
// Import here to avoid circular dependencies
const { SongMatchingService } = await import('./songMatchingService.js');
const { MusicFile } = await import('../models/MusicFile.js');
const { Song } = await import('../models/Song.js');
const matchingService = new SongMatchingService();
this.updateProgress(jobId, {
message: 'Finding unmatched music files...',
current: 0,
total: 0
});
// Get all unmatched music files
const unmatchedMusicFiles = await MusicFile.find({ songId: { $exists: false } });
this.updateProgress(jobId, {
message: `Found ${unmatchedMusicFiles.length} unmatched music files`,
current: 0,
total: unmatchedMusicFiles.length
});
if (unmatchedMusicFiles.length === 0) {
this.updateProgress(jobId, {
message: 'No unmatched music files found',
current: 0,
total: 0
});
this.completeJob(jobId, {
linked: 0,
unmatched: 0,
total: 0
});
return;
}
// Get all songs for matching
const allSongs = await Song.find({});
this.updateProgress(jobId, {
message: `Starting matching process with ${allSongs.length} songs...`,
current: 0,
total: unmatchedMusicFiles.length
});
let linked = 0;
let unmatched = 0;
const batchSize = 50;
const musicFileUpdates: any[] = [];
const songUpdates: any[] = [];
for (let i = 0; i < unmatchedMusicFiles.length; i++) {
const musicFile = unmatchedMusicFiles[i];
this.updateProgress(jobId, {
message: `Matching music files`,
current: i + 1,
total: unmatchedMusicFiles.length
});
try {
// Get matching suggestions
const matches = await matchingService.matchMusicFileToSongs(musicFile, {
minConfidence: options?.minConfidence || 0.7,
enableFuzzyMatching: options?.enableFuzzyMatching !== false,
enablePartialMatching: options?.enablePartialMatching !== false,
maxResults: 1
});
if (matches.length > 0 && matches[0].confidence >= (options?.minConfidence || 0.7)) {
const bestMatch = matches[0];
// Prepare updates
musicFileUpdates.push({
updateOne: {
filter: { _id: musicFile._id },
update: { songId: bestMatch.song._id }
}
});
songUpdates.push({
updateOne: {
filter: { _id: bestMatch.song._id },
update: {
$addToSet: {
s3File: {
musicFileId: musicFile._id,
hasS3File: true
}
}
}
}
});
linked++;
} else {
unmatched++;
}
// Process batch updates
if (musicFileUpdates.length >= batchSize) {
this.updateProgress(jobId, {
message: `Saving to database`,
current: i + 1,
total: unmatchedMusicFiles.length
});
await MusicFile.bulkWrite(musicFileUpdates);
await Song.bulkWrite(songUpdates);
musicFileUpdates.length = 0;
songUpdates.length = 0;
}
} catch (error) {
console.error(`Error matching ${musicFile.originalName}:`, error);
unmatched++;
}
}
// Save remaining updates
if (musicFileUpdates.length > 0) {
this.updateProgress(jobId, {
message: `Saving to database`,
current: unmatchedMusicFiles.length,
total: unmatchedMusicFiles.length
});
await MusicFile.bulkWrite(musicFileUpdates);
await Song.bulkWrite(songUpdates);
}
const result = {
linked,
unmatched,
total: unmatchedMusicFiles.length
};
this.completeJob(jobId, result);
} catch (error) {
this.failJob(jobId, error instanceof Error ? error.message : 'Unknown error');
throw error;
}
}
}
export const backgroundJobService = new BackgroundJobService();

282
packages/backend/src/services/songMatchingService.ts
View File

@ -25,7 +25,7 @@ export class SongMatchingService {
}
/**
* Match a single music file to songs in the library
* Match a single music file to songs in the library with optimized performance
*/
async matchMusicFileToSongs(
musicFile: any,
@ -39,6 +39,8 @@ export class SongMatchingService {
} = options;
const results: MatchResult[] = [];
let exactMatches = 0;
const maxExactMatches = 3; // Limit exact matches for performance
// Get all songs from the library
const songs = await Song.find({});
@ -51,6 +53,15 @@ export class SongMatchingService {
if (matchResult.confidence >= minConfidence) {
results.push(matchResult);
// Early termination for exact matches
if (matchResult.matchType === 'exact') {
exactMatches++;
if (exactMatches >= maxExactMatches) {
console.log(`🎯 Found ${exactMatches} exact matches, stopping early for performance`);
break;
}
}
}
}
@ -97,7 +108,7 @@ export class SongMatchingService {
}
/**
* Auto-match and link music files to songs
* Auto-match and link music files to songs with optimized performance
*/
async autoMatchAndLink(
options: MatchOptions = {}
@ -118,6 +129,8 @@ export class SongMatchingService {
let linked = 0;
let unmatched = 0;
let processedCount = 0;
const batchSize = 50; // Process in batches for better performance
const updates = [];
for (const musicFile of musicFiles) {
processedCount++;
@ -135,12 +148,31 @@ export class SongMatchingService {
if (matches.length > 0 && matches[0].confidence >= minConfidence) {
// Link the music file to the best match
console.log(`🔗 Linking ${musicFile.originalName} to ${matches[0].song.title} (${(matches[0].confidence * 100).toFixed(1)}% confidence)`);
await this.linkMusicFileToSong(musicFile, matches[0].song);
// Prepare batch updates
updates.push({
musicFileId: musicFile._id,
songId: matches[0].song._id,
s3Key: musicFile.s3Key,
s3Url: musicFile.s3Url
});
linked++;
} else {
console.log(`❌ No suitable match found for ${musicFile.originalName} (best confidence: ${matches.length > 0 ? (matches[0].confidence * 100).toFixed(1) : 0}%)`);
unmatched++;
}
// Process batch updates
if (updates.length >= batchSize) {
await this.processBatchUpdates(updates);
updates.length = 0; // Clear the array
}
}
// Process remaining updates
if (updates.length > 0) {
await this.processBatchUpdates(updates);
}
console.log(`🎉 Auto-match and link completed:`);
@ -151,6 +183,41 @@ export class SongMatchingService {
return { linked, unmatched };
}
/**
* Process batch updates for better performance
*/
private async processBatchUpdates(updates: any[]): Promise<void> {
console.log(`💾 Processing batch update for ${updates.length} files...`);
const bulkOps = updates.map(update => ({
updateOne: {
filter: { _id: update.musicFileId },
update: { $set: { songId: update.songId } }
}
}));
const songBulkOps = updates.map(update => ({
updateOne: {
filter: { _id: update.songId },
update: {
$set: {
's3File.musicFileId': update.musicFileId,
's3File.s3Key': update.s3Key,
's3File.s3Url': update.s3Url,
's3File.streamingUrl': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${update.s3Key}`,
's3File.hasS3File': true
}
}
}
}));
// Execute bulk operations
await Promise.all([
MusicFile.bulkWrite(bulkOps),
Song.bulkWrite(songBulkOps)
]);
}
/**
* Link a music file to a song (preserves original location)
*/
@ -206,31 +273,63 @@ export class SongMatchingService {
): MatchResult {
const scores: { score: number; reason: string }[] = [];
// 1. Exact filename match (highest priority)
// 1. Exact filename match (highest priority) - if this matches, it's likely a 1:1 match
const filenameScore = this.matchFilename(musicFile.originalName, song);
if (filenameScore.score >= 0.95) {
// If we have a very high filename match, return immediately
return {
song,
musicFile,
confidence: filenameScore.score,
matchType: 'exact',
matchReason: filenameScore.reason
};
}
if (filenameScore.score > 0) {
scores.push(filenameScore);
}
// 2. Title match
// 2. Original location match (high priority for Rekordbox files)
if (song.location) {
const locationScore = this.matchLocation(musicFile.originalName, song.location);
if (locationScore.score >= 0.9) {
// If we have a very high location match, return immediately
return {
song,
musicFile,
confidence: locationScore.score,
matchType: 'exact',
matchReason: locationScore.reason
};
}
if (locationScore.score > 0) {
scores.push(locationScore);
}
}
// 3. Title match (only if filename didn't match well)
if (filenameScore.score < 0.8) {
const titleScore = this.matchTitle(musicFile.title, song.title);
if (titleScore.score > 0) {
scores.push(titleScore);
}
}
// 3. Artist match
// 4. Artist match (only if filename didn't match well)
if (filenameScore.score < 0.8) {
const artistScore = this.matchArtist(musicFile.artist, song.artist);
if (artistScore.score > 0) {
scores.push(artistScore);
}
}
// 4. Album match
// 5. Album match (lower priority)
const albumScore = this.matchAlbum(musicFile.album, song.album);
if (albumScore.score > 0) {
scores.push(albumScore);
}
// 5. Duration match (if available)
// 6. Duration match (if available, as a tiebreaker)
if (musicFile.duration && song.totalTime) {
const durationScore = this.matchDuration(musicFile.duration, song.totalTime);
if (durationScore.score > 0) {
@ -238,17 +337,27 @@ export class SongMatchingService {
}
}
// 6. Original location match (if available)
if (song.location) {
const locationScore = this.matchLocation(musicFile.originalName, song.location);
if (locationScore.score > 0) {
scores.push(locationScore);
}
// Calculate weighted average score with filename bias
let totalScore = 0;
let totalWeight = 0;
for (const score of scores) {
let weight = 1;
// Give higher weight to filename and location matches
if (score.reason.includes('filename') || score.reason.includes('location')) {
weight = 3;
} else if (score.reason.includes('title')) {
weight = 2;
} else if (score.reason.includes('artist')) {
weight = 1.5;
}
// Calculate weighted average score
const totalScore = scores.reduce((sum, s) => sum + s.score, 0);
const averageScore = scores.length > 0 ? totalScore / scores.length : 0;
totalScore += score.score * weight;
totalWeight += weight;
}
const averageScore = totalWeight > 0 ? totalScore / totalWeight : 0;
// Determine match type
let matchType: 'exact' | 'fuzzy' | 'partial' | 'none' = 'none';
@ -275,44 +384,105 @@ export class SongMatchingService {
}
/**
* Match filename to song
* Match filename to song with comprehensive pattern matching
*/
private matchFilename(filename: string, song: any): { score: number; reason: string } {
if (!filename || !song.title) return { score: 0, reason: '' };
const cleanFilename = this.cleanString(filename.replace(/\.[^/.]+$/, '')); // Remove extension
const cleanTitle = this.cleanString(song.title);
const cleanArtist = song.artist ? this.cleanString(song.artist) : '';
// Exact match
// 1. Exact filename match (highest confidence)
if (cleanFilename === cleanTitle) {
return { score: 1.0, reason: 'Exact filename match' };
}
// Contains match
// 2. Artist - Title pattern matches (very common in music files)
if (cleanArtist) {
const patterns = [
`${cleanArtist} - ${cleanTitle}`,
`${cleanTitle} - ${cleanArtist}`,
`${cleanArtist} feat. ${cleanTitle}`,
`${cleanTitle} feat. ${cleanArtist}`,
`${cleanArtist} ft. ${cleanTitle}`,
`${cleanTitle} ft. ${cleanArtist}`,
`${cleanArtist} featuring ${cleanTitle}`,
`${cleanTitle} featuring ${cleanArtist}`,
`${cleanArtist} & ${cleanTitle}`,
`${cleanTitle} & ${cleanArtist}`,
`${cleanArtist} vs ${cleanTitle}`,
`${cleanTitle} vs ${cleanArtist}`,
`${cleanArtist} x ${cleanTitle}`,
`${cleanTitle} x ${cleanArtist}`
];
for (const pattern of patterns) {
if (cleanFilename === pattern) {
return { score: 1.0, reason: 'Exact Artist-Title pattern match' };
}
}
// Partial pattern matches
for (const pattern of patterns) {
if (cleanFilename.includes(pattern) || pattern.includes(cleanFilename)) {
return { score: 0.95, reason: 'Partial Artist-Title pattern match' };
}
}
}
// 3. Filename contains title (common when filenames have extra info)
if (cleanFilename.includes(cleanTitle) || cleanTitle.includes(cleanFilename)) {
return { score: 0.8, reason: 'Filename contains title' };
return { score: 0.9, reason: 'Filename contains title' };
}
// Artist - Title pattern match
if (song.artist) {
const cleanArtist = this.cleanString(song.artist);
const artistTitlePattern = `${cleanArtist} - ${cleanTitle}`;
const titleArtistPattern = `${cleanTitle} - ${cleanArtist}`;
// 4. Handle common filename variations
const filenameVariations = [
cleanFilename,
cleanFilename.replace(/\([^)]*\)/g, '').trim(), // Remove parentheses content
cleanFilename.replace(/\[[^\]]*\]/g, '').trim(), // Remove bracket content
cleanFilename.replace(/remix|mix|edit|vip|extended|radio|clean|dirty/gi, '').trim(), // Remove common suffixes
cleanFilename.replace(/\s+/g, ' ').trim() // Normalize whitespace
];
if (cleanFilename === artistTitlePattern || cleanFilename === titleArtistPattern) {
return { score: 0.95, reason: 'Artist - Title pattern match' };
for (const variation of filenameVariations) {
if (variation === cleanTitle) {
return { score: 0.95, reason: 'Filename variation matches title' };
}
if (variation.includes(cleanTitle) || cleanTitle.includes(variation)) {
return { score: 0.85, reason: 'Filename variation contains title' };
}
}
if (cleanFilename.includes(artistTitlePattern) || cleanFilename.includes(titleArtistPattern)) {
return { score: 0.85, reason: 'Filename contains Artist - Title pattern' };
// 5. Handle title variations
const titleVariations = [
cleanTitle,
cleanTitle.replace(/\([^)]*\)/g, '').trim(),
cleanTitle.replace(/\[[^\]]*\]/g, '').trim(),
cleanTitle.replace(/remix|mix|edit|vip|extended|radio|clean|dirty/gi, '').trim(),
cleanTitle.replace(/\s+/g, ' ').trim()
];
for (const titleVar of titleVariations) {
if (cleanFilename === titleVar) {
return { score: 0.95, reason: 'Filename matches title variation' };
}
if (cleanFilename.includes(titleVar) || titleVar.includes(cleanFilename)) {
return { score: 0.85, reason: 'Filename contains title variation' };
}
}
// 6. Fuzzy match for similar filenames
const similarity = this.calculateSimilarity(cleanFilename, cleanTitle);
if (similarity > 0.8) {
return { score: similarity * 0.8, reason: 'Fuzzy filename match' };
}
return { score: 0, reason: '' };
}
/**
* Match original location to filename
* Match original location to filename with comprehensive path handling
*/
private matchLocation(filename: string, location: string): { score: number; reason: string } {
if (!filename || !location) return { score: 0, reason: '' };
@ -320,18 +490,52 @@ export class SongMatchingService {
const cleanFilename = this.cleanString(filename);
const cleanLocation = this.cleanString(location);
// Extract filename from location path
const locationFilename = cleanLocation.split('/').pop() || cleanLocation;
// Extract filename from location path (handle different path separators)
const pathParts = cleanLocation.split(/[\/\\]/);
const locationFilename = pathParts[pathParts.length - 1] || cleanLocation;
const locationFilenameNoExt = locationFilename.replace(/\.[^/.]+$/, '');
const filenameNoExt = cleanFilename.replace(/\.[^/.]+$/, '');
// Exact filename match
if (cleanFilename.includes(locationFilenameNoExt) || locationFilenameNoExt.includes(cleanFilename.replace(/\.[^/.]+$/, ''))) {
return { score: 0.9, reason: 'Original location filename match' };
// 1. Exact filename match (highest confidence)
if (filenameNoExt === locationFilenameNoExt) {
return { score: 1.0, reason: 'Exact location filename match' };
}
// Path contains filename
if (cleanLocation.includes(cleanFilename.replace(/\.[^/.]+$/, ''))) {
return { score: 0.7, reason: 'Original location contains filename' };
// 2. Filename contains location filename or vice versa
if (filenameNoExt.includes(locationFilenameNoExt) || locationFilenameNoExt.includes(filenameNoExt)) {
return { score: 0.95, reason: 'Location filename contains match' };
}
// 3. Handle common filename variations in location
const locationVariations = [
locationFilenameNoExt,
locationFilenameNoExt.replace(/\([^)]*\)/g, '').trim(),
locationFilenameNoExt.replace(/\[[^\]]*\]/g, '').trim(),
locationFilenameNoExt.replace(/remix|mix|edit|vip|extended|radio|clean|dirty/gi, '').trim(),
locationFilenameNoExt.replace(/\s+/g, ' ').trim()
];
for (const variation of locationVariations) {
if (filenameNoExt === variation) {
return { score: 0.95, reason: 'Filename matches location variation' };
}
if (filenameNoExt.includes(variation) || variation.includes(filenameNoExt)) {
return { score: 0.9, reason: 'Filename contains location variation' };
}
}
// 4. Check if any part of the path contains the filename
for (const pathPart of pathParts) {
const cleanPathPart = pathPart.replace(/\.[^/.]+$/, ''); // Remove extension
if (cleanPathPart && (filenameNoExt.includes(cleanPathPart) || cleanPathPart.includes(filenameNoExt))) {
return { score: 0.8, reason: 'Path part contains filename' };
}
}
// 5. Fuzzy match for similar filenames
const similarity = this.calculateSimilarity(filenameNoExt, locationFilenameNoExt);
if (similarity > 0.8) {
return { score: similarity * 0.7, reason: 'Fuzzy location filename match' };
}
return { score: 0, reason: '' };

275
packages/backend/src/services/twoPhaseSyncService.ts Normal file
View File

@ -0,0 +1,275 @@
import { S3Service } from './s3Service.js';
import { AudioMetadataService } from './audioMetadataService.js';
import { SongMatchingService } from './songMatchingService.js';
import { MusicFile } from '../models/MusicFile.js';
import { Song } from '../models/Song.js';
export interface SyncResult {
phase1: {
totalFiles: number;
quickMatches: number;
unmatchedFiles: number;
errors: number;
};
phase2: {
processedFiles: number;
complexMatches: number;
stillUnmatched: number;
errors: number;
};
total: {
processed: number;
matched: number;
unmatched: number;
errors: number;
};
}
export class TwoPhaseSyncService {
private s3Service: S3Service;
private audioMetadataService: AudioMetadataService;
private songMatchingService: SongMatchingService;
constructor() {
this.audioMetadataService = new AudioMetadataService();
this.songMatchingService = new SongMatchingService();
}
/**
* Initialize S3 service
*/
async initialize(): Promise<void> {
this.s3Service = await S3Service.createFromConfig();
}
/**
* Extract filename from S3 key
*/
private getFilenameFromS3Key(s3Key: string): string {
return s3Key.split('/').pop() || s3Key;
}
/**
* Extract filename from Rekordbox location path
*/
private getFilenameFromLocation(location: string): string {
// Handle both Windows and Unix paths
const normalizedPath = location.replace(/\\/g, '/');
return normalizedPath.split('/').pop() || location;
}
/**
* Normalize filename for comparison (remove extension, lowercase)
*/
private normalizeFilename(filename: string): string {
return filename.replace(/\.[^/.]+$/, '').toLowerCase();
}
/**
* Phase 1: Quick filename-based matching
*/
async phase1QuickMatch(): Promise<{
quickMatches: MusicFile[];
unmatchedFiles: any[];
errors: any[];
}> {
console.log('🚀 Starting Phase 1: Quick filename-based matching...');
// Get all S3 files
const s3Files = await this.s3Service.listAllFiles();
const audioFiles = s3Files.filter(s3File => {
const filename = this.getFilenameFromS3Key(s3File.key);
return this.audioMetadataService.isAudioFile(filename);
});
console.log(`📁 Found ${audioFiles.length} audio files in S3`);
// Get existing music files to avoid duplicates
const existingFiles = await MusicFile.find({}, { s3Key: 1 });
const existingS3Keys = new Set(existingFiles.map(f => f.s3Key));
const newAudioFiles = audioFiles.filter(s3File => !existingS3Keys.has(s3File.key));
console.log(`🆕 Found ${newAudioFiles.length} new audio files to process`);
// Get all songs from database for filename matching
const allSongs = await Song.find({}, { id: 1, title: 1, artist: 1, location: 1 });
console.log(`🎵 Found ${allSongs.length} songs in database for matching`);
const quickMatches: MusicFile[] = [];
const unmatchedFiles: any[] = [];
const errors: any[] = [];
for (const s3File of newAudioFiles) {
try {
const s3Filename = this.getFilenameFromS3Key(s3File.key);
const normalizedS3Filename = this.normalizeFilename(s3Filename);
// Try to find exact filename match in Rekordbox songs
let matchedSong = null;
for (const song of allSongs) {
if (song.location) {
const rekordboxFilename = this.getFilenameFromLocation(song.location);
const normalizedRekordboxFilename = this.normalizeFilename(rekordboxFilename);
if (normalizedS3Filename === normalizedRekordboxFilename) {
matchedSong = song;
break;
}
}
}
if (matchedSong) {
console.log(`✅ Quick match found: ${s3Filename} -> ${matchedSong.title}`);
// Extract basic metadata from filename (no need to download file)
const basicMetadata = this.audioMetadataService.extractBasicMetadataFromFilename(s3Filename);
const musicFile = new MusicFile({
originalName: s3Filename,
s3Key: s3File.key,
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
contentType: 'audio/mpeg',
size: s3File.size,
...basicMetadata,
songId: matchedSong._id, // Link to the matched song
});
quickMatches.push(musicFile);
} else {
console.log(`❓ No quick match for: ${s3Filename}`);
unmatchedFiles.push(s3File);
}
} catch (error) {
console.error(`❌ Error processing ${s3File.key}:`, error);
errors.push({ file: s3File, error: error.message });
}
}
console.log(`✅ Phase 1 completed: ${quickMatches.length} quick matches, ${unmatchedFiles.length} unmatched files`);
return { quickMatches, unmatchedFiles, errors };
}
/**
* Phase 2: Complex matching for unmatched files
*/
async phase2ComplexMatch(unmatchedFiles: any[]): Promise<{
processedFiles: MusicFile[];
complexMatches: number;
stillUnmatched: number;
errors: any[];
}> {
console.log('🔍 Starting Phase 2: Complex matching for unmatched files...');
const processedFiles: MusicFile[] = [];
let complexMatches = 0;
let stillUnmatched = 0;
const errors: any[] = [];
for (const s3File of unmatchedFiles) {
try {
const filename = this.getFilenameFromS3Key(s3File.key);
console.log(`🔍 Processing unmatched file: ${filename}`);
// Download file and extract metadata
const fileBuffer = await this.s3Service.getFileContent(s3File.key);
const metadata = await this.audioMetadataService.extractMetadata(fileBuffer, filename);
const musicFile = new MusicFile({
originalName: filename,
s3Key: s3File.key,
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
contentType: 'audio/mpeg',
size: s3File.size,
...metadata,
});
processedFiles.push(musicFile);
// Try complex matching
const matchResult = await this.songMatchingService.matchMusicFileToSongs(musicFile, {
minConfidence: 0.7,
enableFuzzyMatching: true,
enablePartialMatching: true,
maxResults: 1
});
if (matchResult.length > 0 && matchResult[0].confidence >= 0.7) {
const bestMatch = matchResult[0];
musicFile.songId = bestMatch.song._id;
complexMatches++;
console.log(`✅ Complex match found: ${filename} -> ${bestMatch.song.title} (${bestMatch.confidence})`);
} else {
stillUnmatched++;
console.log(`❓ No complex match for: ${filename}`);
}
} catch (error) {
console.error(`❌ Error processing ${s3File.key}:`, error);
errors.push({ file: s3File, error: error.message });
}
}
console.log(`✅ Phase 2 completed: ${complexMatches} complex matches, ${stillUnmatched} still unmatched`);
return { processedFiles, complexMatches, stillUnmatched, errors };
}
/**
* Run complete two-phase sync
*/
async runTwoPhaseSync(): Promise<SyncResult> {
console.log('🎯 Starting Two-Phase S3 Sync...');
const startTime = Date.now();
await this.initialize();
// Phase 1: Quick filename matching
const phase1Result = await this.phase1QuickMatch();
// Phase 2: Complex matching for unmatched files
const phase2Result = await this.phase2ComplexMatch(phase1Result.unmatchedFiles);
// Combine all music files
const allMusicFiles = [...phase1Result.quickMatches, ...phase2Result.processedFiles];
// Batch save all music files
if (allMusicFiles.length > 0) {
console.log(`💾 Saving ${allMusicFiles.length} music files to database...`);
await MusicFile.insertMany(allMusicFiles);
}
const endTime = Date.now();
const duration = ((endTime - startTime) / 1000).toFixed(2);
const result: SyncResult = {
phase1: {
totalFiles: phase1Result.quickMatches.length + phase1Result.unmatchedFiles.length,
quickMatches: phase1Result.quickMatches.length,
unmatchedFiles: phase1Result.unmatchedFiles.length,
errors: phase1Result.errors.length,
},
phase2: {
processedFiles: phase2Result.processedFiles.length,
complexMatches: phase2Result.complexMatches,
stillUnmatched: phase2Result.stillUnmatched,
errors: phase2Result.errors.length,
},
total: {
processed: allMusicFiles.length,
matched: phase1Result.quickMatches.length + phase2Result.complexMatches,
unmatched: phase2Result.stillUnmatched,
errors: phase1Result.errors.length + phase2Result.errors.length,
},
};
console.log(`🎉 Two-Phase Sync completed in ${duration}s:`);
console.log(` Phase 1: ${result.phase1.quickMatches} quick matches, ${result.phase1.unmatchedFiles} unmatched`);
console.log(` Phase 2: ${result.phase2.complexMatches} complex matches, ${result.phase2.stillUnmatched} still unmatched`);
console.log(` Total: ${result.total.processed} processed, ${result.total.matched} matched, ${result.total.unmatched} unmatched`);
return result;
}
}

165
packages/backend/test-matching.js Normal file
View File

@ -0,0 +1,165 @@
import { SongMatchingService } from './src/services/songMatchingService.js';
async function testMatchingAlgorithm() {
console.log('🧪 Testing improved matching algorithm...');
const matchingService = new SongMatchingService();
// Test cases for filename matching
const testCases = [
{
musicFile: {
originalName: 'Artist Name - Song Title.mp3',
title: 'Song Title',
artist: 'Artist Name'
},
song: {
title: 'Song Title',
artist: 'Artist Name',
location: '/path/to/Artist Name - Song Title.mp3'
},
expectedScore: 1.0,
description: 'Exact Artist-Title pattern match'
},
{
musicFile: {
originalName: 'Song Title (Remix).mp3',
title: 'Song Title',
artist: 'Artist Name'
},
song: {
title: 'Song Title',
artist: 'Artist Name',
location: '/path/to/Song Title.mp3'
},
expectedScore: 0.95,
description: 'Filename variation match'
},
{
musicFile: {
originalName: 'Artist Name feat. Other Artist - Song Title.mp3',
title: 'Song Title',
artist: 'Artist Name'
},
song: {
title: 'Song Title',
artist: 'Artist Name',
location: '/path/to/Artist Name - Song Title.mp3'
},
expectedScore: 0.95,
description: 'Partial Artist-Title pattern match'
},
{
musicFile: {
originalName: 'Song Title.mp3',
title: 'Song Title',
artist: 'Artist Name'
},
song: {
title: 'Song Title',
artist: 'Artist Name',
location: '/path/to/Song Title.mp3'
},
expectedScore: 1.0,
description: 'Exact filename match'
},
{
musicFile: {
originalName: 'Different Name.mp3',
title: 'Song Title',
artist: 'Artist Name'
},
song: {
title: 'Song Title',
artist: 'Artist Name',
location: '/path/to/Song Title.mp3'
},
expectedScore: 0.9,
description: 'Title match when filename differs'
}
];
console.log('\n📊 Testing filename matching scenarios...');
for (const testCase of testCases) {
console.log(`\n🔍 Testing: ${testCase.description}`);
console.log(` Music File: ${testCase.musicFile.originalName}`);
console.log(` Song: ${testCase.song.title} by ${testCase.song.artist}`);
// Test the calculateMatch method directly
const matchResult = matchingService['calculateMatch'](testCase.musicFile, testCase.song, {
enableFuzzyMatching: true,
enablePartialMatching: true
});
const status = matchResult.confidence >= testCase.expectedScore * 0.9 ? '✅' : '❌';
console.log(` ${status} Confidence: ${(matchResult.confidence * 100).toFixed(1)}% (expected: ${(testCase.expectedScore * 100).toFixed(1)}%)`);
console.log(` Match Type: ${matchResult.matchType}`);
console.log(` Reason: ${matchResult.matchReason}`);
}
// Test filename matching function directly
console.log('\n🔍 Testing filename matching function...');
const filenameTests = [
{
filename: 'Artist - Song.mp3',
song: { title: 'Song', artist: 'Artist' },
expected: 'Exact Artist-Title pattern match'
},
{
filename: 'Song (Remix).mp3',
song: { title: 'Song', artist: 'Artist' },
expected: 'Filename variation matches title'
},
{
filename: 'Artist feat. Other - Song.mp3',
song: { title: 'Song', artist: 'Artist' },
expected: 'Exact Artist-Title pattern match'
}
];
for (const test of filenameTests) {
const result = matchingService['matchFilename'](test.filename, test.song);
const status = result.reason.includes(test.expected) ? '✅' : '❌';
console.log(`${status} "${test.filename}" -> "${test.expected}" (${(result.score * 100).toFixed(1)}%)`);
}
// Test location matching function
console.log('\n🔍 Testing location matching function...');
const locationTests = [
{
filename: 'Song.mp3',
location: '/path/to/Song.mp3',
expected: 'Exact location filename match'
},
{
filename: 'Song (Remix).mp3',
location: '/path/to/Song.mp3',
expected: 'Filename matches location variation'
},
{
filename: 'Artist - Song.mp3',
location: '/path/to/Song.mp3',
expected: 'Location filename contains match'
}
];
for (const test of locationTests) {
const result = matchingService['matchLocation'](test.filename, test.location);
const status = result.reason.includes(test.expected) ? '✅' : '❌';
console.log(`${status} "${test.filename}" in "${test.location}" -> "${test.expected}" (${(result.score * 100).toFixed(1)}%)`);
}
console.log('\n🎉 Matching algorithm tests completed!');
console.log('\n📋 Key improvements:');
console.log(' ✅ Filename matching is now the highest priority');
console.log(' ✅ Early termination for exact matches (95%+ confidence)');
console.log(' ✅ Comprehensive pattern matching for Artist-Title combinations');
console.log(' ✅ Better handling of filename variations (remix, edit, etc.)');
console.log(' ✅ Improved location matching for Rekordbox file paths');
console.log(' ✅ Weighted scoring system with filename bias');
}
testMatchingAlgorithm().catch(console.error);

6
packages/frontend/src/App.tsx
View File

@ -7,6 +7,7 @@ import { PlaylistManager } from "./components/PlaylistManager";
import { SongDetails } from "./components/SongDetails";
import { Configuration } from "./pages/Configuration";
import { PersistentMusicPlayer } from "./components/PersistentMusicPlayer";
import { BackgroundJobProgress } from "./components/BackgroundJobProgress";
import { MusicPlayerProvider, useMusicPlayer } from "./contexts/MusicPlayerContext";
import { useXmlParser } from "./hooks/useXmlParser";
import { usePaginatedSongs } from "./hooks/usePaginatedSongs";
@ -128,7 +129,7 @@ const RekordboxReader: React.FC = () => {
loadNextPage,
searchSongs,
searchQuery
} = usePaginatedSongs({ pageSize: 50, playlistName: currentPlaylist });
} = usePaginatedSongs({ pageSize: 100, playlistName: currentPlaylist });
// Export library to XML
const handleExportLibrary = useCallback(async () => {
@ -657,6 +658,9 @@ const RekordboxReader: React.FC = () => {
currentSong={currentSong}
onClose={handleCloseMusicPlayer}
/>
{/* Background Job Progress */}
<BackgroundJobProgress />
</Box>
);
};

310
packages/frontend/src/components/BackgroundJobProgress.tsx Normal file
View File

@ -0,0 +1,310 @@
import React, { useState, useEffect, useRef } from 'react';
import {
Box,
Flex,
Text,
Progress,
Button,
VStack,
HStack,
Badge,
IconButton,
useDisclosure,
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalBody,
ModalCloseButton,
Table,
Thead,
Tbody,
Tr,
Th,
Td,
Spinner,
} from '@chakra-ui/react';
import { CloseIcon } from '@chakra-ui/icons';
import { api } from '../services/api';
interface JobProgress {
jobId: string;
type: 's3-sync' | 'song-matching';
status: 'running' | 'completed' | 'failed';
progress: number;
current: number;
total: number;
message: string;
startTime: Date;
endTime?: Date;
result?: any;
error?: string;
}
interface BackgroundJobProgressProps {
jobId?: string;
onJobComplete?: (result: any) => void;
onJobError?: (error: string) => void;
}
export const BackgroundJobProgress: React.FC<BackgroundJobProgressProps> = ({
jobId,
onJobComplete,
onJobError,
}) => {
const [jobs, setJobs] = useState<JobProgress[]>([]);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const { isOpen, onOpen, onClose } = useDisclosure();
const intervalRef = useRef<NodeJS.Timeout | null>(null);
// Load all jobs
const loadJobs = async () => {
try {
setLoading(true);
const jobsData = await api.getAllJobs();
setJobs(jobsData);
setError(null);
} catch (err) {
setError(err instanceof Error ? err.message : 'Failed to load jobs');
} finally {
setLoading(false);
}
};
// Update specific job progress
const updateJobProgress = async (jobId: string) => {
try {
const progress = await api.getJobProgress(jobId);
setJobs(prev => prev.map(job =>
job.jobId === jobId ? progress : job
));
// Handle job completion
if (progress.status === 'completed' && onJobComplete) {
onJobComplete(progress.result);
} else if (progress.status === 'failed' && onJobError) {
onJobError(progress.error || 'Job failed');
}
} catch (err) {
console.error('Error updating job progress:', err);
}
};
// Start polling for all active jobs
const startPolling = () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
}
intervalRef.current = setInterval(() => {
// Update all active jobs
const activeJobIds = jobs.filter(job => job.status === 'running').map(job => job.jobId);
activeJobIds.forEach(jobId => {
updateJobProgress(jobId);
});
// Also update specific jobId if provided
if (jobId) {
updateJobProgress(jobId);
}
}, 2000); // Poll every 2 seconds for less frequent updates
};
// Stop polling
const stopPolling = () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
};
// Load jobs on mount
useEffect(() => {
loadJobs();
}, []);
// Start polling for active jobs
useEffect(() => {
const activeJobs = jobs.filter(job => job.status === 'running');
if (activeJobs.length > 0 || jobId) {
startPolling();
}
return () => {
stopPolling();
};
}, [jobs, jobId]);
// Cleanup on unmount
useEffect(() => {
return () => {
stopPolling();
};
}, []);
const getStatusColor = (status: string) => {
switch (status) {
case 'running': return 'blue';
case 'completed': return 'green';
case 'failed': return 'red';
default: return 'gray';
}
};
const getStatusIcon = (status: string) => {
switch (status) {
case 'running': return <Spinner size="sm" />;
case 'completed': return <Text fontSize="sm">✅</Text>;
case 'failed': return <Text fontSize="sm">❌</Text>;
default: return <Text fontSize="sm">⏸️</Text>;
}
};
const formatDuration = (startTime: Date, endTime?: Date) => {
const start = new Date(startTime).getTime();
const end = endTime ? new Date(endTime).getTime() : Date.now();
const duration = Math.floor((end - start) / 1000);
if (duration < 60) return `${duration}s`;
if (duration < 3600) return `${Math.floor(duration / 60)}m ${duration % 60}s`;
return `${Math.floor(duration / 3600)}h ${Math.floor((duration % 3600) / 60)}m`;
};
const activeJobs = jobs.filter(job => job.status === 'running');
const completedJobs = jobs.filter(job => job.status === 'completed' || job.status === 'failed');
return (
<>
{/* Active Jobs Summary */}
{activeJobs.length > 0 && (
<Box
position="fixed"
bottom={4}
right={4}
bg="gray.800"
border="1px solid"
borderColor="gray.600"
borderRadius="lg"
p={4}
boxShadow="xl"
zIndex={1000}
maxW="400px"
>
<Flex justify="space-between" align="center" mb={3}>
<Text fontWeight="bold" fontSize="sm" color="gray.100">
Background Jobs ({activeJobs.length})
</Text>
</Flex>
<VStack spacing={3} align="stretch">
{activeJobs.map(job => (
<Box key={job.jobId} p={3} bg="gray.700" borderRadius="md">
<Flex justify="space-between" align="center" mb={2}>
<Text fontSize="sm" fontWeight="medium" color="gray.100">
{job.type === 's3-sync' ? 'S3 Sync' : 'Song Matching'}
</Text>
<Badge colorScheme={getStatusColor(job.status)} size="sm">
{job.status}
</Badge>
</Flex>
<Text fontSize="xs" color="gray.400" mb={2}>
{job.message}
</Text>
<Progress
value={job.progress}
size="sm"
colorScheme={getStatusColor(job.status)}
mb={2}
/>
<Flex justify="space-between" fontSize="xs" color="gray.400">
<Text>{job.progress}%</Text>
<Text>{formatDuration(job.startTime)}</Text>
</Flex>
</Box>
))}
</VStack>
</Box>
)}
{/* All Jobs Modal */}
<Modal isOpen={isOpen} onClose={onClose} size="xl">
<ModalOverlay />
<ModalContent bg="gray.800" color="gray.100">
<ModalHeader color="gray.100">Background Jobs</ModalHeader>
<ModalCloseButton color="gray.300" />
<ModalBody pb={6}>
<VStack spacing={4} align="stretch">
<HStack justify="space-between">
<Text fontWeight="bold" color="gray.100">All Jobs</Text>
<Button size="sm" onClick={loadJobs} isLoading={loading}>
Refresh
</Button>
</HStack>
{error && (
<Text color="red.500" fontSize="sm">
{error}
</Text>
)}
<Table variant="simple" size="sm">
<Thead>
<Tr>
<Th color="gray.300">Type</Th>
<Th color="gray.300">Status</Th>
<Th color="gray.300">Progress</Th>
<Th color="gray.300">Duration</Th>
<Th color="gray.300">Message</Th>
</Tr>
</Thead>
<Tbody>
{jobs.map(job => (
<Tr key={job.jobId}>
<Td>
<Text fontSize="sm" color="gray.100">
{job.type === 's3-sync' ? 'S3 Sync' : 'Song Matching'}
</Text>
</Td>
<Td>
<HStack spacing={2}>
{getStatusIcon(job.status)}
<Badge colorScheme={getStatusColor(job.status)} size="sm">
{job.status}
</Badge>
</HStack>
</Td>
<Td>
<Text fontSize="sm" color="gray.100">{job.progress}%</Text>
</Td>
<Td>
<Text fontSize="sm" color="gray.100">
{formatDuration(job.startTime, job.endTime)}
</Text>
</Td>
<Td>
<Text fontSize="sm" noOfLines={2} color="gray.100">
{job.message}
</Text>
</Td>
</Tr>
))}
</Tbody>
</Table>
{jobs.length === 0 && !loading && (
<Text textAlign="center" color="gray.500" py={4}>
No background jobs found
</Text>
)}
</VStack>
</ModalBody>
</ModalContent>
</Modal>
</>
);
};

15
packages/frontend/src/components/PlaylistManager.tsx
View File

@ -16,11 +16,13 @@ import {
MenuButton,
MenuList,
MenuItem,
Text,
HStack,
Collapse,
MenuDivider,
MenuGroup,
Icon
Icon,
Badge
} from "@chakra-ui/react";
import { ChevronDownIcon, DeleteIcon, ChevronRightIcon, AddIcon, ViewIcon } from "@chakra-ui/icons";
import React, { useState, useCallback } from "react";
@ -207,7 +209,14 @@ const PlaylistItem: React.FC<PlaylistItemProps> = React.memo(({
borderRadius="1px"
/>
)}
{node.name}
<Flex align="center" justify="space-between" w="100%">
<Text>{node.name}</Text>
{((node.trackCount || 0) + (node.tracks?.length || 0)) > 0 && (
<Badge size="sm" colorScheme="blue" variant="subtle" ml={2}>
{node.trackCount || node.tracks?.length || 0}
</Badge>
)}
</Flex>
</Button>
<Menu>
<MenuButton

2
packages/frontend/src/components/PlaylistSelectionModal.tsx
View File

@ -142,7 +142,7 @@ export const PlaylistSelectionModal: React.FC<PlaylistSelectionModalProps> = ({
{playlist.name}
</Text>
<Text fontSize="sm" color="gray.400">
{playlist.tracks?.length || 0} tracks
{playlist.trackCount || playlist.tracks?.length || 0} tracks
</Text>
</VStack>
</HStack>

2
packages/frontend/src/hooks/usePaginatedSongs.ts
View File

@ -9,7 +9,7 @@ interface UsePaginatedSongsOptions {
}
export const usePaginatedSongs = (options: UsePaginatedSongsOptions = {}) => {
const { pageSize = 50, initialSearch = '', playlistName } = options;
const { pageSize = 100, initialSearch = '', playlistName } = options;
const [songs, setSongs] = useState<Song[]>([]);
const [loading, setLoading] = useState(false);

33
packages/frontend/src/services/api.ts
View File

@ -26,7 +26,7 @@ class Api {
}
// New paginated method for all songs
async getSongsPaginated(page: number = 1, limit: number = 50, search: string = ''): Promise<SongsResponse> {
async getSongsPaginated(page: number = 1, limit: number = 100, search: string = ''): Promise<SongsResponse> {
const params = new URLSearchParams({
page: page.toString(),
limit: limit.toString(),
@ -39,7 +39,7 @@ class Api {
}
// New paginated method for playlist songs
async getPlaylistSongsPaginated(playlistName: string, page: number = 1, limit: number = 50, search: string = ''): Promise<SongsResponse> {
async getPlaylistSongsPaginated(playlistName: string, page: number = 1, limit: number = 100, search: string = ''): Promise<SongsResponse> {
const params = new URLSearchParams({
page: page.toString(),
limit: limit.toString(),
@ -113,6 +113,35 @@ class Api {
throw error;
}
}
// Background job methods
async startBackgroundJob(type: 's3-sync' | 'song-matching', options?: any): Promise<{ jobId: string; type: string }> {
const response = await fetch(`${API_BASE_URL}/background-jobs/start`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ type, options }),
});
if (!response.ok) throw new Error('Failed to start background job');
return response.json();
}
async getJobProgress(jobId: string): Promise<any> {
const response = await fetch(`${API_BASE_URL}/background-jobs/progress/${jobId}`);
if (!response.ok) throw new Error('Failed to get job progress');
return response.json();
}
async getAllJobs(): Promise<any[]> {
const response = await fetch(`${API_BASE_URL}/background-jobs/jobs`);
if (!response.ok) throw new Error('Failed to get jobs');
const data = await response.json();
return data.jobs;
}
}
export const api = new Api();

1
packages/frontend/src/types/interfaces.ts
View File

@ -71,6 +71,7 @@ export interface PlaylistNode {
type: 'folder' | 'playlist';
children?: PlaylistNode[];
tracks?: string[];
trackCount?: number;
}
// Keep the old Playlist interface for backward compatibility during transition