import { S3Service } from './s3Service.js'; import { AudioMetadataService } from './audioMetadataService.js'; import { SongMatchingService } from './songMatchingService.js'; import { MusicFile } from '../models/MusicFile.js'; import { Song } from '../models/Song.js'; export interface SyncResult { phase1: { totalFiles: number; quickMatches: number; unmatchedFiles: number; errors: number; }; phase2: { processedFiles: number; complexMatches: number; stillUnmatched: number; errors: number; }; total: { processed: number; matched: number; unmatched: number; errors: number; }; } export class TwoPhaseSyncService { private s3Service: S3Service; private audioMetadataService: AudioMetadataService; private songMatchingService: SongMatchingService; constructor() { this.audioMetadataService = new AudioMetadataService(); this.songMatchingService = new SongMatchingService(); } /** * Initialize S3 service */ async initialize(): Promise { this.s3Service = await S3Service.createFromConfig(); } /** * Extract filename from S3 key */ private getFilenameFromS3Key(s3Key: string): string { return s3Key.split('/').pop() || s3Key; } /** * Extract filename from Rekordbox location path */ private getFilenameFromLocation(location: string): string { // Handle both Windows and Unix paths const normalizedPath = location.replace(/\\/g, '/'); return normalizedPath.split('/').pop() || location; } /** * Normalize filename for comparison (remove extension, lowercase) */ private normalizeFilename(filename: string): string { return filename.replace(/\.[^/.]+$/, '').toLowerCase(); } /** * Phase 1: Quick filename-based matching */ async phase1QuickMatch(): Promise<{ quickMatches: MusicFile[]; unmatchedFiles: any[]; errors: any[]; }> { console.log('🚀 Starting Phase 1: Quick filename-based matching...'); // Get all S3 files const s3Files = await this.s3Service.listAllFiles(); const audioFiles = s3Files.filter(s3File => { const filename = this.getFilenameFromS3Key(s3File.key); return this.audioMetadataService.isAudioFile(filename); }); console.log(`📁 Found ${audioFiles.length} audio files in S3`); // Get existing music files to avoid duplicates const existingFiles = await MusicFile.find({}, { s3Key: 1 }); const existingS3Keys = new Set(existingFiles.map(f => f.s3Key)); const newAudioFiles = audioFiles.filter(s3File => !existingS3Keys.has(s3File.key)); console.log(`🆕 Found ${newAudioFiles.length} new audio files to process`); // Get all songs from database for filename matching const allSongs = await Song.find({}, { id: 1, title: 1, artist: 1, location: 1 }); console.log(`🎵 Found ${allSongs.length} songs in database for matching`); const quickMatches: MusicFile[] = []; const unmatchedFiles: any[] = []; const errors: any[] = []; for (const s3File of newAudioFiles) { try { const s3Filename = this.getFilenameFromS3Key(s3File.key); const normalizedS3Filename = this.normalizeFilename(s3Filename); // Try to find exact filename match in Rekordbox songs let matchedSong = null; for (const song of allSongs) { if (song.location) { const rekordboxFilename = this.getFilenameFromLocation(song.location); const normalizedRekordboxFilename = this.normalizeFilename(rekordboxFilename); if (normalizedS3Filename === normalizedRekordboxFilename) { matchedSong = song; break; } } } if (matchedSong) { console.log(`✅ Quick match found: ${s3Filename} -> ${matchedSong.title}`); // Extract basic metadata from filename (no need to download file) const basicMetadata = this.audioMetadataService.extractBasicMetadataFromFilename(s3Filename); const musicFile = new MusicFile({ originalName: s3Filename, s3Key: s3File.key, s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`, contentType: 'audio/mpeg', size: s3File.size, ...basicMetadata, songId: matchedSong._id, // Link to the matched song }); quickMatches.push(musicFile); } else { console.log(`❓ No quick match for: ${s3Filename}`); unmatchedFiles.push(s3File); } } catch (error) { console.error(`❌ Error processing ${s3File.key}:`, error); errors.push({ file: s3File, error: error.message }); } } console.log(`✅ Phase 1 completed: ${quickMatches.length} quick matches, ${unmatchedFiles.length} unmatched files`); return { quickMatches, unmatchedFiles, errors }; } /** * Phase 2: Complex matching for unmatched files */ async phase2ComplexMatch(unmatchedFiles: any[]): Promise<{ processedFiles: MusicFile[]; complexMatches: number; stillUnmatched: number; errors: any[]; }> { console.log('🔍 Starting Phase 2: Complex matching for unmatched files...'); const processedFiles: MusicFile[] = []; let complexMatches = 0; let stillUnmatched = 0; const errors: any[] = []; for (const s3File of unmatchedFiles) { try { const filename = this.getFilenameFromS3Key(s3File.key); console.log(`🔍 Processing unmatched file: ${filename}`); // Download file and extract metadata const fileBuffer = await this.s3Service.getFileContent(s3File.key); const metadata = await this.audioMetadataService.extractMetadata(fileBuffer, filename); const musicFile = new MusicFile({ originalName: filename, s3Key: s3File.key, s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`, contentType: 'audio/mpeg', size: s3File.size, ...metadata, }); processedFiles.push(musicFile); // Try complex matching const matchResult = await this.songMatchingService.matchMusicFileToSongs(musicFile, { minConfidence: 0.7, enableFuzzyMatching: true, enablePartialMatching: true, maxResults: 1 }); if (matchResult.length > 0 && matchResult[0].confidence >= 0.7) { const bestMatch = matchResult[0]; musicFile.songId = bestMatch.song._id; complexMatches++; console.log(`✅ Complex match found: ${filename} -> ${bestMatch.song.title} (${bestMatch.confidence})`); } else { stillUnmatched++; console.log(`❓ No complex match for: ${filename}`); } } catch (error) { console.error(`❌ Error processing ${s3File.key}:`, error); errors.push({ file: s3File, error: error.message }); } } console.log(`✅ Phase 2 completed: ${complexMatches} complex matches, ${stillUnmatched} still unmatched`); return { processedFiles, complexMatches, stillUnmatched, errors }; } /** * Run complete two-phase sync */ async runTwoPhaseSync(): Promise { console.log('🎯 Starting Two-Phase S3 Sync...'); const startTime = Date.now(); await this.initialize(); // Phase 1: Quick filename matching const phase1Result = await this.phase1QuickMatch(); // Phase 2: Complex matching for unmatched files const phase2Result = await this.phase2ComplexMatch(phase1Result.unmatchedFiles); // Combine all music files const allMusicFiles = [...phase1Result.quickMatches, ...phase2Result.processedFiles]; // Batch save all music files if (allMusicFiles.length > 0) { console.log(`💾 Saving ${allMusicFiles.length} music files to database...`); await MusicFile.insertMany(allMusicFiles); } const endTime = Date.now(); const duration = ((endTime - startTime) / 1000).toFixed(2); const result: SyncResult = { phase1: { totalFiles: phase1Result.quickMatches.length + phase1Result.unmatchedFiles.length, quickMatches: phase1Result.quickMatches.length, unmatchedFiles: phase1Result.unmatchedFiles.length, errors: phase1Result.errors.length, }, phase2: { processedFiles: phase2Result.processedFiles.length, complexMatches: phase2Result.complexMatches, stillUnmatched: phase2Result.stillUnmatched, errors: phase2Result.errors.length, }, total: { processed: allMusicFiles.length, matched: phase1Result.quickMatches.length + phase2Result.complexMatches, unmatched: phase2Result.stillUnmatched, errors: phase1Result.errors.length + phase2Result.errors.length, }, }; console.log(`🎉 Two-Phase Sync completed in ${duration}s:`); console.log(` Phase 1: ${result.phase1.quickMatches} quick matches, ${result.phase1.unmatchedFiles} unmatched`); console.log(` Phase 2: ${result.phase2.complexMatches} complex matches, ${result.phase2.stillUnmatched} still unmatched`); console.log(` Total: ${result.total.processed} processed, ${result.total.matched} matched, ${result.total.unmatched} unmatched`); return result; } }