chore(refactor): remove unused files (TwoPhaseSyncService, unused frontend types)
This commit is contained in:
Geert Rademakes
parent
f6ecd07d98
commit
940469ba52
@ -1,275 +0,0 @@
|
||||
import { S3Service } from './s3Service.js';
|
||||
import { AudioMetadataService } from './audioMetadataService.js';
|
||||
import { SongMatchingService } from './songMatchingService.js';
|
||||
import { MusicFile } from '../models/MusicFile.js';
|
||||
import { Song } from '../models/Song.js';
|
||||
|
||||
export interface SyncResult {
|
||||
phase1: {
|
||||
totalFiles: number;
|
||||
quickMatches: number;
|
||||
unmatchedFiles: number;
|
||||
errors: number;
|
||||
};
|
||||
phase2: {
|
||||
processedFiles: number;
|
||||
complexMatches: number;
|
||||
stillUnmatched: number;
|
||||
errors: number;
|
||||
};
|
||||
total: {
|
||||
processed: number;
|
||||
matched: number;
|
||||
unmatched: number;
|
||||
errors: number;
|
||||
};
|
||||
}
|
||||
|
||||
export class TwoPhaseSyncService {
|
||||
private s3Service: S3Service;
|
||||
private audioMetadataService: AudioMetadataService;
|
||||
private songMatchingService: SongMatchingService;
|
||||
|
||||
constructor() {
|
||||
this.audioMetadataService = new AudioMetadataService();
|
||||
this.songMatchingService = new SongMatchingService();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize S3 service
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
this.s3Service = await S3Service.createFromConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filename from S3 key
|
||||
*/
|
||||
private getFilenameFromS3Key(s3Key: string): string {
|
||||
return s3Key.split('/').pop() || s3Key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filename from Rekordbox location path
|
||||
*/
|
||||
private getFilenameFromLocation(location: string): string {
|
||||
// Handle both Windows and Unix paths
|
||||
const normalizedPath = location.replace(/\\/g, '/');
|
||||
return normalizedPath.split('/').pop() || location;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize filename for comparison (remove extension, lowercase)
|
||||
*/
|
||||
private normalizeFilename(filename: string): string {
|
||||
return filename.replace(/\.[^/.]+$/, '').toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 1: Quick filename-based matching
|
||||
*/
|
||||
async phase1QuickMatch(): Promise<{
|
||||
quickMatches: MusicFile[];
|
||||
unmatchedFiles: any[];
|
||||
errors: any[];
|
||||
}> {
|
||||
console.log('🚀 Starting Phase 1: Quick filename-based matching...');
|
||||
|
||||
// Get all S3 files
|
||||
const s3Files = await this.s3Service.listAllFiles();
|
||||
const audioFiles = s3Files.filter(s3File => {
|
||||
const filename = this.getFilenameFromS3Key(s3File.key);
|
||||
return this.audioMetadataService.isAudioFile(filename);
|
||||
});
|
||||
|
||||
console.log(`📁 Found ${audioFiles.length} audio files in S3`);
|
||||
|
||||
// Get existing music files to avoid duplicates
|
||||
const existingFiles = await MusicFile.find({}, { s3Key: 1 });
|
||||
const existingS3Keys = new Set(existingFiles.map(f => f.s3Key));
|
||||
const newAudioFiles = audioFiles.filter(s3File => !existingS3Keys.has(s3File.key));
|
||||
|
||||
console.log(`🆕 Found ${newAudioFiles.length} new audio files to process`);
|
||||
|
||||
// Get all songs from database for filename matching
|
||||
const allSongs = await Song.find({}, { id: 1, title: 1, artist: 1, location: 1 });
|
||||
console.log(`🎵 Found ${allSongs.length} songs in database for matching`);
|
||||
|
||||
const quickMatches: MusicFile[] = [];
|
||||
const unmatchedFiles: any[] = [];
|
||||
const errors: any[] = [];
|
||||
|
||||
for (const s3File of newAudioFiles) {
|
||||
try {
|
||||
const s3Filename = this.getFilenameFromS3Key(s3File.key);
|
||||
const normalizedS3Filename = this.normalizeFilename(s3Filename);
|
||||
|
||||
// Try to find exact filename match in Rekordbox songs
|
||||
let matchedSong = null;
|
||||
|
||||
for (const song of allSongs) {
|
||||
if (song.location) {
|
||||
const rekordboxFilename = this.getFilenameFromLocation(song.location);
|
||||
const normalizedRekordboxFilename = this.normalizeFilename(rekordboxFilename);
|
||||
|
||||
if (normalizedS3Filename === normalizedRekordboxFilename) {
|
||||
matchedSong = song;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedSong) {
|
||||
console.log(`✅ Quick match found: ${s3Filename} -> ${matchedSong.title}`);
|
||||
|
||||
// Extract basic metadata from filename (no need to download file)
|
||||
const basicMetadata = this.audioMetadataService.extractBasicMetadataFromFilename(s3Filename);
|
||||
|
||||
const musicFile = new MusicFile({
|
||||
originalName: s3Filename,
|
||||
s3Key: s3File.key,
|
||||
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
||||
contentType: 'audio/mpeg',
|
||||
size: s3File.size,
|
||||
...basicMetadata,
|
||||
songId: matchedSong._id, // Link to the matched song
|
||||
});
|
||||
|
||||
quickMatches.push(musicFile);
|
||||
} else {
|
||||
console.log(`❓ No quick match for: ${s3Filename}`);
|
||||
unmatchedFiles.push(s3File);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing ${s3File.key}:`, error);
|
||||
errors.push({ file: s3File, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Phase 1 completed: ${quickMatches.length} quick matches, ${unmatchedFiles.length} unmatched files`);
|
||||
|
||||
return { quickMatches, unmatchedFiles, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 2: Complex matching for unmatched files
|
||||
*/
|
||||
async phase2ComplexMatch(unmatchedFiles: any[]): Promise<{
|
||||
processedFiles: MusicFile[];
|
||||
complexMatches: number;
|
||||
stillUnmatched: number;
|
||||
errors: any[];
|
||||
}> {
|
||||
console.log('🔍 Starting Phase 2: Complex matching for unmatched files...');
|
||||
|
||||
const processedFiles: MusicFile[] = [];
|
||||
let complexMatches = 0;
|
||||
let stillUnmatched = 0;
|
||||
const errors: any[] = [];
|
||||
|
||||
for (const s3File of unmatchedFiles) {
|
||||
try {
|
||||
const filename = this.getFilenameFromS3Key(s3File.key);
|
||||
console.log(`🔍 Processing unmatched file: ${filename}`);
|
||||
|
||||
// Download file and extract metadata
|
||||
const fileBuffer = await this.s3Service.getFileContent(s3File.key);
|
||||
const metadata = await this.audioMetadataService.extractMetadata(fileBuffer, filename);
|
||||
|
||||
const musicFile = new MusicFile({
|
||||
originalName: filename,
|
||||
s3Key: s3File.key,
|
||||
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
||||
contentType: 'audio/mpeg',
|
||||
size: s3File.size,
|
||||
...metadata,
|
||||
});
|
||||
|
||||
processedFiles.push(musicFile);
|
||||
|
||||
// Try complex matching
|
||||
const matchResult = await this.songMatchingService.matchMusicFileToSongs(musicFile, {
|
||||
minConfidence: 0.7,
|
||||
enableFuzzyMatching: true,
|
||||
enablePartialMatching: true,
|
||||
maxResults: 1
|
||||
});
|
||||
|
||||
if (matchResult.length > 0 && matchResult[0].confidence >= 0.7) {
|
||||
const bestMatch = matchResult[0];
|
||||
musicFile.songId = bestMatch.song._id;
|
||||
complexMatches++;
|
||||
console.log(`✅ Complex match found: ${filename} -> ${bestMatch.song.title} (${bestMatch.confidence})`);
|
||||
} else {
|
||||
stillUnmatched++;
|
||||
console.log(`❓ No complex match for: ${filename}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing ${s3File.key}:`, error);
|
||||
errors.push({ file: s3File, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Phase 2 completed: ${complexMatches} complex matches, ${stillUnmatched} still unmatched`);
|
||||
|
||||
return { processedFiles, complexMatches, stillUnmatched, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run complete two-phase sync
|
||||
*/
|
||||
async runTwoPhaseSync(): Promise<SyncResult> {
|
||||
console.log('🎯 Starting Two-Phase S3 Sync...');
|
||||
const startTime = Date.now();
|
||||
|
||||
await this.initialize();
|
||||
|
||||
// Phase 1: Quick filename matching
|
||||
const phase1Result = await this.phase1QuickMatch();
|
||||
|
||||
// Phase 2: Complex matching for unmatched files
|
||||
const phase2Result = await this.phase2ComplexMatch(phase1Result.unmatchedFiles);
|
||||
|
||||
// Combine all music files
|
||||
const allMusicFiles = [...phase1Result.quickMatches, ...phase2Result.processedFiles];
|
||||
|
||||
// Batch save all music files
|
||||
if (allMusicFiles.length > 0) {
|
||||
console.log(`💾 Saving ${allMusicFiles.length} music files to database...`);
|
||||
await MusicFile.insertMany(allMusicFiles);
|
||||
}
|
||||
|
||||
const endTime = Date.now();
|
||||
const duration = ((endTime - startTime) / 1000).toFixed(2);
|
||||
|
||||
const result: SyncResult = {
|
||||
phase1: {
|
||||
totalFiles: phase1Result.quickMatches.length + phase1Result.unmatchedFiles.length,
|
||||
quickMatches: phase1Result.quickMatches.length,
|
||||
unmatchedFiles: phase1Result.unmatchedFiles.length,
|
||||
errors: phase1Result.errors.length,
|
||||
},
|
||||
phase2: {
|
||||
processedFiles: phase2Result.processedFiles.length,
|
||||
complexMatches: phase2Result.complexMatches,
|
||||
stillUnmatched: phase2Result.stillUnmatched,
|
||||
errors: phase2Result.errors.length,
|
||||
},
|
||||
total: {
|
||||
processed: allMusicFiles.length,
|
||||
matched: phase1Result.quickMatches.length + phase2Result.complexMatches,
|
||||
unmatched: phase2Result.stillUnmatched,
|
||||
errors: phase1Result.errors.length + phase2Result.errors.length,
|
||||
},
|
||||
};
|
||||
|
||||
console.log(`🎉 Two-Phase Sync completed in ${duration}s:`);
|
||||
console.log(` Phase 1: ${result.phase1.quickMatches} quick matches, ${result.phase1.unmatchedFiles} unmatched`);
|
||||
console.log(` Phase 2: ${result.phase2.complexMatches} complex matches, ${result.phase2.stillUnmatched} still unmatched`);
|
||||
console.log(` Total: ${result.total.processed} processed, ${result.total.matched} matched, ${result.total.unmatched} unmatched`);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
export interface Playlist {
|
||||
_id: string;
|
||||
name: string;
|
||||
songs: string[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
export interface Song {
|
||||
_id: string;
|
||||
title: string;
|
||||
artist: string;
|
||||
genre: string;
|
||||
bpm: number;
|
||||
key: string;
|
||||
rating: number;
|
||||
comments: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user