chore(refactor): remove unused files (TwoPhaseSyncService, unused frontend types)
This commit is contained in:
Geert Rademakes
parent
f6ecd07d98
commit
940469ba52
@ -1,275 +0,0 @@
|
|||||||
import { S3Service } from './s3Service.js';
|
|
||||||
import { AudioMetadataService } from './audioMetadataService.js';
|
|
||||||
import { SongMatchingService } from './songMatchingService.js';
|
|
||||||
import { MusicFile } from '../models/MusicFile.js';
|
|
||||||
import { Song } from '../models/Song.js';
|
|
||||||
|
|
||||||
export interface SyncResult {
|
|
||||||
phase1: {
|
|
||||||
totalFiles: number;
|
|
||||||
quickMatches: number;
|
|
||||||
unmatchedFiles: number;
|
|
||||||
errors: number;
|
|
||||||
};
|
|
||||||
phase2: {
|
|
||||||
processedFiles: number;
|
|
||||||
complexMatches: number;
|
|
||||||
stillUnmatched: number;
|
|
||||||
errors: number;
|
|
||||||
};
|
|
||||||
total: {
|
|
||||||
processed: number;
|
|
||||||
matched: number;
|
|
||||||
unmatched: number;
|
|
||||||
errors: number;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export class TwoPhaseSyncService {
|
|
||||||
private s3Service: S3Service;
|
|
||||||
private audioMetadataService: AudioMetadataService;
|
|
||||||
private songMatchingService: SongMatchingService;
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
this.audioMetadataService = new AudioMetadataService();
|
|
||||||
this.songMatchingService = new SongMatchingService();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize S3 service
|
|
||||||
*/
|
|
||||||
async initialize(): Promise<void> {
|
|
||||||
this.s3Service = await S3Service.createFromConfig();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract filename from S3 key
|
|
||||||
*/
|
|
||||||
private getFilenameFromS3Key(s3Key: string): string {
|
|
||||||
return s3Key.split('/').pop() || s3Key;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract filename from Rekordbox location path
|
|
||||||
*/
|
|
||||||
private getFilenameFromLocation(location: string): string {
|
|
||||||
// Handle both Windows and Unix paths
|
|
||||||
const normalizedPath = location.replace(/\\/g, '/');
|
|
||||||
return normalizedPath.split('/').pop() || location;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Normalize filename for comparison (remove extension, lowercase)
|
|
||||||
*/
|
|
||||||
private normalizeFilename(filename: string): string {
|
|
||||||
return filename.replace(/\.[^/.]+$/, '').toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Phase 1: Quick filename-based matching
|
|
||||||
*/
|
|
||||||
async phase1QuickMatch(): Promise<{
|
|
||||||
quickMatches: MusicFile[];
|
|
||||||
unmatchedFiles: any[];
|
|
||||||
errors: any[];
|
|
||||||
}> {
|
|
||||||
console.log('🚀 Starting Phase 1: Quick filename-based matching...');
|
|
||||||
|
|
||||||
// Get all S3 files
|
|
||||||
const s3Files = await this.s3Service.listAllFiles();
|
|
||||||
const audioFiles = s3Files.filter(s3File => {
|
|
||||||
const filename = this.getFilenameFromS3Key(s3File.key);
|
|
||||||
return this.audioMetadataService.isAudioFile(filename);
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`📁 Found ${audioFiles.length} audio files in S3`);
|
|
||||||
|
|
||||||
// Get existing music files to avoid duplicates
|
|
||||||
const existingFiles = await MusicFile.find({}, { s3Key: 1 });
|
|
||||||
const existingS3Keys = new Set(existingFiles.map(f => f.s3Key));
|
|
||||||
const newAudioFiles = audioFiles.filter(s3File => !existingS3Keys.has(s3File.key));
|
|
||||||
|
|
||||||
console.log(`🆕 Found ${newAudioFiles.length} new audio files to process`);
|
|
||||||
|
|
||||||
// Get all songs from database for filename matching
|
|
||||||
const allSongs = await Song.find({}, { id: 1, title: 1, artist: 1, location: 1 });
|
|
||||||
console.log(`🎵 Found ${allSongs.length} songs in database for matching`);
|
|
||||||
|
|
||||||
const quickMatches: MusicFile[] = [];
|
|
||||||
const unmatchedFiles: any[] = [];
|
|
||||||
const errors: any[] = [];
|
|
||||||
|
|
||||||
for (const s3File of newAudioFiles) {
|
|
||||||
try {
|
|
||||||
const s3Filename = this.getFilenameFromS3Key(s3File.key);
|
|
||||||
const normalizedS3Filename = this.normalizeFilename(s3Filename);
|
|
||||||
|
|
||||||
// Try to find exact filename match in Rekordbox songs
|
|
||||||
let matchedSong = null;
|
|
||||||
|
|
||||||
for (const song of allSongs) {
|
|
||||||
if (song.location) {
|
|
||||||
const rekordboxFilename = this.getFilenameFromLocation(song.location);
|
|
||||||
const normalizedRekordboxFilename = this.normalizeFilename(rekordboxFilename);
|
|
||||||
|
|
||||||
if (normalizedS3Filename === normalizedRekordboxFilename) {
|
|
||||||
matchedSong = song;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (matchedSong) {
|
|
||||||
console.log(`✅ Quick match found: ${s3Filename} -> ${matchedSong.title}`);
|
|
||||||
|
|
||||||
// Extract basic metadata from filename (no need to download file)
|
|
||||||
const basicMetadata = this.audioMetadataService.extractBasicMetadataFromFilename(s3Filename);
|
|
||||||
|
|
||||||
const musicFile = new MusicFile({
|
|
||||||
originalName: s3Filename,
|
|
||||||
s3Key: s3File.key,
|
|
||||||
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
|
||||||
contentType: 'audio/mpeg',
|
|
||||||
size: s3File.size,
|
|
||||||
...basicMetadata,
|
|
||||||
songId: matchedSong._id, // Link to the matched song
|
|
||||||
});
|
|
||||||
|
|
||||||
quickMatches.push(musicFile);
|
|
||||||
} else {
|
|
||||||
console.log(`❓ No quick match for: ${s3Filename}`);
|
|
||||||
unmatchedFiles.push(s3File);
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`❌ Error processing ${s3File.key}:`, error);
|
|
||||||
errors.push({ file: s3File, error: error.message });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`✅ Phase 1 completed: ${quickMatches.length} quick matches, ${unmatchedFiles.length} unmatched files`);
|
|
||||||
|
|
||||||
return { quickMatches, unmatchedFiles, errors };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Phase 2: Complex matching for unmatched files
|
|
||||||
*/
|
|
||||||
async phase2ComplexMatch(unmatchedFiles: any[]): Promise<{
|
|
||||||
processedFiles: MusicFile[];
|
|
||||||
complexMatches: number;
|
|
||||||
stillUnmatched: number;
|
|
||||||
errors: any[];
|
|
||||||
}> {
|
|
||||||
console.log('🔍 Starting Phase 2: Complex matching for unmatched files...');
|
|
||||||
|
|
||||||
const processedFiles: MusicFile[] = [];
|
|
||||||
let complexMatches = 0;
|
|
||||||
let stillUnmatched = 0;
|
|
||||||
const errors: any[] = [];
|
|
||||||
|
|
||||||
for (const s3File of unmatchedFiles) {
|
|
||||||
try {
|
|
||||||
const filename = this.getFilenameFromS3Key(s3File.key);
|
|
||||||
console.log(`🔍 Processing unmatched file: ${filename}`);
|
|
||||||
|
|
||||||
// Download file and extract metadata
|
|
||||||
const fileBuffer = await this.s3Service.getFileContent(s3File.key);
|
|
||||||
const metadata = await this.audioMetadataService.extractMetadata(fileBuffer, filename);
|
|
||||||
|
|
||||||
const musicFile = new MusicFile({
|
|
||||||
originalName: filename,
|
|
||||||
s3Key: s3File.key,
|
|
||||||
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
|
||||||
contentType: 'audio/mpeg',
|
|
||||||
size: s3File.size,
|
|
||||||
...metadata,
|
|
||||||
});
|
|
||||||
|
|
||||||
processedFiles.push(musicFile);
|
|
||||||
|
|
||||||
// Try complex matching
|
|
||||||
const matchResult = await this.songMatchingService.matchMusicFileToSongs(musicFile, {
|
|
||||||
minConfidence: 0.7,
|
|
||||||
enableFuzzyMatching: true,
|
|
||||||
enablePartialMatching: true,
|
|
||||||
maxResults: 1
|
|
||||||
});
|
|
||||||
|
|
||||||
if (matchResult.length > 0 && matchResult[0].confidence >= 0.7) {
|
|
||||||
const bestMatch = matchResult[0];
|
|
||||||
musicFile.songId = bestMatch.song._id;
|
|
||||||
complexMatches++;
|
|
||||||
console.log(`✅ Complex match found: ${filename} -> ${bestMatch.song.title} (${bestMatch.confidence})`);
|
|
||||||
} else {
|
|
||||||
stillUnmatched++;
|
|
||||||
console.log(`❓ No complex match for: ${filename}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`❌ Error processing ${s3File.key}:`, error);
|
|
||||||
errors.push({ file: s3File, error: error.message });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`✅ Phase 2 completed: ${complexMatches} complex matches, ${stillUnmatched} still unmatched`);
|
|
||||||
|
|
||||||
return { processedFiles, complexMatches, stillUnmatched, errors };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run complete two-phase sync
|
|
||||||
*/
|
|
||||||
async runTwoPhaseSync(): Promise<SyncResult> {
|
|
||||||
console.log('🎯 Starting Two-Phase S3 Sync...');
|
|
||||||
const startTime = Date.now();
|
|
||||||
|
|
||||||
await this.initialize();
|
|
||||||
|
|
||||||
// Phase 1: Quick filename matching
|
|
||||||
const phase1Result = await this.phase1QuickMatch();
|
|
||||||
|
|
||||||
// Phase 2: Complex matching for unmatched files
|
|
||||||
const phase2Result = await this.phase2ComplexMatch(phase1Result.unmatchedFiles);
|
|
||||||
|
|
||||||
// Combine all music files
|
|
||||||
const allMusicFiles = [...phase1Result.quickMatches, ...phase2Result.processedFiles];
|
|
||||||
|
|
||||||
// Batch save all music files
|
|
||||||
if (allMusicFiles.length > 0) {
|
|
||||||
console.log(`💾 Saving ${allMusicFiles.length} music files to database...`);
|
|
||||||
await MusicFile.insertMany(allMusicFiles);
|
|
||||||
}
|
|
||||||
|
|
||||||
const endTime = Date.now();
|
|
||||||
const duration = ((endTime - startTime) / 1000).toFixed(2);
|
|
||||||
|
|
||||||
const result: SyncResult = {
|
|
||||||
phase1: {
|
|
||||||
totalFiles: phase1Result.quickMatches.length + phase1Result.unmatchedFiles.length,
|
|
||||||
quickMatches: phase1Result.quickMatches.length,
|
|
||||||
unmatchedFiles: phase1Result.unmatchedFiles.length,
|
|
||||||
errors: phase1Result.errors.length,
|
|
||||||
},
|
|
||||||
phase2: {
|
|
||||||
processedFiles: phase2Result.processedFiles.length,
|
|
||||||
complexMatches: phase2Result.complexMatches,
|
|
||||||
stillUnmatched: phase2Result.stillUnmatched,
|
|
||||||
errors: phase2Result.errors.length,
|
|
||||||
},
|
|
||||||
total: {
|
|
||||||
processed: allMusicFiles.length,
|
|
||||||
matched: phase1Result.quickMatches.length + phase2Result.complexMatches,
|
|
||||||
unmatched: phase2Result.stillUnmatched,
|
|
||||||
errors: phase1Result.errors.length + phase2Result.errors.length,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log(`🎉 Two-Phase Sync completed in ${duration}s:`);
|
|
||||||
console.log(` Phase 1: ${result.phase1.quickMatches} quick matches, ${result.phase1.unmatchedFiles} unmatched`);
|
|
||||||
console.log(` Phase 2: ${result.phase2.complexMatches} complex matches, ${result.phase2.stillUnmatched} still unmatched`);
|
|
||||||
console.log(` Total: ${result.total.processed} processed, ${result.total.matched} matched, ${result.total.unmatched} unmatched`);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,7 +0,0 @@
|
|||||||
export interface Playlist {
|
|
||||||
_id: string;
|
|
||||||
name: string;
|
|
||||||
songs: string[];
|
|
||||||
createdAt: string;
|
|
||||||
updatedAt: string;
|
|
||||||
}
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
export interface Song {
|
|
||||||
_id: string;
|
|
||||||
title: string;
|
|
||||||
artist: string;
|
|
||||||
genre: string;
|
|
||||||
bpm: number;
|
|
||||||
key: string;
|
|
||||||
rating: number;
|
|
||||||
comments: string;
|
|
||||||
createdAt: string;
|
|
||||||
updatedAt: string;
|
|
||||||
}
|
|
||||||
Loading…
x
Reference in New Issue
Block a user