master/rekordbox-viewer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve S3 sync and song matching with better FLAC support, NaN validation, and enhanced logging

Browse Source 
- Enhanced AudioMetadataService with comprehensive NaN handling for all numeric fields
- Added validation for year, duration, bitrate, sampleRate, and channels
- Improved FLAC format detection and error handling with fallback support
- Added detailed logging for S3 sync process with progress tracking and file statistics
- Enhanced song matching service with progress indicators and confidence scoring
- Added comprehensive logging for auto-match and link operations
- Improved error handling and graceful degradation for metadata extraction
- Added test scripts for metadata service validation
- Updated S3 service to use configuration from s3-config.json file
- Added automatic S3 service reload when configuration is updated

The S3 importer now provides much better visibility into file processing
and song matching operations, making it easier to debug issues and
monitor performance. FLAC files are properly handled and invalid
metadata values are filtered out to prevent database corruption.
This commit is contained in:
Geert Rademakes 2025-08-07 17:14:57 +02:00
parent 050e31288a
commit 3cd83ff2b5
10 changed files with 480 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
packages/backend/s3-config.json
View File

@ -1,8 +1,8 @@
{ {
"endpoint": "http://localhost:9000", "endpoint": "https://garage.geertrademakers.nl",
"region": "us-east-1", "region": "garage",
"accessKeyId": "minioadmin", "accessKeyId": "GK1c1a4a30946eb1e7f8d60847",
"secretAccessKey": "minioadmin", "secretAccessKey": "2ed6673f0e3c42d347adeb54ba6b95a1ebc6414750f2a95e1d3d89758f1add63",
"bucketName": "music-files", "bucketName": "music",
"useSSL": true "useSSL": true
} }

9
packages/backend/src/routes/config.ts
View File

@ -2,6 +2,7 @@ import express from 'express';
import { S3Client, ListBucketsCommand, HeadBucketCommand } from '@aws-sdk/client-s3'; import { S3Client, ListBucketsCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
import fs from 'fs/promises'; import fs from 'fs/promises';
import path from 'path'; import path from 'path';
import { reloadS3Service } from './music.js';
const router = express.Router(); const router = express.Router();
@ -79,7 +80,13 @@ router.post('/s3', async (req, res) => {
process.env.S3_BUCKET_NAME = config.bucketName; process.env.S3_BUCKET_NAME = config.bucketName;
process.env.S3_USE_SSL = config.useSSL.toString(); process.env.S3_USE_SSL = config.useSSL.toString();
res.json({ message: 'S3 configuration saved successfully' }); // Reload S3 service with new configuration
const reloadSuccess = await reloadS3Service();
res.json({
message: 'S3 configuration saved successfully',
s3ServiceReloaded: reloadSuccess
});
} catch (error) { } catch (error) {
console.error('Error saving S3 config:', error); console.error('Error saving S3 config:', error);
res.status(500).json({ error: 'Failed to save S3 configuration' }); res.status(500).json({ error: 'Failed to save S3 configuration' });

32
packages/backend/src/routes/matching.ts
View File

@ -41,6 +41,9 @@ router.get('/music-file/:id/suggestions', async (req, res) => {
*/ */
router.get('/suggestions', async (req, res) => { router.get('/suggestions', async (req, res) => {
try { try {
console.log('🔍 Getting matching suggestions for all unmatched music files...');
const startTime = Date.now();
const options = { const options = {
minConfidence: parseFloat(req.query.minConfidence as string) || 0.3, minConfidence: parseFloat(req.query.minConfidence as string) || 0.3,
enableFuzzyMatching: req.query.enableFuzzyMatching !== 'false', enableFuzzyMatching: req.query.enableFuzzyMatching !== 'false',
@ -48,15 +51,24 @@ router.get('/suggestions', async (req, res) => {
maxResults: parseInt(req.query.maxResults as string) || 3 maxResults: parseInt(req.query.maxResults as string) || 3
}; };
console.log('⚙️ Matching options:', options);
const results = await matchingService.matchAllMusicFilesToSongs(options); const results = await matchingService.matchAllMusicFilesToSongs(options);
const endTime = Date.now();
const duration = ((endTime - startTime) / 1000).toFixed(2);
console.log(`✅ Matching suggestions completed in ${duration} seconds`);
console.log(`📊 Found suggestions for ${results.length} unmatched files`);
res.json({ res.json({
results, results,
options, options,
totalUnmatched: results.length totalUnmatched: results.length,
duration: `${duration}s`
}); });
} catch (error) { } catch (error) {
console.error('Error getting all matching suggestions:', error); console.error('❌ Error getting all matching suggestions:', error);
res.status(500).json({ error: 'Failed to get matching suggestions' }); res.status(500).json({ error: 'Failed to get matching suggestions' });
} }
}); });
@ -66,21 +78,33 @@ router.get('/suggestions', async (req, res) => {
*/ */
router.post('/auto-link', async (req, res) => { router.post('/auto-link', async (req, res) => {
try { try {
console.log('🚀 Starting auto-match and link request...');
const startTime = Date.now();
const options = { const options = {
minConfidence: parseFloat(req.body.minConfidence as string) || 0.7, minConfidence: parseFloat(req.body.minConfidence as string) || 0.7,
enableFuzzyMatching: req.body.enableFuzzyMatching !== false, enableFuzzyMatching: req.body.enableFuzzyMatching !== false,
enablePartialMatching: req.body.enablePartialMatching !== false enablePartialMatching: req.body.enablePartialMatching !== false
}; };
console.log('⚙️ Auto-linking options:', options);
const result = await matchingService.autoMatchAndLink(options); const result = await matchingService.autoMatchAndLink(options);
const endTime = Date.now();
const duration = ((endTime - startTime) / 1000).toFixed(2);
console.log(`✅ Auto-linking completed in ${duration} seconds`);
console.log(`📊 Results: ${result.linked} linked, ${result.unmatched} unmatched`);
res.json({ res.json({
message: 'Auto-linking completed', message: 'Auto-linking completed',
result, result,
options options,
duration: `${duration}s`
}); });
} catch (error) { } catch (error) {
console.error('Error during auto-linking:', error); console.error('❌ Error during auto-linking:', error);
res.status(500).json({ error: 'Failed to auto-link music files' }); res.status(500).json({ error: 'Failed to auto-link music files' });
} }
}); });

90
packages/backend/src/routes/music.ts
View File

@ -24,13 +24,43 @@ const upload = multer({
}); });
// Initialize services // Initialize services
const s3Service = new S3Service({ let s3Service: S3Service;
// Initialize S3 service with configuration from file
async function initializeS3Service() {
try {
s3Service = await S3Service.createFromConfig();
console.log('✅ S3 service initialized with configuration from s3-config.json');
} catch (error) {
console.error('❌ Failed to initialize S3 service:', error);
// Fallback to environment variables
s3Service = new S3Service({
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000', endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin', accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin', secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
bucketName: process.env.S3_BUCKET_NAME || 'music-files', bucketName: process.env.S3_BUCKET_NAME || 'music-files',
region: process.env.S3_REGION || 'us-east-1', region: process.env.S3_REGION || 'us-east-1',
}); });
console.log('⚠️ S3 service initialized with environment variables as fallback');
}
}
// Initialize S3 service on startup
initializeS3Service();
/**
* Reload S3 service with updated configuration
*/
export async function reloadS3Service() {
try {
s3Service = await S3Service.createFromConfig();
console.log('✅ S3 service reloaded with updated configuration');
return true;
} catch (error) {
console.error('❌ Failed to reload S3 service:', error);
return false;
}
}
const audioMetadataService = new AudioMetadataService(); const audioMetadataService = new AudioMetadataService();
@ -84,7 +114,7 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => {
const results = []; const results = [];
for (const file of req.files) { for (const file of req.files as Express.Multer.File[]) {
try { try {
const { buffer, originalname, mimetype } = file; const { buffer, originalname, mimetype } = file;
@ -108,7 +138,7 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => {
results.push({ success: true, musicFile }); results.push({ success: true, musicFile });
} catch (error) { } catch (error) {
console.error(`Error uploading ${file.originalname}:`, error); console.error(`Error uploading ${file.originalname}:`, error);
results.push({ success: false, fileName: file.originalname, error: error.message }); results.push({ success: false, fileName: file.originalname, error: (error as Error).message });
} }
} }
@ -140,26 +170,39 @@ router.get('/files', async (req, res) => {
*/ */
router.post('/sync-s3', async (req, res) => { router.post('/sync-s3', async (req, res) => {
try { try {
console.log('Starting S3 sync...'); console.log('🔄 Starting S3 sync process...');
const startTime = Date.now();
// Get all files from S3 recursively // Get all files from S3 recursively
console.log('📁 Fetching files from S3 bucket...');
const s3Files = await s3Service.listAllFiles(); const s3Files = await s3Service.listAllFiles();
console.log(`Found ${s3Files.length} files in S3 bucket`); console.log(`✅ Found ${s3Files.length} files in S3 bucket`);
const results = { const results = {
total: s3Files.length, total: s3Files.length,
synced: 0, synced: 0,
skipped: 0, skipped: 0,
errors: 0, errors: 0,
newFiles: 0 newFiles: 0,
audioFiles: 0,
nonAudioFiles: 0
}; };
console.log('🔍 Processing files...');
let processedCount = 0;
for (const s3File of s3Files) { for (const s3File of s3Files) {
processedCount++;
const progress = ((processedCount / s3Files.length) * 100).toFixed(1);
try { try {
console.log(`📄 [${progress}%] Processing: ${s3File.key} (${audioMetadataService.formatFileSize(s3File.size)})`);
// Check if file already exists in database // Check if file already exists in database
const existingFile = await MusicFile.findOne({ s3Key: s3File.key }); const existingFile = await MusicFile.findOne({ s3Key: s3File.key });
if (existingFile) { if (existingFile) {
console.log(`⏭️ Skipping existing file: ${s3File.key}`);
results.skipped++; results.skipped++;
continue; continue;
} }
@ -169,16 +212,24 @@ router.post('/sync-s3', async (req, res) => {
// Check if it's an audio file // Check if it's an audio file
if (!audioMetadataService.isAudioFile(filename)) { if (!audioMetadataService.isAudioFile(filename)) {
console.log(`🚫 Skipping non-audio file: ${filename}`);
results.skipped++; results.skipped++;
results.nonAudioFiles++;
continue; continue;
} }
results.audioFiles++;
console.log(`🎵 Processing audio file: ${filename}`);
// Get file content to extract metadata // Get file content to extract metadata
try { try {
console.log(`⬇️ Downloading file content: ${s3File.key}`);
const fileBuffer = await s3Service.getFileContent(s3File.key); const fileBuffer = await s3Service.getFileContent(s3File.key);
console.log(`📊 Extracting metadata: ${filename}`);
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename); const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
// Save to database // Save to database
console.log(`💾 Saving to database: ${filename}`);
const musicFile = new MusicFile({ const musicFile = new MusicFile({
originalName: filename, originalName: filename,
s3Key: s3File.key, s3Key: s3File.key,
@ -191,9 +242,12 @@ router.post('/sync-s3', async (req, res) => {
await musicFile.save(); await musicFile.save();
results.synced++; results.synced++;
results.newFiles++; results.newFiles++;
console.log(`✅ Successfully synced: ${filename}`);
} catch (metadataError) { } catch (metadataError) {
console.error(`Error extracting metadata for ${s3File.key}:`, metadataError); console.error(`❌ Error extracting metadata for ${s3File.key}:`, metadataError);
console.log(`🔄 Saving file without metadata: ${filename}`);
// Still save the file without metadata // Still save the file without metadata
const musicFile = new MusicFile({ const musicFile = new MusicFile({
originalName: filename, originalName: filename,
@ -205,22 +259,36 @@ router.post('/sync-s3', async (req, res) => {
await musicFile.save(); await musicFile.save();
results.synced++; results.synced++;
results.newFiles++; results.newFiles++;
console.log(`✅ Saved without metadata: ${filename}`);
} }
} catch (error) { } catch (error) {
console.error(`Error processing ${s3File.key}:`, error); console.error(`❌ Error processing ${s3File.key}:`, error);
results.errors++; results.errors++;
} }
} }
console.log('S3 sync completed:', results); const endTime = Date.now();
const duration = ((endTime - startTime) / 1000).toFixed(2);
console.log('🎉 S3 sync completed!');
console.log(`⏱️ Duration: ${duration} seconds`);
console.log(`📊 Results:`, results);
console.log(` Total files: ${results.total}`);
console.log(` Audio files: ${results.audioFiles}`);
console.log(` Non-audio files: ${results.nonAudioFiles}`);
console.log(` New files synced: ${results.newFiles}`);
console.log(` Files skipped: ${results.skipped}`);
console.log(` Errors: ${results.errors}`);
res.json({ res.json({
message: 'S3 sync completed', message: 'S3 sync completed',
results results,
duration: `${duration}s`
}); });
} catch (error) { } catch (error) {
console.error('S3 sync error:', error); console.error('❌ S3 sync error:', error);
res.status(500).json({ error: 'Failed to sync S3 files' }); res.status(500).json({ error: 'Failed to sync S3 files' });
} }
}); });

104
packages/backend/src/services/audioMetadataService.ts
View File

@ -54,33 +54,103 @@ export class AudioMetadataService {
return container.toUpperCase(); return container.toUpperCase();
} }
/**
* Validate and sanitize numeric values
*/
private sanitizeNumericValue(value: any, fieldName: string): number | undefined {
if (value === null || value === undefined) {
return undefined;
}
const numValue = typeof value === 'number' ? value : Number(value);
if (isNaN(numValue) || !isFinite(numValue)) {
console.warn(`⚠️ Invalid ${fieldName} value: ${value}, setting to undefined`);
return undefined;
}
// Additional validation for specific fields
if (fieldName === 'year' && (numValue < 1900 || numValue > new Date().getFullYear() + 1)) {
console.warn(`⚠️ Invalid year value: ${numValue}, setting to undefined`);
return undefined;
}
if (fieldName === 'duration' && numValue < 0) {
console.warn(`⚠️ Invalid duration value: ${numValue}, setting to undefined`);
return undefined;
}
if (fieldName === 'bitrate' && numValue < 0) {
console.warn(`⚠️ Invalid bitrate value: ${numValue}, setting to undefined`);
return undefined;
}
if (fieldName === 'sampleRate' && numValue < 0) {
console.warn(`⚠️ Invalid sampleRate value: ${numValue}, setting to undefined`);
return undefined;
}
if (fieldName === 'channels' && (numValue < 1 || numValue > 8)) {
console.warn(`⚠️ Invalid channels value: ${numValue}, setting to undefined`);
return undefined;
}
return numValue;
}
/** /**
* Extract metadata from audio file buffer * Extract metadata from audio file buffer
*/ */
async extractMetadata(fileBuffer: Buffer, fileName: string): Promise<AudioMetadata> { async extractMetadata(fileBuffer: Buffer, fileName: string): Promise<AudioMetadata> {
console.log(`🎵 Extracting metadata for: ${fileName} (${this.formatFileSize(fileBuffer.length)})`);
try { try {
const metadata = await parseBuffer(fileBuffer, fileName); const metadata = await parseBuffer(fileBuffer, fileName);
console.log(`✅ Successfully parsed metadata for ${fileName}`);
return { // Extract and sanitize metadata
title: metadata.common.title, const extractedMetadata: AudioMetadata = {
artist: metadata.common.artist, title: metadata.common.title || undefined,
album: metadata.common.album, artist: metadata.common.artist || undefined,
year: metadata.common.year, album: metadata.common.album || undefined,
genre: metadata.common.genre?.[0], year: this.sanitizeNumericValue(metadata.common.year, 'year'),
duration: metadata.format.duration, genre: metadata.common.genre?.[0] || undefined,
bitrate: metadata.format.bitrate, duration: this.sanitizeNumericValue(metadata.format.duration, 'duration'),
sampleRate: metadata.format.sampleRate, bitrate: this.sanitizeNumericValue(metadata.format.bitrate, 'bitrate'),
channels: metadata.format.numberOfChannels, sampleRate: this.sanitizeNumericValue(metadata.format.sampleRate, 'sampleRate'),
format: this.mapFormatToDisplayName(metadata.format.container, fileName), channels: this.sanitizeNumericValue(metadata.format.numberOfChannels, 'channels'),
format: this.mapFormatToDisplayName(metadata.format.container || '', fileName),
size: fileBuffer.length, size: fileBuffer.length,
}; };
// Log extracted metadata (without sensitive info)
console.log(`📊 Metadata extracted for ${fileName}:`);
console.log(` Format: ${extractedMetadata.format}`);
console.log(` Duration: ${extractedMetadata.duration ? this.formatDuration(extractedMetadata.duration) : 'Unknown'}`);
console.log(` Bitrate: ${extractedMetadata.bitrate ? `${Math.round(extractedMetadata.bitrate / 1000)}kbps` : 'Unknown'}`);
console.log(` Sample Rate: ${extractedMetadata.sampleRate ? `${extractedMetadata.sampleRate}Hz` : 'Unknown'}`);
console.log(` Channels: ${extractedMetadata.channels || 'Unknown'}`);
console.log(` Title: ${extractedMetadata.title || 'Unknown'}`);
console.log(` Artist: ${extractedMetadata.artist || 'Unknown'}`);
console.log(` Album: ${extractedMetadata.album || 'Unknown'}`);
console.log(` Year: ${extractedMetadata.year || 'Unknown'}`);
console.log(` Genre: ${extractedMetadata.genre || 'Unknown'}`);
return extractedMetadata;
} catch (error) { } catch (error) {
console.error('Error extracting audio metadata:', error); console.error(`❌ Error extracting metadata for ${fileName}:`, error);
// Try to determine format from file extension as fallback
const extension = fileName.split('.').pop()?.toLowerCase();
const fallbackFormat = extension ? this.mapFormatToDisplayName(extension, fileName) : 'UNKNOWN';
console.log(`🔄 Using fallback metadata for ${fileName}`);
// Return basic metadata if extraction fails // Return basic metadata if extraction fails
return { return {
title: fileName.replace(/\.[^/.]+$/, ''), // Remove file extension title: fileName.replace(/\.[^/.]+$/, ''), // Remove file extension
format: this.mapFormatToDisplayName(fileName.split('.').pop()?.toLowerCase() || '', fileName), format: fallbackFormat,
size: fileBuffer.length, size: fileBuffer.length,
}; };
} }
@ -95,7 +165,13 @@ export class AudioMetadataService {
]; ];
const extension = fileName.split('.').pop()?.toLowerCase(); const extension = fileName.split('.').pop()?.toLowerCase();
return extension ? supportedFormats.includes(extension) : false; const isSupported = extension ? supportedFormats.includes(extension) : false;
if (!isSupported) {
console.log(`⚠️ Unsupported audio format: ${extension} for file: ${fileName}`);
}
return isSupported;
} }
/** /**

32
packages/backend/src/services/s3Service.ts
View File

@ -1,6 +1,8 @@
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'; import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import fs from 'fs/promises';
import path from 'path';
export interface S3Config { export interface S3Config {
endpoint: string; endpoint: string;
@ -8,6 +10,7 @@ export interface S3Config {
secretAccessKey: string; secretAccessKey: string;
bucketName: string; bucketName: string;
region: string; region: string;
useSSL?: boolean;
} }
export interface UploadResult { export interface UploadResult {
@ -41,6 +44,35 @@ export class S3Service {
this.bucketName = config.bucketName; this.bucketName = config.bucketName;
} }
/**
* Load S3 configuration from s3-config.json file
*/
static async loadConfig(): Promise<S3Config> {
try {
const configPath = path.join(process.cwd(), 's3-config.json');
const configData = await fs.readFile(configPath, 'utf-8');
return JSON.parse(configData);
} catch (error) {
console.warn('Failed to load s3-config.json, using environment variables as fallback');
return {
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
bucketName: process.env.S3_BUCKET_NAME || 'music-files',
region: process.env.S3_REGION || 'us-east-1',
useSSL: process.env.S3_USE_SSL !== 'false',
};
}
}
/**
* Create S3Service instance with configuration from file
*/
static async createFromConfig(): Promise<S3Service> {
const config = await this.loadConfig();
return new S3Service(config);
}
/** /**
* Upload a file to S3 * Upload a file to S3
*/ */

38
packages/backend/src/services/songMatchingService.ts
View File

@ -66,14 +66,33 @@ export class SongMatchingService {
async matchAllMusicFilesToSongs( async matchAllMusicFilesToSongs(
options: MatchOptions = {} options: MatchOptions = {}
): Promise<{ musicFile: any; matches: MatchResult[] }[]> { ): Promise<{ musicFile: any; matches: MatchResult[] }[]> {
console.log('🔍 Starting song matching for all unmatched music files...');
const musicFiles = await MusicFile.find({ songId: { $exists: false } }); const musicFiles = await MusicFile.find({ songId: { $exists: false } });
console.log(`📁 Found ${musicFiles.length} unmatched music files`);
const results = []; const results = [];
let processedCount = 0;
for (const musicFile of musicFiles) { for (const musicFile of musicFiles) {
processedCount++;
const progress = ((processedCount / musicFiles.length) * 100).toFixed(1);
console.log(`🎵 [${progress}%] Matching: ${musicFile.originalName}`);
const matches = await this.matchMusicFileToSongs(musicFile, options); const matches = await this.matchMusicFileToSongs(musicFile, options);
if (matches.length > 0) {
const bestMatch = matches[0];
console.log(`✅ Best match for ${musicFile.originalName}: ${bestMatch.song.title} (${(bestMatch.confidence * 100).toFixed(1)}% confidence)`);
} else {
console.log(`❌ No matches found for ${musicFile.originalName}`);
}
results.push({ musicFile, matches }); results.push({ musicFile, matches });
} }
console.log(`🎉 Song matching completed for ${musicFiles.length} files`);
return results; return results;
} }
@ -83,17 +102,29 @@ export class SongMatchingService {
async autoMatchAndLink( async autoMatchAndLink(
options: MatchOptions = {} options: MatchOptions = {}
): Promise<{ linked: number; unmatched: number }> { ): Promise<{ linked: number; unmatched: number }> {
console.log('🔗 Starting auto-match and link process...');
const { const {
minConfidence = 0.7, // Higher threshold for auto-linking minConfidence = 0.7, // Higher threshold for auto-linking
enableFuzzyMatching = true, enableFuzzyMatching = true,
enablePartialMatching = false // Disable partial matching for auto-linking enablePartialMatching = false // Disable partial matching for auto-linking
} = options; } = options;
console.log(`⚙️ Auto-linking options: minConfidence=${minConfidence}, enableFuzzyMatching=${enableFuzzyMatching}, enablePartialMatching=${enablePartialMatching}`);
const musicFiles = await MusicFile.find({ songId: { $exists: false } }); const musicFiles = await MusicFile.find({ songId: { $exists: false } });
console.log(`📁 Found ${musicFiles.length} unmatched music files to process`);
let linked = 0; let linked = 0;
let unmatched = 0; let unmatched = 0;
let processedCount = 0;
for (const musicFile of musicFiles) { for (const musicFile of musicFiles) {
processedCount++;
const progress = ((processedCount / musicFiles.length) * 100).toFixed(1);
console.log(`🔍 [${progress}%] Auto-matching: ${musicFile.originalName}`);
const matches = await this.matchMusicFileToSongs(musicFile, { const matches = await this.matchMusicFileToSongs(musicFile, {
minConfidence, minConfidence,
enableFuzzyMatching, enableFuzzyMatching,
@ -103,13 +134,20 @@ export class SongMatchingService {
if (matches.length > 0 && matches[0].confidence >= minConfidence) { if (matches.length > 0 && matches[0].confidence >= minConfidence) {
// Link the music file to the best match // Link the music file to the best match
console.log(`🔗 Linking ${musicFile.originalName} to ${matches[0].song.title} (${(matches[0].confidence * 100).toFixed(1)}% confidence)`);
await this.linkMusicFileToSong(musicFile, matches[0].song); await this.linkMusicFileToSong(musicFile, matches[0].song);
linked++; linked++;
} else { } else {
console.log(`❌ No suitable match found for ${musicFile.originalName} (best confidence: ${matches.length > 0 ? (matches[0].confidence * 100).toFixed(1) : 0}%)`);
unmatched++; unmatched++;
} }
} }
console.log(`🎉 Auto-match and link completed:`);
console.log(` Linked: ${linked} files`);
console.log(` Unmatched: ${unmatched} files`);
console.log(` Success rate: ${musicFiles.length > 0 ? ((linked / musicFiles.length) * 100).toFixed(1) : 0}%`);
return { linked, unmatched }; return { linked, unmatched };
} }

105
packages/backend/test-metadata.js Normal file
View File

@ -0,0 +1,105 @@
import { AudioMetadataService } from './src/services/audioMetadataService.js';
async function testMetadataService() {
console.log('🧪 Testing AudioMetadataService...');
const audioService = new AudioMetadataService();
// Test 1: NaN handling for numeric values
console.log('\n📊 Testing NaN handling...');
const testCases = [
{ value: 2023, field: 'year', expected: 2023 },
{ value: NaN, field: 'year', expected: undefined },
{ value: 'invalid', field: 'year', expected: undefined },
{ value: 300, field: 'duration', expected: 300 },
{ value: -1, field: 'duration', expected: undefined },
{ value: 320000, field: 'bitrate', expected: 320000 },
{ value: 44100, field: 'sampleRate', expected: 44100 },
{ value: 2, field: 'channels', expected: 2 },
{ value: 10, field: 'channels', expected: undefined }, // Too many channels
{ value: 1800, field: 'year', expected: undefined }, // Too old
{ value: 2030, field: 'year', expected: undefined }, // Too future
];
for (const testCase of testCases) {
const result = audioService['sanitizeNumericValue'](testCase.value, testCase.field);
const status = result === testCase.expected ? '✅' : '❌';
console.log(`${status} ${testCase.field}: ${testCase.value} -> ${result} (expected: ${testCase.expected})`);
}
// Test 2: Audio file format detection
console.log('\n🎵 Testing audio file format detection...');
const audioFiles = [
'song.mp3',
'track.wav',
'music.flac',
'audio.aac',
'sound.ogg',
'file.m4a',
'test.txt',
'image.jpg',
'document.pdf'
];
for (const file of audioFiles) {
const isAudio = audioService.isAudioFile(file);
const status = isAudio ? '✅' : '❌';
console.log(`${status} ${file}: ${isAudio ? 'Audio' : 'Not audio'}`);
}
// Test 3: Format mapping
console.log('\n📋 Testing format mapping...');
const formatTests = [
{ container: 'MPEG', fileName: 'song.mp3', expected: 'MP3' },
{ container: 'FLAC', fileName: 'track.flac', expected: 'FLAC' },
{ container: 'WAVE', fileName: 'music.wav', expected: 'WAV' },
{ container: 'unknown', fileName: 'audio.ogg', expected: 'OGG' },
{ container: 'unknown', fileName: 'file.xyz', expected: 'UNKNOWN' }
];
for (const test of formatTests) {
const result = audioService['mapFormatToDisplayName'](test.container, test.fileName);
const status = result === test.expected ? '✅' : '❌';
console.log(`${status} ${test.container} + ${test.fileName} -> ${result} (expected: ${test.expected})`);
}
// Test 4: File size formatting
console.log('\n📏 Testing file size formatting...');
const sizeTests = [
0,
1024,
1048576,
1073741824,
1099511627776
];
for (const size of sizeTests) {
const formatted = audioService.formatFileSize(size);
console.log(`📄 ${size} bytes -> ${formatted}`);
}
// Test 5: Duration formatting
console.log('\n⏱️ Testing duration formatting...');
const durationTests = [
0,
61,
125.5,
3600,
NaN,
null
];
for (const duration of durationTests) {
const formatted = audioService.formatDuration(duration);
console.log(`⏱️ ${duration} seconds -> ${formatted}`);
}
console.log('\n🎉 AudioMetadataService tests completed!');
}
testMetadataService().catch(console.error);

42
packages/backend/test-s3-service.js Normal file
View File

@ -0,0 +1,42 @@
import { S3Service } from './src/services/s3Service.js';
async function testS3ServiceConfig() {
try {
console.log('🧪 Testing S3Service configuration loading...');
// Test loading configuration from file
const config = await S3Service.loadConfig();
console.log('✅ Configuration loaded from s3-config.json:');
console.log(` Endpoint: ${config.endpoint}`);
console.log(` Region: ${config.region}`);
console.log(` Bucket: ${config.bucketName}`);
console.log(` UseSSL: ${config.useSSL}`);
console.log(` AccessKeyId: ${config.accessKeyId ? '***' : 'not set'}`);
console.log(` SecretAccessKey: ${config.secretAccessKey ? '***' : 'not set'}`);
// Test creating S3Service instance
const s3Service = await S3Service.createFromConfig();
console.log('✅ S3Service instance created successfully');
// Test listing files (this will verify the connection works)
console.log('📁 Testing file listing...');
const files = await s3Service.listAllFiles();
console.log(`✅ Found ${files.length} files in bucket`);
if (files.length > 0) {
console.log(' Sample files:');
files.slice(0, 3).forEach(file => {
console.log(` - ${file.key} (${file.size} bytes)`);
});
}
console.log('\n🎉 S3Service configuration test passed!');
console.log(' The S3 sync is now using the correct configuration from s3-config.json');
} catch (error) {
console.error('❌ S3Service configuration test failed:', error.message);
console.log('\n💡 This means the S3 sync will fall back to environment variables');
}
}
testS3ServiceConfig();

45
packages/backend/test-s3.js
View File

@ -1,20 +1,57 @@
import { S3Client, ListBucketsCommand, CreateBucketCommand, HeadBucketCommand } from '@aws-sdk/client-s3'; import { S3Client, ListBucketsCommand, CreateBucketCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
import fs from 'fs/promises';
import path from 'path';
// Test S3 service configuration // Load S3 configuration from s3-config.json
const s3Client = new S3Client({ async function loadS3Config() {
try {
const configPath = path.join(process.cwd(), 's3-config.json');
const configData = await fs.readFile(configPath, 'utf-8');
return JSON.parse(configData);
} catch (error) {
console.warn('Failed to load s3-config.json, using default MinIO configuration');
return {
endpoint: 'http://localhost:9000', endpoint: 'http://localhost:9000',
region: 'us-east-1', region: 'us-east-1',
credentials: {
accessKeyId: 'minioadmin', accessKeyId: 'minioadmin',
secretAccessKey: 'minioadmin', secretAccessKey: 'minioadmin',
bucketName: 'music-files',
useSSL: false,
};
}
}
// Test S3 service configuration
let s3Client;
let bucketName;
async function initializeS3Client() {
const config = await loadS3Config();
s3Client = new S3Client({
endpoint: config.endpoint,
region: config.region,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
}, },
forcePathStyle: true, forcePathStyle: true,
}); });
const bucketName = 'music-files'; bucketName = config.bucketName;
console.log(`🔧 Using S3 configuration:`);
console.log(` Endpoint: ${config.endpoint}`);
console.log(` Region: ${config.region}`);
console.log(` Bucket: ${config.bucketName}`);
console.log(` UseSSL: ${config.useSSL}`);
}
async function testS3Connection() { async function testS3Connection() {
try { try {
// Initialize S3 client with configuration from file
await initializeS3Client();
console.log('🧪 Testing S3/MinIO connection...'); console.log('🧪 Testing S3/MinIO connection...');
// Test connection by listing buckets // Test connection by listing buckets