feat: improve S3 sync and song matching with better FLAC support, NaN validation, and enhanced logging
- Enhanced AudioMetadataService with comprehensive NaN handling for all numeric fields - Added validation for year, duration, bitrate, sampleRate, and channels - Improved FLAC format detection and error handling with fallback support - Added detailed logging for S3 sync process with progress tracking and file statistics - Enhanced song matching service with progress indicators and confidence scoring - Added comprehensive logging for auto-match and link operations - Improved error handling and graceful degradation for metadata extraction - Added test scripts for metadata service validation - Updated S3 service to use configuration from s3-config.json file - Added automatic S3 service reload when configuration is updated The S3 importer now provides much better visibility into file processing and song matching operations, making it easier to debug issues and monitor performance. FLAC files are properly handled and invalid metadata values are filtered out to prevent database corruption.
This commit is contained in:
Geert Rademakes
parent
050e31288a
commit
3cd83ff2b5
@ -1,8 +1,8 @@
|
||||
{
|
||||
"endpoint": "http://localhost:9000",
|
||||
"region": "us-east-1",
|
||||
"accessKeyId": "minioadmin",
|
||||
"secretAccessKey": "minioadmin",
|
||||
"bucketName": "music-files",
|
||||
"endpoint": "https://garage.geertrademakers.nl",
|
||||
"region": "garage",
|
||||
"accessKeyId": "GK1c1a4a30946eb1e7f8d60847",
|
||||
"secretAccessKey": "2ed6673f0e3c42d347adeb54ba6b95a1ebc6414750f2a95e1d3d89758f1add63",
|
||||
"bucketName": "music",
|
||||
"useSSL": true
|
||||
}
|
||||
@ -2,6 +2,7 @@ import express from 'express';
|
||||
import { S3Client, ListBucketsCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { reloadS3Service } from './music.js';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@ -79,7 +80,13 @@ router.post('/s3', async (req, res) => {
|
||||
process.env.S3_BUCKET_NAME = config.bucketName;
|
||||
process.env.S3_USE_SSL = config.useSSL.toString();
|
||||
|
||||
res.json({ message: 'S3 configuration saved successfully' });
|
||||
// Reload S3 service with new configuration
|
||||
const reloadSuccess = await reloadS3Service();
|
||||
|
||||
res.json({
|
||||
message: 'S3 configuration saved successfully',
|
||||
s3ServiceReloaded: reloadSuccess
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error saving S3 config:', error);
|
||||
res.status(500).json({ error: 'Failed to save S3 configuration' });
|
||||
|
||||
@ -41,6 +41,9 @@ router.get('/music-file/:id/suggestions', async (req, res) => {
|
||||
*/
|
||||
router.get('/suggestions', async (req, res) => {
|
||||
try {
|
||||
console.log('🔍 Getting matching suggestions for all unmatched music files...');
|
||||
const startTime = Date.now();
|
||||
|
||||
const options = {
|
||||
minConfidence: parseFloat(req.query.minConfidence as string) || 0.3,
|
||||
enableFuzzyMatching: req.query.enableFuzzyMatching !== 'false',
|
||||
@ -48,15 +51,24 @@ router.get('/suggestions', async (req, res) => {
|
||||
maxResults: parseInt(req.query.maxResults as string) || 3
|
||||
};
|
||||
|
||||
console.log('⚙️ Matching options:', options);
|
||||
|
||||
const results = await matchingService.matchAllMusicFilesToSongs(options);
|
||||
|
||||
const endTime = Date.now();
|
||||
const duration = ((endTime - startTime) / 1000).toFixed(2);
|
||||
|
||||
console.log(`✅ Matching suggestions completed in ${duration} seconds`);
|
||||
console.log(`📊 Found suggestions for ${results.length} unmatched files`);
|
||||
|
||||
res.json({
|
||||
results,
|
||||
options,
|
||||
totalUnmatched: results.length
|
||||
totalUnmatched: results.length,
|
||||
duration: `${duration}s`
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error getting all matching suggestions:', error);
|
||||
console.error('❌ Error getting all matching suggestions:', error);
|
||||
res.status(500).json({ error: 'Failed to get matching suggestions' });
|
||||
}
|
||||
});
|
||||
@ -66,21 +78,33 @@ router.get('/suggestions', async (req, res) => {
|
||||
*/
|
||||
router.post('/auto-link', async (req, res) => {
|
||||
try {
|
||||
console.log('🚀 Starting auto-match and link request...');
|
||||
const startTime = Date.now();
|
||||
|
||||
const options = {
|
||||
minConfidence: parseFloat(req.body.minConfidence as string) || 0.7,
|
||||
enableFuzzyMatching: req.body.enableFuzzyMatching !== false,
|
||||
enablePartialMatching: req.body.enablePartialMatching !== false
|
||||
};
|
||||
|
||||
console.log('⚙️ Auto-linking options:', options);
|
||||
|
||||
const result = await matchingService.autoMatchAndLink(options);
|
||||
|
||||
const endTime = Date.now();
|
||||
const duration = ((endTime - startTime) / 1000).toFixed(2);
|
||||
|
||||
console.log(`✅ Auto-linking completed in ${duration} seconds`);
|
||||
console.log(`📊 Results: ${result.linked} linked, ${result.unmatched} unmatched`);
|
||||
|
||||
res.json({
|
||||
message: 'Auto-linking completed',
|
||||
result,
|
||||
options
|
||||
options,
|
||||
duration: `${duration}s`
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error during auto-linking:', error);
|
||||
console.error('❌ Error during auto-linking:', error);
|
||||
res.status(500).json({ error: 'Failed to auto-link music files' });
|
||||
}
|
||||
});
|
||||
|
||||
@ -24,13 +24,43 @@ const upload = multer({
|
||||
});
|
||||
|
||||
// Initialize services
|
||||
const s3Service = new S3Service({
|
||||
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
|
||||
bucketName: process.env.S3_BUCKET_NAME || 'music-files',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
});
|
||||
let s3Service: S3Service;
|
||||
|
||||
// Initialize S3 service with configuration from file
|
||||
async function initializeS3Service() {
|
||||
try {
|
||||
s3Service = await S3Service.createFromConfig();
|
||||
console.log('✅ S3 service initialized with configuration from s3-config.json');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize S3 service:', error);
|
||||
// Fallback to environment variables
|
||||
s3Service = new S3Service({
|
||||
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
|
||||
bucketName: process.env.S3_BUCKET_NAME || 'music-files',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
});
|
||||
console.log('⚠️ S3 service initialized with environment variables as fallback');
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize S3 service on startup
|
||||
initializeS3Service();
|
||||
|
||||
/**
|
||||
* Reload S3 service with updated configuration
|
||||
*/
|
||||
export async function reloadS3Service() {
|
||||
try {
|
||||
s3Service = await S3Service.createFromConfig();
|
||||
console.log('✅ S3 service reloaded with updated configuration');
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to reload S3 service:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const audioMetadataService = new AudioMetadataService();
|
||||
|
||||
@ -84,7 +114,7 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => {
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const file of req.files) {
|
||||
for (const file of req.files as Express.Multer.File[]) {
|
||||
try {
|
||||
const { buffer, originalname, mimetype } = file;
|
||||
|
||||
@ -108,7 +138,7 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => {
|
||||
results.push({ success: true, musicFile });
|
||||
} catch (error) {
|
||||
console.error(`Error uploading ${file.originalname}:`, error);
|
||||
results.push({ success: false, fileName: file.originalname, error: error.message });
|
||||
results.push({ success: false, fileName: file.originalname, error: (error as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
@ -140,26 +170,39 @@ router.get('/files', async (req, res) => {
|
||||
*/
|
||||
router.post('/sync-s3', async (req, res) => {
|
||||
try {
|
||||
console.log('Starting S3 sync...');
|
||||
console.log('🔄 Starting S3 sync process...');
|
||||
const startTime = Date.now();
|
||||
|
||||
// Get all files from S3 recursively
|
||||
console.log('📁 Fetching files from S3 bucket...');
|
||||
const s3Files = await s3Service.listAllFiles();
|
||||
console.log(`Found ${s3Files.length} files in S3 bucket`);
|
||||
console.log(`✅ Found ${s3Files.length} files in S3 bucket`);
|
||||
|
||||
const results = {
|
||||
total: s3Files.length,
|
||||
synced: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
newFiles: 0
|
||||
newFiles: 0,
|
||||
audioFiles: 0,
|
||||
nonAudioFiles: 0
|
||||
};
|
||||
|
||||
console.log('🔍 Processing files...');
|
||||
let processedCount = 0;
|
||||
|
||||
for (const s3File of s3Files) {
|
||||
processedCount++;
|
||||
const progress = ((processedCount / s3Files.length) * 100).toFixed(1);
|
||||
|
||||
try {
|
||||
console.log(`📄 [${progress}%] Processing: ${s3File.key} (${audioMetadataService.formatFileSize(s3File.size)})`);
|
||||
|
||||
// Check if file already exists in database
|
||||
const existingFile = await MusicFile.findOne({ s3Key: s3File.key });
|
||||
|
||||
if (existingFile) {
|
||||
console.log(`⏭️ Skipping existing file: ${s3File.key}`);
|
||||
results.skipped++;
|
||||
continue;
|
||||
}
|
||||
@ -169,16 +212,24 @@ router.post('/sync-s3', async (req, res) => {
|
||||
|
||||
// Check if it's an audio file
|
||||
if (!audioMetadataService.isAudioFile(filename)) {
|
||||
console.log(`🚫 Skipping non-audio file: ${filename}`);
|
||||
results.skipped++;
|
||||
results.nonAudioFiles++;
|
||||
continue;
|
||||
}
|
||||
|
||||
results.audioFiles++;
|
||||
console.log(`🎵 Processing audio file: ${filename}`);
|
||||
|
||||
// Get file content to extract metadata
|
||||
try {
|
||||
console.log(`⬇️ Downloading file content: ${s3File.key}`);
|
||||
const fileBuffer = await s3Service.getFileContent(s3File.key);
|
||||
console.log(`📊 Extracting metadata: ${filename}`);
|
||||
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
|
||||
|
||||
// Save to database
|
||||
console.log(`💾 Saving to database: ${filename}`);
|
||||
const musicFile = new MusicFile({
|
||||
originalName: filename,
|
||||
s3Key: s3File.key,
|
||||
@ -191,9 +242,12 @@ router.post('/sync-s3', async (req, res) => {
|
||||
await musicFile.save();
|
||||
results.synced++;
|
||||
results.newFiles++;
|
||||
console.log(`✅ Successfully synced: ${filename}`);
|
||||
|
||||
} catch (metadataError) {
|
||||
console.error(`Error extracting metadata for ${s3File.key}:`, metadataError);
|
||||
console.error(`❌ Error extracting metadata for ${s3File.key}:`, metadataError);
|
||||
console.log(`🔄 Saving file without metadata: ${filename}`);
|
||||
|
||||
// Still save the file without metadata
|
||||
const musicFile = new MusicFile({
|
||||
originalName: filename,
|
||||
@ -205,22 +259,36 @@ router.post('/sync-s3', async (req, res) => {
|
||||
await musicFile.save();
|
||||
results.synced++;
|
||||
results.newFiles++;
|
||||
console.log(`✅ Saved without metadata: ${filename}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Error processing ${s3File.key}:`, error);
|
||||
console.error(`❌ Error processing ${s3File.key}:`, error);
|
||||
results.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('S3 sync completed:', results);
|
||||
const endTime = Date.now();
|
||||
const duration = ((endTime - startTime) / 1000).toFixed(2);
|
||||
|
||||
console.log('🎉 S3 sync completed!');
|
||||
console.log(`⏱️ Duration: ${duration} seconds`);
|
||||
console.log(`📊 Results:`, results);
|
||||
console.log(` Total files: ${results.total}`);
|
||||
console.log(` Audio files: ${results.audioFiles}`);
|
||||
console.log(` Non-audio files: ${results.nonAudioFiles}`);
|
||||
console.log(` New files synced: ${results.newFiles}`);
|
||||
console.log(` Files skipped: ${results.skipped}`);
|
||||
console.log(` Errors: ${results.errors}`);
|
||||
|
||||
res.json({
|
||||
message: 'S3 sync completed',
|
||||
results
|
||||
results,
|
||||
duration: `${duration}s`
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('S3 sync error:', error);
|
||||
console.error('❌ S3 sync error:', error);
|
||||
res.status(500).json({ error: 'Failed to sync S3 files' });
|
||||
}
|
||||
});
|
||||
|
||||
@ -54,33 +54,103 @@ export class AudioMetadataService {
|
||||
return container.toUpperCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and sanitize numeric values
|
||||
*/
|
||||
private sanitizeNumericValue(value: any, fieldName: string): number | undefined {
|
||||
if (value === null || value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const numValue = typeof value === 'number' ? value : Number(value);
|
||||
|
||||
if (isNaN(numValue) || !isFinite(numValue)) {
|
||||
console.warn(`⚠️ Invalid ${fieldName} value: ${value}, setting to undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Additional validation for specific fields
|
||||
if (fieldName === 'year' && (numValue < 1900 || numValue > new Date().getFullYear() + 1)) {
|
||||
console.warn(`⚠️ Invalid year value: ${numValue}, setting to undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (fieldName === 'duration' && numValue < 0) {
|
||||
console.warn(`⚠️ Invalid duration value: ${numValue}, setting to undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (fieldName === 'bitrate' && numValue < 0) {
|
||||
console.warn(`⚠️ Invalid bitrate value: ${numValue}, setting to undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (fieldName === 'sampleRate' && numValue < 0) {
|
||||
console.warn(`⚠️ Invalid sampleRate value: ${numValue}, setting to undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (fieldName === 'channels' && (numValue < 1 || numValue > 8)) {
|
||||
console.warn(`⚠️ Invalid channels value: ${numValue}, setting to undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return numValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract metadata from audio file buffer
|
||||
*/
|
||||
async extractMetadata(fileBuffer: Buffer, fileName: string): Promise<AudioMetadata> {
|
||||
console.log(`🎵 Extracting metadata for: ${fileName} (${this.formatFileSize(fileBuffer.length)})`);
|
||||
|
||||
try {
|
||||
const metadata = await parseBuffer(fileBuffer, fileName);
|
||||
console.log(`✅ Successfully parsed metadata for ${fileName}`);
|
||||
|
||||
return {
|
||||
title: metadata.common.title,
|
||||
artist: metadata.common.artist,
|
||||
album: metadata.common.album,
|
||||
year: metadata.common.year,
|
||||
genre: metadata.common.genre?.[0],
|
||||
duration: metadata.format.duration,
|
||||
bitrate: metadata.format.bitrate,
|
||||
sampleRate: metadata.format.sampleRate,
|
||||
channels: metadata.format.numberOfChannels,
|
||||
format: this.mapFormatToDisplayName(metadata.format.container, fileName),
|
||||
// Extract and sanitize metadata
|
||||
const extractedMetadata: AudioMetadata = {
|
||||
title: metadata.common.title || undefined,
|
||||
artist: metadata.common.artist || undefined,
|
||||
album: metadata.common.album || undefined,
|
||||
year: this.sanitizeNumericValue(metadata.common.year, 'year'),
|
||||
genre: metadata.common.genre?.[0] || undefined,
|
||||
duration: this.sanitizeNumericValue(metadata.format.duration, 'duration'),
|
||||
bitrate: this.sanitizeNumericValue(metadata.format.bitrate, 'bitrate'),
|
||||
sampleRate: this.sanitizeNumericValue(metadata.format.sampleRate, 'sampleRate'),
|
||||
channels: this.sanitizeNumericValue(metadata.format.numberOfChannels, 'channels'),
|
||||
format: this.mapFormatToDisplayName(metadata.format.container || '', fileName),
|
||||
size: fileBuffer.length,
|
||||
};
|
||||
|
||||
// Log extracted metadata (without sensitive info)
|
||||
console.log(`📊 Metadata extracted for ${fileName}:`);
|
||||
console.log(` Format: ${extractedMetadata.format}`);
|
||||
console.log(` Duration: ${extractedMetadata.duration ? this.formatDuration(extractedMetadata.duration) : 'Unknown'}`);
|
||||
console.log(` Bitrate: ${extractedMetadata.bitrate ? `${Math.round(extractedMetadata.bitrate / 1000)}kbps` : 'Unknown'}`);
|
||||
console.log(` Sample Rate: ${extractedMetadata.sampleRate ? `${extractedMetadata.sampleRate}Hz` : 'Unknown'}`);
|
||||
console.log(` Channels: ${extractedMetadata.channels || 'Unknown'}`);
|
||||
console.log(` Title: ${extractedMetadata.title || 'Unknown'}`);
|
||||
console.log(` Artist: ${extractedMetadata.artist || 'Unknown'}`);
|
||||
console.log(` Album: ${extractedMetadata.album || 'Unknown'}`);
|
||||
console.log(` Year: ${extractedMetadata.year || 'Unknown'}`);
|
||||
console.log(` Genre: ${extractedMetadata.genre || 'Unknown'}`);
|
||||
|
||||
return extractedMetadata;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error extracting audio metadata:', error);
|
||||
console.error(`❌ Error extracting metadata for ${fileName}:`, error);
|
||||
|
||||
// Try to determine format from file extension as fallback
|
||||
const extension = fileName.split('.').pop()?.toLowerCase();
|
||||
const fallbackFormat = extension ? this.mapFormatToDisplayName(extension, fileName) : 'UNKNOWN';
|
||||
|
||||
console.log(`🔄 Using fallback metadata for ${fileName}`);
|
||||
|
||||
// Return basic metadata if extraction fails
|
||||
return {
|
||||
title: fileName.replace(/\.[^/.]+$/, ''), // Remove file extension
|
||||
format: this.mapFormatToDisplayName(fileName.split('.').pop()?.toLowerCase() || '', fileName),
|
||||
format: fallbackFormat,
|
||||
size: fileBuffer.length,
|
||||
};
|
||||
}
|
||||
@ -95,7 +165,13 @@ export class AudioMetadataService {
|
||||
];
|
||||
|
||||
const extension = fileName.split('.').pop()?.toLowerCase();
|
||||
return extension ? supportedFormats.includes(extension) : false;
|
||||
const isSupported = extension ? supportedFormats.includes(extension) : false;
|
||||
|
||||
if (!isSupported) {
|
||||
console.log(`⚠️ Unsupported audio format: ${extension} for file: ${fileName}`);
|
||||
}
|
||||
|
||||
return isSupported;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
export interface S3Config {
|
||||
endpoint: string;
|
||||
@ -8,6 +10,7 @@ export interface S3Config {
|
||||
secretAccessKey: string;
|
||||
bucketName: string;
|
||||
region: string;
|
||||
useSSL?: boolean;
|
||||
}
|
||||
|
||||
export interface UploadResult {
|
||||
@ -41,6 +44,35 @@ export class S3Service {
|
||||
this.bucketName = config.bucketName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load S3 configuration from s3-config.json file
|
||||
*/
|
||||
static async loadConfig(): Promise<S3Config> {
|
||||
try {
|
||||
const configPath = path.join(process.cwd(), 's3-config.json');
|
||||
const configData = await fs.readFile(configPath, 'utf-8');
|
||||
return JSON.parse(configData);
|
||||
} catch (error) {
|
||||
console.warn('Failed to load s3-config.json, using environment variables as fallback');
|
||||
return {
|
||||
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
|
||||
bucketName: process.env.S3_BUCKET_NAME || 'music-files',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
useSSL: process.env.S3_USE_SSL !== 'false',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create S3Service instance with configuration from file
|
||||
*/
|
||||
static async createFromConfig(): Promise<S3Service> {
|
||||
const config = await this.loadConfig();
|
||||
return new S3Service(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to S3
|
||||
*/
|
||||
|
||||
@ -66,14 +66,33 @@ export class SongMatchingService {
|
||||
async matchAllMusicFilesToSongs(
|
||||
options: MatchOptions = {}
|
||||
): Promise<{ musicFile: any; matches: MatchResult[] }[]> {
|
||||
console.log('🔍 Starting song matching for all unmatched music files...');
|
||||
|
||||
const musicFiles = await MusicFile.find({ songId: { $exists: false } });
|
||||
console.log(`📁 Found ${musicFiles.length} unmatched music files`);
|
||||
|
||||
const results = [];
|
||||
let processedCount = 0;
|
||||
|
||||
for (const musicFile of musicFiles) {
|
||||
processedCount++;
|
||||
const progress = ((processedCount / musicFiles.length) * 100).toFixed(1);
|
||||
|
||||
console.log(`🎵 [${progress}%] Matching: ${musicFile.originalName}`);
|
||||
|
||||
const matches = await this.matchMusicFileToSongs(musicFile, options);
|
||||
|
||||
if (matches.length > 0) {
|
||||
const bestMatch = matches[0];
|
||||
console.log(`✅ Best match for ${musicFile.originalName}: ${bestMatch.song.title} (${(bestMatch.confidence * 100).toFixed(1)}% confidence)`);
|
||||
} else {
|
||||
console.log(`❌ No matches found for ${musicFile.originalName}`);
|
||||
}
|
||||
|
||||
results.push({ musicFile, matches });
|
||||
}
|
||||
|
||||
console.log(`🎉 Song matching completed for ${musicFiles.length} files`);
|
||||
return results;
|
||||
}
|
||||
|
||||
@ -83,17 +102,29 @@ export class SongMatchingService {
|
||||
async autoMatchAndLink(
|
||||
options: MatchOptions = {}
|
||||
): Promise<{ linked: number; unmatched: number }> {
|
||||
console.log('🔗 Starting auto-match and link process...');
|
||||
|
||||
const {
|
||||
minConfidence = 0.7, // Higher threshold for auto-linking
|
||||
enableFuzzyMatching = true,
|
||||
enablePartialMatching = false // Disable partial matching for auto-linking
|
||||
} = options;
|
||||
|
||||
console.log(`⚙️ Auto-linking options: minConfidence=${minConfidence}, enableFuzzyMatching=${enableFuzzyMatching}, enablePartialMatching=${enablePartialMatching}`);
|
||||
|
||||
const musicFiles = await MusicFile.find({ songId: { $exists: false } });
|
||||
console.log(`📁 Found ${musicFiles.length} unmatched music files to process`);
|
||||
|
||||
let linked = 0;
|
||||
let unmatched = 0;
|
||||
let processedCount = 0;
|
||||
|
||||
for (const musicFile of musicFiles) {
|
||||
processedCount++;
|
||||
const progress = ((processedCount / musicFiles.length) * 100).toFixed(1);
|
||||
|
||||
console.log(`🔍 [${progress}%] Auto-matching: ${musicFile.originalName}`);
|
||||
|
||||
const matches = await this.matchMusicFileToSongs(musicFile, {
|
||||
minConfidence,
|
||||
enableFuzzyMatching,
|
||||
@ -103,13 +134,20 @@ export class SongMatchingService {
|
||||
|
||||
if (matches.length > 0 && matches[0].confidence >= minConfidence) {
|
||||
// Link the music file to the best match
|
||||
console.log(`🔗 Linking ${musicFile.originalName} to ${matches[0].song.title} (${(matches[0].confidence * 100).toFixed(1)}% confidence)`);
|
||||
await this.linkMusicFileToSong(musicFile, matches[0].song);
|
||||
linked++;
|
||||
} else {
|
||||
console.log(`❌ No suitable match found for ${musicFile.originalName} (best confidence: ${matches.length > 0 ? (matches[0].confidence * 100).toFixed(1) : 0}%)`);
|
||||
unmatched++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`🎉 Auto-match and link completed:`);
|
||||
console.log(` Linked: ${linked} files`);
|
||||
console.log(` Unmatched: ${unmatched} files`);
|
||||
console.log(` Success rate: ${musicFiles.length > 0 ? ((linked / musicFiles.length) * 100).toFixed(1) : 0}%`);
|
||||
|
||||
return { linked, unmatched };
|
||||
}
|
||||
|
||||
|
||||
105
packages/backend/test-metadata.js
Normal file
105
packages/backend/test-metadata.js
Normal file
@ -0,0 +1,105 @@
|
||||
import { AudioMetadataService } from './src/services/audioMetadataService.js';
|
||||
|
||||
async function testMetadataService() {
|
||||
console.log('🧪 Testing AudioMetadataService...');
|
||||
|
||||
const audioService = new AudioMetadataService();
|
||||
|
||||
// Test 1: NaN handling for numeric values
|
||||
console.log('\n📊 Testing NaN handling...');
|
||||
|
||||
const testCases = [
|
||||
{ value: 2023, field: 'year', expected: 2023 },
|
||||
{ value: NaN, field: 'year', expected: undefined },
|
||||
{ value: 'invalid', field: 'year', expected: undefined },
|
||||
{ value: 300, field: 'duration', expected: 300 },
|
||||
{ value: -1, field: 'duration', expected: undefined },
|
||||
{ value: 320000, field: 'bitrate', expected: 320000 },
|
||||
{ value: 44100, field: 'sampleRate', expected: 44100 },
|
||||
{ value: 2, field: 'channels', expected: 2 },
|
||||
{ value: 10, field: 'channels', expected: undefined }, // Too many channels
|
||||
{ value: 1800, field: 'year', expected: undefined }, // Too old
|
||||
{ value: 2030, field: 'year', expected: undefined }, // Too future
|
||||
];
|
||||
|
||||
for (const testCase of testCases) {
|
||||
const result = audioService['sanitizeNumericValue'](testCase.value, testCase.field);
|
||||
const status = result === testCase.expected ? '✅' : '❌';
|
||||
console.log(`${status} ${testCase.field}: ${testCase.value} -> ${result} (expected: ${testCase.expected})`);
|
||||
}
|
||||
|
||||
// Test 2: Audio file format detection
|
||||
console.log('\n🎵 Testing audio file format detection...');
|
||||
|
||||
const audioFiles = [
|
||||
'song.mp3',
|
||||
'track.wav',
|
||||
'music.flac',
|
||||
'audio.aac',
|
||||
'sound.ogg',
|
||||
'file.m4a',
|
||||
'test.txt',
|
||||
'image.jpg',
|
||||
'document.pdf'
|
||||
];
|
||||
|
||||
for (const file of audioFiles) {
|
||||
const isAudio = audioService.isAudioFile(file);
|
||||
const status = isAudio ? '✅' : '❌';
|
||||
console.log(`${status} ${file}: ${isAudio ? 'Audio' : 'Not audio'}`);
|
||||
}
|
||||
|
||||
// Test 3: Format mapping
|
||||
console.log('\n📋 Testing format mapping...');
|
||||
|
||||
const formatTests = [
|
||||
{ container: 'MPEG', fileName: 'song.mp3', expected: 'MP3' },
|
||||
{ container: 'FLAC', fileName: 'track.flac', expected: 'FLAC' },
|
||||
{ container: 'WAVE', fileName: 'music.wav', expected: 'WAV' },
|
||||
{ container: 'unknown', fileName: 'audio.ogg', expected: 'OGG' },
|
||||
{ container: 'unknown', fileName: 'file.xyz', expected: 'UNKNOWN' }
|
||||
];
|
||||
|
||||
for (const test of formatTests) {
|
||||
const result = audioService['mapFormatToDisplayName'](test.container, test.fileName);
|
||||
const status = result === test.expected ? '✅' : '❌';
|
||||
console.log(`${status} ${test.container} + ${test.fileName} -> ${result} (expected: ${test.expected})`);
|
||||
}
|
||||
|
||||
// Test 4: File size formatting
|
||||
console.log('\n📏 Testing file size formatting...');
|
||||
|
||||
const sizeTests = [
|
||||
0,
|
||||
1024,
|
||||
1048576,
|
||||
1073741824,
|
||||
1099511627776
|
||||
];
|
||||
|
||||
for (const size of sizeTests) {
|
||||
const formatted = audioService.formatFileSize(size);
|
||||
console.log(`📄 ${size} bytes -> ${formatted}`);
|
||||
}
|
||||
|
||||
// Test 5: Duration formatting
|
||||
console.log('\n⏱️ Testing duration formatting...');
|
||||
|
||||
const durationTests = [
|
||||
0,
|
||||
61,
|
||||
125.5,
|
||||
3600,
|
||||
NaN,
|
||||
null
|
||||
];
|
||||
|
||||
for (const duration of durationTests) {
|
||||
const formatted = audioService.formatDuration(duration);
|
||||
console.log(`⏱️ ${duration} seconds -> ${formatted}`);
|
||||
}
|
||||
|
||||
console.log('\n🎉 AudioMetadataService tests completed!');
|
||||
}
|
||||
|
||||
testMetadataService().catch(console.error);
|
||||
42
packages/backend/test-s3-service.js
Normal file
42
packages/backend/test-s3-service.js
Normal file
@ -0,0 +1,42 @@
|
||||
import { S3Service } from './src/services/s3Service.js';
|
||||
|
||||
async function testS3ServiceConfig() {
|
||||
try {
|
||||
console.log('🧪 Testing S3Service configuration loading...');
|
||||
|
||||
// Test loading configuration from file
|
||||
const config = await S3Service.loadConfig();
|
||||
console.log('✅ Configuration loaded from s3-config.json:');
|
||||
console.log(` Endpoint: ${config.endpoint}`);
|
||||
console.log(` Region: ${config.region}`);
|
||||
console.log(` Bucket: ${config.bucketName}`);
|
||||
console.log(` UseSSL: ${config.useSSL}`);
|
||||
console.log(` AccessKeyId: ${config.accessKeyId ? '***' : 'not set'}`);
|
||||
console.log(` SecretAccessKey: ${config.secretAccessKey ? '***' : 'not set'}`);
|
||||
|
||||
// Test creating S3Service instance
|
||||
const s3Service = await S3Service.createFromConfig();
|
||||
console.log('✅ S3Service instance created successfully');
|
||||
|
||||
// Test listing files (this will verify the connection works)
|
||||
console.log('📁 Testing file listing...');
|
||||
const files = await s3Service.listAllFiles();
|
||||
console.log(`✅ Found ${files.length} files in bucket`);
|
||||
|
||||
if (files.length > 0) {
|
||||
console.log(' Sample files:');
|
||||
files.slice(0, 3).forEach(file => {
|
||||
console.log(` - ${file.key} (${file.size} bytes)`);
|
||||
});
|
||||
}
|
||||
|
||||
console.log('\n🎉 S3Service configuration test passed!');
|
||||
console.log(' The S3 sync is now using the correct configuration from s3-config.json');
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ S3Service configuration test failed:', error.message);
|
||||
console.log('\n💡 This means the S3 sync will fall back to environment variables');
|
||||
}
|
||||
}
|
||||
|
||||
testS3ServiceConfig();
|
||||
@ -1,20 +1,57 @@
|
||||
import { S3Client, ListBucketsCommand, CreateBucketCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
// Load S3 configuration from s3-config.json
|
||||
async function loadS3Config() {
|
||||
try {
|
||||
const configPath = path.join(process.cwd(), 's3-config.json');
|
||||
const configData = await fs.readFile(configPath, 'utf-8');
|
||||
return JSON.parse(configData);
|
||||
} catch (error) {
|
||||
console.warn('Failed to load s3-config.json, using default MinIO configuration');
|
||||
return {
|
||||
endpoint: 'http://localhost:9000',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: 'minioadmin',
|
||||
secretAccessKey: 'minioadmin',
|
||||
bucketName: 'music-files',
|
||||
useSSL: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Test S3 service configuration
|
||||
const s3Client = new S3Client({
|
||||
endpoint: 'http://localhost:9000',
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: 'minioadmin',
|
||||
secretAccessKey: 'minioadmin',
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
let s3Client;
|
||||
let bucketName;
|
||||
|
||||
const bucketName = 'music-files';
|
||||
async function initializeS3Client() {
|
||||
const config = await loadS3Config();
|
||||
|
||||
s3Client = new S3Client({
|
||||
endpoint: config.endpoint,
|
||||
region: config.region,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
bucketName = config.bucketName;
|
||||
|
||||
console.log(`🔧 Using S3 configuration:`);
|
||||
console.log(` Endpoint: ${config.endpoint}`);
|
||||
console.log(` Region: ${config.region}`);
|
||||
console.log(` Bucket: ${config.bucketName}`);
|
||||
console.log(` UseSSL: ${config.useSSL}`);
|
||||
}
|
||||
|
||||
async function testS3Connection() {
|
||||
try {
|
||||
// Initialize S3 client with configuration from file
|
||||
await initializeS3Client();
|
||||
|
||||
console.log('🧪 Testing S3/MinIO connection...');
|
||||
|
||||
// Test connection by listing buckets
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user