Geert Rademakes 3cd83ff2b5 feat: improve S3 sync and song matching with better FLAC support, NaN validation, and enhanced logging
- Enhanced AudioMetadataService with comprehensive NaN handling for all numeric fields
- Added validation for year, duration, bitrate, sampleRate, and channels
- Improved FLAC format detection and error handling with fallback support
- Added detailed logging for S3 sync process with progress tracking and file statistics
- Enhanced song matching service with progress indicators and confidence scoring
- Added comprehensive logging for auto-match and link operations
- Improved error handling and graceful degradation for metadata extraction
- Added test scripts for metadata service validation
- Updated S3 service to use configuration from s3-config.json file
- Added automatic S3 service reload when configuration is updated

The S3 importer now provides much better visibility into file processing
and song matching operations, making it easier to debug issues and
monitor performance. FLAC files are properly handled and invalid
metadata values are filtered out to prevent database corruption.
2025-08-07 17:14:57 +02:00

132 lines
4.6 KiB
JavaScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { S3Client, ListBucketsCommand, CreateBucketCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
import fs from 'fs/promises';
import path from 'path';
// Load S3 configuration from s3-config.json
async function loadS3Config() {
try {
const configPath = path.join(process.cwd(), 's3-config.json');
const configData = await fs.readFile(configPath, 'utf-8');
return JSON.parse(configData);
} catch (error) {
console.warn('Failed to load s3-config.json, using default MinIO configuration');
return {
endpoint: 'http://localhost:9000',
region: 'us-east-1',
accessKeyId: 'minioadmin',
secretAccessKey: 'minioadmin',
bucketName: 'music-files',
useSSL: false,
};
}
}
// Test S3 service configuration
let s3Client;
let bucketName;
async function initializeS3Client() {
const config = await loadS3Config();
s3Client = new S3Client({
endpoint: config.endpoint,
region: config.region,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
forcePathStyle: true,
});
bucketName = config.bucketName;
console.log(`🔧 Using S3 configuration:`);
console.log(` Endpoint: ${config.endpoint}`);
console.log(` Region: ${config.region}`);
console.log(` Bucket: ${config.bucketName}`);
console.log(` UseSSL: ${config.useSSL}`);
}
async function testS3Connection() {
try {
// Initialize S3 client with configuration from file
await initializeS3Client();
console.log('🧪 Testing S3/MinIO connection...');
// Test connection by listing buckets
const buckets = await s3Client.send(new ListBucketsCommand({}));
console.log('✅ S3 connection successful');
console.log(' Available buckets:', buckets.Buckets?.map(b => b.Name).join(', ') || 'none');
// Test if our bucket exists
try {
await s3Client.send(new HeadBucketCommand({ Bucket: bucketName }));
console.log(`✅ Bucket '${bucketName}' exists`);
} catch (error) {
if (error.name === 'NotFound') {
console.log(`⚠️ Bucket '${bucketName}' doesn't exist, creating...`);
await s3Client.send(new CreateBucketCommand({ Bucket: bucketName }));
console.log(`✅ Bucket '${bucketName}' created successfully`);
} else {
throw error;
}
}
// Test audio file validation (simple regex test)
const audioExtensions = ['.mp3', '.wav', '.flac', '.m4a', '.aac'];
const testFiles = ['song.mp3', 'track.wav', 'music.flac', 'test.txt'];
console.log('\n🎵 Testing audio file validation:');
testFiles.forEach(file => {
const extension = file.toLowerCase().substring(file.lastIndexOf('.'));
const isValidAudio = audioExtensions.includes(extension);
console.log(` ${isValidAudio ? '✅' : '❌'} ${file}: ${isValidAudio ? 'Valid audio' : 'Not audio'}`);
});
// Test file size formatting
const formatFileSize = (bytes) => {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
};
console.log('\n📏 Testing file size formatting:');
const testSizes = [1024, 5242880, 1073741824];
testSizes.forEach(size => {
console.log(` ${formatFileSize(size)} (${size} bytes)`);
});
// Test duration formatting
const formatDuration = (seconds) => {
const minutes = Math.floor(seconds / 60);
const remainingSeconds = Math.floor(seconds % 60);
return `${minutes}:${remainingSeconds.toString().padStart(2, '0')}`;
};
console.log('\n⏱ Testing duration formatting:');
const testDurations = [61, 125.5, 3600];
testDurations.forEach(duration => {
console.log(` ${formatDuration(duration)} (${duration} seconds)`);
});
console.log('\n🎉 All tests passed! S3 storage is ready to use.');
console.log('\n📋 Next steps:');
console.log(' 1. Start backend: npm run dev');
console.log(' 2. Start frontend: cd ../frontend && npm run dev');
console.log(' 3. Open browser: http://localhost:5173');
} catch (error) {
console.error('❌ Test failed:', error.message);
console.log('\n💡 Troubleshooting:');
console.log(' 1. Make sure MinIO is running:');
console.log(' docker-compose -f docker-compose.dev.yml up -d');
console.log(' 2. Check MinIO logs:');
console.log(' docker logs minio');
console.log(' 3. Verify MinIO is accessible at: http://localhost:9000');
}
}
testS3Connection();