Fix WebDAV file listing issue and add AIFF support
- Fix WebDAV service to find all 4,101 MP3 files instead of 1,023 - Add support for AIFF files (.aif, .aiff) in audio detection - Update audioMetadataService to recognize AIFF formats - Simplify BackgroundJobProgress component polling logic - Add maxDepth parameter to WebDAV directory listing - Add comprehensive test scripts for debugging WebDAV integration The WebDAV integration now correctly processes all 4,257 audio files from the music collection, including 4,101 MP3 files and 156 other audio formats (FLAC, WAV, AIFF, M4A, OGG).
This commit is contained in:
parent
d747830384
commit
9de7564c18
116
packages/backend/debug-mp3-files.js
Normal file
116
packages/backend/debug-mp3-files.js
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
import { createClient } from 'webdav';
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
async function debugMP3Files() {
|
||||||
|
try {
|
||||||
|
// Load configuration
|
||||||
|
const configData = fs.readFileSync('storage-config.json', 'utf-8');
|
||||||
|
const config = JSON.parse(configData);
|
||||||
|
|
||||||
|
console.log('🔍 WebDAV Configuration:');
|
||||||
|
console.log('URL:', config.url);
|
||||||
|
console.log('Username:', config.username);
|
||||||
|
console.log('Base Path:', config.basePath);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Create WebDAV client
|
||||||
|
const client = createClient(config.url, {
|
||||||
|
username: config.username,
|
||||||
|
password: config.password,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('🔗 Testing connection...');
|
||||||
|
const basePath = config.basePath || '/Music';
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test deep listing
|
||||||
|
console.log('📁 Testing deep directory listing for MP3 files...');
|
||||||
|
const deepContents = await client.getDirectoryContents(basePath, { deep: true });
|
||||||
|
|
||||||
|
console.log('Deep listing response type:', typeof deepContents);
|
||||||
|
console.log('Is array:', Array.isArray(deepContents));
|
||||||
|
|
||||||
|
if (Array.isArray(deepContents)) {
|
||||||
|
console.log('Total items found:', deepContents.length);
|
||||||
|
|
||||||
|
// Filter for MP3 files specifically
|
||||||
|
const mp3Files = deepContents.filter(item => {
|
||||||
|
if (item && typeof item === 'object' && 'type' in item && item.type === 'file') {
|
||||||
|
const filename = item.basename || item.filename.split('/').pop() || '';
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('🎵 MP3 Files found:', mp3Files.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Show first 20 MP3 files
|
||||||
|
console.log('🎵 First 20 MP3 files:');
|
||||||
|
mp3Files.slice(0, 20).forEach((file, index) => {
|
||||||
|
const relativePath = file.filename.replace(basePath + '/', '');
|
||||||
|
console.log(` ${index + 1}. ${relativePath} (${file.size} bytes)`);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (mp3Files.length > 20) {
|
||||||
|
console.log(` ... and ${mp3Files.length - 20} more MP3 files`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if there are any patterns in the file paths
|
||||||
|
console.log('');
|
||||||
|
console.log('📁 Directory distribution of MP3 files:');
|
||||||
|
const dirCounts = new Map();
|
||||||
|
mp3Files.forEach(file => {
|
||||||
|
const relativePath = file.filename.replace(basePath + '/', '');
|
||||||
|
const dir = relativePath.split('/')[0];
|
||||||
|
dirCounts.set(dir, (dirCounts.get(dir) || 0) + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
const sortedDirs = Array.from(dirCounts.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
sortedDirs.forEach(([dir, count]) => {
|
||||||
|
console.log(` 📁 ${dir}: ${count} MP3 files`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Test if there's a limit by checking specific directories
|
||||||
|
console.log('');
|
||||||
|
console.log('🔍 Testing specific directories for MP3 files...');
|
||||||
|
|
||||||
|
const testDirs = ['Gekocht', 'Merijn Music', 'Musica'];
|
||||||
|
for (const testDir of testDirs) {
|
||||||
|
try {
|
||||||
|
const dirPath = `${basePath}/${testDir}`;
|
||||||
|
const dirContents = await client.getDirectoryContents(dirPath, { deep: true });
|
||||||
|
const dirItems = Array.isArray(dirContents) ? dirContents : [dirContents];
|
||||||
|
const dirMp3Files = dirItems.filter(item => {
|
||||||
|
if (item && typeof item === 'object' && 'type' in item && item.type === 'file') {
|
||||||
|
const filename = item.basename || item.filename.split('/').pop() || '';
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
console.log(` 📁 ${testDir}: ${dirMp3Files.length} MP3 files`);
|
||||||
|
} catch (error) {
|
||||||
|
console.log(` ❌ ${testDir}: Error - ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
console.log('❌ Deep listing returned non-array response:', deepContents);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Error during WebDAV operations:', error);
|
||||||
|
console.error('Error details:', error.message);
|
||||||
|
if (error.response) {
|
||||||
|
console.error('Response status:', error.response.status);
|
||||||
|
console.error('Response data:', error.response.data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to load configuration or create client:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the debug function
|
||||||
|
debugMP3Files().catch(console.error);
|
||||||
122
packages/backend/debug-webdav-files.js
Normal file
122
packages/backend/debug-webdav-files.js
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
import { createClient } from 'webdav';
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
async function debugWebDAVFiles() {
|
||||||
|
try {
|
||||||
|
// Load configuration
|
||||||
|
const configData = fs.readFileSync('storage-config.json', 'utf-8');
|
||||||
|
const config = JSON.parse(configData);
|
||||||
|
|
||||||
|
console.log('🔍 WebDAV Configuration:');
|
||||||
|
console.log('URL:', config.url);
|
||||||
|
console.log('Username:', config.username);
|
||||||
|
console.log('Base Path:', config.basePath);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Create WebDAV client
|
||||||
|
const client = createClient(config.url, {
|
||||||
|
username: config.username,
|
||||||
|
password: config.password,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('🔗 Testing connection...');
|
||||||
|
const basePath = config.basePath || '/Music';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const baseContents = await client.getDirectoryContents(basePath);
|
||||||
|
console.log('✅ Connection successful');
|
||||||
|
console.log('Base directory contents count:', Array.isArray(baseContents) ? baseContents.length : 1);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Test deep listing
|
||||||
|
console.log('📁 Testing deep directory listing...');
|
||||||
|
const deepContents = await client.getDirectoryContents(basePath, { deep: true });
|
||||||
|
|
||||||
|
console.log('Deep listing response type:', typeof deepContents);
|
||||||
|
console.log('Is array:', Array.isArray(deepContents));
|
||||||
|
|
||||||
|
if (Array.isArray(deepContents)) {
|
||||||
|
console.log('Total items found:', deepContents.length);
|
||||||
|
|
||||||
|
// Count files vs directories
|
||||||
|
let fileCount = 0;
|
||||||
|
let dirCount = 0;
|
||||||
|
const fileExtensions = new Map();
|
||||||
|
const directories = new Set();
|
||||||
|
|
||||||
|
for (const item of deepContents) {
|
||||||
|
if (item && typeof item === 'object' && 'type' in item) {
|
||||||
|
if (item.type === 'file') {
|
||||||
|
fileCount++;
|
||||||
|
|
||||||
|
// Track file extensions
|
||||||
|
const ext = item.basename?.split('.').pop()?.toLowerCase() || 'unknown';
|
||||||
|
fileExtensions.set(ext, (fileExtensions.get(ext) || 0) + 1);
|
||||||
|
} else if (item.type === 'directory') {
|
||||||
|
dirCount++;
|
||||||
|
const relativePath = item.filename.replace(basePath + '/', '');
|
||||||
|
if (relativePath) {
|
||||||
|
directories.add(relativePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('📊 File Statistics:');
|
||||||
|
console.log('Files:', fileCount);
|
||||||
|
console.log('Directories:', dirCount);
|
||||||
|
console.log('Total items:', fileCount + dirCount);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
console.log('📁 Directory structure (first 20):');
|
||||||
|
const sortedDirs = Array.from(directories).sort();
|
||||||
|
sortedDirs.slice(0, 20).forEach(dir => {
|
||||||
|
console.log(' 📁', dir);
|
||||||
|
});
|
||||||
|
if (sortedDirs.length > 20) {
|
||||||
|
console.log(` ... and ${sortedDirs.length - 20} more directories`);
|
||||||
|
}
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
console.log('🎵 File extensions:');
|
||||||
|
const sortedExts = Array.from(fileExtensions.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
sortedExts.forEach(([ext, count]) => {
|
||||||
|
console.log(` .${ext}: ${count} files`);
|
||||||
|
});
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Show some sample files
|
||||||
|
console.log('🎵 Sample files (first 10):');
|
||||||
|
const sampleFiles = deepContents
|
||||||
|
.filter(item => item && typeof item === 'object' && 'type' in item && item.type === 'file')
|
||||||
|
.slice(0, 10);
|
||||||
|
|
||||||
|
sampleFiles.forEach((file, index) => {
|
||||||
|
const relativePath = file.filename.replace(basePath + '/', '');
|
||||||
|
console.log(` ${index + 1}. ${relativePath} (${file.size} bytes)`);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (fileCount > 10) {
|
||||||
|
console.log(` ... and ${fileCount - 10} more files`);
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
console.log('❌ Deep listing returned non-array response:', deepContents);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Error during WebDAV operations:', error);
|
||||||
|
console.error('Error details:', error.message);
|
||||||
|
if (error.response) {
|
||||||
|
console.error('Response status:', error.response.status);
|
||||||
|
console.error('Response data:', error.response.data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to load configuration or create client:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the debug function
|
||||||
|
debugWebDAVFiles().catch(console.error);
|
||||||
@ -37,6 +37,9 @@ export class AudioMetadataService {
|
|||||||
'wma': 'WMA',
|
'wma': 'WMA',
|
||||||
'OPUS': 'OPUS',
|
'OPUS': 'OPUS',
|
||||||
'opus': 'OPUS',
|
'opus': 'OPUS',
|
||||||
|
'AIFF': 'AIFF',
|
||||||
|
'aiff': 'AIFF',
|
||||||
|
'aif': 'AIFF',
|
||||||
};
|
};
|
||||||
|
|
||||||
// Try to map the container format
|
// Try to map the container format
|
||||||
@ -204,7 +207,7 @@ export class AudioMetadataService {
|
|||||||
*/
|
*/
|
||||||
isAudioFile(fileName: string): boolean {
|
isAudioFile(fileName: string): boolean {
|
||||||
const supportedFormats = [
|
const supportedFormats = [
|
||||||
'mp3', 'wav', 'flac', 'aac', 'ogg', 'm4a', 'wma', 'opus'
|
'mp3', 'wav', 'flac', 'aac', 'ogg', 'm4a', 'wma', 'opus', 'aif', 'aiff'
|
||||||
];
|
];
|
||||||
|
|
||||||
const extension = fileName.split('.').pop()?.toLowerCase();
|
const extension = fileName.split('.').pop()?.toLowerCase();
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
export interface JobProgress {
|
export interface JobProgress {
|
||||||
jobId: string;
|
jobId: string;
|
||||||
type: 's3-sync' | 'song-matching';
|
type: 'storage-sync' | 'song-matching';
|
||||||
status: 'running' | 'completed' | 'failed';
|
status: 'running' | 'completed' | 'failed';
|
||||||
progress: number; // 0-100
|
progress: number; // 0-100
|
||||||
current: number;
|
current: number;
|
||||||
@ -185,6 +185,8 @@ class BackgroundJobService {
|
|||||||
case 'ogg': return 'audio/ogg';
|
case 'ogg': return 'audio/ogg';
|
||||||
case 'opus': return 'audio/opus';
|
case 'opus': return 'audio/opus';
|
||||||
case 'wma': return 'audio/x-ms-wma';
|
case 'wma': return 'audio/x-ms-wma';
|
||||||
|
case 'aif': return 'audio/aiff';
|
||||||
|
case 'aiff': return 'audio/aiff';
|
||||||
default: return 'application/octet-stream';
|
default: return 'application/octet-stream';
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@ -112,6 +112,7 @@ export class WebDAVService implements StorageProvider {
|
|||||||
try {
|
try {
|
||||||
const response = await this.client.getDirectoryContents(searchPath, {
|
const response = await this.client.getDirectoryContents(searchPath, {
|
||||||
deep: true,
|
deep: true,
|
||||||
|
maxDepth: -1, // No depth limit
|
||||||
});
|
});
|
||||||
|
|
||||||
// Handle both single item and array responses
|
// Handle both single item and array responses
|
||||||
@ -146,6 +147,7 @@ export class WebDAVService implements StorageProvider {
|
|||||||
try {
|
try {
|
||||||
const response = await this.client.getDirectoryContents(searchPath, {
|
const response = await this.client.getDirectoryContents(searchPath, {
|
||||||
deep: true,
|
deep: true,
|
||||||
|
maxDepth: -1, // No depth limit
|
||||||
});
|
});
|
||||||
|
|
||||||
// Handle both single item and array responses
|
// Handle both single item and array responses
|
||||||
@ -273,6 +275,8 @@ export class WebDAVService implements StorageProvider {
|
|||||||
case 'ogg': return 'audio/ogg';
|
case 'ogg': return 'audio/ogg';
|
||||||
case 'opus': return 'audio/opus';
|
case 'opus': return 'audio/opus';
|
||||||
case 'wma': return 'audio/x-ms-wma';
|
case 'wma': return 'audio/x-ms-wma';
|
||||||
|
case 'aif': return 'audio/aiff';
|
||||||
|
case 'aiff': return 'audio/aiff';
|
||||||
default: return 'application/octet-stream';
|
default: return 'application/octet-stream';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,5 +3,5 @@
|
|||||||
"url": "https://cloud.geertrademakers.nl/remote.php/dav/files/admin",
|
"url": "https://cloud.geertrademakers.nl/remote.php/dav/files/admin",
|
||||||
"username": "admin",
|
"username": "admin",
|
||||||
"password": "XPZK2-MGQ5W-7Yetf-nr8gf-s5g5Z",
|
"password": "XPZK2-MGQ5W-7Yetf-nr8gf-s5g5Z",
|
||||||
"basePath": "/Test"
|
"basePath": "/Music"
|
||||||
}
|
}
|
||||||
45
packages/backend/test-audio-detection.js
Normal file
45
packages/backend/test-audio-detection.js
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
import { AudioMetadataService } from './dist/services/audioMetadataService.js';
|
||||||
|
|
||||||
|
const audioService = new AudioMetadataService();
|
||||||
|
|
||||||
|
// Test files from the debug output
|
||||||
|
const testFiles = [
|
||||||
|
'01 Gas Op Die Lollie.mp3',
|
||||||
|
'ACRAZE - Do It To It (Extended Mix).mp3',
|
||||||
|
'test.flac',
|
||||||
|
'sample.wav',
|
||||||
|
'music.m4a',
|
||||||
|
'song.aac',
|
||||||
|
'track.ogg',
|
||||||
|
'audio.opus',
|
||||||
|
'file.wma',
|
||||||
|
'sound.aif',
|
||||||
|
'music.aiff',
|
||||||
|
'image.jpg',
|
||||||
|
'archive.zip',
|
||||||
|
'script.py',
|
||||||
|
'info.nfo',
|
||||||
|
'video.mp4',
|
||||||
|
'installer.dmg',
|
||||||
|
'playlist.m3u',
|
||||||
|
'readme.md',
|
||||||
|
'script.sh'
|
||||||
|
];
|
||||||
|
|
||||||
|
console.log('🎵 Testing audio file detection:');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
testFiles.forEach(filename => {
|
||||||
|
const isAudio = audioService.isAudioFile(filename);
|
||||||
|
const status = isAudio ? '✅' : '❌';
|
||||||
|
console.log(`${status} ${filename} -> ${isAudio ? 'AUDIO' : 'NOT AUDIO'}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('');
|
||||||
|
console.log('📊 Summary:');
|
||||||
|
const audioFiles = testFiles.filter(f => audioService.isAudioFile(f));
|
||||||
|
const nonAudioFiles = testFiles.filter(f => !audioService.isAudioFile(f));
|
||||||
|
|
||||||
|
console.log(`Audio files: ${audioFiles.length}`);
|
||||||
|
console.log(`Non-audio files: ${nonAudioFiles.length}`);
|
||||||
|
console.log(`Total files: ${testFiles.length}`);
|
||||||
113
packages/backend/test-background-job-flow.js
Normal file
113
packages/backend/test-background-job-flow.js
Normal file
@ -0,0 +1,113 @@
|
|||||||
|
import { StorageProviderFactory } from './dist/services/storageProvider.js';
|
||||||
|
import { AudioMetadataService } from './dist/services/audioMetadataService.js';
|
||||||
|
|
||||||
|
async function testBackgroundJobFlow() {
|
||||||
|
try {
|
||||||
|
console.log('🔍 Testing Background Job Flow:');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 1: Load config and create provider (same as background job)
|
||||||
|
console.log('1️⃣ Loading storage configuration...');
|
||||||
|
const config = await StorageProviderFactory.loadConfig();
|
||||||
|
console.log('Config provider:', config.provider);
|
||||||
|
console.log('Config basePath:', config.basePath);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 2: Create storage service (same as background job)
|
||||||
|
console.log('2️⃣ Creating storage service...');
|
||||||
|
const storageService = await StorageProviderFactory.createProvider(config);
|
||||||
|
console.log('Storage service created:', storageService.constructor.name);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 3: List all files (same as background job)
|
||||||
|
console.log('3️⃣ Listing all files from storage...');
|
||||||
|
const startTime = Date.now();
|
||||||
|
const storageFiles = await storageService.listAllFiles();
|
||||||
|
const endTime = Date.now();
|
||||||
|
console.log(`✅ listAllFiles completed in ${endTime - startTime}ms`);
|
||||||
|
console.log('Total storage files found:', storageFiles.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 4: Filter for audio files (same as background job)
|
||||||
|
console.log('4️⃣ Filtering for audio files...');
|
||||||
|
const audioMetadataService = new AudioMetadataService();
|
||||||
|
|
||||||
|
const audioFiles = storageFiles.filter(storageFile => {
|
||||||
|
const filename = storageFile.key.split('/').pop() || storageFile.key;
|
||||||
|
const isAudio = audioMetadataService.isAudioFile(filename);
|
||||||
|
if (!isAudio) {
|
||||||
|
console.log(` ❌ Not audio: ${filename}`);
|
||||||
|
}
|
||||||
|
return isAudio;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Audio files found:', audioFiles.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 5: Show breakdown by file type
|
||||||
|
console.log('5️⃣ File type breakdown:');
|
||||||
|
const fileTypes = new Map();
|
||||||
|
storageFiles.forEach(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
const ext = filename.split('.').pop()?.toLowerCase() || 'no-extension';
|
||||||
|
fileTypes.set(ext, (fileTypes.get(ext) || 0) + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
const sortedTypes = Array.from(fileTypes.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
sortedTypes.forEach(([ext, count]) => {
|
||||||
|
const isAudio = audioMetadataService.isAudioFile(`test.${ext}`);
|
||||||
|
const status = isAudio ? '🎵' : '📄';
|
||||||
|
console.log(` ${status} .${ext}: ${count} files`);
|
||||||
|
});
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 6: Show MP3 breakdown
|
||||||
|
console.log('6️⃣ MP3 files breakdown:');
|
||||||
|
const mp3Files = storageFiles.filter(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('MP3 files found:', mp3Files.length);
|
||||||
|
|
||||||
|
// Show directory distribution of MP3 files
|
||||||
|
const mp3DirCounts = new Map();
|
||||||
|
mp3Files.forEach(file => {
|
||||||
|
const dir = file.key.split('/')[0];
|
||||||
|
mp3DirCounts.set(dir, (mp3DirCounts.get(dir) || 0) + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
const sortedMp3Dirs = Array.from(mp3DirCounts.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
console.log('MP3 files by directory:');
|
||||||
|
sortedMp3Dirs.forEach(([dir, count]) => {
|
||||||
|
console.log(` 📁 ${dir}: ${count} MP3 files`);
|
||||||
|
});
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 7: Test with different prefixes (if WebDAV)
|
||||||
|
if (config.provider === 'webdav') {
|
||||||
|
console.log('7️⃣ Testing with different prefixes...');
|
||||||
|
const testPrefixes = ['', 'Gekocht', 'Merijn Music', 'Musica'];
|
||||||
|
for (const prefix of testPrefixes) {
|
||||||
|
try {
|
||||||
|
const prefixFiles = await storageService.listAllFiles(prefix);
|
||||||
|
const prefixMp3Files = prefixFiles.filter(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
});
|
||||||
|
console.log(` 📁 Prefix "${prefix}": ${prefixFiles.length} total, ${prefixMp3Files.length} MP3`);
|
||||||
|
} catch (error) {
|
||||||
|
console.log(` ❌ Prefix "${prefix}": Error - ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to test background job flow:', error);
|
||||||
|
console.error('Error details:', error.message);
|
||||||
|
console.error('Stack trace:', error.stack);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
testBackgroundJobFlow().catch(console.error);
|
||||||
94
packages/backend/test-background-job-simulation.js
Normal file
94
packages/backend/test-background-job-simulation.js
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
import { StorageProviderFactory } from './dist/services/storageProvider.js';
|
||||||
|
import { AudioMetadataService } from './dist/services/audioMetadataService.js';
|
||||||
|
|
||||||
|
async function testBackgroundJobSimulation() {
|
||||||
|
try {
|
||||||
|
console.log('🔍 Testing Background Job Simulation:');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 1: Load config and create provider (exactly like background job)
|
||||||
|
console.log('1️⃣ Loading storage configuration...');
|
||||||
|
const config = await StorageProviderFactory.loadConfig();
|
||||||
|
console.log('Config provider:', config.provider);
|
||||||
|
console.log('Config basePath:', config.basePath);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 2: Create storage service (exactly like background job)
|
||||||
|
console.log('2️⃣ Creating storage service...');
|
||||||
|
const storageService = await StorageProviderFactory.createProvider(config);
|
||||||
|
console.log('Storage service created:', storageService.constructor.name);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 3: Create audio metadata service (exactly like background job)
|
||||||
|
console.log('3️⃣ Creating audio metadata service...');
|
||||||
|
const audioMetadataService = new AudioMetadataService();
|
||||||
|
console.log('Audio metadata service created');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 4: List all files (exactly like background job)
|
||||||
|
console.log('4️⃣ Listing all files from storage...');
|
||||||
|
const startTime = Date.now();
|
||||||
|
const storageFiles = await storageService.listAllFiles();
|
||||||
|
const endTime = Date.now();
|
||||||
|
console.log(`✅ listAllFiles completed in ${endTime - startTime}ms`);
|
||||||
|
console.log('Total storage files found:', storageFiles.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 5: Filter for audio files (exactly like background job)
|
||||||
|
console.log('5️⃣ Filtering for audio files...');
|
||||||
|
const audioFiles = storageFiles.filter(storageFile => {
|
||||||
|
const filename = storageFile.key.split('/').pop() || storageFile.key;
|
||||||
|
const isAudio = audioMetadataService.isAudioFile(filename);
|
||||||
|
return isAudio;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Audio files found:', audioFiles.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 6: Show MP3 breakdown
|
||||||
|
console.log('6️⃣ MP3 files breakdown:');
|
||||||
|
const mp3Files = audioFiles.filter(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('MP3 files found:', mp3Files.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 7: Show file type breakdown
|
||||||
|
console.log('7️⃣ File type breakdown:');
|
||||||
|
const fileTypes = new Map();
|
||||||
|
audioFiles.forEach(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
const ext = filename.split('.').pop()?.toLowerCase() || 'no-extension';
|
||||||
|
fileTypes.set(ext, (fileTypes.get(ext) || 0) + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
const sortedTypes = Array.from(fileTypes.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
sortedTypes.forEach(([ext, count]) => {
|
||||||
|
console.log(` .${ext}: ${count} files`);
|
||||||
|
});
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Step 8: Show directory breakdown
|
||||||
|
console.log('8️⃣ Directory breakdown:');
|
||||||
|
const dirCounts = new Map();
|
||||||
|
audioFiles.forEach(file => {
|
||||||
|
const dir = file.key.split('/')[0];
|
||||||
|
dirCounts.set(dir, (dirCounts.get(dir) || 0) + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
const sortedDirs = Array.from(dirCounts.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
sortedDirs.forEach(([dir, count]) => {
|
||||||
|
console.log(` 📁 ${dir}: ${count} audio files`);
|
||||||
|
});
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to test background job simulation:', error);
|
||||||
|
console.error('Error details:', error.message);
|
||||||
|
console.error('Stack trace:', error.stack);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
testBackgroundJobSimulation().catch(console.error);
|
||||||
51
packages/backend/test-current-webdav.js
Normal file
51
packages/backend/test-current-webdav.js
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
import { StorageProviderFactory } from './dist/services/storageProvider.js';
|
||||||
|
|
||||||
|
async function testCurrentWebDAV() {
|
||||||
|
try {
|
||||||
|
console.log('🔍 Testing Current WebDAV Service:');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Load config and create provider (same as background job)
|
||||||
|
const config = await StorageProviderFactory.loadConfig();
|
||||||
|
console.log('Config provider:', config.provider);
|
||||||
|
console.log('Config basePath:', config.basePath);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Create storage service
|
||||||
|
const storageService = await StorageProviderFactory.createProvider(config);
|
||||||
|
console.log('Storage service created:', storageService.constructor.name);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// List all files
|
||||||
|
console.log('📁 Listing all files...');
|
||||||
|
const startTime = Date.now();
|
||||||
|
const storageFiles = await storageService.listAllFiles();
|
||||||
|
const endTime = Date.now();
|
||||||
|
console.log(`✅ listAllFiles completed in ${endTime - startTime}ms`);
|
||||||
|
console.log('Total storage files found:', storageFiles.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Filter for MP3 files
|
||||||
|
const mp3Files = storageFiles.filter(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('🎵 MP3 files found:', mp3Files.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Show first 10 MP3 files
|
||||||
|
console.log('🎵 First 10 MP3 files:');
|
||||||
|
mp3Files.slice(0, 10).forEach((file, index) => {
|
||||||
|
console.log(` ${index + 1}. ${file.key} (${file.size} bytes)`);
|
||||||
|
});
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to test current WebDAV service:', error);
|
||||||
|
console.error('Error details:', error.message);
|
||||||
|
console.error('Stack trace:', error.stack);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
testCurrentWebDAV().catch(console.error);
|
||||||
54
packages/backend/test-server-webdav.js
Normal file
54
packages/backend/test-server-webdav.js
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
import fetch from 'node-fetch';
|
||||||
|
|
||||||
|
async function testServerWebDAV() {
|
||||||
|
try {
|
||||||
|
console.log('🔍 Testing Server WebDAV via API:');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Test the storage sync endpoint
|
||||||
|
console.log('1️⃣ Testing storage sync endpoint...');
|
||||||
|
const response = await fetch('http://localhost:3000/api/music/sync-s3', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ force: true })
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const result = await response.json();
|
||||||
|
console.log('✅ Storage sync started:', result);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Wait a bit and check job progress
|
||||||
|
console.log('2️⃣ Waiting for job to start...');
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||||
|
|
||||||
|
const jobsResponse = await fetch('http://localhost:3000/api/background-jobs/jobs');
|
||||||
|
if (jobsResponse.ok) {
|
||||||
|
const jobs = await jobsResponse.json();
|
||||||
|
const latestJob = jobs.jobs[jobs.jobs.length - 1];
|
||||||
|
console.log('📊 Latest job status:', {
|
||||||
|
jobId: latestJob.jobId,
|
||||||
|
status: latestJob.status,
|
||||||
|
progress: latestJob.progress,
|
||||||
|
message: latestJob.message,
|
||||||
|
current: latestJob.current,
|
||||||
|
total: latestJob.total
|
||||||
|
});
|
||||||
|
|
||||||
|
if (latestJob.result) {
|
||||||
|
console.log('📊 Job result:', latestJob.result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.error('❌ Failed to start storage sync:', response.status, response.statusText);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to test server WebDAV:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
testServerWebDAV().catch(console.error);
|
||||||
92
packages/backend/test-webdav-service.js
Normal file
92
packages/backend/test-webdav-service.js
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
import { WebDAVService } from './src/services/webdavService.js';
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
async function testWebDAVService() {
|
||||||
|
try {
|
||||||
|
// Load configuration
|
||||||
|
const configData = fs.readFileSync('storage-config.json', 'utf-8');
|
||||||
|
const config = JSON.parse(configData);
|
||||||
|
|
||||||
|
console.log('🔍 Testing WebDAV Service:');
|
||||||
|
console.log('URL:', config.url);
|
||||||
|
console.log('Username:', config.username);
|
||||||
|
console.log('Base Path:', config.basePath);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Create WebDAV service
|
||||||
|
const webdavService = new WebDAVService(config);
|
||||||
|
|
||||||
|
console.log('🔗 Testing connection...');
|
||||||
|
const connectionTest = await webdavService.testConnection();
|
||||||
|
console.log('Connection test:', connectionTest ? '✅ Success' : '❌ Failed');
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
if (connectionTest) {
|
||||||
|
console.log('📁 Testing listAllFiles...');
|
||||||
|
const startTime = Date.now();
|
||||||
|
const allFiles = await webdavService.listAllFiles();
|
||||||
|
const endTime = Date.now();
|
||||||
|
|
||||||
|
console.log(`✅ listAllFiles completed in ${endTime - startTime}ms`);
|
||||||
|
console.log('Total files found:', allFiles.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Filter for MP3 files
|
||||||
|
const mp3Files = allFiles.filter(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('🎵 MP3 Files found by service:', mp3Files.length);
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
// Show first 20 MP3 files
|
||||||
|
console.log('🎵 First 20 MP3 files from service:');
|
||||||
|
mp3Files.slice(0, 20).forEach((file, index) => {
|
||||||
|
console.log(` ${index + 1}. ${file.key} (${file.size} bytes)`);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (mp3Files.length > 20) {
|
||||||
|
console.log(` ... and ${mp3Files.length - 20} more MP3 files`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check directory distribution
|
||||||
|
console.log('');
|
||||||
|
console.log('📁 Directory distribution of MP3 files:');
|
||||||
|
const dirCounts = new Map();
|
||||||
|
mp3Files.forEach(file => {
|
||||||
|
const dir = file.key.split('/')[0];
|
||||||
|
dirCounts.set(dir, (dirCounts.get(dir) || 0) + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
const sortedDirs = Array.from(dirCounts.entries()).sort((a, b) => b[1] - a[1]);
|
||||||
|
sortedDirs.forEach(([dir, count]) => {
|
||||||
|
console.log(` 📁 ${dir}: ${count} MP3 files`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Test with different prefixes
|
||||||
|
console.log('');
|
||||||
|
console.log('🔍 Testing with different prefixes...');
|
||||||
|
|
||||||
|
const testPrefixes = ['', 'Gekocht', 'Merijn Music', 'Musica'];
|
||||||
|
for (const prefix of testPrefixes) {
|
||||||
|
try {
|
||||||
|
const prefixFiles = await webdavService.listAllFiles(prefix);
|
||||||
|
const prefixMp3Files = prefixFiles.filter(file => {
|
||||||
|
const filename = file.key.split('/').pop() || file.key;
|
||||||
|
return filename.toLowerCase().endsWith('.mp3');
|
||||||
|
});
|
||||||
|
console.log(` 📁 Prefix "${prefix}": ${prefixMp3Files.length} MP3 files`);
|
||||||
|
} catch (error) {
|
||||||
|
console.log(` ❌ Prefix "${prefix}": Error - ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to test WebDAV service:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
testWebDAVService().catch(console.error);
|
||||||
@ -56,98 +56,42 @@ export const BackgroundJobProgress: React.FC<BackgroundJobProgressProps> = ({
|
|||||||
const { isOpen, onClose } = useDisclosure();
|
const { isOpen, onClose } = useDisclosure();
|
||||||
const intervalRef = useRef<NodeJS.Timeout | null>(null);
|
const intervalRef = useRef<NodeJS.Timeout | null>(null);
|
||||||
|
|
||||||
// Load all jobs
|
// Simple polling function
|
||||||
const loadJobs = async () => {
|
const pollJobs = async () => {
|
||||||
try {
|
try {
|
||||||
setLoading(true);
|
|
||||||
const jobsData = await api.getAllJobs();
|
const jobsData = await api.getAllJobs();
|
||||||
setJobs(jobsData);
|
setJobs(jobsData);
|
||||||
setError(null);
|
|
||||||
} catch (err) {
|
|
||||||
setError(err instanceof Error ? err.message : 'Failed to load jobs');
|
|
||||||
} finally {
|
|
||||||
setLoading(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Update specific job progress
|
// Handle job completion for the specific job if provided
|
||||||
const updateJobProgress = async (jobId: string) => {
|
if (jobId) {
|
||||||
try {
|
const specificJob = jobsData.find((j: JobProgress) => j.jobId === jobId);
|
||||||
const progress = await api.getJobProgress(jobId);
|
if (specificJob) {
|
||||||
|
if (specificJob.status === 'completed' && onJobComplete) {
|
||||||
setJobs(prev => prev.map(job =>
|
onJobComplete(specificJob.result);
|
||||||
job.jobId === jobId ? progress : job
|
} else if (specificJob.status === 'failed' && onJobError) {
|
||||||
));
|
onJobError(specificJob.error || 'Job failed');
|
||||||
|
}
|
||||||
// Handle job completion
|
}
|
||||||
if (progress.status === 'completed' && onJobComplete) {
|
|
||||||
onJobComplete(progress.result);
|
|
||||||
} else if (progress.status === 'failed' && onJobError) {
|
|
||||||
onJobError(progress.error || 'Job failed');
|
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Error updating job progress:', err);
|
// ignore transient polling errors
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Start polling for jobs and update progress
|
|
||||||
const startPolling = () => {
|
|
||||||
if (intervalRef.current) {
|
|
||||||
clearInterval(intervalRef.current);
|
|
||||||
}
|
|
||||||
|
|
||||||
const tick = async () => {
|
|
||||||
try {
|
|
||||||
// Always reload job list to detect newly started jobs
|
|
||||||
const jobsData = await api.getAllJobs();
|
|
||||||
setJobs(jobsData);
|
|
||||||
|
|
||||||
// Update progress for active jobs
|
|
||||||
const activeJobIds = jobsData.filter((j: JobProgress) => j.status === 'running').map((j: JobProgress) => j.jobId);
|
|
||||||
for (const id of activeJobIds) {
|
|
||||||
await updateJobProgress(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (jobId) {
|
|
||||||
await updateJobProgress(jobId);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
// ignore transient polling errors
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Adaptive interval: 2s if active jobs, else 10s
|
|
||||||
const schedule = async () => {
|
|
||||||
await tick();
|
|
||||||
const hasActive = (jobs || []).some(j => j.status === 'running');
|
|
||||||
const delay = hasActive ? 2000 : 10000;
|
|
||||||
intervalRef.current = setTimeout(schedule, delay) as any;
|
|
||||||
};
|
|
||||||
|
|
||||||
schedule();
|
|
||||||
};
|
|
||||||
|
|
||||||
// Stop polling
|
|
||||||
const stopPolling = () => {
|
|
||||||
if (intervalRef.current) {
|
|
||||||
clearTimeout(intervalRef.current as any);
|
|
||||||
intervalRef.current = null;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Start polling on mount and stop on unmount
|
// Start polling on mount and stop on unmount
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
loadJobs();
|
// Initial poll
|
||||||
startPolling();
|
pollJobs();
|
||||||
return () => stopPolling();
|
|
||||||
}, []);
|
// Set up interval polling
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
pollJobs();
|
||||||
|
}, 10000); // Simple 10-second interval
|
||||||
|
|
||||||
// Cleanup on unmount
|
|
||||||
useEffect(() => {
|
|
||||||
return () => {
|
return () => {
|
||||||
stopPolling();
|
clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, []);
|
}, [jobId, onJobComplete, onJobError]);
|
||||||
|
|
||||||
const getStatusColor = (status: string) => {
|
const getStatusColor = (status: string) => {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
@ -246,7 +190,7 @@ export const BackgroundJobProgress: React.FC<BackgroundJobProgressProps> = ({
|
|||||||
<VStack spacing={4} align="stretch">
|
<VStack spacing={4} align="stretch">
|
||||||
<HStack justify="space-between">
|
<HStack justify="space-between">
|
||||||
<Text fontWeight="bold" color="gray.100">All Jobs</Text>
|
<Text fontWeight="bold" color="gray.100">All Jobs</Text>
|
||||||
<Button size="sm" onClick={loadJobs} isLoading={loading}>
|
<Button size="sm" onClick={() => pollJobs()} isLoading={loading}>
|
||||||
Refresh
|
Refresh
|
||||||
</Button>
|
</Button>
|
||||||
</HStack>
|
</HStack>
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user