More webdav compatiblity
This commit is contained in:
parent
218046ec4f
commit
7065247277
@ -10,8 +10,8 @@ router.post('/start', async (req, res) => {
|
|||||||
try {
|
try {
|
||||||
const { type, options } = req.body;
|
const { type, options } = req.body;
|
||||||
|
|
||||||
if (!type || !['s3-sync', 'song-matching'].includes(type)) {
|
if (!type || !['storage-sync', 'song-matching'].includes(type)) {
|
||||||
return res.status(400).json({ error: 'Invalid job type. Must be "s3-sync" or "song-matching"' });
|
return res.status(400).json({ error: 'Invalid job type. Must be "storage-sync" or "song-matching"' });
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`🚀 Starting background job: ${type}`);
|
console.log(`🚀 Starting background job: ${type}`);
|
||||||
|
|||||||
@ -342,12 +342,12 @@ router.post('/sync-s3', async (req, res) => {
|
|||||||
|
|
||||||
// Start the background job
|
// Start the background job
|
||||||
const jobId = await backgroundJobService.startJob({
|
const jobId = await backgroundJobService.startJob({
|
||||||
type: 's3-sync',
|
type: 'storage-sync',
|
||||||
options: req.body
|
options: req.body
|
||||||
});
|
});
|
||||||
|
|
||||||
res.json({
|
res.json({
|
||||||
message: 'S3 sync started as background job',
|
message: 'Storage sync started as background job',
|
||||||
jobId,
|
jobId,
|
||||||
status: 'started'
|
status: 'started'
|
||||||
});
|
});
|
||||||
@ -479,31 +479,8 @@ router.get('/', async (req, res) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
/**
|
// DELETE endpoint removed to keep WebDAV integration read-only
|
||||||
* Delete a music file
|
// Music files cannot be deleted to prevent accidental data loss from WebDAV
|
||||||
*/
|
|
||||||
router.delete('/:id', async (req, res) => {
|
|
||||||
try {
|
|
||||||
const musicFile = await MusicFile.findById(req.params.id);
|
|
||||||
if (!musicFile) {
|
|
||||||
return res.status(404).json({ error: 'Music file not found' });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete from storage
|
|
||||||
await storageService.deleteFile(musicFile.s3Key);
|
|
||||||
|
|
||||||
// Delete from database
|
|
||||||
await MusicFile.findByIdAndDelete(req.params.id);
|
|
||||||
|
|
||||||
// Invalidate folder cache since we removed a file
|
|
||||||
invalidateFolderCache();
|
|
||||||
|
|
||||||
res.json({ message: 'Music file deleted successfully' });
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Delete error:', error);
|
|
||||||
res.status(500).json({ error: 'Failed to delete music file' });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Link music file to existing song
|
* Link music file to existing song
|
||||||
|
|||||||
@ -13,7 +13,7 @@ export interface JobProgress {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface JobOptions {
|
export interface JobOptions {
|
||||||
type: 's3-sync' | 'song-matching';
|
type: 'storage-sync' | 'song-matching';
|
||||||
options?: any;
|
options?: any;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -140,8 +140,8 @@ class BackgroundJobService {
|
|||||||
private async runJob(jobId: string, jobOptions: JobOptions): Promise<void> {
|
private async runJob(jobId: string, jobOptions: JobOptions): Promise<void> {
|
||||||
try {
|
try {
|
||||||
switch (jobOptions.type) {
|
switch (jobOptions.type) {
|
||||||
case 's3-sync':
|
case 'storage-sync':
|
||||||
await this.runS3SyncJob(jobId, jobOptions.options);
|
await this.runStorageSyncJob(jobId, jobOptions.options);
|
||||||
break;
|
break;
|
||||||
case 'song-matching':
|
case 'song-matching':
|
||||||
await this.runSongMatchingJob(jobId, jobOptions.options);
|
await this.runSongMatchingJob(jobId, jobOptions.options);
|
||||||
@ -156,18 +156,20 @@ class BackgroundJobService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run S3 sync job
|
* Run storage sync job (works with any storage provider)
|
||||||
*/
|
*/
|
||||||
private async runS3SyncJob(jobId: string, options?: any): Promise<void> {
|
private async runStorageSyncJob(jobId: string, options?: any): Promise<void> {
|
||||||
try {
|
try {
|
||||||
// Import here to avoid circular dependencies
|
// Import here to avoid circular dependencies
|
||||||
const { S3Service } = await import('./s3Service.js');
|
const { StorageProviderFactory } = await import('./storageProvider.js');
|
||||||
const { AudioMetadataService } = await import('./audioMetadataService.js');
|
const { AudioMetadataService } = await import('./audioMetadataService.js');
|
||||||
const { SongMatchingService } = await import('./songMatchingService.js');
|
const { SongMatchingService } = await import('./songMatchingService.js');
|
||||||
const { MusicFile } = await import('../models/MusicFile.js');
|
const { MusicFile } = await import('../models/MusicFile.js');
|
||||||
const { Song } = await import('../models/Song.js');
|
const { Song } = await import('../models/Song.js');
|
||||||
|
|
||||||
const s3Service = await S3Service.createFromConfig();
|
// Get the configured storage provider
|
||||||
|
const config = await StorageProviderFactory.loadConfig();
|
||||||
|
const storageService = await StorageProviderFactory.createProvider(config);
|
||||||
const audioMetadataService = new AudioMetadataService();
|
const audioMetadataService = new AudioMetadataService();
|
||||||
const songMatchingService = new SongMatchingService();
|
const songMatchingService = new SongMatchingService();
|
||||||
|
|
||||||
@ -213,14 +215,14 @@ class BackgroundJobService {
|
|||||||
|
|
||||||
// Phase 1: Quick filename matching
|
// Phase 1: Quick filename matching
|
||||||
this.updateProgress(jobId, {
|
this.updateProgress(jobId, {
|
||||||
message: 'Phase 1: Fetching files from S3...',
|
message: `Phase 1: Fetching files from ${config.provider.toUpperCase()}...`,
|
||||||
current: 0,
|
current: 0,
|
||||||
total: 0
|
total: 0
|
||||||
});
|
});
|
||||||
|
|
||||||
const s3Files = await s3Service.listAllFiles();
|
const storageFiles = await storageService.listAllFiles();
|
||||||
const audioFiles = s3Files.filter(s3File => {
|
const audioFiles = storageFiles.filter(storageFile => {
|
||||||
const filename = s3File.key.split('/').pop() || s3File.key;
|
const filename = storageFile.key.split('/').pop() || storageFile.key;
|
||||||
return audioMetadataService.isAudioFile(filename);
|
return audioMetadataService.isAudioFile(filename);
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -232,8 +234,8 @@ class BackgroundJobService {
|
|||||||
|
|
||||||
// Get existing files
|
// Get existing files
|
||||||
const existingFiles = await MusicFile.find({}, { s3Key: 1 });
|
const existingFiles = await MusicFile.find({}, { s3Key: 1 });
|
||||||
const existingS3Keys = new Set(existingFiles.map(f => f.s3Key));
|
const existingStorageKeys = new Set(existingFiles.map(f => f.s3Key));
|
||||||
const newAudioFiles = options?.force ? audioFiles : audioFiles.filter(s3File => !existingS3Keys.has(s3File.key));
|
const newAudioFiles = options?.force ? audioFiles : audioFiles.filter(storageFile => !existingStorageKeys.has(storageFile.key));
|
||||||
|
|
||||||
this.updateProgress(jobId, {
|
this.updateProgress(jobId, {
|
||||||
message: `Phase 1: Processing ${newAudioFiles.length} new audio files...`,
|
message: `Phase 1: Processing ${newAudioFiles.length} new audio files...`,
|
||||||
@ -249,11 +251,11 @@ class BackgroundJobService {
|
|||||||
let processedCount = 0;
|
let processedCount = 0;
|
||||||
let phase1Errors = 0;
|
let phase1Errors = 0;
|
||||||
|
|
||||||
for (const s3File of newAudioFiles) {
|
for (const storageFile of newAudioFiles) {
|
||||||
processedCount++;
|
processedCount++;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const filename = s3File.key.split('/').pop() || s3File.key;
|
const filename = storageFile.key.split('/').pop() || storageFile.key;
|
||||||
|
|
||||||
this.updateProgress(jobId, {
|
this.updateProgress(jobId, {
|
||||||
message: `Phase 1: Quick filename matching`,
|
message: `Phase 1: Quick filename matching`,
|
||||||
@ -265,7 +267,7 @@ class BackgroundJobService {
|
|||||||
// Decode URL-encoded sequences so %20, %27 etc. are compared correctly
|
// Decode URL-encoded sequences so %20, %27 etc. are compared correctly
|
||||||
const safeDecode = (s: string): string => { try { return decodeURIComponent(s); } catch { return s; } };
|
const safeDecode = (s: string): string => { try { return decodeURIComponent(s); } catch { return s; } };
|
||||||
const stripDiacritics = (s: string) => s.normalize('NFKD').replace(/[\u0300-\u036f]/g, '');
|
const stripDiacritics = (s: string) => s.normalize('NFKD').replace(/[\u0300-\u036f]/g, '');
|
||||||
const normalizedS3Filename = stripDiacritics(safeDecode(filename)).replace(/\.[^/.]+$/, '').toLowerCase();
|
const normalizedStorageFilename = stripDiacritics(safeDecode(filename)).replace(/\.[^/.]+$/, '').toLowerCase();
|
||||||
let matchedSong = null;
|
let matchedSong = null;
|
||||||
|
|
||||||
for (const song of allSongs) {
|
for (const song of allSongs) {
|
||||||
@ -273,7 +275,7 @@ class BackgroundJobService {
|
|||||||
const rekordboxFilename = song.location.split(/[/\\]/).pop() || song.location;
|
const rekordboxFilename = song.location.split(/[/\\]/).pop() || song.location;
|
||||||
const normalizedRekordboxFilename = stripDiacritics(safeDecode(rekordboxFilename)).replace(/\.[^/.]+$/, '').toLowerCase();
|
const normalizedRekordboxFilename = stripDiacritics(safeDecode(rekordboxFilename)).replace(/\.[^/.]+$/, '').toLowerCase();
|
||||||
|
|
||||||
if (normalizedS3Filename === normalizedRekordboxFilename) {
|
if (normalizedStorageFilename === normalizedRekordboxFilename) {
|
||||||
matchedSong = song;
|
matchedSong = song;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -283,30 +285,31 @@ class BackgroundJobService {
|
|||||||
if (matchedSong) {
|
if (matchedSong) {
|
||||||
const basicMetadata = audioMetadataService.extractBasicMetadataFromFilename(filename);
|
const basicMetadata = audioMetadataService.extractBasicMetadataFromFilename(filename);
|
||||||
// Reuse existing MusicFile if present (force mode), otherwise create new
|
// Reuse existing MusicFile if present (force mode), otherwise create new
|
||||||
let musicFile = await MusicFile.findOne({ s3Key: s3File.key });
|
let musicFile = await MusicFile.findOne({ s3Key: storageFile.key });
|
||||||
if (!musicFile) {
|
if (!musicFile) {
|
||||||
musicFile = new MusicFile({ s3Key: s3File.key });
|
musicFile = new MusicFile({ s3Key: storageFile.key });
|
||||||
}
|
}
|
||||||
musicFile.originalName = filename;
|
musicFile.originalName = filename;
|
||||||
musicFile.s3Key = s3File.key;
|
musicFile.s3Key = storageFile.key;
|
||||||
musicFile.s3Url = `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`;
|
musicFile.s3Url = await storageService.getPresignedUrl(storageFile.key);
|
||||||
musicFile.contentType = guessContentType(filename);
|
musicFile.contentType = guessContentType(filename);
|
||||||
musicFile.size = s3File.size;
|
musicFile.size = storageFile.size;
|
||||||
Object.assign(musicFile, basicMetadata);
|
Object.assign(musicFile, basicMetadata);
|
||||||
musicFile.songId = matchedSong._id;
|
musicFile.songId = matchedSong._id;
|
||||||
|
|
||||||
await musicFile.save();
|
await musicFile.save();
|
||||||
quickMatches.push(musicFile);
|
quickMatches.push(musicFile);
|
||||||
|
|
||||||
// Update the Song document to indicate it has an S3 file
|
// Update the Song document to indicate it has a storage file
|
||||||
|
const storageUrl = await storageService.getPresignedUrl(storageFile.key);
|
||||||
await Song.updateOne(
|
await Song.updateOne(
|
||||||
{ _id: matchedSong._id },
|
{ _id: matchedSong._id },
|
||||||
{
|
{
|
||||||
$set: {
|
$set: {
|
||||||
's3File.musicFileId': musicFile._id,
|
's3File.musicFileId': musicFile._id,
|
||||||
's3File.s3Key': s3File.key,
|
's3File.s3Key': storageFile.key,
|
||||||
's3File.s3Url': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
's3File.s3Url': storageUrl,
|
||||||
's3File.streamingUrl': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
's3File.streamingUrl': storageUrl,
|
||||||
's3File.hasS3File': true
|
's3File.hasS3File': true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -314,14 +317,14 @@ class BackgroundJobService {
|
|||||||
|
|
||||||
console.log(`✅ Quick match saved immediately: ${filename}`);
|
console.log(`✅ Quick match saved immediately: ${filename}`);
|
||||||
} else {
|
} else {
|
||||||
unmatchedFiles.push(s3File);
|
unmatchedFiles.push(storageFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error in quick matching ${s3File.key}:`, error);
|
console.error(`Error in quick matching ${storageFile.key}:`, error);
|
||||||
unmatchedFiles.push(s3File);
|
unmatchedFiles.push(storageFile);
|
||||||
phase1Errors++;
|
phase1Errors++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -346,10 +349,10 @@ class BackgroundJobService {
|
|||||||
const processedFiles: any[] = [];
|
const processedFiles: any[] = [];
|
||||||
|
|
||||||
for (let i = 0; i < unmatchedFiles.length; i++) {
|
for (let i = 0; i < unmatchedFiles.length; i++) {
|
||||||
const s3File = unmatchedFiles[i];
|
const storageFile = unmatchedFiles[i];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const filename = s3File.key.split('/').pop() || s3File.key;
|
const filename = storageFile.key.split('/').pop() || storageFile.key;
|
||||||
|
|
||||||
this.updateProgress(jobId, {
|
this.updateProgress(jobId, {
|
||||||
message: `Phase 2: Complex matching`,
|
message: `Phase 2: Complex matching`,
|
||||||
@ -358,19 +361,19 @@ class BackgroundJobService {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Download file and extract metadata
|
// Download file and extract metadata
|
||||||
const fileBuffer = await s3Service.getFileContent(s3File.key);
|
const fileBuffer = await storageService.getFileContent(storageFile.key);
|
||||||
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
|
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
|
||||||
|
|
||||||
// Reuse existing MusicFile document if present to avoid duplicate key errors
|
// Reuse existing MusicFile document if present to avoid duplicate key errors
|
||||||
let musicFile = await MusicFile.findOne({ s3Key: s3File.key });
|
let musicFile = await MusicFile.findOne({ s3Key: storageFile.key });
|
||||||
if (!musicFile) {
|
if (!musicFile) {
|
||||||
musicFile = new MusicFile({ s3Key: s3File.key });
|
musicFile = new MusicFile({ s3Key: storageFile.key });
|
||||||
}
|
}
|
||||||
musicFile.originalName = filename;
|
musicFile.originalName = filename;
|
||||||
musicFile.s3Key = s3File.key;
|
musicFile.s3Key = storageFile.key;
|
||||||
musicFile.s3Url = `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`;
|
musicFile.s3Url = await storageService.getPresignedUrl(storageFile.key);
|
||||||
musicFile.contentType = guessContentType(filename);
|
musicFile.contentType = guessContentType(filename);
|
||||||
musicFile.size = s3File.size;
|
musicFile.size = storageFile.size;
|
||||||
Object.assign(musicFile, metadata);
|
Object.assign(musicFile, metadata);
|
||||||
|
|
||||||
// Try complex matching
|
// Try complex matching
|
||||||
@ -386,15 +389,16 @@ class BackgroundJobService {
|
|||||||
musicFile.songId = bestMatch.song._id;
|
musicFile.songId = bestMatch.song._id;
|
||||||
complexMatches++;
|
complexMatches++;
|
||||||
|
|
||||||
// Update the Song document to indicate it has an S3 file
|
// Update the Song document to indicate it has a storage file
|
||||||
|
const storageUrl = await storageService.getPresignedUrl(storageFile.key);
|
||||||
await Song.updateOne(
|
await Song.updateOne(
|
||||||
{ _id: bestMatch.song._id },
|
{ _id: bestMatch.song._id },
|
||||||
{
|
{
|
||||||
$set: {
|
$set: {
|
||||||
's3File.musicFileId': musicFile._id,
|
's3File.musicFileId': musicFile._id,
|
||||||
's3File.s3Key': s3File.key,
|
's3File.s3Key': storageFile.key,
|
||||||
's3File.s3Url': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
's3File.s3Url': storageUrl,
|
||||||
's3File.streamingUrl': `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
's3File.streamingUrl': storageUrl,
|
||||||
's3File.hasS3File': true
|
's3File.hasS3File': true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -412,7 +416,7 @@ class BackgroundJobService {
|
|||||||
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error processing ${s3File.key}:`, error);
|
console.error(`Error processing ${storageFile.key}:`, error);
|
||||||
stillUnmatched++;
|
stillUnmatched++;
|
||||||
phase2Errors++;
|
phase2Errors++;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -43,6 +43,7 @@ export class S3Service implements StorageProvider {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('Failed to load s3-config.json, using environment variables as fallback');
|
console.warn('Failed to load s3-config.json, using environment variables as fallback');
|
||||||
return {
|
return {
|
||||||
|
provider: 's3' as const,
|
||||||
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
||||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
|
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
|
||||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
|
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
|
||||||
@ -70,16 +71,22 @@ export class S3Service implements StorageProvider {
|
|||||||
contentType: string,
|
contentType: string,
|
||||||
targetFolder?: string
|
targetFolder?: string
|
||||||
): Promise<UploadResult> {
|
): Promise<UploadResult> {
|
||||||
const fileExtension = originalName.split('.').pop();
|
// Sanitize filename to be safe for S3
|
||||||
|
const sanitizedFilename = this.sanitizeFilename(originalName);
|
||||||
const cleaned = typeof targetFolder === 'string' ? targetFolder.replace(/^\/+|\/+$/g, '') : '';
|
const cleaned = typeof targetFolder === 'string' ? targetFolder.replace(/^\/+|\/+$/g, '') : '';
|
||||||
const safeFolder = cleaned;
|
const safeFolder = cleaned;
|
||||||
|
|
||||||
|
// Use original filename instead of UUID
|
||||||
const key = safeFolder
|
const key = safeFolder
|
||||||
? `${safeFolder}/${uuidv4()}.${fileExtension}`
|
? `${safeFolder}/${sanitizedFilename}`
|
||||||
: `${uuidv4()}.${fileExtension}`;
|
: sanitizedFilename;
|
||||||
|
|
||||||
|
// Check if file already exists and handle conflicts
|
||||||
|
const finalKey = await this.handleFilenameConflict(key);
|
||||||
|
|
||||||
const command = new PutObjectCommand({
|
const command = new PutObjectCommand({
|
||||||
Bucket: this.bucketName,
|
Bucket: this.bucketName,
|
||||||
Key: key,
|
Key: finalKey,
|
||||||
Body: file,
|
Body: file,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
Metadata: {
|
Metadata: {
|
||||||
@ -91,8 +98,8 @@ export class S3Service implements StorageProvider {
|
|||||||
await this.client.send(command);
|
await this.client.send(command);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
key,
|
key: finalKey,
|
||||||
url: `${this.bucketName}/${key}`,
|
url: `${this.bucketName}/${finalKey}`,
|
||||||
size: file.length,
|
size: file.length,
|
||||||
contentType,
|
contentType,
|
||||||
};
|
};
|
||||||
@ -184,12 +191,7 @@ export class S3Service implements StorageProvider {
|
|||||||
* Delete a file from S3
|
* Delete a file from S3
|
||||||
*/
|
*/
|
||||||
async deleteFile(key: string): Promise<void> {
|
async deleteFile(key: string): Promise<void> {
|
||||||
const command = new DeleteObjectCommand({
|
throw new Error('File deletion is disabled to prevent accidental data loss');
|
||||||
Bucket: this.bucketName,
|
|
||||||
Key: key,
|
|
||||||
});
|
|
||||||
|
|
||||||
await this.client.send(command);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -269,4 +271,56 @@ export class S3Service implements StorageProvider {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sanitize filename to be safe for S3
|
||||||
|
*/
|
||||||
|
private sanitizeFilename(filename: string): string {
|
||||||
|
// Remove or replace characters that might cause issues in S3
|
||||||
|
return filename
|
||||||
|
.replace(/[<>:"|?*]/g, '_') // Replace problematic characters
|
||||||
|
.replace(/[\x00-\x1f\x80-\x9f]/g, '') // Remove control characters
|
||||||
|
.replace(/\s+/g, ' ') // Normalize whitespace
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle filename conflicts by adding a number suffix
|
||||||
|
*/
|
||||||
|
private async handleFilenameConflict(key: string): Promise<string> {
|
||||||
|
try {
|
||||||
|
// Check if file exists
|
||||||
|
const command = new HeadObjectCommand({
|
||||||
|
Bucket: this.bucketName,
|
||||||
|
Key: key,
|
||||||
|
});
|
||||||
|
await this.client.send(command);
|
||||||
|
|
||||||
|
// File exists, generate a new name with number suffix
|
||||||
|
const pathParts = key.split('/');
|
||||||
|
const filename = pathParts.pop() || '';
|
||||||
|
const dir = pathParts.join('/');
|
||||||
|
const nameWithoutExt = filename.substring(0, filename.lastIndexOf('.'));
|
||||||
|
const extension = filename.substring(filename.lastIndexOf('.'));
|
||||||
|
|
||||||
|
let counter = 1;
|
||||||
|
let newKey: string;
|
||||||
|
|
||||||
|
do {
|
||||||
|
const newFilename = `${nameWithoutExt} (${counter})${extension}`;
|
||||||
|
newKey = dir ? `${dir}/${newFilename}` : newFilename;
|
||||||
|
counter++;
|
||||||
|
|
||||||
|
// Prevent infinite loop
|
||||||
|
if (counter > 1000) {
|
||||||
|
throw new Error('Too many filename conflicts');
|
||||||
|
}
|
||||||
|
} while (await this.fileExists(newKey));
|
||||||
|
|
||||||
|
return newKey;
|
||||||
|
} catch (error) {
|
||||||
|
// File doesn't exist, use original key
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@ -13,8 +13,10 @@ export interface WebDAVConfig extends StorageConfig {
|
|||||||
export class WebDAVService implements StorageProvider {
|
export class WebDAVService implements StorageProvider {
|
||||||
private client: WebDAVClient;
|
private client: WebDAVClient;
|
||||||
private basePath: string;
|
private basePath: string;
|
||||||
|
private config: WebDAVConfig;
|
||||||
|
|
||||||
constructor(config: WebDAVConfig) {
|
constructor(config: WebDAVConfig) {
|
||||||
|
this.config = config;
|
||||||
this.client = createClient(config.url, {
|
this.client = createClient(config.url, {
|
||||||
username: config.username,
|
username: config.username,
|
||||||
password: config.password,
|
password: config.password,
|
||||||
@ -65,12 +67,15 @@ export class WebDAVService implements StorageProvider {
|
|||||||
contentType: string,
|
contentType: string,
|
||||||
targetFolder?: string
|
targetFolder?: string
|
||||||
): Promise<UploadResult> {
|
): Promise<UploadResult> {
|
||||||
const fileExtension = originalName.split('.').pop();
|
// Sanitize filename to be safe for WebDAV
|
||||||
|
const sanitizedFilename = this.sanitizeFilename(originalName);
|
||||||
const cleaned = typeof targetFolder === 'string' ? targetFolder.replace(/^\/+|\/+$/g, '') : '';
|
const cleaned = typeof targetFolder === 'string' ? targetFolder.replace(/^\/+|\/+$/g, '') : '';
|
||||||
const safeFolder = cleaned;
|
const safeFolder = cleaned;
|
||||||
|
|
||||||
|
// Use original filename instead of UUID
|
||||||
const key = safeFolder
|
const key = safeFolder
|
||||||
? `${safeFolder}/${uuidv4()}.${fileExtension}`
|
? `${safeFolder}/${sanitizedFilename}`
|
||||||
: `${uuidv4()}.${fileExtension}`;
|
: sanitizedFilename;
|
||||||
|
|
||||||
const remotePath = `${this.basePath}/${key}`;
|
const remotePath = `${this.basePath}/${key}`;
|
||||||
|
|
||||||
@ -78,8 +83,11 @@ export class WebDAVService implements StorageProvider {
|
|||||||
const dirPath = remotePath.substring(0, remotePath.lastIndexOf('/'));
|
const dirPath = remotePath.substring(0, remotePath.lastIndexOf('/'));
|
||||||
await this.ensureDirectoryExists(dirPath);
|
await this.ensureDirectoryExists(dirPath);
|
||||||
|
|
||||||
|
// Check if file already exists and handle conflicts
|
||||||
|
const finalKey = await this.handleFilenameConflict(key, remotePath);
|
||||||
|
|
||||||
// Upload the file
|
// Upload the file
|
||||||
await this.client.putFileContents(remotePath, file, {
|
await this.client.putFileContents(`${this.basePath}/${finalKey}`, file, {
|
||||||
overwrite: true,
|
overwrite: true,
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': contentType,
|
'Content-Type': contentType,
|
||||||
@ -87,8 +95,8 @@ export class WebDAVService implements StorageProvider {
|
|||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
key,
|
key: finalKey,
|
||||||
url: remotePath,
|
url: `${this.basePath}/${finalKey}`,
|
||||||
size: file.length,
|
size: file.length,
|
||||||
contentType,
|
contentType,
|
||||||
};
|
};
|
||||||
@ -170,11 +178,10 @@ export class WebDAVService implements StorageProvider {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a file from WebDAV
|
* Delete a file from WebDAV - DISABLED for read-only safety
|
||||||
*/
|
*/
|
||||||
async deleteFile(key: string): Promise<void> {
|
async deleteFile(key: string): Promise<void> {
|
||||||
const remotePath = `${this.basePath}/${key}`;
|
throw new Error('File deletion is disabled for WebDAV integration to prevent accidental data loss');
|
||||||
await this.client.deleteFile(remotePath);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -268,6 +275,56 @@ export class WebDAVService implements StorageProvider {
|
|||||||
default: return 'application/octet-stream';
|
default: return 'application/octet-stream';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sanitize filename to be safe for WebDAV
|
||||||
|
*/
|
||||||
|
private sanitizeFilename(filename: string): string {
|
||||||
|
// Remove or replace characters that might cause issues in WebDAV
|
||||||
|
return filename
|
||||||
|
.replace(/[<>:"|?*]/g, '_') // Replace problematic characters
|
||||||
|
.replace(/[\x00-\x1f\x80-\x9f]/g, '') // Remove control characters
|
||||||
|
.replace(/\s+/g, ' ') // Normalize whitespace
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle filename conflicts by adding a number suffix
|
||||||
|
*/
|
||||||
|
private async handleFilenameConflict(key: string, remotePath: string): Promise<string> {
|
||||||
|
try {
|
||||||
|
// Check if file exists
|
||||||
|
await this.client.stat(remotePath);
|
||||||
|
|
||||||
|
// File exists, generate a new name with number suffix
|
||||||
|
const pathParts = key.split('/');
|
||||||
|
const filename = pathParts.pop() || '';
|
||||||
|
const dir = pathParts.join('/');
|
||||||
|
const nameWithoutExt = filename.substring(0, filename.lastIndexOf('.'));
|
||||||
|
const extension = filename.substring(filename.lastIndexOf('.'));
|
||||||
|
|
||||||
|
let counter = 1;
|
||||||
|
let newKey: string;
|
||||||
|
let newRemotePath: string;
|
||||||
|
|
||||||
|
do {
|
||||||
|
const newFilename = `${nameWithoutExt} (${counter})${extension}`;
|
||||||
|
newKey = dir ? `${dir}/${newFilename}` : newFilename;
|
||||||
|
newRemotePath = `${this.basePath}/${newKey}`;
|
||||||
|
counter++;
|
||||||
|
|
||||||
|
// Prevent infinite loop
|
||||||
|
if (counter > 1000) {
|
||||||
|
throw new Error('Too many filename conflicts');
|
||||||
|
}
|
||||||
|
} while (await this.fileExists(newKey));
|
||||||
|
|
||||||
|
return newKey;
|
||||||
|
} catch (error) {
|
||||||
|
// File doesn't exist, use original key
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Import required modules
|
// Import required modules
|
||||||
|
|||||||
@ -27,7 +27,7 @@ import { api } from '../services/api';
|
|||||||
|
|
||||||
interface JobProgress {
|
interface JobProgress {
|
||||||
jobId: string;
|
jobId: string;
|
||||||
type: 's3-sync' | 'song-matching';
|
type: 'storage-sync' | 'song-matching';
|
||||||
status: 'running' | 'completed' | 'failed';
|
status: 'running' | 'completed' | 'failed';
|
||||||
progress: number;
|
progress: number;
|
||||||
current: number;
|
current: number;
|
||||||
@ -208,7 +208,7 @@ export const BackgroundJobProgress: React.FC<BackgroundJobProgressProps> = ({
|
|||||||
<Box key={job.jobId} p={3} bg="gray.700" borderRadius="md">
|
<Box key={job.jobId} p={3} bg="gray.700" borderRadius="md">
|
||||||
<Flex justify="space-between" align="center" mb={2}>
|
<Flex justify="space-between" align="center" mb={2}>
|
||||||
<Text fontSize="sm" fontWeight="medium" color="gray.100">
|
<Text fontSize="sm" fontWeight="medium" color="gray.100">
|
||||||
{job.type === 's3-sync' ? 'S3 Sync' : 'Song Matching'}
|
{job.type === 'storage-sync' ? 'Storage Sync' : 'Song Matching'}
|
||||||
</Text>
|
</Text>
|
||||||
<Badge colorScheme={getStatusColor(job.status)} size="sm">
|
<Badge colorScheme={getStatusColor(job.status)} size="sm">
|
||||||
{job.status}
|
{job.status}
|
||||||
@ -272,7 +272,7 @@ export const BackgroundJobProgress: React.FC<BackgroundJobProgressProps> = ({
|
|||||||
<Tr key={job.jobId}>
|
<Tr key={job.jobId}>
|
||||||
<Td>
|
<Td>
|
||||||
<Text fontSize="sm" color="gray.100">
|
<Text fontSize="sm" color="gray.100">
|
||||||
{job.type === 's3-sync' ? 'S3 Sync' : 'Song Matching'}
|
{job.type === 'storage-sync' ? 'Storage Sync' : 'Song Matching'}
|
||||||
</Text>
|
</Text>
|
||||||
</Td>
|
</Td>
|
||||||
<Td>
|
<Td>
|
||||||
|
|||||||
@ -162,12 +162,12 @@ export const DuplicatesViewer: React.FC = () => {
|
|||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
<Tooltip label="Delete other duplicates (optionally remove music files)">
|
<Tooltip label="Merge duplicates (keeps target, removes others from playlists only)">
|
||||||
<IconButton
|
<IconButton
|
||||||
aria-label="Delete duplicates"
|
aria-label="Merge duplicates"
|
||||||
icon={<FiTrash2 />}
|
icon={<FiCheck />}
|
||||||
size="sm"
|
size="sm"
|
||||||
colorScheme="red"
|
colorScheme="blue"
|
||||||
variant="outline"
|
variant="outline"
|
||||||
isLoading={processingGroupKey === group.key}
|
isLoading={processingGroupKey === group.key}
|
||||||
onClick={async () => {
|
onClick={async () => {
|
||||||
@ -175,7 +175,7 @@ export const DuplicatesViewer: React.FC = () => {
|
|||||||
try {
|
try {
|
||||||
const targetId = it.songId;
|
const targetId = it.songId;
|
||||||
const others = group.items.map(x => x.songId).filter(id => id !== targetId);
|
const others = group.items.map(x => x.songId).filter(id => id !== targetId);
|
||||||
// First merge playlists (safe), then delete redundant songs and optionally their music files
|
// Merge playlists (safe), but don't delete songs or music files
|
||||||
const allPlaylists = await api.getPlaylists();
|
const allPlaylists = await api.getPlaylists();
|
||||||
const updated = allPlaylists.map(p => {
|
const updated = allPlaylists.map(p => {
|
||||||
if (p.type === 'playlist') {
|
if (p.type === 'playlist') {
|
||||||
@ -196,7 +196,7 @@ export const DuplicatesViewer: React.FC = () => {
|
|||||||
return p;
|
return p;
|
||||||
});
|
});
|
||||||
await api.savePlaylists(updated as any);
|
await api.savePlaylists(updated as any);
|
||||||
await api.deleteDuplicateSongs(targetId, others, true);
|
// Note: We don't call deleteDuplicateSongs anymore to keep it read-only
|
||||||
await loadDuplicates(minGroupSize);
|
await loadDuplicates(minGroupSize);
|
||||||
} finally {
|
} finally {
|
||||||
setProcessingGroupKey(null);
|
setProcessingGroupKey(null);
|
||||||
|
|||||||
@ -249,6 +249,9 @@ export const MusicUpload: React.FC<MusicUploadProps> = ({ onUploadComplete }) =>
|
|||||||
<Text fontSize="sm" color="gray.400">
|
<Text fontSize="sm" color="gray.400">
|
||||||
Supports MP3, WAV, FLAC, AAC, OGG, WMA, Opus (max 100MB per file)
|
Supports MP3, WAV, FLAC, AAC, OGG, WMA, Opus (max 100MB per file)
|
||||||
</Text>
|
</Text>
|
||||||
|
<Text fontSize="xs" color="gray.500">
|
||||||
|
Original filenames and metadata will be preserved
|
||||||
|
</Text>
|
||||||
</VStack>
|
</VStack>
|
||||||
</Box>
|
</Box>
|
||||||
|
|
||||||
|
|||||||
@ -261,7 +261,7 @@ export const SongMatching: React.FC = () => {
|
|||||||
<CardBody>
|
<CardBody>
|
||||||
<VStack spacing={4}>
|
<VStack spacing={4}>
|
||||||
<Text color="gray.300" textAlign="center">
|
<Text color="gray.300" textAlign="center">
|
||||||
Try linking any remaining unmatched files. The main S3 sync already performs matching; use this for leftovers.
|
Try linking any remaining unmatched files. The main storage sync already performs matching; use this for leftovers.
|
||||||
</Text>
|
</Text>
|
||||||
<Button
|
<Button
|
||||||
leftIcon={<FiZap />}
|
leftIcon={<FiZap />}
|
||||||
|
|||||||
@ -34,7 +34,7 @@ import { MusicUpload } from "../components/MusicUpload";
|
|||||||
import { SongMatching } from "../components/SongMatching";
|
import { SongMatching } from "../components/SongMatching";
|
||||||
import { api } from "../services/api";
|
import { api } from "../services/api";
|
||||||
import { DuplicatesViewer } from "../components/DuplicatesViewer.tsx";
|
import { DuplicatesViewer } from "../components/DuplicatesViewer.tsx";
|
||||||
import { useState, useMemo } from "react";
|
import { useState, useMemo, useEffect } from "react";
|
||||||
|
|
||||||
interface MusicFile {
|
interface MusicFile {
|
||||||
_id: string;
|
_id: string;
|
||||||
@ -63,10 +63,27 @@ export function Configuration() {
|
|||||||
return stored ? parseInt(stored, 10) : 0;
|
return stored ? parseInt(stored, 10) : 0;
|
||||||
}, []);
|
}, []);
|
||||||
const [tabIndex, setTabIndex] = useState<number>(initialTabIndex);
|
const [tabIndex, setTabIndex] = useState<number>(initialTabIndex);
|
||||||
|
const [storageProvider, setStorageProvider] = useState<string>('Storage');
|
||||||
|
|
||||||
// No explicit tab index enum needed
|
// No explicit tab index enum needed
|
||||||
|
|
||||||
// S3 config fetch removed; Sync buttons remain available in the panel
|
// Load current storage provider for dynamic button labels
|
||||||
|
useEffect(() => {
|
||||||
|
const loadStorageProvider = async () => {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/config/storage');
|
||||||
|
if (response.ok) {
|
||||||
|
const data = await response.json();
|
||||||
|
setStorageProvider(data.config.provider.toUpperCase());
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to load storage provider:', error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
loadStorageProvider();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Storage config fetch removed; Sync buttons remain available in the panel
|
||||||
|
|
||||||
// Removed Music Library sync handlers from Config (moved to Sync & Matching panel)
|
// Removed Music Library sync handlers from Config (moved to Sync & Matching panel)
|
||||||
|
|
||||||
@ -255,25 +272,25 @@ export function Configuration() {
|
|||||||
leftIcon={<FiRefreshCw />}
|
leftIcon={<FiRefreshCw />}
|
||||||
colorScheme="blue"
|
colorScheme="blue"
|
||||||
variant="solid"
|
variant="solid"
|
||||||
onClick={() => api.startS3Sync()}
|
onClick={() => api.startStorageSync()}
|
||||||
>
|
>
|
||||||
Sync S3 (incremental)
|
Sync {storageProvider} (incremental)
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
leftIcon={<FiRefreshCw />}
|
leftIcon={<FiRefreshCw />}
|
||||||
colorScheme="orange"
|
colorScheme="orange"
|
||||||
variant="outline"
|
variant="outline"
|
||||||
onClick={() => api.startS3Sync({ force: true })}
|
onClick={() => api.startStorageSync({ force: true })}
|
||||||
>
|
>
|
||||||
Force Sync (rescan all)
|
Force {storageProvider} Sync (rescan all)
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
leftIcon={<FiTrash2 />}
|
leftIcon={<FiTrash2 />}
|
||||||
colorScheme="red"
|
colorScheme="red"
|
||||||
variant="outline"
|
variant="outline"
|
||||||
onClick={() => api.startS3Sync({ clearLinks: true, force: true })}
|
onClick={() => api.startStorageSync({ clearLinks: true, force: true })}
|
||||||
>
|
>
|
||||||
Clear Links + Force Sync
|
Clear Links + Force {storageProvider} Sync
|
||||||
</Button>
|
</Button>
|
||||||
</HStack>
|
</HStack>
|
||||||
<SongMatching />
|
<SongMatching />
|
||||||
|
|||||||
@ -142,7 +142,7 @@ class Api {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Background job methods
|
// Background job methods
|
||||||
async startBackgroundJob(type: 's3-sync' | 'song-matching', options?: any): Promise<{ jobId: string; type: string }> {
|
async startBackgroundJob(type: 'storage-sync' | 'song-matching', options?: any): Promise<{ jobId: string; type: string }> {
|
||||||
const response = await fetch(`${API_BASE_URL}/background-jobs/start`, {
|
const response = await fetch(`${API_BASE_URL}/background-jobs/start`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
@ -155,13 +155,13 @@ class Api {
|
|||||||
return response.json();
|
return response.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
async startS3Sync(options?: { force?: boolean; clearLinks?: boolean }): Promise<{ jobId: string; type: string }> {
|
async startStorageSync(options?: { force?: boolean; clearLinks?: boolean }): Promise<{ jobId: string; type: string }> {
|
||||||
const response = await fetch(`${API_BASE_URL}/music/sync-s3`, {
|
const response = await fetch(`${API_BASE_URL}/music/sync-s3`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify(options || {})
|
body: JSON.stringify(options || {})
|
||||||
});
|
});
|
||||||
if (!response.ok) throw new Error('Failed to start S3 sync');
|
if (!response.ok) throw new Error('Failed to start storage sync');
|
||||||
return response.json();
|
return response.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -205,15 +205,7 @@ class Api {
|
|||||||
return response.json();
|
return response.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteDuplicateSongs(targetSongId: string, redundantSongIds: string[], deleteMusicFiles: boolean): Promise<{ deletedSongs: number; unlinkedOrDeletedMusicFiles: number; }>{
|
// deleteDuplicateSongs method removed to keep WebDAV integration read-only
|
||||||
const response = await fetch(`${API_BASE_URL}/songs/delete-duplicates`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
body: JSON.stringify({ targetSongId, redundantSongIds, deleteMusicFiles })
|
|
||||||
});
|
|
||||||
if (!response.ok) throw new Error('Failed to delete duplicates');
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export const api = new Api();
|
export const api = new Api();
|
||||||
Loading…
x
Reference in New Issue
Block a user