From 7f186c6337a11c312945fb6ab9e4c9855cad7ea4 Mon Sep 17 00:00:00 2001 From: Geert Rademakes Date: Wed, 6 Aug 2025 14:57:46 +0200 Subject: [PATCH] feat: Add recursive S3 file discovery and sync functionality - Add listAllFiles() method to S3Service to recursively list all files in S3 bucket - Add getFileContent() method to download file content for metadata extraction - Add /api/music/sync-s3 endpoint to sync S3 files with database - Add 'Sync S3' button to Music Storage page - Support for files in subdirectories like /Disco/ - Automatic metadata extraction for synced files - Recursive file discovery finds all files regardless of directory structure This allows the system to find and sync music files that were uploaded directly to S3/MinIO, including files in subdirectories, and make them available for song matching and browser playback. --- packages/backend/src/routes/music.ts | 103 +++++++++++++++++++ packages/backend/src/services/s3Service.ts | 75 ++++++++++++-- packages/frontend/src/pages/MusicStorage.tsx | 78 ++++++++++++-- 3 files changed, 242 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/routes/music.ts b/packages/backend/src/routes/music.ts index bb7f285..b6c11d4 100644 --- a/packages/backend/src/routes/music.ts +++ b/packages/backend/src/routes/music.ts @@ -122,6 +122,109 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => { } }); +/** + * Get all music files (from database) + */ +router.get('/files', async (req, res) => { + try { + const musicFiles = await MusicFile.find({}).sort({ uploadedAt: -1 }); + res.json({ musicFiles }); + } catch (error) { + console.error('Error fetching music files:', error); + res.status(500).json({ error: 'Failed to fetch music files' }); + } +}); + +/** + * Sync S3 files with database - recursively list all files in S3 and sync + */ +router.post('/sync-s3', async (req, res) => { + try { + console.log('Starting S3 sync...'); + + // Get all files from S3 recursively + const s3Files = await s3Service.listAllFiles(); + console.log(`Found ${s3Files.length} files in S3 bucket`); + + const results = { + total: s3Files.length, + synced: 0, + skipped: 0, + errors: 0, + newFiles: 0 + }; + + for (const s3File of s3Files) { + try { + // Check if file already exists in database + const existingFile = await MusicFile.findOne({ s3Key: s3File.key }); + + if (existingFile) { + results.skipped++; + continue; + } + + // Extract filename from S3 key + const filename = s3File.key.split('/').pop() || s3File.key; + + // Check if it's an audio file + if (!audioMetadataService.isAudioFile(filename)) { + results.skipped++; + continue; + } + + // Get file content to extract metadata + try { + const fileBuffer = await s3Service.getFileContent(s3File.key); + const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename); + + // Save to database + const musicFile = new MusicFile({ + originalName: filename, + s3Key: s3File.key, + s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`, + contentType: 'audio/mpeg', // Default, will be updated by metadata + size: s3File.size, + ...metadata, + }); + + await musicFile.save(); + results.synced++; + results.newFiles++; + + } catch (metadataError) { + console.error(`Error extracting metadata for ${s3File.key}:`, metadataError); + // Still save the file without metadata + const musicFile = new MusicFile({ + originalName: filename, + s3Key: s3File.key, + s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`, + contentType: 'audio/mpeg', + size: s3File.size, + }); + await musicFile.save(); + results.synced++; + results.newFiles++; + } + + } catch (error) { + console.error(`Error processing ${s3File.key}:`, error); + results.errors++; + } + } + + console.log('S3 sync completed:', results); + res.json({ + message: 'S3 sync completed', + results + }); + + } catch (error) { + console.error('S3 sync error:', error); + res.status(500).json({ error: 'Failed to sync S3 files' }); + } +}); + /** * Get streaming URL for a music file */ diff --git a/packages/backend/src/services/s3Service.ts b/packages/backend/src/services/s3Service.ts index ad296a9..fd97e20 100644 --- a/packages/backend/src/services/s3Service.ts +++ b/packages/backend/src/services/s3Service.ts @@ -1,4 +1,4 @@ -import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; +import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { v4 as uuidv4 } from 'uuid'; @@ -17,6 +17,13 @@ export interface UploadResult { contentType: string; } +export interface S3FileInfo { + key: string; + size: number; + lastModified: Date; + contentType?: string; +} + export class S3Service { private client: S3Client; private bucketName: string; @@ -66,6 +73,40 @@ export class S3Service { }; } + /** + * Recursively list all files in the S3 bucket + */ + async listAllFiles(prefix: string = ''): Promise { + const files: S3FileInfo[] = []; + let continuationToken: string | undefined; + + do { + const command = new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: prefix, + ContinuationToken: continuationToken, + }); + + const response = await this.client.send(command); + + if (response.Contents) { + for (const object of response.Contents) { + if (object.Key && !object.Key.endsWith('/')) { // Skip directories + files.push({ + key: object.Key, + size: object.Size || 0, + lastModified: object.LastModified || new Date(), + }); + } + } + } + + continuationToken = response.NextContinuationToken; + } while (continuationToken); + + return files; + } + /** * Generate a presigned URL for secure file access */ @@ -99,7 +140,6 @@ export class S3Service { Bucket: this.bucketName, Key: key, }); - await this.client.send(command); return true; } catch (error) { @@ -115,16 +155,39 @@ export class S3Service { Bucket: this.bucketName, Key: key, }); - return await this.client.send(command); } /** - * Create a streaming URL for audio playback + * Get file content as buffer + */ + async getFileContent(key: string): Promise { + const command = new GetObjectCommand({ + Bucket: this.bucketName, + Key: key, + }); + + const response = await this.client.send(command); + + if (!response.Body) { + throw new Error('File has no content'); + } + + // Convert stream to buffer + const chunks: Uint8Array[] = []; + const stream = response.Body as any; + + return new Promise((resolve, reject) => { + stream.on('data', (chunk: Uint8Array) => chunks.push(chunk)); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(chunks))); + }); + } + + /** + * Get streaming URL for a file */ async getStreamingUrl(key: string): Promise { - // For MinIO, we can create a direct URL if the bucket is public - // Otherwise, use presigned URL return `${process.env.S3_ENDPOINT}/${this.bucketName}/${key}`; } } \ No newline at end of file diff --git a/packages/frontend/src/pages/MusicStorage.tsx b/packages/frontend/src/pages/MusicStorage.tsx index 08f6ca3..9ff135d 100644 --- a/packages/frontend/src/pages/MusicStorage.tsx +++ b/packages/frontend/src/pages/MusicStorage.tsx @@ -19,8 +19,10 @@ import { useToast, Alert, AlertIcon, + Button, + Spinner, } from '@chakra-ui/react'; -import { FiPlay, FiTrash2, FiMusic } from 'react-icons/fi'; +import { FiPlay, FiTrash2, FiMusic, FiRefreshCw } from 'react-icons/fi'; import { MusicUpload } from '../components/MusicUpload'; import { MusicPlayer } from '../components/MusicPlayer'; import { SongMatching } from '../components/SongMatching'; @@ -42,6 +44,7 @@ export const MusicStorage: React.FC = () => { const [musicFiles, setMusicFiles] = useState([]); const [selectedFile, setSelectedFile] = useState(null); const [isLoading, setIsLoading] = useState(false); + const [isSyncing, setIsSyncing] = useState(false); const toast = useToast(); // Load music files on component mount @@ -52,7 +55,7 @@ export const MusicStorage: React.FC = () => { const loadMusicFiles = async () => { setIsLoading(true); try { - const response = await fetch('/api/music'); + const response = await fetch('/api/music/files'); if (response.ok) { const data = await response.json(); setMusicFiles(data.musicFiles || []); @@ -73,6 +76,42 @@ export const MusicStorage: React.FC = () => { } }; + const handleSyncS3 = async () => { + setIsSyncing(true); + try { + const response = await fetch('/api/music/sync-s3', { + method: 'POST', + }); + + if (response.ok) { + const data = await response.json(); + toast({ + title: 'S3 Sync Complete', + description: `Found ${data.results.total} files, synced ${data.results.newFiles} new files`, + status: 'success', + duration: 5000, + isClosable: true, + }); + + // Reload music files to show the new ones + await loadMusicFiles(); + } else { + throw new Error('Failed to sync S3'); + } + } catch (error) { + console.error('Error syncing S3:', error); + toast({ + title: 'Error', + description: 'Failed to sync S3 files', + status: 'error', + duration: 3000, + isClosable: true, + }); + } finally { + setIsSyncing(false); + } + }; + const handleUploadComplete = (files: MusicFile[]) => { setMusicFiles(prev => [...files, ...prev]); toast({ @@ -179,9 +218,21 @@ export const MusicStorage: React.FC = () => { Music Library - - {musicFiles.length} file{musicFiles.length !== 1 ? 's' : ''} - + + + {musicFiles.length} file{musicFiles.length !== 1 ? 's' : ''} + + + {isLoading ? ( @@ -189,9 +240,20 @@ export const MusicStorage: React.FC = () => { Loading music files... ) : musicFiles.length === 0 ? ( - - No music files uploaded yet. Upload some files in the Upload tab. - + + No music files found in the database. + + Try uploading files in the Upload tab, or click "Sync S3" to find files already in your S3 bucket. + + + ) : ( {musicFiles.map((file) => (