feat: Add recursive S3 file discovery and sync functionality

- Add listAllFiles() method to S3Service to recursively list all files in S3 bucket
- Add getFileContent() method to download file content for metadata extraction
- Add /api/music/sync-s3 endpoint to sync S3 files with database
- Add 'Sync S3' button to Music Storage page
- Support for files in subdirectories like /Disco/
- Automatic metadata extraction for synced files
- Recursive file discovery finds all files regardless of directory structure

This allows the system to find and sync music files that were uploaded
directly to S3/MinIO, including files in subdirectories, and make them
available for song matching and browser playback.

packages/backend/src/routes/music.ts

@@ -122,6 +122,109 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => {
   }
 });
 
+/**
+ * Get all music files (from database)
+ */
+router.get('/files', async (req, res) => {
+  try {
+    const musicFiles = await MusicFile.find({}).sort({ uploadedAt: -1 });
+    res.json({ musicFiles });
+  } catch (error) {
+    console.error('Error fetching music files:', error);
+    res.status(500).json({ error: 'Failed to fetch music files' });
+  }
+});
+
+/**
+ * Sync S3 files with database - recursively list all files in S3 and sync
+ */
+router.post('/sync-s3', async (req, res) => {
+  try {
+    console.log('Starting S3 sync...');
+    
+    // Get all files from S3 recursively
+    const s3Files = await s3Service.listAllFiles();
+    console.log(`Found ${s3Files.length} files in S3 bucket`);
+    
+    const results = {
+      total: s3Files.length,
+      synced: 0,
+      skipped: 0,
+      errors: 0,
+      newFiles: 0
+    };
+
+    for (const s3File of s3Files) {
+      try {
+        // Check if file already exists in database
+        const existingFile = await MusicFile.findOne({ s3Key: s3File.key });
+        
+        if (existingFile) {
+          results.skipped++;
+          continue;
+        }
+
+        // Extract filename from S3 key
+        const filename = s3File.key.split('/').pop() || s3File.key;
+        
+        // Check if it's an audio file
+        if (!audioMetadataService.isAudioFile(filename)) {
+          results.skipped++;
+          continue;
+        }
+
+        // Get file content to extract metadata
+        try {
+          const fileBuffer = await s3Service.getFileContent(s3File.key);
+          const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
+          
+          // Save to database
+          const musicFile = new MusicFile({
+            originalName: filename,
+            s3Key: s3File.key,
+            s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
+            contentType: 'audio/mpeg', // Default, will be updated by metadata
+            size: s3File.size,
+            ...metadata,
+          });
+
+          await musicFile.save();
+          results.synced++;
+          results.newFiles++;
+          
+        } catch (metadataError) {
+          console.error(`Error extracting metadata for ${s3File.key}:`, metadataError);
+          // Still save the file without metadata
+          const musicFile = new MusicFile({
+            originalName: filename,
+            s3Key: s3File.key,
+            s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
+            contentType: 'audio/mpeg',
+            size: s3File.size,
+          });
+          await musicFile.save();
+          results.synced++;
+          results.newFiles++;
+        }
+        
+      } catch (error) {
+        console.error(`Error processing ${s3File.key}:`, error);
+        results.errors++;
+      }
+    }
+
+    console.log('S3 sync completed:', results);
+    res.json({
+      message: 'S3 sync completed',
+      results
+    });
+    
+  } catch (error) {
+    console.error('S3 sync error:', error);
+    res.status(500).json({ error: 'Failed to sync S3 files' });
+  }
+});
+
 /**
  * Get streaming URL for a music file
  */

packages/backend/src/services/s3Service.ts

@@ -1,4 +1,4 @@
-import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
+import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { v4 as uuidv4 } from 'uuid';
 
@@ -17,6 +17,13 @@ export interface UploadResult {
   contentType: string;
 }
 
+export interface S3FileInfo {
+  key: string;
+  size: number;
+  lastModified: Date;
+  contentType?: string;
+}
+
 export class S3Service {
   private client: S3Client;
   private bucketName: string;
@@ -66,6 +73,40 @@ export class S3Service {
     };
   }
 
+  /**
+   * Recursively list all files in the S3 bucket
+   */
+  async listAllFiles(prefix: string = ''): Promise<S3FileInfo[]> {
+    const files: S3FileInfo[] = [];
+    let continuationToken: string | undefined;
+
+    do {
+      const command = new ListObjectsV2Command({
+        Bucket: this.bucketName,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      });
+
+      const response = await this.client.send(command);
+      
+      if (response.Contents) {
+        for (const object of response.Contents) {
+          if (object.Key && !object.Key.endsWith('/')) { // Skip directories
+            files.push({
+              key: object.Key,
+              size: object.Size || 0,
+              lastModified: object.LastModified || new Date(),
+            });
+          }
+        }
+      }
+
+      continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+
+    return files;
+  }
+
   /**
    * Generate a presigned URL for secure file access
    */
@@ -99,7 +140,6 @@ export class S3Service {
         Bucket: this.bucketName,
         Key: key,
       });
-
       await this.client.send(command);
       return true;
     } catch (error) {
@@ -115,16 +155,39 @@ export class S3Service {
       Bucket: this.bucketName,
       Key: key,
     });
-
     return await this.client.send(command);
   }
 
   /**
-   * Create a streaming URL for audio playback
+   * Get file content as buffer
+   */
+  async getFileContent(key: string): Promise<Buffer> {
+    const command = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: key,
+    });
+
+    const response = await this.client.send(command);
+    
+    if (!response.Body) {
+      throw new Error('File has no content');
+    }
+
+    // Convert stream to buffer
+    const chunks: Uint8Array[] = [];
+    const stream = response.Body as any;
+    
+    return new Promise((resolve, reject) => {
+      stream.on('data', (chunk: Uint8Array) => chunks.push(chunk));
+      stream.on('error', reject);
+      stream.on('end', () => resolve(Buffer.concat(chunks)));
+    });
+  }
+
+  /**
+   * Get streaming URL for a file
    */
   async getStreamingUrl(key: string): Promise<string> {
-    // For MinIO, we can create a direct URL if the bucket is public
-    // Otherwise, use presigned URL
     return `${process.env.S3_ENDPOINT}/${this.bucketName}/${key}`;
   }
 } 

packages/frontend/src/pages/MusicStorage.tsx

@@ -19,8 +19,10 @@ import {
   useToast,
   Alert,
   AlertIcon,
+  Button,
+  Spinner,
 } from '@chakra-ui/react';
-import { FiPlay, FiTrash2, FiMusic } from 'react-icons/fi';
+import { FiPlay, FiTrash2, FiMusic, FiRefreshCw } from 'react-icons/fi';
 import { MusicUpload } from '../components/MusicUpload';
 import { MusicPlayer } from '../components/MusicPlayer';
 import { SongMatching } from '../components/SongMatching';
@@ -42,6 +44,7 @@ export const MusicStorage: React.FC = () => {
   const [musicFiles, setMusicFiles] = useState<MusicFile[]>([]);
   const [selectedFile, setSelectedFile] = useState<MusicFile | null>(null);
   const [isLoading, setIsLoading] = useState(false);
+  const [isSyncing, setIsSyncing] = useState(false);
   const toast = useToast();
 
   // Load music files on component mount
@@ -52,7 +55,7 @@ export const MusicStorage: React.FC = () => {
   const loadMusicFiles = async () => {
     setIsLoading(true);
     try {
-      const response = await fetch('/api/music');
+      const response = await fetch('/api/music/files');
       if (response.ok) {
         const data = await response.json();
         setMusicFiles(data.musicFiles || []);
@@ -73,6 +76,42 @@ export const MusicStorage: React.FC = () => {
     }
   };
 
+  const handleSyncS3 = async () => {
+    setIsSyncing(true);
+    try {
+      const response = await fetch('/api/music/sync-s3', {
+        method: 'POST',
+      });
+
+      if (response.ok) {
+        const data = await response.json();
+        toast({
+          title: 'S3 Sync Complete',
+          description: `Found ${data.results.total} files, synced ${data.results.newFiles} new files`,
+          status: 'success',
+          duration: 5000,
+          isClosable: true,
+        });
+        
+        // Reload music files to show the new ones
+        await loadMusicFiles();
+      } else {
+        throw new Error('Failed to sync S3');
+      }
+    } catch (error) {
+      console.error('Error syncing S3:', error);
+      toast({
+        title: 'Error',
+        description: 'Failed to sync S3 files',
+        status: 'error',
+        duration: 3000,
+        isClosable: true,
+      });
+    } finally {
+      setIsSyncing(false);
+    }
+  };
+
   const handleUploadComplete = (files: MusicFile[]) => {
     setMusicFiles(prev => [...files, ...prev]);
     toast({
@@ -179,9 +218,21 @@ export const MusicStorage: React.FC = () => {
               <VStack spacing={4} align="stretch">
                 <HStack justify="space-between">
                   <Heading size="md">Music Library</Heading>
-                  <Text color="gray.600">
+                  <HStack spacing={2}>
-                    {musicFiles.length} file{musicFiles.length !== 1 ? 's' : ''}
+                    <Text color="gray.600">
-                  </Text>
+                      {musicFiles.length} file{musicFiles.length !== 1 ? 's' : ''}
+                    </Text>
+                    <Button
+                      leftIcon={isSyncing ? <Spinner size="sm" /> : <FiRefreshCw />}
+                      size="sm"
+                      variant="outline"
+                      onClick={handleSyncS3}
+                      isLoading={isSyncing}
+                      loadingText="Syncing..."
+                    >
+                      Sync S3
+                    </Button>
+                  </HStack>
                 </HStack>
 
                 {isLoading ? (
@@ -189,9 +240,20 @@ export const MusicStorage: React.FC = () => {
                     Loading music files...
                   </Text>
                 ) : musicFiles.length === 0 ? (
-                  <Text textAlign="center" color="gray.500">
+                  <VStack spacing={4} textAlign="center" color="gray.500">
-                    No music files uploaded yet. Upload some files in the Upload tab.
+                    <Text>No music files found in the database.</Text>
-                  </Text>
+                    <Text fontSize="sm">
+                      Try uploading files in the Upload tab, or click "Sync S3" to find files already in your S3 bucket.
+                    </Text>
+                    <Button
+                      leftIcon={<FiRefreshCw />}
+                      onClick={handleSyncS3}
+                      isLoading={isSyncing}
+                      loadingText="Syncing..."
+                    >
+                      Sync S3 Bucket
+                    </Button>
+                  </VStack>
                 ) : (
                   <SimpleGrid columns={{ base: 1, md: 2, lg: 3 }} spacing={4}>
                     {musicFiles.map((file) => (