feat: Add recursive S3 file discovery and sync functionality
- Add listAllFiles() method to S3Service to recursively list all files in S3 bucket - Add getFileContent() method to download file content for metadata extraction - Add /api/music/sync-s3 endpoint to sync S3 files with database - Add 'Sync S3' button to Music Storage page - Support for files in subdirectories like /Disco/ - Automatic metadata extraction for synced files - Recursive file discovery finds all files regardless of directory structure This allows the system to find and sync music files that were uploaded directly to S3/MinIO, including files in subdirectories, and make them available for song matching and browser playback.
This commit is contained in:
Geert Rademakes
parent
72dfa951b4
commit
7f186c6337
@ -122,6 +122,109 @@ router.post('/batch-upload', upload.array('files', 10), async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get all music files (from database)
|
||||
*/
|
||||
router.get('/files', async (req, res) => {
|
||||
try {
|
||||
const musicFiles = await MusicFile.find({}).sort({ uploadedAt: -1 });
|
||||
res.json({ musicFiles });
|
||||
} catch (error) {
|
||||
console.error('Error fetching music files:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch music files' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Sync S3 files with database - recursively list all files in S3 and sync
|
||||
*/
|
||||
router.post('/sync-s3', async (req, res) => {
|
||||
try {
|
||||
console.log('Starting S3 sync...');
|
||||
|
||||
// Get all files from S3 recursively
|
||||
const s3Files = await s3Service.listAllFiles();
|
||||
console.log(`Found ${s3Files.length} files in S3 bucket`);
|
||||
|
||||
const results = {
|
||||
total: s3Files.length,
|
||||
synced: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
newFiles: 0
|
||||
};
|
||||
|
||||
for (const s3File of s3Files) {
|
||||
try {
|
||||
// Check if file already exists in database
|
||||
const existingFile = await MusicFile.findOne({ s3Key: s3File.key });
|
||||
|
||||
if (existingFile) {
|
||||
results.skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract filename from S3 key
|
||||
const filename = s3File.key.split('/').pop() || s3File.key;
|
||||
|
||||
// Check if it's an audio file
|
||||
if (!audioMetadataService.isAudioFile(filename)) {
|
||||
results.skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get file content to extract metadata
|
||||
try {
|
||||
const fileBuffer = await s3Service.getFileContent(s3File.key);
|
||||
const metadata = await audioMetadataService.extractMetadata(fileBuffer, filename);
|
||||
|
||||
// Save to database
|
||||
const musicFile = new MusicFile({
|
||||
originalName: filename,
|
||||
s3Key: s3File.key,
|
||||
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
||||
contentType: 'audio/mpeg', // Default, will be updated by metadata
|
||||
size: s3File.size,
|
||||
...metadata,
|
||||
});
|
||||
|
||||
await musicFile.save();
|
||||
results.synced++;
|
||||
results.newFiles++;
|
||||
|
||||
} catch (metadataError) {
|
||||
console.error(`Error extracting metadata for ${s3File.key}:`, metadataError);
|
||||
// Still save the file without metadata
|
||||
const musicFile = new MusicFile({
|
||||
originalName: filename,
|
||||
s3Key: s3File.key,
|
||||
s3Url: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET_NAME}/${s3File.key}`,
|
||||
contentType: 'audio/mpeg',
|
||||
size: s3File.size,
|
||||
});
|
||||
await musicFile.save();
|
||||
results.synced++;
|
||||
results.newFiles++;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Error processing ${s3File.key}:`, error);
|
||||
results.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('S3 sync completed:', results);
|
||||
res.json({
|
||||
message: 'S3 sync completed',
|
||||
results
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('S3 sync error:', error);
|
||||
res.status(500).json({ error: 'Failed to sync S3 files' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get streaming URL for a music file
|
||||
*/
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
@ -17,6 +17,13 @@ export interface UploadResult {
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
export interface S3FileInfo {
|
||||
key: string;
|
||||
size: number;
|
||||
lastModified: Date;
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
export class S3Service {
|
||||
private client: S3Client;
|
||||
private bucketName: string;
|
||||
@ -66,6 +73,40 @@ export class S3Service {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively list all files in the S3 bucket
|
||||
*/
|
||||
async listAllFiles(prefix: string = ''): Promise<S3FileInfo[]> {
|
||||
const files: S3FileInfo[] = [];
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const command = new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
});
|
||||
|
||||
const response = await this.client.send(command);
|
||||
|
||||
if (response.Contents) {
|
||||
for (const object of response.Contents) {
|
||||
if (object.Key && !object.Key.endsWith('/')) { // Skip directories
|
||||
files.push({
|
||||
key: object.Key,
|
||||
size: object.Size || 0,
|
||||
lastModified: object.LastModified || new Date(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
continuationToken = response.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a presigned URL for secure file access
|
||||
*/
|
||||
@ -99,7 +140,6 @@ export class S3Service {
|
||||
Bucket: this.bucketName,
|
||||
Key: key,
|
||||
});
|
||||
|
||||
await this.client.send(command);
|
||||
return true;
|
||||
} catch (error) {
|
||||
@ -115,16 +155,39 @@ export class S3Service {
|
||||
Bucket: this.bucketName,
|
||||
Key: key,
|
||||
});
|
||||
|
||||
return await this.client.send(command);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a streaming URL for audio playback
|
||||
* Get file content as buffer
|
||||
*/
|
||||
async getFileContent(key: string): Promise<Buffer> {
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: key,
|
||||
});
|
||||
|
||||
const response = await this.client.send(command);
|
||||
|
||||
if (!response.Body) {
|
||||
throw new Error('File has no content');
|
||||
}
|
||||
|
||||
// Convert stream to buffer
|
||||
const chunks: Uint8Array[] = [];
|
||||
const stream = response.Body as any;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk: Uint8Array) => chunks.push(chunk));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get streaming URL for a file
|
||||
*/
|
||||
async getStreamingUrl(key: string): Promise<string> {
|
||||
// For MinIO, we can create a direct URL if the bucket is public
|
||||
// Otherwise, use presigned URL
|
||||
return `${process.env.S3_ENDPOINT}/${this.bucketName}/${key}`;
|
||||
}
|
||||
}
|
||||
@ -19,8 +19,10 @@ import {
|
||||
useToast,
|
||||
Alert,
|
||||
AlertIcon,
|
||||
Button,
|
||||
Spinner,
|
||||
} from '@chakra-ui/react';
|
||||
import { FiPlay, FiTrash2, FiMusic } from 'react-icons/fi';
|
||||
import { FiPlay, FiTrash2, FiMusic, FiRefreshCw } from 'react-icons/fi';
|
||||
import { MusicUpload } from '../components/MusicUpload';
|
||||
import { MusicPlayer } from '../components/MusicPlayer';
|
||||
import { SongMatching } from '../components/SongMatching';
|
||||
@ -42,6 +44,7 @@ export const MusicStorage: React.FC = () => {
|
||||
const [musicFiles, setMusicFiles] = useState<MusicFile[]>([]);
|
||||
const [selectedFile, setSelectedFile] = useState<MusicFile | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [isSyncing, setIsSyncing] = useState(false);
|
||||
const toast = useToast();
|
||||
|
||||
// Load music files on component mount
|
||||
@ -52,7 +55,7 @@ export const MusicStorage: React.FC = () => {
|
||||
const loadMusicFiles = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const response = await fetch('/api/music');
|
||||
const response = await fetch('/api/music/files');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setMusicFiles(data.musicFiles || []);
|
||||
@ -73,6 +76,42 @@ export const MusicStorage: React.FC = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const handleSyncS3 = async () => {
|
||||
setIsSyncing(true);
|
||||
try {
|
||||
const response = await fetch('/api/music/sync-s3', {
|
||||
method: 'POST',
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
toast({
|
||||
title: 'S3 Sync Complete',
|
||||
description: `Found ${data.results.total} files, synced ${data.results.newFiles} new files`,
|
||||
status: 'success',
|
||||
duration: 5000,
|
||||
isClosable: true,
|
||||
});
|
||||
|
||||
// Reload music files to show the new ones
|
||||
await loadMusicFiles();
|
||||
} else {
|
||||
throw new Error('Failed to sync S3');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error syncing S3:', error);
|
||||
toast({
|
||||
title: 'Error',
|
||||
description: 'Failed to sync S3 files',
|
||||
status: 'error',
|
||||
duration: 3000,
|
||||
isClosable: true,
|
||||
});
|
||||
} finally {
|
||||
setIsSyncing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleUploadComplete = (files: MusicFile[]) => {
|
||||
setMusicFiles(prev => [...files, ...prev]);
|
||||
toast({
|
||||
@ -179,9 +218,21 @@ export const MusicStorage: React.FC = () => {
|
||||
<VStack spacing={4} align="stretch">
|
||||
<HStack justify="space-between">
|
||||
<Heading size="md">Music Library</Heading>
|
||||
<Text color="gray.600">
|
||||
{musicFiles.length} file{musicFiles.length !== 1 ? 's' : ''}
|
||||
</Text>
|
||||
<HStack spacing={2}>
|
||||
<Text color="gray.600">
|
||||
{musicFiles.length} file{musicFiles.length !== 1 ? 's' : ''}
|
||||
</Text>
|
||||
<Button
|
||||
leftIcon={isSyncing ? <Spinner size="sm" /> : <FiRefreshCw />}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={handleSyncS3}
|
||||
isLoading={isSyncing}
|
||||
loadingText="Syncing..."
|
||||
>
|
||||
Sync S3
|
||||
</Button>
|
||||
</HStack>
|
||||
</HStack>
|
||||
|
||||
{isLoading ? (
|
||||
@ -189,9 +240,20 @@ export const MusicStorage: React.FC = () => {
|
||||
Loading music files...
|
||||
</Text>
|
||||
) : musicFiles.length === 0 ? (
|
||||
<Text textAlign="center" color="gray.500">
|
||||
No music files uploaded yet. Upload some files in the Upload tab.
|
||||
</Text>
|
||||
<VStack spacing={4} textAlign="center" color="gray.500">
|
||||
<Text>No music files found in the database.</Text>
|
||||
<Text fontSize="sm">
|
||||
Try uploading files in the Upload tab, or click "Sync S3" to find files already in your S3 bucket.
|
||||
</Text>
|
||||
<Button
|
||||
leftIcon={<FiRefreshCw />}
|
||||
onClick={handleSyncS3}
|
||||
isLoading={isSyncing}
|
||||
loadingText="Syncing..."
|
||||
>
|
||||
Sync S3 Bucket
|
||||
</Button>
|
||||
</VStack>
|
||||
) : (
|
||||
<SimpleGrid columns={{ base: 1, md: 2, lg: 3 }} spacing={4}>
|
||||
{musicFiles.map((file) => (
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user