- Enhanced AudioMetadataService with comprehensive NaN handling for all numeric fields - Added validation for year, duration, bitrate, sampleRate, and channels - Improved FLAC format detection and error handling with fallback support - Added detailed logging for S3 sync process with progress tracking and file statistics - Enhanced song matching service with progress indicators and confidence scoring - Added comprehensive logging for auto-match and link operations - Improved error handling and graceful degradation for metadata extraction - Added test scripts for metadata service validation - Updated S3 service to use configuration from s3-config.json file - Added automatic S3 service reload when configuration is updated The S3 importer now provides much better visibility into file processing and song matching operations, making it easier to debug issues and monitor performance. FLAC files are properly handled and invalid metadata values are filtered out to prevent database corruption.
225 lines
5.6 KiB
TypeScript
225 lines
5.6 KiB
TypeScript
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
|
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
import fs from 'fs/promises';
|
|
import path from 'path';
|
|
|
|
export interface S3Config {
|
|
endpoint: string;
|
|
accessKeyId: string;
|
|
secretAccessKey: string;
|
|
bucketName: string;
|
|
region: string;
|
|
useSSL?: boolean;
|
|
}
|
|
|
|
export interface UploadResult {
|
|
key: string;
|
|
url: string;
|
|
size: number;
|
|
contentType: string;
|
|
}
|
|
|
|
export interface S3FileInfo {
|
|
key: string;
|
|
size: number;
|
|
lastModified: Date;
|
|
contentType?: string;
|
|
}
|
|
|
|
export class S3Service {
|
|
private client: S3Client;
|
|
private bucketName: string;
|
|
|
|
constructor(config: S3Config) {
|
|
this.client = new S3Client({
|
|
endpoint: config.endpoint,
|
|
region: config.region,
|
|
credentials: {
|
|
accessKeyId: config.accessKeyId,
|
|
secretAccessKey: config.secretAccessKey,
|
|
},
|
|
forcePathStyle: true, // Required for MinIO
|
|
});
|
|
this.bucketName = config.bucketName;
|
|
}
|
|
|
|
/**
|
|
* Load S3 configuration from s3-config.json file
|
|
*/
|
|
static async loadConfig(): Promise<S3Config> {
|
|
try {
|
|
const configPath = path.join(process.cwd(), 's3-config.json');
|
|
const configData = await fs.readFile(configPath, 'utf-8');
|
|
return JSON.parse(configData);
|
|
} catch (error) {
|
|
console.warn('Failed to load s3-config.json, using environment variables as fallback');
|
|
return {
|
|
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
|
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minioadmin',
|
|
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'minioadmin',
|
|
bucketName: process.env.S3_BUCKET_NAME || 'music-files',
|
|
region: process.env.S3_REGION || 'us-east-1',
|
|
useSSL: process.env.S3_USE_SSL !== 'false',
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create S3Service instance with configuration from file
|
|
*/
|
|
static async createFromConfig(): Promise<S3Service> {
|
|
const config = await this.loadConfig();
|
|
return new S3Service(config);
|
|
}
|
|
|
|
/**
|
|
* Upload a file to S3
|
|
*/
|
|
async uploadFile(
|
|
file: Buffer,
|
|
originalName: string,
|
|
contentType: string
|
|
): Promise<UploadResult> {
|
|
const fileExtension = originalName.split('.').pop();
|
|
const key = `music/${uuidv4()}.${fileExtension}`;
|
|
|
|
const command = new PutObjectCommand({
|
|
Bucket: this.bucketName,
|
|
Key: key,
|
|
Body: file,
|
|
ContentType: contentType,
|
|
Metadata: {
|
|
originalName,
|
|
uploadedAt: new Date().toISOString(),
|
|
},
|
|
});
|
|
|
|
await this.client.send(command);
|
|
|
|
return {
|
|
key,
|
|
url: `${this.bucketName}/${key}`,
|
|
size: file.length,
|
|
contentType,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Recursively list all files in the S3 bucket
|
|
*/
|
|
async listAllFiles(prefix: string = ''): Promise<S3FileInfo[]> {
|
|
const files: S3FileInfo[] = [];
|
|
let continuationToken: string | undefined;
|
|
|
|
do {
|
|
const command = new ListObjectsV2Command({
|
|
Bucket: this.bucketName,
|
|
Prefix: prefix,
|
|
ContinuationToken: continuationToken,
|
|
});
|
|
|
|
const response = await this.client.send(command);
|
|
|
|
if (response.Contents) {
|
|
for (const object of response.Contents) {
|
|
if (object.Key && !object.Key.endsWith('/')) { // Skip directories
|
|
files.push({
|
|
key: object.Key,
|
|
size: object.Size || 0,
|
|
lastModified: object.LastModified || new Date(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
continuationToken = response.NextContinuationToken;
|
|
} while (continuationToken);
|
|
|
|
return files;
|
|
}
|
|
|
|
/**
|
|
* Generate a presigned URL for secure file access
|
|
*/
|
|
async getPresignedUrl(key: string, expiresIn: number = 3600): Promise<string> {
|
|
const command = new GetObjectCommand({
|
|
Bucket: this.bucketName,
|
|
Key: key,
|
|
});
|
|
|
|
return await getSignedUrl(this.client, command, { expiresIn });
|
|
}
|
|
|
|
/**
|
|
* Delete a file from S3
|
|
*/
|
|
async deleteFile(key: string): Promise<void> {
|
|
const command = new DeleteObjectCommand({
|
|
Bucket: this.bucketName,
|
|
Key: key,
|
|
});
|
|
|
|
await this.client.send(command);
|
|
}
|
|
|
|
/**
|
|
* Check if a file exists
|
|
*/
|
|
async fileExists(key: string): Promise<boolean> {
|
|
try {
|
|
const command = new HeadObjectCommand({
|
|
Bucket: this.bucketName,
|
|
Key: key,
|
|
});
|
|
await this.client.send(command);
|
|
return true;
|
|
} catch (error) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get file metadata
|
|
*/
|
|
async getFileMetadata(key: string) {
|
|
const command = new HeadObjectCommand({
|
|
Bucket: this.bucketName,
|
|
Key: key,
|
|
});
|
|
return await this.client.send(command);
|
|
}
|
|
|
|
/**
|
|
* Get file content as buffer
|
|
*/
|
|
async getFileContent(key: string): Promise<Buffer> {
|
|
const command = new GetObjectCommand({
|
|
Bucket: this.bucketName,
|
|
Key: key,
|
|
});
|
|
|
|
const response = await this.client.send(command);
|
|
|
|
if (!response.Body) {
|
|
throw new Error('File has no content');
|
|
}
|
|
|
|
// Convert stream to buffer
|
|
const chunks: Uint8Array[] = [];
|
|
const stream = response.Body as any;
|
|
|
|
return new Promise((resolve, reject) => {
|
|
stream.on('data', (chunk: Uint8Array) => chunks.push(chunk));
|
|
stream.on('error', reject);
|
|
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Get streaming URL for a file
|
|
*/
|
|
async getStreamingUrl(key: string): Promise<string> {
|
|
return `${process.env.S3_ENDPOINT}/${this.bucketName}/${key}`;
|
|
}
|
|
}
|