perf: Implement streaming XML export for large libraries - Add streamToXml function that processes data in batches - Stream songs in chunks of 100 to avoid memory issues - Write XML directly to response stream instead of building in memory - Add progress logging for large exports - Proper XML escaping for special characters - Much better performance for libraries with thousands of songs

packages/backend/src/routes/songs.ts

@@ -195,34 +195,28 @@ router.get('/count', async (req: Request, res: Response) => {
   }
 });
 
-// Export library to XML format
+// Export library to XML format with streaming
 router.get('/export', async (req: Request, res: Response) => {
   try {
-    console.log('Exporting library to XML format...');
-    
-    // Get all songs from database
-    const allSongs = await Song.find({}).lean();
-    console.log(`Found ${allSongs.length} songs to export`);
-    
-    // Get all playlists from database
-    const allPlaylists = await Playlist.find({}).lean();
-    console.log(`Found ${allPlaylists.length} playlists to export`);
-    
-    // Import the XML generation function
-    const { exportToXml } = await import('../services/xmlService.js');
-    
-    // Generate XML content
-    const xmlContent = exportToXml(allSongs, allPlaylists);
+    console.log('Starting streaming XML export...');
     
     // Set response headers for file download
     res.setHeader('Content-Type', 'application/xml');
     res.setHeader('Content-Disposition', `attachment; filename="rekordbox-library-${new Date().toISOString().split('T')[0]}.xml"`);
+    res.setHeader('Transfer-Encoding', 'chunked');
     
-    console.log('XML export completed successfully');
-    res.send(xmlContent);
+    // Import the streaming XML generation function
+    const { streamToXml } = await import('../services/xmlService.js');
+    
+    // Stream XML generation to response
+    await streamToXml(res);
+    
+    console.log('Streaming XML export completed successfully');
   } catch (error) {
     console.error('Error exporting library:', error);
-    res.status(500).json({ message: 'Error exporting library', error });
+    if (!res.headersSent) {
+      res.status(500).json({ message: 'Error exporting library', error });
+    }
   }
 });
 

packages/backend/src/services/xmlService.ts

@@ -29,6 +29,97 @@ const buildXmlNode = (node: any): any => {
   return xmlNode;
 };
 
+export const streamToXml = async (res: any) => {
+  // Write XML header
+  res.write('<?xml version="1.0" encoding="UTF-8"?>\n');
+  res.write('<DJ_PLAYLISTS Version="1.0.0">\n');
+  
+  // Start COLLECTION section
+  const songCount = await Song.countDocuments();
+  res.write(`  <COLLECTION Entries="${songCount}">\n`);
+  
+  // Stream songs in batches to avoid memory issues
+  const batchSize = 100;
+  let processedSongs = 0;
+  
+  while (processedSongs < songCount) {
+    const songs = await Song.find({})
+      .skip(processedSongs)
+      .limit(batchSize)
+      .lean();
+    
+    for (const song of songs) {
+      res.write(`    <TRACK TrackID="${song.id}" Name="${escapeXml(song.title || '')}" Artist="${escapeXml(song.artist || '')}" Composer="${escapeXml(song.composer || '')}" Album="${escapeXml(song.album || '')}" Grouping="${escapeXml(song.grouping || '')}" Genre="${escapeXml(song.genre || '')}" Kind="${escapeXml(song.kind || '')}" Size="${song.size || ''}" TotalTime="${song.totalTime || ''}" DiscNumber="${song.discNumber || ''}" TrackNumber="${song.trackNumber || ''}" Year="${song.year || ''}" AverageBpm="${song.averageBpm || ''}" DateAdded="${song.dateAdded || ''}" BitRate="${song.bitRate || ''}" SampleRate="${song.sampleRate || ''}" Comments="${escapeXml(song.comments || '')}" PlayCount="${song.playCount || ''}" Rating="${song.rating || ''}" Location="${escapeXml(song.location || '')}" Remixer="${escapeXml(song.remixer || '')}" Tonality="${escapeXml(song.tonality || '')}" Label="${escapeXml(song.label || '')}" Mix="${escapeXml(song.mix || '')}"`);
+      
+      if (song.tempo) {
+        res.write(`>\n      <TEMPO Inizio="${song.tempo.inizio}" Bpm="${song.tempo.bpm}" Metro="${song.tempo.metro}" Battito="${song.tempo.battito}"/>\n    </TRACK>\n`);
+      } else {
+        res.write('/>\n');
+      }
+    }
+    
+    processedSongs += songs.length;
+    console.log(`Streamed ${processedSongs}/${songCount} songs...`);
+  }
+  
+  res.write('  </COLLECTION>\n');
+  
+  // Start PLAYLISTS section
+  res.write('  <PLAYLISTS>\n');
+  res.write('    <NODE Type="0" Name="ROOT" Count="0">\n');
+  
+  // Stream playlists
+  const playlists = await Playlist.find({}).lean();
+  for (const playlist of playlists) {
+    await streamPlaylistNode(res, playlist, 6);
+  }
+  
+  res.write('    </NODE>\n');
+  res.write('  </PLAYLISTS>\n');
+  res.write('</DJ_PLAYLISTS>');
+  
+  res.end();
+};
+
+const streamPlaylistNode = async (res: any, node: any, indent: number) => {
+  const spaces = ' '.repeat(indent);
+  const nodeType = node.type === 'folder' ? '0' : '1';
+  
+  if (node.type === 'folder') {
+    const childCount = node.children ? node.children.length : 0;
+    res.write(`${spaces}<NODE Type="${nodeType}" Name="${escapeXml(node.name)}" Count="${childCount}">\n`);
+    
+    if (node.children && node.children.length > 0) {
+      for (const child of node.children) {
+        await streamPlaylistNode(res, child, indent + 2);
+      }
+    }
+    
+    res.write(`${spaces}</NODE>\n`);
+  } else {
+    const trackCount = node.tracks ? node.tracks.length : 0;
+    res.write(`${spaces}<NODE Type="${nodeType}" Name="${escapeXml(node.name)}" KeyType="0" Entries="${trackCount}">\n`);
+    
+    if (node.tracks && node.tracks.length > 0) {
+      for (const trackId of node.tracks) {
+        res.write(`${spaces}  <TRACK Key="${trackId}"/>\n`);
+      }
+    }
+    
+    res.write(`${spaces}</NODE>\n`);
+  }
+};
+
+const escapeXml = (text: string): string => {
+  if (!text) return '';
+  return text
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+};
+
 export const exportToXml = (songs: any[], playlists: any[]): string => {
   const xml = create({ 
     DJ_PLAYLISTS: {