diff --git a/qbittorrent_performer_sorter.html b/qbittorrent_performer_sorter.html index cebeff2..b935ed6 100644 --- a/qbittorrent_performer_sorter.html +++ b/qbittorrent_performer_sorter.html @@ -2155,6 +2155,93 @@ + +
+
+
+
+ ๐Ÿ”€ Stash Transcode Correlator & Storage Saver (Replace Originals) +
+

+ Correlate generated Stash web-optimized transcodes with original video files, preview storage savings, and safely replace original files with direct-stream transcodes. +

+
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + + +
+
+ + + + + +
+
+
๐Ÿ”€
+

No transcode scan executed yet

+

Click "Scan & Correlate Transcodes" to index all generated Stash transcodes and match them to original video files.

+
+
+
+ @@ -7547,6 +7634,315 @@ Return ONLY a valid JSON object with an array named "results": const btnScanTagDupes = document.getElementById('btn-scan-tag-dupes'); if (btnScanTagDupes) btnScanTagDupes.addEventListener('click', scanTagDuplicates); + // ==================== SECTION D: TRANSCODE CORRELATOR & STORAGE SAVER ==================== + let transcodeItems = []; + let selectedTranscodeHashes = new Set(); + + async function scanStashTranscodes() { + const btn = document.getElementById('btn-scan-stash-transcodes'); + const container = document.getElementById('stash-transcodes-container'); + const summaryBar = document.getElementById('transcode-summary-bar'); + const batchBtn = document.getElementById('btn-batch-replace-transcodes'); + + const transcodeDir = document.getElementById('stash-transcode-dir')?.value.trim() || '/mnt/isolation/stashapp/generated/transcodes'; + const stashPrefix = document.getElementById('stash-container-prefix')?.value.trim() || '/data'; + const localPrefix = document.getElementById('stash-local-prefix')?.value.trim() || '/mnt/isolation/videos'; + + try { + if (btn) { btn.disabled = true; btn.textContent = 'Correlating Transcodes...'; } + if (container) { + container.innerHTML = ` +
+
โณ
+

Scanning Stash library and matching generated transcodes to original video files...

+
+ `; + } + + const queryParams = new URLSearchParams({ + transcode_dir: transcodeDir, + stash_prefix: stashPrefix, + local_prefix: localPrefix + }); + + const res = await fetch(`/api/stash/transcodes/scan?${queryParams.toString()}`); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + transcodeItems = data.items || []; + selectedTranscodeHashes.clear(); + + if (summaryBar) summaryBar.style.display = 'flex'; + if (batchBtn) batchBtn.style.display = 'inline-flex'; + + const statMatches = document.getElementById('stat-transcode-matches'); + if (statMatches) statMatches.textContent = `${transcodeItems.length} matched (${data.total_transcodes_on_disk || 0} on disk)`; + + const statSavings = document.getElementById('stat-transcode-savings'); + if (statSavings) { + const gb = (data.total_potential_savings_bytes || 0) / (1024**3); + statSavings.textContent = `${gb.toFixed(2)} GB`; + } + + renderStashTranscodes(); + } catch (err) { + showToast(`Transcode scan error: ${err.message}`, 'error'); + if (container) { + container.innerHTML = `
Failed to correlate transcodes: ${err.message}
`; + } + } finally { + if (btn) { btn.disabled = false; btn.textContent = '๐Ÿ” Scan & Correlate Transcodes'; } + } + } + + function renderStashTranscodes() { + const container = document.getElementById('stash-transcodes-container'); + const filterMode = document.getElementById('transcode-filter-mode')?.value || 'savers'; + const searchQ = (document.getElementById('transcode-search-input')?.value || '').toLowerCase().trim(); + + if (!container) return; + + let filtered = transcodeItems.filter(item => { + if (searchQ && !item.basename.toLowerCase().includes(searchQ) && !item.orig_local_path.toLowerCase().includes(searchQ)) { + return false; + } + if (filterMode === 'savers') return item.saved_bytes > 0; + if (filterMode === 'large') return item.saved_bytes >= 500 * 1024 * 1024; + return true; + }); + + if (filtered.length === 0) { + container.innerHTML = ` +
+
โœจ
+

No matching transcodes found

+

Try changing the filter mode or search term above.

+
+ `; + return; + } + + container.innerHTML = ` +
+ + + + + + + + + + + + + + ${filtered.map(item => { + const origMb = (item.orig_size / (1024 * 1024)).toFixed(1); + const transMb = (item.transcode_size / (1024 * 1024)).toFixed(1); + const savedMb = (item.saved_bytes / (1024 * 1024)).toFixed(1); + const isPositive = item.saved_bytes > 0; + const isChecked = selectedTranscodeHashes.has(item.oshash); + + return ` + + + + + + + + + + `; + }).join('')} + +
+ + Original Video File & Local PathTranscode Cache (Stash)Original SizeTranscoded SizeStorage SavedActions
+ + +
+ ${escapeHtml(item.basename)} +
+
+ ๐Ÿ“ ${escapeHtml(item.orig_local_path)} + ${!item.orig_exists ? `Original Missing on Disk` : ''} +
+ 		+ + ${escapeHtml(item.transcode_filename)} + + + ${origMb} MB + + ${transMb} MB + + + ${isPositive ? `๐ŸŸข -${savedMb} MB (-${item.pct_change}%)` : `๐Ÿ”ด +${Math.abs(savedMb)} MB (+${Math.abs(item.pct_change)}%)`} + + +
+ + +
+
+
+ `; + + // Select all listener + const thSelectAll = document.getElementById('th-select-all-transcodes'); + if (thSelectAll) { + thSelectAll.addEventListener('change', () => { + filtered.forEach(item => { + if (thSelectAll.checked) { + selectedTranscodeHashes.add(item.oshash); + } else { + selectedTranscodeHashes.delete(item.oshash); + } + }); + document.querySelectorAll('.transcode-checkbox').forEach(cb => cb.checked = thSelectAll.checked); + updateSelectedTranscodeCount(); + }); + } + + // Individual checkbox listeners + container.querySelectorAll('.transcode-checkbox').forEach(cb => { + cb.addEventListener('change', () => { + const h = cb.getAttribute('data-hash'); + if (cb.checked) selectedTranscodeHashes.add(h); + else selectedTranscodeHashes.delete(h); + updateSelectedTranscodeCount(); + }); + }); + + // Preview listener + container.querySelectorAll('.btn-transcode-preview').forEach(btn => { + btn.addEventListener('click', () => { + const p = btn.getAttribute('data-path'); + const t = btn.getAttribute('data-title'); + if (p) openVideoPreview(p, t, null); + }); + }); + + // Single Replace listener + container.querySelectorAll('.btn-transcode-replace-single').forEach(btn => { + btn.addEventListener('click', async () => { + const h = btn.getAttribute('data-hash'); + const item = transcodeItems.find(x => x.oshash === h); + if (!item) return; + + if (confirm(`Safely replace original video file with Stash transcode?\n\nFile: ${item.basename}\nOriginal Size: ${(item.orig_size/(1024*1024)).toFixed(1)} MB\nTranscode Size: ${(item.transcode_size/(1024*1024)).toFixed(1)} MB\nSpace Saved: ${(item.saved_bytes/(1024*1024)).toFixed(1)} MB`)) { + await executeSingleTranscodeReplace(item, btn); + } + }); + }); + } + + function updateSelectedTranscodeCount() { + const countEl = document.getElementById('transcode-selected-count'); + if (countEl) countEl.textContent = selectedTranscodeHashes.size; + } + + async function executeSingleTranscodeReplace(item, btnEl) { + const deleteCache = document.getElementById('transcode-opt-delete-cache')?.checked; + const triggerRescan = document.getElementById('transcode-opt-trigger-rescan')?.checked; + + try { + btnEl.disabled = true; + btnEl.textContent = 'Replacing...'; + const res = await fetch('/api/stash/transcodes/replace', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + orig_local_path: item.orig_local_path, + transcode_path: item.transcode_path, + delete_transcode: deleteCache, + trigger_rescan: triggerRescan + }) + }); + const data = await res.json(); + if (data.success) { + showToast(`โœ“ Replaced "${item.basename}" (Saved ${(data.saved_bytes/(1024*1024)).toFixed(1)} MB)!`, 'success'); + const row = document.getElementById(`transcode-row-${item.oshash}`); + if (row) { + row.style.opacity = '0.4'; + row.style.background = 'rgba(16, 185, 129, 0.1)'; + btnEl.textContent = 'โœ… Replaced'; + } + } else { + showToast(`Replacement failed: ${data.error}`, 'error'); + btnEl.disabled = false; + btnEl.textContent = 'โšก Replace'; + } + } catch (err) { + showToast(`Replace error: ${err.message}`, 'error'); + btnEl.disabled = false; + btnEl.textContent = 'โšก Replace'; + } + } + + // Batch Replace Action + const btnBatchReplaceTranscodes = document.getElementById('btn-batch-replace-transcodes'); + if (btnBatchReplaceTranscodes) { + btnBatchReplaceTranscodes.addEventListener('click', async () => { + if (selectedTranscodeHashes.size === 0) { + showToast('No transcodes selected for replacement', 'error'); + return; + } + + const selectedItems = transcodeItems.filter(item => selectedTranscodeHashes.has(item.oshash)); + const totalSavedMb = (selectedItems.reduce((acc, x) => acc + x.saved_bytes, 0) / (1024*1024)).toFixed(1); + + if (!confirm(`Batch replace ${selectedItems.length} original video files with their Stash transcoded files?\n\nTotal Estimated Space Saved: ${totalSavedMb} MB\nOriginal files will be atomically overwritten with the transcoded MP4s.`)) { + return; + } + + try { + btnBatchReplaceTranscodes.disabled = true; + btnBatchReplaceTranscodes.textContent = `Replacing ${selectedItems.length}...`; + + const deleteCache = document.getElementById('transcode-opt-delete-cache')?.checked; + const triggerRescan = document.getElementById('transcode-opt-trigger-rescan')?.checked; + + const res = await fetch('/api/stash/transcodes/batch-replace', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + items: selectedItems, + delete_transcode: deleteCache, + trigger_rescan: triggerRescan + }) + }); + const data = await res.json(); + if (data.success) { + showToast(`โœ“ Batch replaced ${data.replaced_count} files (Saved ${(data.total_saved_bytes/(1024*1024)).toFixed(1)} MB)!`, 'success'); + await scanStashTranscodes(); + } else { + showToast(`Batch error: ${data.error}`, 'error'); + } + } catch (err) { + showToast(`Batch execution error: ${err.message}`, 'error'); + } finally { + btnBatchReplaceTranscodes.disabled = false; + btnBatchReplaceTranscodes.textContent = `โšก Batch Replace Selected (${selectedTranscodeHashes.size})`; + } + }); + } + + // Filter & Search Event Listeners + const transcodeFilterMode = document.getElementById('transcode-filter-mode'); + if (transcodeFilterMode) transcodeFilterMode.addEventListener('change', renderStashTranscodes); + + const transcodeSearchInput = document.getElementById('transcode-search-input'); + if (transcodeSearchInput) transcodeSearchInput.addEventListener('input', renderStashTranscodes); + + const btnScanStashTranscodes = document.getElementById('btn-scan-stash-transcodes'); + if (btnScanStashTranscodes) btnScanStashTranscodes.addEventListener('click', scanStashTranscodes); + // ==================== POST-MOVE PIPELINE AUTOMATION ==================== async function triggerPostMoveAutomation(targetDir) { const autoPipeline = document.getElementById('stash-auto-pipeline')?.value; @@ -7613,6 +8009,7 @@ Return ONLY a valid JSON object with an array named "results": { title: 'Stash: Find Performer Duplicates & Merge', icon: '๐Ÿ‘ฅ', action: () => { switchTab('stash-tasks'); scanPerformerDuplicates(); } }, { title: 'Stash: Missing Metadata Radar', icon: '๐Ÿ“ก', action: () => { switchTab('stash-tasks'); loadMissingMetadataRadar(); } }, { title: 'Stash: Tag Normalizer & Deduplication', icon: '๐Ÿท๏ธ', action: () => { switchTab('stash-tasks'); scanTagDuplicates(); } }, + { title: 'Stash: Correlate Transcodes & Replace Originals', icon: '๐Ÿ”€', action: () => { switchTab('stash-tasks'); scanStashTranscodes(); } }, { title: 'Stash: Clean Database & Generated Files', icon: '๐Ÿงน', action: () => { switchTab('stash-tasks'); executeStashCleanTask(); } }, { title: 'Stash: Generate Previews & Sprites', icon: 'โšก', action: () => { switchTab('stash-tasks'); executeStashGenerateTask(); } }, { title: 'Scan SMB Files (/videodownloader)', icon: '๐Ÿ”', action: () => { switchTab('smb'); document.getElementById('btn-fetch-smb')?.click(); } }, diff --git a/server.py b/server.py index 6bae6e1..9f8dab9 100644 --- a/server.py +++ b/server.py @@ -809,6 +809,8 @@ class RobustProxyHandler(http.server.SimpleHTTPRequestHandler): self.handle_stash_missing_metadata() elif self.path.startswith('/api/stash/tags/duplicates'): self.handle_stash_tag_duplicates() + elif self.path.startswith('/api/stash/transcodes/scan'): + self.handle_stash_transcode_scan() # SMB API elif self.path.startswith('/api/smb/status'): self.handle_smb_status() @@ -884,6 +886,10 @@ class RobustProxyHandler(http.server.SimpleHTTPRequestHandler): self.handle_stash_performer_merge() elif self.path == '/api/stash/tags/merge': self.handle_stash_tag_merge() + elif self.path == '/api/stash/transcodes/replace': + self.handle_stash_transcode_replace() + elif self.path == '/api/stash/transcodes/batch-replace': + self.handle_stash_transcode_batch_replace() # SMB API Moves, Deletes & Cleanups elif self.path == '/api/smb/move': self.handle_smb_move() @@ -2612,6 +2618,281 @@ class RobustProxyHandler(http.server.SimpleHTTPRequestHandler): except Exception as e: self.send_json_response(500, {"error": str(e)}) + # ==================== Stash Transcode Correlator & Storage Saver Handlers ==================== + def handle_stash_transcode_scan(self): + parsed = urllib.parse.urlparse(self.path) + params = urllib.parse.parse_qs(parsed.query) + stash_url = params.get('stash_url', [DEFAULT_STASH_URL])[0] + transcode_dir = params.get('transcode_dir', ['/mnt/isolation/stashapp/generated/transcodes'])[0] + stash_prefix = params.get('stash_prefix', ['/data'])[0] + local_prefix = params.get('local_prefix', ['/mnt/isolation/videos'])[0] + + if not os.path.exists(transcode_dir): + self.send_json_response(400, {"error": f"Transcode directory not found: {transcode_dir}", "items": []}) + return + + # Build dictionary of all transcode files on disk + transcode_files = {} + for f in os.listdir(transcode_dir): + if f.endswith('.mp4'): + h = f[:-4] + full_p = os.path.join(transcode_dir, f) + try: + transcode_files[h] = { + "filename": f, + "path": full_p, + "size": os.path.getsize(full_p) + } + except OSError: + pass + + # Scan Stash findFiles in batches of 5000 until all matched or end + matched_items = [] + page = 1 + page_size = 5000 + unmatched_transcodes = set(transcode_files.keys()) + + while unmatched_transcodes: + q = f''' + query GetFilesBatch {{ + findFiles(filter: {{ per_page: {page_size}, page: {page} }}) {{ + files {{ + id + path + basename + size + fingerprints {{ + type + value + }} + ... on VideoFile {{ + width + height + duration + video_codec + }} + }} + }} + }} + ''' + try: + res = execute_stash_graphql(q, stash_url, timeout=30) + files = res.get('data', {}).get('findFiles', {}).get('files', []) + if not files: + break + + for f in files: + fps = {fp['type']: fp['value'] for fp in f.get('fingerprints', [])} + oshash = fps.get('oshash') + if oshash and oshash in transcode_files: + tinfo = transcode_files[oshash] + orig_stash_path = f.get('path', '') + orig_local_path = orig_stash_path.replace(stash_prefix, local_prefix) if stash_prefix and orig_stash_path.startswith(stash_prefix) else orig_stash_path + + orig_size = f.get('size') or 0 + transcode_size = tinfo['size'] + saved_bytes = orig_size - transcode_size + pct_change = ((saved_bytes / orig_size) * 100) if orig_size > 0 else 0 + + matched_items.append({ + "file_id": f['id'], + "oshash": oshash, + "basename": f.get('basename', ''), + "orig_stash_path": orig_stash_path, + "orig_local_path": orig_local_path, + "orig_exists": os.path.exists(orig_local_path), + "orig_size": orig_size, + "orig_resolution": f"{f.get('width', '')}x{f.get('height', '')}", + "orig_codec": f.get('video_codec', ''), + "transcode_filename": tinfo['filename'], + "transcode_path": tinfo['path'], + "transcode_size": transcode_size, + "saved_bytes": saved_bytes, + "pct_change": round(pct_change, 1) + }) + unmatched_transcodes.discard(oshash) + + if len(files) < page_size: + break + page += 1 + except Exception as e: + print(f"Error querying Stash findFiles page {page}: {e}") + break + + # Sort by space saved descending + matched_items.sort(key=lambda x: x['saved_bytes'], reverse=True) + total_saved_bytes = sum(x['saved_bytes'] for x in matched_items if x['saved_bytes'] > 0) + + self.send_json_response(200, { + "items": matched_items, + "total_matched": len(matched_items), + "total_transcodes_on_disk": len(transcode_files), + "total_potential_savings_bytes": total_saved_bytes + }) + + def handle_stash_transcode_replace(self): + content_length = int(self.headers.get('Content-Length', 0)) + post_data = self.rfile.read(content_length) + try: + payload = json.loads(post_data.decode('utf-8')) + orig_local_path = payload.get('orig_local_path') + transcode_path = payload.get('transcode_path') + delete_transcode = bool(payload.get('delete_transcode', True)) + trigger_rescan = bool(payload.get('trigger_rescan', True)) + stash_url = payload.get('stash_url') or DEFAULT_STASH_URL + + if not orig_local_path or not transcode_path: + self.send_json_response(400, {"error": "Missing orig_local_path or transcode_path"}) + return + + if not os.path.exists(transcode_path): + self.send_json_response(400, {"error": f"Transcode file does not exist: {transcode_path}"}) + return + + if not os.path.exists(orig_local_path): + self.send_json_response(400, {"error": f"Original file does not exist at local path: {orig_local_path}"}) + return + + orig_size = os.path.getsize(orig_local_path) + transcode_size = os.path.getsize(transcode_path) + if transcode_size == 0: + self.send_json_response(400, {"error": "Transcode file is 0 bytes (corrupted), aborting replacement"}) + return + + # Step 1: Copy transcode to temporary staging file next to original + tmp_staging_path = f"{orig_local_path}.transcode.tmp" + if os.path.exists(tmp_staging_path): + try: + os.remove(tmp_staging_path) + except OSError: + pass + + shutil.copy2(transcode_path, tmp_staging_path) + + # Step 2: Verify size of staging file + staged_size = os.path.getsize(tmp_staging_path) + if staged_size != transcode_size: + if os.path.exists(tmp_staging_path): + os.remove(tmp_staging_path) + self.send_json_response(500, {"error": f"Size mismatch during staging: expected {transcode_size}, got {staged_size}"}) + return + + # Step 3: Atomically overwrite original file with staged transcode + os.replace(tmp_staging_path, orig_local_path) + + # Step 4: If delete_transcode is requested, remove original transcode cache + if delete_transcode: + try: + os.remove(transcode_path) + except OSError as e: + print(f"Warning: Could not remove transcode cache file {transcode_path}: {e}") + + saved_bytes = orig_size - transcode_size + + # Step 5: Record to Relocation History audit log + record_relocation_history( + source_path=transcode_path, + target_path=orig_local_path, + performer_name="Transcode Optimization", + status="completed", + size_bytes=transcode_size, + action_type="transcode_replace" + ) + + # Step 6: Trigger Stash metadata scan on parent folder if requested + if trigger_rescan: + try: + parent_dir = os.path.dirname(orig_local_path) + stash_parent = parent_dir.replace('/mnt/isolation/videos', '/data') + mutation = ''' + mutation MetadataScan($input: ScanMetadataInput!) { + metadataScan(input: $input) + } + ''' + execute_stash_graphql(mutation, stash_url, variables={"input": {"paths": [stash_parent], "rescan": True}}, timeout=10) + except Exception as e: + print(f"Background Stash rescan trigger error: {e}") + + self.send_json_response(200, { + "success": True, + "orig_local_path": orig_local_path, + "orig_size": orig_size, + "new_size": transcode_size, + "saved_bytes": saved_bytes + }) + except Exception as e: + self.send_json_response(500, {"error": str(e)}) + + def handle_stash_transcode_batch_replace(self): + content_length = int(self.headers.get('Content-Length', 0)) + post_data = self.rfile.read(content_length) + try: + payload = json.loads(post_data.decode('utf-8')) + items = payload.get('items', []) + delete_transcode = bool(payload.get('delete_transcode', True)) + trigger_rescan = bool(payload.get('trigger_rescan', True)) + stash_url = payload.get('stash_url') or DEFAULT_STASH_URL + + replaced_count = 0 + total_saved_bytes = 0 + errors = [] + rescanned_dirs = set() + + for item in items: + orig_local_path = item.get('orig_local_path') + transcode_path = item.get('transcode_path') + if not orig_local_path or not transcode_path or not os.path.exists(transcode_path) or not os.path.exists(orig_local_path): + continue + + try: + orig_size = os.path.getsize(orig_local_path) + transcode_size = os.path.getsize(transcode_path) + if transcode_size == 0: + continue + + tmp_staging_path = f"{orig_local_path}.transcode.tmp" + shutil.copy2(transcode_path, tmp_staging_path) + if os.path.getsize(tmp_staging_path) != transcode_size: + if os.path.exists(tmp_staging_path): + os.remove(tmp_staging_path) + continue + + os.replace(tmp_staging_path, orig_local_path) + if delete_transcode: + try: + os.remove(transcode_path) + except OSError: + pass + + replaced_count += 1 + saved = orig_size - transcode_size + total_saved_bytes += saved + rescanned_dirs.add(os.path.dirname(orig_local_path)) + except Exception as e: + errors.append({"path": orig_local_path, "error": str(e)}) + + # Trigger Stash scan for all touched directories + if trigger_rescan and rescanned_dirs: + try: + stash_dirs = [d.replace('/mnt/isolation/videos', '/data') for d in rescanned_dirs] + mutation = ''' + mutation MetadataScan($input: ScanMetadataInput!) { + metadataScan(input: $input) + } + ''' + execute_stash_graphql(mutation, stash_url, variables={"input": {"paths": stash_dirs, "rescan": True}}, timeout=15) + except Exception as e: + print(f"Batch Stash rescan trigger error: {e}") + + self.send_json_response(200, { + "success": True, + "replaced_count": replaced_count, + "total_saved_bytes": total_saved_bytes, + "errors": errors + }) + except Exception as e: + self.send_json_response(500, {"error": str(e)}) + # ==================== SMB Maintenance & Cleanup Handlers ==================== def handle_smb_scan_cleanup(self): parsed = urllib.parse.urlparse(self.path)