Implement advanced AI enhancements UI tracking, styling, requirements, and documentation modal

app/main.py

@@ -98,7 +98,7 @@ def load_jobs_db():
                     if job.status == "queued":
                         job_queue.put(job_id)
                     # Mark active items as interrupted so they can be resumed
-                    elif job.status in ["analyzing", "extracting", "upscaling", "assembling"]:
+                    elif job.status in ["analyzing", "extracting", "upscaling", "restoring_faces", "interpolating", "assembling"]:
                         job.status = "interrupted"
                         job.eta = "Interrupted"
                     jobs_db[job_id] = job
@@ -232,6 +232,9 @@ class StartUpscaleRequest(BaseModel):
     webhook_url: str | None = None
     transcode_format: str = "mp4"
     is_preview: bool = False
+    ai_face_restoration: bool = False
+    ai_rife_interpolation: bool = False
+    ai_audio_denoise: bool = False
 
 class PreviewRequest(BaseModel):
     file_id: str
@@ -416,7 +419,10 @@ def start_upscale(req: StartUpscaleRequest):
         interpolation=req.interpolation,
         webhook_url=req.webhook_url,
         transcode_format=req.transcode_format,
-        is_preview=req.is_preview
+        is_preview=req.is_preview,
+        ai_face_restoration=req.ai_face_restoration,
+        ai_rife_interpolation=req.ai_rife_interpolation,
+        ai_audio_denoise=req.ai_audio_denoise
     )
     
     jobs_db[job_id] = job

app/upscaler.py

@@ -23,13 +23,18 @@ class UpscaleJob:
                  unsharp: bool = False, double_fps: bool = False, preserve_subtitles: bool = True,
                  start_sec: float = None, end_sec: float = None, crf: int = 18, preset: str = "medium",
                  denoise: bool = False, sharpen: bool = False, interpolation: bool = False, 
-                 webhook_url: str = None, transcode_format: str = "mp4", is_preview: bool = False):
+                 webhook_url: str = None, transcode_format: str = "mp4", is_preview: bool = False,
+                 ai_face_restoration: bool = False, ai_rife_interpolation: bool = False,
+                 ai_audio_denoise: bool = False):
         self.job_id = job_id
         self.video_path = video_path
         self.model = model
         self.scale = scale
         self.tile_size = tile_size
         self.preserve_audio = preserve_audio
+        self.ai_face_restoration = ai_face_restoration
+        self.ai_rife_interpolation = ai_rife_interpolation
+        self.ai_audio_denoise = ai_audio_denoise
         
         # Trim mapping
         if ss is not None:
@@ -448,6 +453,87 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
         processed_files = len(os.listdir(output_frames_dir))
         job.update_status("upscaling", progress=80.0, current_frame=processed_files)
         
+        # Step 2.6: AI Face Restoration (GFPGAN)
+        if getattr(job, "ai_face_restoration", False):
+            job.update_status("restoring_faces", progress=81.0)
+            on_progress_update(job.job_id, {"status": "restoring_faces", "progress": 81.0})
+            
+            import importlib.util
+            gfpgan_installed = importlib.util.find_spec("gfpgan") is not None
+            
+            if gfpgan_installed:
+                print(f"Job {job.job_id}: GFPGAN detected. Running Face Restoration...")
+                restored_dir = os.path.join(job_temp_dir, "restored_frames")
+                os.makedirs(restored_dir, exist_ok=True)
+                
+                gfpgan_cmd = [
+                    sys.executable, "-m", "gfpgan.inference_gfpgan",
+                    "-i", output_frames_dir,
+                    "-o", restored_dir,
+                    "-v", "1.4",
+                    "-s", "1",
+                    "--bg_upsampler", "None"
+                ]
+                
+                p_gfp = job.run_command(gfpgan_cmd)
+                stdout, stderr = p_gfp.communicate()
+                job.cleanup_process(p_gfp)
+                
+                if p_gfp.returncode == 0:
+                    gfp_output_path = os.path.join(restored_dir, "restored_imgs")
+                    if os.path.exists(gfp_output_path) and len(os.listdir(gfp_output_path)) > 0:
+                        for filename in os.listdir(gfp_output_path):
+                            src_f = os.path.join(gfp_output_path, filename)
+                            dst_f = os.path.join(output_frames_dir, filename)
+                            try:
+                                shutil.copy2(src_f, dst_f)
+                            except Exception as e:
+                                print(f"Error copying restored face frame: {e}")
+                        print(f"Job {job.job_id}: Face Restoration completed successfully.")
+                    else:
+                        print(f"Job {job.job_id}: GFPGAN did not generate outputs in restored_imgs.")
+                else:
+                    print(f"Job {job.job_id}: GFPGAN failed (exit code {p_gfp.returncode}). Continuing with normal upscale.")
+            else:
+                print(f"Job {job.job_id}: 'gfpgan' package is not installed in the virtual environment. Skipping face restoration. To enable, run: pip install gfpgan realesrgan")
+
+        # Step 2.7: AI Frame Interpolation (RIFE)
+        rife_frames_dir = os.path.join(job_temp_dir, "rife_frames")
+        use_rife = False
+        
+        if getattr(job, "ai_rife_interpolation", False):
+            rife_bin = os.path.join(BASE_DIR, "rife-bin", "rife-ncnn-vulkan")
+            if os.path.isfile(rife_bin):
+                job.update_status("interpolating", progress=83.0)
+                on_progress_update(job.job_id, {"status": "interpolating", "progress": 83.0})
+                os.makedirs(rife_frames_dir, exist_ok=True)
+                
+                try:
+                    os.chmod(rife_bin, 0o755)
+                except Exception:
+                    pass
+                
+                rife_cmd = [
+                    rife_bin,
+                    "-i", output_frames_dir,
+                    "-o", rife_frames_dir,
+                    "-f", "jpg"
+                ]
+                if getattr(job, "gpu_ids", None) is not None:
+                    rife_cmd.extend(["-g", str(job.gpu_ids)])
+                    
+                p_rife = job.run_command(rife_cmd)
+                stdout, stderr = p_rife.communicate()
+                job.cleanup_process(p_rife)
+                
+                if p_rife.returncode == 0:
+                    use_rife = True
+                    print(f"Job {job.job_id}: Successfully ran RIFE frame interpolation.")
+                else:
+                    print(f"Job {job.job_id}: RIFE failed (exit code {p_rife.returncode}). Falling back to FFmpeg interpolation.")
+            else:
+                print(f"Job {job.job_id}: RIFE binary not found at {rife_bin}. Falling back to FFmpeg interpolation.")
+
         # Step 3: Reassemble video
         job.update_status("assembling", progress=85.0)
         on_progress_update(job.job_id, {"status": "assembling", "progress": 85.0})
@@ -463,12 +549,15 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
         if transcode_fmt == "webm":
             vcodec = "libvpx-vp9"
             acodec = "libvorbis"
+            
+        assemble_frames_dir = rife_frames_dir if use_rife else output_frames_dir
+        assemble_fps = fps * 2 if (use_rife or getattr(job, "double_fps", False) or getattr(job, "interpolation", False)) else fps
         
         # Construct ffmpeg reassembly command
         assemble_cmd = [
             "ffmpeg", "-y",
-            "-framerate", str(fps),
+            "-framerate", str(assemble_fps),
-            "-i", os.path.join(output_frames_dir, "frame_%08d.jpg")
+            "-i", os.path.join(assemble_frames_dir, "frame_%08d.jpg")
         ]
         
         # We need the original video as the second input (index 1) if we preserve audio or subtitles
@@ -485,10 +574,15 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
         assemble_cmd.extend(["-map", "0:v:0"])
         
         if job.preserve_audio:
-            assemble_cmd.extend([
+            assemble_cmd.extend(["-map", "1:a:0?"])
-                "-map", "1:a:0?",
+            if getattr(job, "ai_audio_denoise", False):
-                "-c:a", acodec
+                acodec_denoise = "libvorbis" if transcode_fmt == "webm" else "aac"
-            ])
+                assemble_cmd.extend([
+                    "-af", "arnnoise",
+                    "-c:a", acodec_denoise
+                ])
+            else:
+                assemble_cmd.extend(["-c:a", acodec])
             
         if getattr(job, "preserve_subtitles", True):
             assemble_cmd.extend([
@@ -502,7 +596,7 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
             vf_filters.append("hqdn3d")
         if getattr(job, "sharpen", False) or getattr(job, "unsharp", False):
             vf_filters.append("unsharp=3:3:0.5:3:3:0.5")
-        if getattr(job, "interpolation", False) or getattr(job, "double_fps", False):
+        if (getattr(job, "double_fps", False) or getattr(job, "interpolation", False) or getattr(job, "ai_rife_interpolation", False)) and not use_rife:
             target_fps = fps * 2 if getattr(job, "double_fps", False) else 60
             if target_fps < fps:
                 target_fps = fps

context/README.md

@@ -56,3 +56,9 @@ A typical upscale job follows these sequential steps:
 - **Self-Sufficiency**: Running `python3 start.py` automatically checks for a local virtual environment (`venv/` or `.venv/`).
 - **Auto-Provisioning**: If no virtual environment is found, `start.py` will initialize one in `venv/`, upgrade `pip`, install all dependencies listed in `requirements.txt`, mark the Real-ESRGAN binary as executable (`chmod +x`), and create necessary folders (`uploads/`, `outputs/`, `temp/`).
 - **Seamless Launch**: It then automatically launches the server process using the newly created environment interpreter.
+
+### 4. Advanced AI Enhancements
+- **AI Face Restoration (GFPGAN)**: Runs as a python subprocess invoking `gfpgan.inference_gfpgan` to restore and clear up human faces in low-resolution video frames. Results are copied directly back into the frame output folder before motion interpolation and final video assembly.
+- **AI Frame Interpolation (RIFE)**: Runs using the `rife-ncnn-vulkan` binary (expected in `rife-bin/`). Smooths motion by generating and inserting intermediate frames, doubling the framerate. Falls back to FFmpeg's `minterpolate` optical flow filter if the Vulkan binary is not present.
+- **AI Audio Denoising (RNNoise)**: Transports and filters audio using the deep-learning-based `arnnoise` FFmpeg filter, eliminating background noise from output tracks during assembly.
+

requirements.txt

@@ -7,3 +7,6 @@ python-multipart>=0.0.9
 websockets>=13.0
 pydantic>=2.0
 torch>=2.0
+gfpgan>=1.3.8
+realesrgan>=0.3.0
+

static/app.js

@@ -600,6 +600,10 @@ generateVideoPreviewBtn.addEventListener("click", async () => {
     const filterSharpen = document.getElementById("filter-sharpen");
     const filterFps = document.getElementById("filter-fps");
     
+    const aiFaceRestoration = document.getElementById("ai-face-restoration");
+    const aiRifeInterpolation = document.getElementById("ai-rife-interpolation");
+    const aiAudioDenoise = document.getElementById("ai-audio-denoise");
+    
     const transcodeFormat = transcodeSelect.value;
     
     try {
@@ -626,7 +630,10 @@ generateVideoPreviewBtn.addEventListener("click", async () => {
                 interpolation: filterFps.checked,
                 webhook_url: null,
                 transcode_format: transcodeFormat,
-                is_preview: true
+                is_preview: true,
+                ai_face_restoration: aiFaceRestoration ? aiFaceRestoration.checked : false,
+                ai_rife_interpolation: aiRifeInterpolation ? aiRifeInterpolation.checked : false,
+                ai_audio_denoise: aiAudioDenoise ? aiAudioDenoise.checked : false
             })
         });
         
@@ -711,6 +718,10 @@ startUpscaleBtn.addEventListener("click", async () => {
     const filterSharpen = document.getElementById("filter-sharpen");
     const filterFps = document.getElementById("filter-fps");
     const webhookUrl = document.getElementById("webhook-url");
+    
+    const aiFaceRestoration = document.getElementById("ai-face-restoration");
+    const aiRifeInterpolation = document.getElementById("ai-rife-interpolation");
+    const aiAudioDenoise = document.getElementById("ai-audio-denoise");
 
     try {
         const res = await fetch("/api/upscale/start", {
@@ -733,7 +744,10 @@ startUpscaleBtn.addEventListener("click", async () => {
                 denoise: filterDenoise.checked,
                 sharpen: filterSharpen.checked,
                 interpolation: filterFps.checked,
-                webhook_url: webhookUrl.value.trim() || null
+                webhook_url: webhookUrl.value.trim() || null,
+                ai_face_restoration: aiFaceRestoration ? aiFaceRestoration.checked : false,
+                ai_rife_interpolation: aiRifeInterpolation ? aiRifeInterpolation.checked : false,
+                ai_audio_denoise: aiAudioDenoise ? aiAudioDenoise.checked : false
             })
         });
         
@@ -816,6 +830,10 @@ function updateProgressUI(data) {
             statusText = "Queued in pipeline. Waiting for GPU lock...";
         } else if (data.status === "extracting") {
             statusText = "Extracting video frames...";
+        } else if (data.status === "restoring_faces") {
+            statusText = "GFPGAN: Running Face Restoration on upscaled frames...";
+        } else if (data.status === "interpolating") {
+            statusText = "RIFE: Interpolating frames for smooth motion...";
         } else if (data.status === "assembling") {
             statusText = "Assembling video preview and audio tracks...";
         }
@@ -891,6 +909,10 @@ function updateProgressUI(data) {
         appendLogLine("FFmpeg: Splitting video stream to high-fidelity JPG frames...", "info");
     } else if (data.status === "upscaling") {
         appendLogLine(`Vulkan GPU: Processing frame ${data.current_frame} of ${data.total_frames} (ETA: ${data.eta})...`, "info");
+    } else if (data.status === "restoring_faces") {
+        appendLogLine("GFPGAN: Running Face Restoration on upscaled frames...", "info");
+    } else if (data.status === "interpolating") {
+        appendLogLine("RIFE: Interpolating frames for smooth motion...", "info");
     } else if (data.status === "assembling") {
         appendLogLine("FFmpeg: Reassembling enhanced frames & merging audio tracks...", "info");
     } else if (data.status === "completed") {
@@ -1021,7 +1043,7 @@ async function checkActiveJobs() {
         
         const jobs = await res.json();
         const activeJob = jobs.find(job => 
-            ["queued", "analyzing", "extracting", "upscaling", "assembling"].includes(job.status)
+            ["queued", "analyzing", "extracting", "upscaling", "restoring_faces", "interpolating", "assembling"].includes(job.status)
         );
         
         if (activeJob) {
@@ -1318,7 +1340,7 @@ async function loadQueue() {
         const jobs = await res.json();
         
         const displayJobs = jobs.filter(job => 
-            ["queued", "analyzing", "extracting", "upscaling", "assembling", "completed", "failed", "cancelled", "interrupted"].includes(job.status)
+            ["queued", "analyzing", "extracting", "upscaling", "restoring_faces", "interpolating", "assembling", "completed", "failed", "cancelled", "interrupted"].includes(job.status)
         );
         
         if (displayJobs.length === 0) {
@@ -1380,7 +1402,7 @@ async function loadQueue() {
                         <i class="fa-solid fa-trash"></i> Clear
                     </button>
                 `;
-            } else if (["queued", "analyzing", "extracting", "upscaling", "assembling"].includes(job.status)) {
+            } else if (["queued", "analyzing", "extracting", "upscaling", "restoring_faces", "interpolating", "assembling"].includes(job.status)) {
                 actionsHtml = `
                     ${reorderHtml}
                     <button class="btn btn-primary btn-small btn-view-job" data-id="${job.job_id}">
@@ -1401,7 +1423,7 @@ async function loadQueue() {
                 `;
             }
             
-            const showProgress = ["analyzing", "extracting", "upscaling", "assembling"].includes(job.status);
+            const showProgress = ["analyzing", "extracting", "upscaling", "restoring_faces", "interpolating", "assembling"].includes(job.status);
             const progressStyle = showProgress ? "display: block;" : "display: none;";
             
             item.innerHTML = `
@@ -1757,6 +1779,32 @@ async function initializeApp() {
             loadUploadsBrowser();
         });
     }
+
+    // Register documentation modal handlers
+    const docsBtn = document.getElementById("docs-btn");
+    const docsModal = document.getElementById("docs-modal");
+    const docsCloseBtn = document.getElementById("docs-modal-close-btn");
+    
+    if (docsBtn && docsModal) {
+        docsBtn.addEventListener("click", () => {
+            docsModal.style.display = "flex";
+            document.body.style.overflow = "hidden";
+        });
+    }
+    
+    if (docsCloseBtn && docsModal) {
+        docsCloseBtn.addEventListener("click", () => {
+            docsModal.style.display = "none";
+            document.body.style.overflow = "";
+        });
+        
+        docsModal.addEventListener("click", (e) => {
+            if (e.target === docsModal) {
+                docsModal.style.display = "none";
+                document.body.style.overflow = "";
+            }
+        });
+    }
 }
 
 initializeApp();

static/index.html

@@ -20,6 +20,9 @@
                 </div>
             </div>
             <div class="header-actions">
+                <button type="button" id="docs-btn" class="btn btn-secondary" style="height: 38px; border-radius: 50px; padding: 0 1.25rem; display: inline-flex; align-items: center; gap: 0.5rem; font-size: 0.85rem; box-shadow: none; border: 1px solid var(--border-color);">
+                    <i class="fa-solid fa-book-open"></i> Docs
+                </button>
                 <div class="theme-selector">
                     <label for="theme-select"><i class="fa-solid fa-palette"></i> Theme</label>
                     <select id="theme-select">
@@ -280,6 +283,22 @@
                                     </div>
                                 </div>
 
+                                <!-- AI Enhancements Section -->
+                                <div class="filter-group" style="margin-top: 1rem;">
+                                    <span style="font-weight: 600; font-size: 0.85rem; display: block; color: var(--primary);">Advanced AI Enhancements</span>
+                                    <div class="filter-checkboxes">
+                                        <label class="checkbox-container">
+                                            <input type="checkbox" id="ai-face-restoration"> AI Face Restoration (GFPGAN)
+                                        </label>
+                                        <label class="checkbox-container">
+                                            <input type="checkbox" id="ai-rife-interpolation"> AI Frame Interpolation (RIFE)
+                                        </label>
+                                        <label class="checkbox-container">
+                                            <input type="checkbox" id="ai-audio-denoise"> AI Audio Denoising (RNNoise)
+                                        </label>
+                                    </div>
+                                </div>
+
                                 <!-- Webhook Notify Input -->
                                 <div class="form-group">
                                     <label for="webhook-url">Webhook Notification Callback</label>
@@ -590,6 +609,47 @@
         </div>
     </div>
 
+    <!-- Documentation Modal -->
+    <div id="docs-modal" class="modal-overlay" style="display: none;">
+        <div class="modal-card" style="max-width: 800px; max-height: 85vh; display: flex; flex-direction: column;">
+            <div class="modal-header">
+                <h3><i class="fa-solid fa-book-open"></i> Help & System Documentation</h3>
+                <button type="button" class="close-btn" id="docs-modal-close-btn">&times;</button>
+            </div>
+            <div class="modal-body" style="overflow-y: auto; padding: 2rem; font-family: var(--font-body); line-height: 1.6; color: var(--text-main);">
+                <div class="docs-markdown-body">
+                    <h2>🎬 AI Video Upscaler Guide</h2>
+                    <p>Welcome to the AI Video Upscaler! This application leverages neural network models running locally to upscale and enhance video footage.</p>
+                    
+                    <h3>💡 Quick Start</h3>
+                    <ol>
+                        <li><strong>Upload:</strong> Drag & drop a video file onto the upload zone, or browse to select one.</li>
+                        <li><strong>Configure:</strong> Choose your AI model, scale factor, and adjust parameters.</li>
+                        <li><strong>Preview:</strong> Select a frame using the timeline and render a single-frame preview or a 5-second video preview to test your options.</li>
+                        <li><strong>Run:</strong> Click "Start Full Upscale" to add the job to the GPU queue.</li>
+                    </ol>
+
+                    <h3>⚙️ AI Enhancement Features</h3>
+                    <ul>
+                        <li><strong>AI Face Restoration (GFPGAN):</strong> Restores human faces in blurry or low-res footage. Highly recommended for family home videos or film restoration. Requires python <code>gfpgan</code> package.</li>
+                        <li><strong>AI Frame Interpolation (RIFE):</strong> Smooths motion by inserting high-quality AI generated frames (e.g. converting 24/30 FPS to 60 FPS). Requires <code>rife-ncnn-vulkan</code> binary.</li>
+                        <li><strong>AI Audio Denoising (RNNoise):</strong> Eliminates steady background noise, wind, and camera hiss directly from audio tracks using a deep learning RNN model.</li>
+                    </ul>
+
+                    <h3>📊 Advanced Settings</h3>
+                    <ul>
+                        <li><strong>CRF (Constant Rate Factor):</strong> Controls output quality/compression. Lower values (like 18) are near-lossless, higher values compress more.</li>
+                        <li><strong>Tile Size:</strong> Decreasing this (e.g., to 128 or 64) helps run upscaling on low-end GPUs with limited VRAM.</li>
+                        <li><strong>Video Trimming:</strong> Trim start and end times to only upscale the parts you need.</li>
+                    </ul>
+
+                    <h3>🔄 Job Resumption</h3>
+                    <p>If the application stops or the server crashes while upscaling, the job will be marked as <strong>interrupted</strong> in the queue. Simply click <strong>Resume</strong> to pick up exactly where you left off. Already upscaled frames will be skipped, saving time and energy.</p>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <!-- Javascript Orchestration -->
     <script src="app.js"></script>
 </body>

static/styles.css

@@ -653,11 +653,14 @@ input:checked + .slider:before {
 .status-badge.queued { background: rgba(255, 184, 0, 0.1); border: 1px solid var(--warning); color: var(--warning); }
 .status-badge.extracting { background: rgba(189, 0, 255, 0.15); border: 1px solid var(--secondary); color: #e499ff; }
 .status-badge.upscaling { background: rgba(0, 240, 255, 0.1); border: 1px solid var(--primary); color: var(--primary); }
+.status-badge.restoring_faces { background: rgba(255, 0, 255, 0.1); border: 1px solid var(--secondary); color: var(--secondary); }
+.status-badge.interpolating { background: rgba(0, 255, 184, 0.1); border: 1px solid var(--primary); color: var(--primary); }
 .status-badge.assembling { background: rgba(0, 240, 255, 0.1); border: 1px solid var(--primary); color: var(--primary); }
 .status-badge.completed { background: rgba(0, 255, 135, 0.1); border: 1px solid var(--success); color: var(--success); }
 .status-badge.failed { background: rgba(255, 0, 85, 0.1); border: 1px solid var(--danger); color: var(--danger); }
 .status-badge.interrupted { background: rgba(255, 255, 255, 0.08); border: 1px solid var(--text-muted); color: var(--text-muted); }
 
+
 .bar-container {
     width: 100%;
     height: 16px;