Fix subprocess pipe deadlock in upscaler and update static assets

app/upscaler.py

@@ -136,15 +136,15 @@ class UpscaleJob:
                         pass
             self._processes.clear()
 
-    def run_command(self, cmd: list, shell=False) -> subprocess.Popen:
+    def run_command(self, cmd: list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) -> subprocess.Popen:
         with self._lock:
             if self._is_cancelled:
                 raise InterruptedError("Job was cancelled")
             
             p = subprocess.Popen(
                 cmd,
-                stdout=subprocess.PIPE,
+                stdout=stdout,
-                stderr=subprocess.PIPE,
+                stderr=stderr,
                 text=True,
                 shell=shell
             )
@@ -379,7 +379,11 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
                     upscale_cmd.append("-x")
                     
                 upscale_start_time = time.time()
-                p_upscale = job.run_command(upscale_cmd)
+                upscale_stdout_path = os.path.join(job_temp_dir, "upscale_stdout.log")
+                upscale_stderr_path = os.path.join(job_temp_dir, "upscale_stderr.log")
+                
+                with open(upscale_stdout_path, "w") as f_out, open(upscale_stderr_path, "w") as f_err:
+                    p_upscale = job.run_command(upscale_cmd, stdout=f_out, stderr=f_err)
                     
                     # Monitor thread for output files
                     while p_upscale.poll() is None:
@@ -415,7 +419,19 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
                         })
                         time.sleep(0.5)
                 
-                stdout, stderr = p_upscale.communicate()
+                # Read stdout/stderr from files
+                if os.path.exists(upscale_stdout_path):
+                    with open(upscale_stdout_path, "r") as f_out:
+                        stdout = f_out.read()
+                else:
+                    stdout = ""
+                    
+                if os.path.exists(upscale_stderr_path):
+                    with open(upscale_stderr_path, "r") as f_err:
+                        stderr = f_err.read()
+                else:
+                    stderr = ""
+                
                 job.cleanup_process(p_upscale)
                 
                 if job._is_cancelled:
@@ -636,10 +652,13 @@ def run_upscale_pipeline(job: UpscaleJob, on_progress_update: Callable[[str, Dic
             job.update_status("failed", error=str(e))
             on_progress_update(job.job_id, {"status": "failed", "error": str(e)})
     finally:
-        # Clean up temp frames to save space
+        # Clean up temp frames to save space only if completed or cancelled
         try:
+            if job.status in ["completed", "cancelled"]:
                 if os.path.exists(job_temp_dir):
                     shutil.rmtree(job_temp_dir)
+            else:
+                print(f"Job {job.job_id} finished with status {job.status}. Preserving temp directory {job_temp_dir} for potential resume.")
         except Exception as cleanup_err:
             print(f"Error during temp cleanup: {cleanup_err}")