feat: implement job resumption, custom queue reordering, and auto-venv start script setup

app/main.py

@@ -27,21 +27,97 @@ app.add_middleware(
 )
 
 # In-memory databases
+class CustomJobQueue:
+    def __init__(self):
+        self.queue = []
+        self.lock = threading.Lock()
+        self.condition = threading.Condition(self.lock)
+        
+    def put(self, job_id: str):
+        with self.lock:
+            if job_id not in self.queue:
+                self.queue.append(job_id)
+                self.condition.notify()
+                
+    def get(self) -> str:
+        with self.lock:
+            while not self.queue:
+                self.condition.wait()
+            return self.queue.pop(0)
+            
+    def remove(self, job_id: str) -> bool:
+        with self.lock:
+            if job_id in self.queue:
+                self.queue.remove(job_id)
+                return True
+            return False
+            
+    def get_all(self) -> List[str]:
+        with self.lock:
+            return list(self.queue)
+            
+    def reorder(self, job_ids: List[str]):
+        with self.lock:
+            valid_ids = [jid for jid in job_ids if jid in self.queue]
+            missing_ids = [jid for jid in self.queue if jid not in valid_ids]
+            self.queue = valid_ids + missing_ids
+
+    def task_done(self):
+        pass
+
+    def empty(self) -> bool:
+        with self.lock:
+            return len(self.queue) == 0
+
+    def qsize(self) -> int:
+        with self.lock:
+            return len(self.queue)
+
 jobs_db: Dict[str, upscaler.UpscaleJob] = {}
 ws_connections: Dict[str, List[WebSocket]] = {}
 preview_db: Dict[str, Dict[str, str]] = {} # preview_id -> {orig, upscaled}
 
-# FIFO queue for upscaling jobs to prevent GPU memory overload
-job_queue = queue.Queue()
+# Custom thread-safe queue for upscaling jobs to support reordering & cancellation
+job_queue = CustomJobQueue()
 queue_lock = threading.Lock()
 current_running_job_id = None
 
 main_loop = None
 
+JOBS_FILE = os.path.join(upscaler.BASE_DIR, "jobs.json")
+
+def load_jobs_db():
+    global jobs_db
+    if os.path.exists(JOBS_FILE):
+        try:
+            with open(JOBS_FILE, "r") as f:
+                data = json.load(f)
+                for job_id, job_data in data.items():
+                    job = upscaler.UpscaleJob.from_dict(job_data)
+                    # Automatically put queued items back in the queue
+                    if job.status == "queued":
+                        job_queue.put(job_id)
+                    # Mark active items as interrupted so they can be resumed
+                    elif job.status in ["analyzing", "extracting", "upscaling", "assembling"]:
+                        job.status = "interrupted"
+                        job.eta = "Interrupted"
+                    jobs_db[job_id] = job
+        except Exception as e:
+            print(f"Error loading jobs database: {e}")
+
+def save_jobs_db():
+    try:
+        with open(JOBS_FILE, "w") as f:
+            data = {job_id: job.to_dict() for job_id, job in jobs_db.items()}
+            json.dump(data, f, indent=4)
+    except Exception as e:
+        print(f"Error saving jobs database: {e}")
+
 @app.on_event("startup")
 def startup_event():
     global main_loop
     main_loop = asyncio.get_event_loop()
+    load_jobs_db()
 
 global_webhook_url = None
 
@@ -64,6 +140,7 @@ def send_webhook_notification(url: str, payload: dict):
 
 # Broadcast updates to websockets and webhooks
 def broadcast_progress(job_id: str, data: dict):
+    save_jobs_db()
     job = jobs_db.get(job_id)
     if job:
         data["is_preview"] = getattr(job, "is_preview", False)
@@ -344,6 +421,7 @@ def start_upscale(req: StartUpscaleRequest):
     
     jobs_db[job_id] = job
     job_queue.put(job_id)
+    save_jobs_db()
     
     # Broadcast initial queued progress
     broadcast_progress(job_id, {
@@ -386,6 +464,9 @@ def cancel_job(job_id: str):
     if not job:
         raise HTTPException(status_code=404, detail="Job not found.")
         
+    # Remove from queue if it was queued
+    job_queue.remove(job_id)
+    
     job.cancel()
     # Broadcast cancellation status
     broadcast_progress(job_id, {
@@ -395,6 +476,7 @@ def cancel_job(job_id: str):
         "total_frames": job.total_frames,
         "eta": "N/A"
     })
+    save_jobs_db()
     return {"job_id": job_id, "status": "cancelled"}
 
 @app.post("/api/preview/generate")
@@ -486,7 +568,22 @@ def safe_delete_file(file_path: str):
 
 @app.get("/api/jobs")
 def list_jobs():
-    """List details of all submitted jobs"""
+    """List details of all submitted jobs in queue-sorted order"""
+    active_id = current_running_job_id
+    queued_ids = job_queue.get_all()
+    
+    # Sort active first, then queued in order, then history by start time descending
+    def get_sort_key(job):
+        if job.job_id == active_id:
+            return (0, 0)
+        elif job.job_id in queued_ids:
+            return (1, queued_ids.index(job.job_id))
+        else:
+            t = job.start_time if job.start_time is not None else 0
+            return (2, -t)
+            
+    sorted_jobs = sorted(jobs_db.values(), key=get_sort_key)
+    
     return [
         {
             "job_id": job.job_id,
@@ -500,9 +597,10 @@ def list_jobs():
             "scale": job.scale,
             "output_file": os.path.basename(job.output_file) if job.output_file else None,
             "video_path": job.video_path,
-            "is_preview": getattr(job, "is_preview", False)
+            "is_preview": getattr(job, "is_preview", False),
+            "queue_position": queued_ids.index(job.job_id) if job.job_id in queued_ids else -1 if job.job_id == active_id else None
         }
-        for job in jobs_db.values()
+        for job in sorted_jobs
     ]
 
 @app.delete("/api/jobs/{job_id}")
@@ -512,6 +610,9 @@ def delete_job(job_id: str):
     if not job:
         raise HTTPException(status_code=404, detail="Job not found.")
         
+    # Remove from queue if it is queued
+    job_queue.remove(job_id)
+        
     # Safely delete original preview video if present
     for ext in [".mp4", ".mkv", ".avi", ".mov", ".webm"]:
         orig_prev_path = os.path.join(upscaler.OUTPUT_DIR, f"original_{job_id}{ext}")
@@ -534,6 +635,8 @@ def delete_job(job_id: str):
     # Delete from in-memory db
     if job_id in jobs_db:
         del jobs_db[job_id]
+        
+    save_jobs_db()
     
     return {"job_id": job_id, "status": "purged"}
 
@@ -562,11 +665,57 @@ def purge_all_jobs():
     # Reset in-memory database
     jobs_db.clear()
     
+    # Re-initialize custom queue
+    global job_queue
+    job_queue = CustomJobQueue()
+    
     # Reset upload metadata file
     save_upload_metadata({})
     
+    save_jobs_db()
+    
     return {"status": "all purged"}
 
+class ReorderQueueRequest(BaseModel):
+    job_ids: List[str]
+
+@app.post("/api/queue/reorder")
+def reorder_queue(req: ReorderQueueRequest):
+    """Reorder the job queue"""
+    job_queue.reorder(req.job_ids)
+    save_jobs_db()
+    return {"status": "success", "queue": job_queue.get_all()}
+
+@app.get("/api/queue")
+def get_queue():
+    """Get the current job queue order"""
+    return {"queue": job_queue.get_all()}
+
+@app.post("/api/upscale/resume/{job_id}")
+def resume_job(job_id: str):
+    """Resume an interrupted/failed upscale job"""
+    job = jobs_db.get(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found.")
+        
+    # Re-queue the job
+    job.status = "queued"
+    job.error = None
+    job.eta = "Queued for resume..."
+    
+    job_queue.put(job_id)
+    save_jobs_db()
+    
+    broadcast_progress(job_id, {
+        "status": "queued",
+        "progress": job.progress,
+        "current_frame": job.current_frame,
+        "total_frames": job.total_frames,
+        "eta": "Queued for resume..."
+    })
+    
+    return {"job_id": job_id, "status": "queued"}
+
 # Websocket endpoint for real-time progress updates
 @app.websocket("/ws/progress/{job_id}")
 async def websocket_progress(websocket: WebSocket, job_id: str):