Added a significant overhaul to the server manager.
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
from .samba_manager import SambaManager
|
||||
from .config import Settings
|
||||
from smb.smb_structs import OperationFailure
|
||||
|
||||
# --- Dedicated logger for ComicsManager ---
|
||||
comics_logger = logging.getLogger('comics_manager')
|
||||
comics_logger.setLevel(logging.INFO)
|
||||
comics_logger.propagate = False
|
||||
if not comics_logger.handlers:
|
||||
comics_log_handler = logging.FileHandler("logs/comics_organization.log", mode='a')
|
||||
comics_log_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
||||
comics_logger.addHandler(comics_log_handler)
|
||||
comics_logger.addHandler(logging.StreamHandler())
|
||||
|
||||
class ComicsManager:
|
||||
|
||||
@staticmethod
|
||||
def _get_series_name(filename):
|
||||
base_name, _ = os.path.splitext(filename)
|
||||
pattern = re.compile(r'[-_\s]*(v(ol)?|c(h)?|chapter|issue|ep|episode)[-_\s]*\d+.*|[-_\s]+\d+$', re.IGNORECASE)
|
||||
match = pattern.search(base_name)
|
||||
if match:
|
||||
return base_name[:match.start()].strip('-_ ')
|
||||
return base_name.strip('-_ ')
|
||||
|
||||
def cleanup_toberead(self, samba_manager: SambaManager):
|
||||
comics_logger.info("Starting cleanup of 'toberead' directory...")
|
||||
comics_root = "/comics/toberead"
|
||||
try:
|
||||
items_in_toberead = samba_manager.list_path(comics_root)
|
||||
if "error" in items_in_toberead:
|
||||
raise Exception(f"Failed to list files in 'toberead': {items_in_toberead['error']}")
|
||||
|
||||
for item in items_in_toberead:
|
||||
item_path = item["path"]
|
||||
item_name = item["name"]
|
||||
if item["is_directory"]:
|
||||
comics_logger.info(f"Deleting directory '{item_name}' and all its contents.")
|
||||
try:
|
||||
samba_manager.delete_directory_recursive(item_path)
|
||||
comics_logger.info(f"Successfully deleted directory: {item_name}")
|
||||
except OperationFailure as e:
|
||||
comics_logger.error(f"Failed to delete directory {item_name}: {e}")
|
||||
else:
|
||||
comics_logger.info(f"Deleting file '{item_name}'.")
|
||||
try:
|
||||
samba_manager.delete_file(item_path)
|
||||
comics_logger.info(f"Successfully deleted file: {item_name}")
|
||||
except OperationFailure as e:
|
||||
comics_logger.error(f"Failed to delete file {item_name}: {e}")
|
||||
|
||||
comics_logger.info("Cleanup of 'toberead' finished.")
|
||||
|
||||
except Exception as e:
|
||||
comics_logger.error(f"An error occurred during 'toberead' cleanup: {e}", exc_info=True)
|
||||
|
||||
def organize_comics(self, samba_manager: SambaManager):
|
||||
comics_logger.info("Starting comics organization process...")
|
||||
comics_root = "/comics/toberead"
|
||||
comics_dest_root = "/comics/manga"
|
||||
try:
|
||||
items_in_toberead = samba_manager.list_path(comics_root)
|
||||
if "error" in items_in_toberead:
|
||||
raise Exception(f"Failed to list files in 'toberead': {items_in_toberead['error']}")
|
||||
|
||||
series_chapters = defaultdict(list)
|
||||
for item in items_in_toberead:
|
||||
if not item["is_directory"]:
|
||||
series_name = self._get_series_name(item["name"])
|
||||
series_chapters[series_name].append(item)
|
||||
|
||||
comics_logger.info(f"Found {len(series_chapters)} series to process.")
|
||||
|
||||
for series_name, chapters in series_chapters.items():
|
||||
comics_logger.info(f"Processing series: {series_name}")
|
||||
final_series_folder_path = f"{comics_dest_root}/{series_name}".replace("\\", "/")
|
||||
|
||||
try:
|
||||
samba_manager.create_directory(final_series_folder_path)
|
||||
except OperationFailure as e:
|
||||
if "FILE_OBJECT_NAME_COLLISION" not in str(e):
|
||||
comics_logger.error(f"Could not create directory {final_series_folder_path}: {e}")
|
||||
|
||||
for chapter_item in chapters:
|
||||
original_path = chapter_item["path"].replace("\\", "/")
|
||||
original_filename = chapter_item["name"]
|
||||
|
||||
path_to_move = ""
|
||||
filename_to_move = ""
|
||||
|
||||
if original_filename.lower().endswith('.zip'):
|
||||
cbz_filename = os.path.splitext(original_filename)[0] + '.cbz'
|
||||
temp_cbz_path = f"{comics_root}/{cbz_filename}".replace("\\", "/")
|
||||
|
||||
try:
|
||||
samba_manager.rename_file(original_path, temp_cbz_path)
|
||||
path_to_move = temp_cbz_path
|
||||
filename_to_move = cbz_filename
|
||||
comics_logger.info(f"Renamed {original_filename} to {cbz_filename}")
|
||||
except OperationFailure as e:
|
||||
comics_logger.error(f"Failed to rename ZIP {original_filename}: {e}")
|
||||
continue
|
||||
|
||||
elif original_filename.lower().endswith('.cbz'):
|
||||
path_to_move = original_path
|
||||
filename_to_move = original_filename
|
||||
else:
|
||||
continue
|
||||
|
||||
final_cbz_path = f"{final_series_folder_path}/{filename_to_move}".replace("\\", "/")
|
||||
|
||||
try:
|
||||
samba_manager.rename_file(path_to_move, final_cbz_path)
|
||||
comics_logger.info(f"Moved {filename_to_move} to {final_series_folder_path}")
|
||||
except OperationFailure as e:
|
||||
comics_logger.error(f"Failed to move {filename_to_move} to {final_series_folder_path}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
comics_logger.error(f"An error occurred during comics organization: {e}", exc_info=True)
|
||||
|
||||
comics_logger.info("Comics organization process finished.")
|
||||
|
||||
return
|
||||
@@ -0,0 +1,40 @@
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
import subprocess
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configure logging early
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler() # Log to console
|
||||
]
|
||||
)
|
||||
|
||||
def check_gpu_support():
|
||||
try:
|
||||
logger.debug("Checking ffmpeg for CUDA support...")
|
||||
result = subprocess.run("ffmpeg -hwaccels", shell=True, capture_output=True, text=True, check=True)
|
||||
if "cuda" in result.stdout:
|
||||
logger.info("ffmpeg with CUDA support detected. Enabling GPU acceleration.")
|
||||
return True
|
||||
except (FileNotFoundError, subprocess.CalledProcessError) as e:
|
||||
logger.debug(f"GPU detection failed: {e}")
|
||||
pass
|
||||
|
||||
logger.info("No ffmpeg with CUDA support detected. Falling back to CPU-based processing.")
|
||||
return False
|
||||
|
||||
class Settings(BaseSettings):
|
||||
samba_server_ip: str
|
||||
samba_username: str
|
||||
samba_password: str
|
||||
gpu_enabled: bool = check_gpu_support()
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,20 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
|
||||
SQLALCHEMY_DATABASE_URL = "sqlite:///./sql_app.db"
|
||||
|
||||
engine = create_engine(
|
||||
SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
|
||||
)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
# Dependency
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,266 @@
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from PIL import Image
|
||||
import imagehash
|
||||
from sqlalchemy.orm import Session
|
||||
from .samba_manager import SambaManager
|
||||
from .config import Settings
|
||||
from . import models
|
||||
import itertools
|
||||
import logging
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DuplicatesManager:
|
||||
def __init__(self, settings: Settings, db: Session):
|
||||
self.settings = settings
|
||||
self.db = db
|
||||
self.videos_root = "/videos"
|
||||
|
||||
def _get_video_duration(self, filepath):
|
||||
logger.debug(f"Running ffprobe for duration of {filepath}")
|
||||
try:
|
||||
command = [
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
filepath,
|
||||
]
|
||||
logger.debug(f"ffprobe command: {' '.join(command)}")
|
||||
result = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
logger.debug(f"ffprobe stdout: {result.stdout.strip()}")
|
||||
logger.debug(f"ffprobe stderr: {result.stderr.strip()}")
|
||||
return float(result.stdout)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||||
logger.error(f"ffprobe failed for {filepath}: {e}")
|
||||
return None
|
||||
|
||||
def _get_frame_hash(self, filepath):
|
||||
logger.debug(f"Running ffmpeg for frame hash of {filepath}")
|
||||
tmp_frame_path = ""
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp_frame:
|
||||
tmp_frame_path = tmp_frame.name
|
||||
command = ["ffmpeg"]
|
||||
if self.settings.gpu_enabled:
|
||||
command.extend(["-hwaccel", "cuda"])
|
||||
|
||||
command.extend([
|
||||
"-i",
|
||||
filepath,
|
||||
"-ss",
|
||||
"00:00:10",
|
||||
"-vframes",
|
||||
"1",
|
||||
"-y",
|
||||
tmp_frame_path,
|
||||
])
|
||||
|
||||
logger.debug(f"ffmpeg command: {' '.join(command)}")
|
||||
result = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
logger.debug(f"ffmpeg stdout: {result.stdout.strip()}")
|
||||
logger.debug(f"ffmpeg stderr: {result.stderr.strip()}")
|
||||
|
||||
if os.path.exists(tmp_frame_path):
|
||||
logger.debug(f"Temporary frame file exists: {tmp_frame_path}, size: {os.path.getsize(tmp_frame_path)} bytes")
|
||||
phash = imagehash.phash(Image.open(tmp_frame_path))
|
||||
return str(phash)
|
||||
else:
|
||||
logger.warning(f"Temporary frame file was not created: {tmp_frame_path}")
|
||||
return None
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||||
logger.error(f"ffmpeg failed for {filepath}: {e}")
|
||||
return None
|
||||
except Image.UnidentifiedImageError as e:
|
||||
logger.error(f"PIL.UnidentifiedImageError for {filepath} with temp file {tmp_frame_path}: {e}")
|
||||
return None
|
||||
finally:
|
||||
if os.path.exists(tmp_frame_path):
|
||||
os.remove(tmp_frame_path)
|
||||
|
||||
def _is_video_file(self, filename):
|
||||
video_extensions = ['.mp4', '.mkv', '.avi', '.mov', '.wmv', '.flv', '.webm']
|
||||
return any(filename.lower().endswith(ext) for ext in video_extensions)
|
||||
|
||||
def _process_video_file(self, samba_manager: SambaManager, filepath, filename, size):
|
||||
logger.info(f"Processing video: {filepath} ({filename})")
|
||||
existing_video = self.db.query(models.VideoFile).filter_by(filepath=filepath).first()
|
||||
if existing_video:
|
||||
if existing_video.size == size:
|
||||
logger.info(f"Skipping already processed and unaltered video: {filepath}")
|
||||
return
|
||||
else:
|
||||
logger.info(f"File {filepath} has altered size ({existing_video.size} -> {size}). Re-processing.")
|
||||
self.db.delete(existing_video)
|
||||
self.db.commit()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False) as tmp_video:
|
||||
try:
|
||||
samba_manager.download_file(filepath, tmp_video)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download {filepath}: {e}")
|
||||
return
|
||||
|
||||
duration = self._get_video_duration(tmp_video.name)
|
||||
phash = self._get_frame_hash(tmp_video.name)
|
||||
|
||||
os.remove(tmp_video.name)
|
||||
|
||||
if duration is not None and phash is not None:
|
||||
video_file = models.VideoFile(
|
||||
filepath=filepath,
|
||||
filename=filename,
|
||||
size=size,
|
||||
duration=duration,
|
||||
phash=phash,
|
||||
)
|
||||
self.db.add(video_file)
|
||||
self.db.commit()
|
||||
logger.info(f"Processed video: {filepath}")
|
||||
else:
|
||||
logger.warning(f"Could not get duration or hash for {filepath}")
|
||||
|
||||
def scan_videos(self, samba_manager: SambaManager):
|
||||
state_file = "logs/duplicates_scan.state"
|
||||
progress_file = "logs/duplicates_scan.progress"
|
||||
pause_file = "logs/duplicates_scan.pause"
|
||||
|
||||
dirs_to_scan = []
|
||||
|
||||
try:
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r') as f:
|
||||
last_processed_dir = f.read().strip()
|
||||
logger.info(f"Resuming scan from last in-progress directory: {last_processed_dir}")
|
||||
dirs_to_scan.append(last_processed_dir)
|
||||
|
||||
if os.path.exists(state_file):
|
||||
with open(state_file, 'r') as f:
|
||||
dirs_to_scan.extend([line.strip() for line in f if line.strip()])
|
||||
logger.info(f"Loaded {len(dirs_to_scan)} directories from state file.")
|
||||
|
||||
if not dirs_to_scan:
|
||||
dirs_to_scan = [self.videos_root]
|
||||
|
||||
logger.info(f"Starting scan for videos in {self.videos_root} on share 'isolation'")
|
||||
|
||||
while dirs_to_scan:
|
||||
while os.path.exists(pause_file):
|
||||
logger.info("Scan is paused. Waiting for resume signal...")
|
||||
time.sleep(5)
|
||||
|
||||
current_path = dirs_to_scan.pop(0)
|
||||
|
||||
with open(progress_file, 'w') as f:
|
||||
f.write(current_path)
|
||||
|
||||
logger.info(f"Scanning directory: {current_path}")
|
||||
files_and_dirs = samba_manager.list_path(current_path)
|
||||
|
||||
if "error" in files_and_dirs:
|
||||
logger.error(f"Failed to list path {current_path}: {files_and_dirs['error']}")
|
||||
continue
|
||||
|
||||
subdirs = []
|
||||
for item in files_and_dirs:
|
||||
while os.path.exists(pause_file):
|
||||
logger.info("Scan is paused. Waiting for resume signal...")
|
||||
time.sleep(5)
|
||||
|
||||
if item["is_directory"]:
|
||||
subdirs.append(item["path"])
|
||||
elif self._is_video_file(item["name"]):
|
||||
self._process_video_file(samba_manager, item["path"], item["name"], item["size"])
|
||||
|
||||
dirs_to_scan = subdirs + dirs_to_scan
|
||||
|
||||
if os.path.exists(progress_file):
|
||||
os.remove(progress_file)
|
||||
|
||||
with open(state_file, 'w') as f:
|
||||
for d in dirs_to_scan:
|
||||
f.write(d + '\n')
|
||||
|
||||
if os.path.exists(state_file):
|
||||
os.remove(state_file)
|
||||
|
||||
logger.info("Video scan complete.")
|
||||
return {"status": "Scan complete"}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred during video scan: {e}", exc_info=True)
|
||||
return {"status": "Scan failed", "error": str(e)}
|
||||
|
||||
def _hamming_distance(self, s1, s2):
|
||||
return sum(c1 != c2 for c1, c2 in zip(s1, s2))
|
||||
|
||||
def _calculate_similarity(self, file1: models.VideoFile, file2: models.VideoFile):
|
||||
size_similarity = 1 - (abs(file1.size - file2.size) / max(file1.size, file2.size))
|
||||
duration_similarity = 1 - (abs(file1.duration - file2.duration) / max(file1.duration, file2.duration))
|
||||
hash_similarity = 1 - (self._hamming_distance(file1.phash, file2.phash) / len(file1.phash))
|
||||
return (size_similarity * 0.2) + (duration_similarity * 0.3) + (hash_similarity * 0.5)
|
||||
|
||||
def find_duplicates(self, threshold=0.95):
|
||||
report = models.DuplicateReport(status="running")
|
||||
self.db.add(report)
|
||||
self.db.commit()
|
||||
|
||||
videos = self.db.query(models.VideoFile).all()
|
||||
groups = []
|
||||
processed_videos = set()
|
||||
|
||||
for video1, video2 in itertools.combinations(videos, 2):
|
||||
if video1.id in processed_videos or video2.id in processed_videos:
|
||||
continue
|
||||
|
||||
score = self._calculate_similarity(video1, video2)
|
||||
if score >= threshold:
|
||||
existing_group = None
|
||||
for group in groups:
|
||||
if video1.id in group["video_ids"] or video2.id in group["video_ids"]:
|
||||
existing_group = group
|
||||
break
|
||||
|
||||
if existing_group:
|
||||
existing_group["video_ids"].add(video1.id)
|
||||
existing_group["video_ids"].add(video2.id)
|
||||
existing_group["scores"].append(score)
|
||||
else:
|
||||
groups.append({"video_ids": {video1.id, video2.id}, "scores": [score]})
|
||||
|
||||
processed_videos.add(video1.id)
|
||||
processed_videos.add(video2.id)
|
||||
|
||||
for group_data in groups:
|
||||
avg_score = sum(group_data["scores"]) / len(group_data["scores"])
|
||||
db_group = models.DuplicateFileGroup(report_id=report.id, score=avg_score)
|
||||
self.db.add(db_group)
|
||||
self.db.commit()
|
||||
|
||||
for video_id in group_data["video_ids"]:
|
||||
db_file = models.DuplicateFile(group_id=db_group.id, video_file_id=video_id)
|
||||
self.db.add(db_file)
|
||||
|
||||
report.status = "completed"
|
||||
self.db.commit()
|
||||
|
||||
return {"report_id": report.id, "status": "completed"}
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
from fastapi import FastAPI, Depends
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from . import models
|
||||
from .database import SessionLocal, engine, get_db
|
||||
from .routers import samba, comics, duplicates
|
||||
|
||||
models.Base.metadata.create_all(bind=engine)
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
origins = [
|
||||
"http://localhost:5173", # Assuming frontend runs on this port during development
|
||||
"http://127.0.0.1:5173",
|
||||
# Add your production frontend URL(s) here when deploying
|
||||
]
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(samba.router, prefix="/samba", tags=["samba"])
|
||||
app.include_router(comics.router, prefix="/comics", tags=["comics"])
|
||||
app.include_router(duplicates.router, prefix="/duplicates", tags=["duplicates"])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def read_root():
|
||||
return {"Hello": "World"}
|
||||
@@ -0,0 +1,44 @@
|
||||
from sqlalchemy import Column, Integer, String, DateTime, Float, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
from .database import Base
|
||||
import datetime
|
||||
|
||||
|
||||
class DuplicateReport(Base):
|
||||
__tablename__ = "duplicate_reports"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
status = Column(String)
|
||||
groups = relationship("DuplicateFileGroup", back_populates="report")
|
||||
|
||||
|
||||
class VideoFile(Base):
|
||||
__tablename__ = "video_files"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
filepath = Column(String, unique=True, index=True)
|
||||
filename = Column(String)
|
||||
size = Column(Integer)
|
||||
duration = Column(Float)
|
||||
phash = Column(String)
|
||||
|
||||
|
||||
class DuplicateFileGroup(Base):
|
||||
__tablename__ = "duplicate_file_groups"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
report_id = Column(Integer, ForeignKey("duplicate_reports.id"))
|
||||
score = Column(Float)
|
||||
report = relationship("DuplicateReport", back_populates="groups")
|
||||
files = relationship("DuplicateFile", back_populates="group")
|
||||
|
||||
|
||||
class DuplicateFile(Base):
|
||||
__tablename__ = "duplicate_files"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
group_id = Column(Integer, ForeignKey("duplicate_file_groups.id"))
|
||||
video_file_id = Column(Integer, ForeignKey("video_files.id"))
|
||||
group = relationship("DuplicateFileGroup", back_populates="files")
|
||||
video_file = relationship("VideoFile")
|
||||
@@ -0,0 +1,40 @@
|
||||
from fastapi import APIRouter, BackgroundTasks
|
||||
from ..comics_manager import ComicsManager
|
||||
from ..config import settings
|
||||
from ..samba_manager import SambaManager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def run_organize_comics_background():
|
||||
app_settings = settings
|
||||
manager = ComicsManager()
|
||||
samba_manager = SambaManager(
|
||||
app_settings.samba_server_ip,
|
||||
"isolation",
|
||||
app_settings.samba_username,
|
||||
app_settings.samba_password,
|
||||
)
|
||||
manager.organize_comics(samba_manager)
|
||||
samba_manager.close()
|
||||
|
||||
@router.post("/organize")
|
||||
def organize_comics(background_tasks: BackgroundTasks):
|
||||
background_tasks.add_task(run_organize_comics_background)
|
||||
return {"message": "Comics organization started in the background."}
|
||||
|
||||
def run_cleanup_toberead_background():
|
||||
app_settings = settings
|
||||
manager = ComicsManager()
|
||||
samba_manager = SambaManager(
|
||||
app_settings.samba_server_ip,
|
||||
"isolation",
|
||||
app_settings.samba_username,
|
||||
app_settings.samba_password,
|
||||
)
|
||||
manager.cleanup_toberead(samba_manager)
|
||||
samba_manager.close()
|
||||
|
||||
@router.delete("/cleanup_toberead")
|
||||
def cleanup_toberead(background_tasks: BackgroundTasks):
|
||||
background_tasks.add_task(run_cleanup_toberead_background)
|
||||
return {"message": "Comics cleanup started in the background."}
|
||||
@@ -0,0 +1,57 @@
|
||||
from fastapi import APIRouter, Depends, BackgroundTasks
|
||||
from sqlalchemy.orm import Session
|
||||
from ..duplicates_manager import DuplicatesManager
|
||||
from ..config import settings, Settings
|
||||
from ..database import get_db, SessionLocal
|
||||
from ..samba_manager import SambaManager
|
||||
import os
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def run_scan_videos_background():
|
||||
db = SessionLocal()
|
||||
app_settings = Settings()
|
||||
manager = DuplicatesManager(app_settings, db)
|
||||
samba_manager = SambaManager(
|
||||
app_settings.samba_server_ip,
|
||||
"isolation",
|
||||
app_settings.samba_username,
|
||||
app_settings.samba_password,
|
||||
)
|
||||
try:
|
||||
manager.scan_videos(samba_manager)
|
||||
finally:
|
||||
samba_manager.close()
|
||||
db.close()
|
||||
|
||||
@router.post("/scan")
|
||||
def scan_videos(background_tasks: BackgroundTasks):
|
||||
background_tasks.add_task(run_scan_videos_background)
|
||||
return {"message": "Video scan started in the background."}
|
||||
|
||||
@router.post("/scan/pause")
|
||||
def pause_scan():
|
||||
with open("logs/duplicates_scan.pause", 'w') as f:
|
||||
pass
|
||||
return {"message": "Video scan paused."}
|
||||
|
||||
@router.post("/scan/resume")
|
||||
def resume_scan():
|
||||
if os.path.exists("logs/duplicates_scan.pause"):
|
||||
os.remove("logs/duplicates_scan.pause")
|
||||
return {"message": "Video scan resumed."}
|
||||
|
||||
@router.get("/scan/status")
|
||||
def get_scan_status():
|
||||
if os.path.exists("logs/duplicates_scan.pause"):
|
||||
return {"status": "Paused"}
|
||||
if os.path.exists("logs/duplicates_scan.progress") or os.path.exists("logs/duplicates_scan.state"):
|
||||
return {"status": "Scanning"}
|
||||
return {"status": "Idle"}
|
||||
|
||||
|
||||
@router.post("/find")
|
||||
def find_duplicates(threshold: float = 0.95, db: Session = Depends(get_db)):
|
||||
manager = DuplicatesManager(settings, db)
|
||||
result = manager.find_duplicates(threshold)
|
||||
return result
|
||||
@@ -0,0 +1,178 @@
|
||||
from fastapi import APIRouter, UploadFile, File, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from ..samba_manager import SambaManager
|
||||
from ..config import settings
|
||||
import os
|
||||
import io
|
||||
import mimetypes
|
||||
import zipfile
|
||||
|
||||
class RenameRequest(BaseModel):
|
||||
old_path: str
|
||||
new_path: str
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/list/{share_name}")
|
||||
def list_files(share_name: str, path: str = "/"):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
files = manager.list_path(path)
|
||||
manager.close()
|
||||
return files
|
||||
|
||||
|
||||
@router.delete("/files/{share_name}")
|
||||
def delete_file(share_name: str, path: str):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
result = manager.delete_file(path)
|
||||
manager.close()
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/files/upload/{share_name}")
|
||||
def upload_file(share_name: str, path: str, file: UploadFile = File(...)):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
# The path in storeFile is service_name + path
|
||||
remote_path = os.path.join(path, file.filename)
|
||||
result = manager.upload_file(remote_path, file.file)
|
||||
manager.close()
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/files/download/{share_name}")
|
||||
def download_file(share_name: str, path: str):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
file_obj = io.BytesIO()
|
||||
manager.download_file(path, file_obj)
|
||||
manager.close()
|
||||
file_obj.seek(0)
|
||||
return StreamingResponse(file_obj, media_type="application/octet-stream")
|
||||
|
||||
|
||||
@router.get("/view/{share_name}")
|
||||
def view_file(share_name: str, path: str, request: Request):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
|
||||
try:
|
||||
attributes = manager.get_attributes(path)
|
||||
if isinstance(attributes, dict) and "error" in attributes:
|
||||
return Response(status_code=404)
|
||||
file_size = attributes.file_size
|
||||
|
||||
range_header = request.headers.get('Range')
|
||||
|
||||
media_type, _ = mimetypes.guess_type(path)
|
||||
if media_type is None:
|
||||
media_type = "application/octet-stream"
|
||||
|
||||
if range_header:
|
||||
start_str, end_str = range_header.replace('bytes=', '').split('-')
|
||||
start = int(start_str)
|
||||
end = int(end_str) if end_str else file_size - 1
|
||||
|
||||
if start >= file_size:
|
||||
return Response(status_code=416)
|
||||
|
||||
length = end - start + 1
|
||||
file_obj = io.BytesIO()
|
||||
manager.download_file_range(path, file_obj, offset=start, max_length=length)
|
||||
file_obj.seek(0)
|
||||
|
||||
headers = {
|
||||
'Content-Range': f'bytes {start}-{end}/{file_size}',
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Length': str(length),
|
||||
}
|
||||
return Response(file_obj.read(), status_code=206, headers=headers, media_type=media_type)
|
||||
else:
|
||||
file_obj = io.BytesIO()
|
||||
manager.download_file(path, file_obj)
|
||||
file_obj.seek(0)
|
||||
return StreamingResponse(file_obj, media_type=media_type, headers={"Accept-Ranges": "bytes", "Content-Length": str(file_size)})
|
||||
finally:
|
||||
manager.close()
|
||||
|
||||
|
||||
@router.get("/cbz/list/{share_name}")
|
||||
def list_cbz_contents(share_name: str, path: str):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
file_obj = io.BytesIO()
|
||||
manager.download_file(path, file_obj)
|
||||
manager.close()
|
||||
file_obj.seek(0)
|
||||
|
||||
image_files = []
|
||||
with zipfile.ZipFile(file_obj, 'r') as zip_ref:
|
||||
for file_info in zip_ref.infolist():
|
||||
if file_info.filename.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp')):
|
||||
image_files.append(file_info.filename)
|
||||
|
||||
return sorted(image_files)
|
||||
|
||||
@router.get("/cbz/image/{share_name}")
|
||||
def get_cbz_image(share_name: str, path: str, image_name: str):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
file_obj = io.BytesIO()
|
||||
manager.download_file(path, file_obj)
|
||||
manager.close()
|
||||
file_obj.seek(0)
|
||||
|
||||
with zipfile.ZipFile(file_obj, 'r') as zip_ref:
|
||||
with zip_ref.open(image_name) as image_file:
|
||||
image_data = io.BytesIO(image_file.read())
|
||||
|
||||
media_type, _ = mimetypes.guess_type(image_name)
|
||||
if media_type is None:
|
||||
media_type = "application/octet-stream"
|
||||
|
||||
return StreamingResponse(image_data, media_type=media_type)
|
||||
|
||||
|
||||
@router.put("/files/rename/{share_name}")
|
||||
def rename_file(share_name: str, request: RenameRequest):
|
||||
manager = SambaManager(
|
||||
settings.samba_server_ip,
|
||||
share_name,
|
||||
username=settings.samba_username,
|
||||
password=settings.samba_password,
|
||||
)
|
||||
result = manager.rename_file(request.old_path, request.new_path)
|
||||
manager.close()
|
||||
return result
|
||||
@@ -0,0 +1,109 @@
|
||||
from smb.SMBConnection import SMBConnection
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class SambaManager:
|
||||
def __init__(self, server_ip, share_name, username, password, client_machine_name="smbclient"):
|
||||
self.server_ip = server_ip
|
||||
self.share_name = share_name
|
||||
self.conn = SMBConnection(username, password, client_machine_name, "server_name", use_ntlm_v2=True)
|
||||
self.conn.connect(server_ip, 139)
|
||||
|
||||
def list_path(self, current_dir_path="/"):
|
||||
try:
|
||||
files = self.conn.listPath(self.share_name, current_dir_path)
|
||||
items = []
|
||||
for f in files:
|
||||
if f.filename in ['.', '..']:
|
||||
continue
|
||||
|
||||
normalized_current_path = current_dir_path
|
||||
if normalized_current_path != '/' and not normalized_current_path.endswith('/'):
|
||||
normalized_current_path += '/'
|
||||
|
||||
full_item_path = f"{normalized_current_path}{f.filename}".replace("\\\\", "/")
|
||||
|
||||
items.append({
|
||||
"name": f.filename,
|
||||
"is_directory": f.isDirectory,
|
||||
"size": f.file_size,
|
||||
"path": full_item_path
|
||||
})
|
||||
return items
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing path '{current_dir_path}' on share '{self.share_name}': {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
|
||||
def delete_file(self, path):
|
||||
try:
|
||||
self.conn.deleteFiles(self.share_name, path)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def upload_file(self, path, file_obj):
|
||||
try:
|
||||
self.conn.storeFile(self.share_name, path, file_obj)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def download_file(self, path, file_obj):
|
||||
try:
|
||||
self.conn.retrieveFile(self.share_name, path, file_obj)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def download_file_range(self, path, file_obj, offset, max_length):
|
||||
try:
|
||||
self.conn.retrieveFileFromOffset(self.share_name, path, file_obj, offset, max_length)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def get_attributes(self, path):
|
||||
try:
|
||||
return self.conn.getAttributes(self.share_name, path)
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def rename_file(self, old_path, new_path):
|
||||
try:
|
||||
self.conn.rename(self.share_name, old_path, new_path)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def create_directory(self, path):
|
||||
try:
|
||||
self.conn.createDirectory(self.share_name, path)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def delete_directory(self, path):
|
||||
try:
|
||||
self.conn.deleteDirectory(self.share_name, path)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def delete_directory_recursive(self, path):
|
||||
try:
|
||||
items = self.list_path(path)
|
||||
for item in items:
|
||||
item_path = item["path"]
|
||||
if item["is_directory"]:
|
||||
self.delete_directory_recursive(item_path)
|
||||
else:
|
||||
self.delete_file(item_path)
|
||||
self.delete_directory(path)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
def close(self):
|
||||
self.conn.close()
|
||||
Reference in New Issue
Block a user