david/personal_development
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

syncing latest revisions - dk123

Browse Source
This commit is contained in:
david
2026-01-12 15:15:37 -05:00
parent 56494208bd
commit e471d3e1f8
11 changed files with 100 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video_transcription/ai_transcriber_v2/__pycache__/extractor.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/main.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/tracker.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/transcriber.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/translator.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/utils.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/extractor.py
+15 -19
View File
@@ -2,6 +2,7 @@ import os
import subprocess import subprocess
import sys import sys
import json import json
import tracker
from utils import verify_file_not_empty from utils import verify_file_not_empty
def run_ffmpeg(args): def run_ffmpeg(args):
@@ -15,10 +16,10 @@ def run_ffmpeg(args):
subprocess.run(cmd, check=True) subprocess.run(cmd, check=True)
return True return True
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print(f"FFmpeg Error: {e}") tracker.logger.error(f"FFmpeg Error: {e}")
return False return False
except FileNotFoundError: except FileNotFoundError:
print("Error: 'ffmpeg' command not found. Please ensure it is installed on your host system.") tracker.logger.error("Error: 'ffmpeg' command not found. Please ensure it is installed on your host system.")
return False return False
def get_video_duration(file_path): def get_video_duration(file_path):
@@ -54,10 +55,10 @@ def extract_audio(video_path, output_path=None):
output_path = f"{base_name}.wav" output_path = f"{base_name}.wav"
if verify_file_not_empty(output_path): if verify_file_not_empty(output_path):
print(f"Audio file already exists: {output_path}") tracker.logger.info(f"Audio file already exists: {output_path}")
return output_path return output_path
print(f"Extracting audio from {video_path}...") tracker.logger.info(f"Extracting audio from {video_path}...")
args = [ args = [
"-i", video_path, "-i", video_path,
@@ -72,7 +73,7 @@ def extract_audio(video_path, output_path=None):
if run_ffmpeg(args): if run_ffmpeg(args):
if not verify_file_not_empty(output_path): if not verify_file_not_empty(output_path):
raise Exception("FFmpeg succeeded but output is empty.") raise Exception("FFmpeg succeeded but output is empty.")
print(f"Audio extracted to: {output_path}") tracker.logger.info(f"Audio extracted to: {output_path}")
return output_path return output_path
else: else:
sys.exit(1) sys.exit(1)
@@ -83,7 +84,7 @@ def embed_subtitles(video_path, srt_path, output_path=None):
Includes strict safety checks (Size & Duration) to prevent replacing videos with corrupted files. Includes strict safety checks (Size & Duration) to prevent replacing videos with corrupted files.
""" """
if not os.path.exists(video_path) or not os.path.exists(srt_path): if not os.path.exists(video_path) or not os.path.exists(srt_path):
print("Error: Video or SRT file not found for embedding.") tracker.logger.error("Error: Video or SRT file not found for embedding.")
return False return False
# Capture original stats # Capture original stats
@@ -91,14 +92,14 @@ def embed_subtitles(video_path, srt_path, output_path=None):
original_duration = get_video_duration(video_path) original_duration = get_video_duration(video_path)
if original_size == 0: if original_size == 0:
print("Error: Source video is 0 bytes.") tracker.logger.error("Error: Source video is 0 bytes.")
return False return False
if output_path is None: if output_path is None:
base, ext = os.path.splitext(video_path) base, ext = os.path.splitext(video_path)
output_path = f"{base}.subbed{ext}" output_path = f"{base}.subbed{ext}"
print(f"Embedding subtitles into: {output_path}...") tracker.logger.info(f"Embedding subtitles into: {output_path}...")
sub_codec = "mov_text" if video_path.lower().endswith(".mp4") else "srt" sub_codec = "mov_text" if video_path.lower().endswith(".mp4") else "srt"
@@ -124,7 +125,7 @@ def embed_subtitles(video_path, srt_path, output_path=None):
if run_ffmpeg(args): if run_ffmpeg(args):
# --- Safety Checks --- # --- Safety Checks ---
if not os.path.exists(output_path): if not os.path.exists(output_path):
print("Error: Output file was not created.") tracker.logger.error("Error: Output file was not created.")
return False return False
new_size = os.path.getsize(output_path) new_size = os.path.getsize(output_path)
@@ -132,29 +133,24 @@ def embed_subtitles(video_path, srt_path, output_path=None):
# 1. Zero Byte Check # 1. Zero Byte Check
if new_size == 0: if new_size == 0:
print("❌ CRITICAL: Output file is 0 bytes. Deleting corrupted output.") tracker.logger.error("❌ CRITICAL: Output file is 0 bytes. Deleting corrupted output.")
os.remove(output_path) os.remove(output_path)
raise Exception("Embedding failed: Output is empty.") raise Exception("Embedding failed: Output is empty.")
# 2. Significant Size Drop Check # 2. Significant Size Drop Check
if new_size < (original_size * 0.8): if new_size < (original_size * 0.8):
print(f"❌ CRITICAL: Output file is significantly smaller than source!") tracker.logger.error(f"❌ CRITICAL: Output file is significantly smaller than source!")
print(f" Original: {original_size/1024/1024:.2f} MB")
print(f" New: {new_size/1024/1024:.2f} MB")
os.remove(output_path) os.remove(output_path)
raise Exception("Embedding failed: Suspicious file size reduction.") raise Exception("Embedding failed: Suspicious file size reduction.")
# 3. Duration Mismatch Check (New) # 3. Duration Mismatch Check (New)
if abs(original_duration - new_duration) > 1.0: if abs(original_duration - new_duration) > 1.0:
print(f"❌ CRITICAL: Duration mismatch detected!") tracker.logger.error(f"❌ CRITICAL: Duration mismatch detected!")
print(f" Original: {original_duration:.2f}s")
print(f" New: {new_duration:.2f}s")
print(" This indicates truncated video stream. Aborting.")
os.remove(output_path) os.remove(output_path)
raise Exception("Embedding failed: Duration mismatch > 1s.") raise Exception("Embedding failed: Duration mismatch > 1s.")
print(f"Subtitles embedded successfully: {output_path}") tracker.logger.info(f"Subtitles embedded successfully: {output_path}")
return True return True
else: else:
print("Error: Embedding failed.") tracker.logger.error("Error: Embedding failed.")
return False return False
video_transcription/ai_transcriber_v2/main.py
+25 -22
View File
@@ -27,6 +27,8 @@ from diarizer import diarize_audio, merge_diarization_with_transcript
import tracker import tracker
from tracker import JobStatus from tracker import JobStatus
from tqdm import tqdm
def save_srt_with_speakers(segments, output_path): def save_srt_with_speakers(segments, output_path):
"""Helper to save SRT with speaker labels prepended to text.""" """Helper to save SRT with speaker labels prepended to text."""
def format_timestamp(seconds: float): def format_timestamp(seconds: float):
@@ -50,7 +52,7 @@ def save_srt_with_speakers(segments, output_path):
f.write(f"{i}\n") f.write(f"{i}\n")
f.write(f"{start} --> {end}\n") f.write(f"{start} --> {end}\n")
f.write(f"{text}\n\n") f.write(f"{text}\n\n")
print(f"SRT saved to: {output_path}") tracker.logger.info(f"SRT saved to: {output_path}")
def process_file(file_path, args, source_lang=None, loaded_model=None, service_status=None): def process_file(file_path, args, source_lang=None, loaded_model=None, service_status=None):
tracker.logger.info(f"=== Processing: {file_path} ===") tracker.logger.info(f"=== Processing: {file_path} ===")
@@ -324,34 +326,35 @@ def main():
loaded_model = load_whisper_model(args.model) loaded_model = load_whisper_model(args.model)
# ----------------------- # -----------------------
# Initialize Graceful Exit Handler # --- Collect All Files ---
killer = GracefulKiller() all_files = []
video_extensions = ('.mp4', '.mkv', '.mov', '.avi', '.webm', '.flv', '.wmv', '.m4v') video_extensions = ('.mp4', '.mkv', '.mov', '.avi', '.webm', '.flv', '.wmv', '.m4v')
for input_path in valid_inputs: for input_path in valid_inputs:
if os.path.isfile(input_path):
all_files.append(input_path)
elif os.path.isdir(input_path):
for root, dirs, files in os.walk(input_path):
for file in files:
if file.lower().endswith(video_extensions):
all_files.append(os.path.join(root, file))
if not all_files:
print("No video files found to process.")
return
# --- Batch Process with Progress Bar ---
pbar = tqdm(all_files, desc="Batch Progress", unit="file", dynamic_ncols=True, leave=True)
for file_path in pbar:
if killer.kill_now: if killer.kill_now:
break break
if os.path.isfile(input_path): # Update progress bar description with current file
process_file(input_path, args, source_lang, loaded_model=loaded_model, service_status=service_status) filename = os.path.basename(file_path)
elif os.path.isdir(input_path): pbar.set_description(f"File: {filename[:30]}")
found = False
for root, dirs, files in os.walk(input_path):
if killer.kill_now:
break
for file in files: process_file(file_path, args, source_lang, loaded_model=loaded_model, service_status=service_status)
if killer.kill_now:
break
if file.lower().endswith(video_extensions):
found = True
process_file(os.path.join(root, file), args, source_lang, loaded_model=loaded_model, service_status=service_status)
if not found:
print(f"No video files found in {input_path}")
else:
print(f"Error: Invalid input path '{input_path}'")
if killer.kill_now: if killer.kill_now:
print("\n🛑 Process stopped by user. Progress saved in database.") print("\n🛑 Process stopped by user. Progress saved in database.")
video_transcription/ai_transcriber_v2/tracker.py
+14 -5
View File
@@ -14,13 +14,22 @@ log_dir = "logs"
os.makedirs(log_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True)
log_file = os.path.join(log_dir, f"transcriber_{HOSTNAME}_{datetime.now().strftime('%Y%m%d')}.log") log_file = os.path.join(log_dir, f"transcriber_{HOSTNAME}_{datetime.now().strftime('%Y%m%d')}.log")
# Create formatters
log_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
# File Handler (Full Detail)
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(log_formatter)
file_handler.setLevel(logging.INFO)
# Stream Handler (Quiet Detail for Terminal)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(log_formatter)
stream_handler.setLevel(logging.WARNING) # Only warnings/errors to terminal
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s', handlers=[file_handler, stream_handler]
handlers=[
logging.FileHandler(log_file),
logging.StreamHandler()
]
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
video_transcription/ai_transcriber_v2/transcriber.py
+38 -35
View File
@@ -3,18 +3,21 @@ import os
import sys import sys
import subprocess import subprocess
import torch import torch
import tracker
def check_gpu_health(): def check_gpu_health():
""" """
Performs a robust check for GPU availability and prints detailed troubleshooting Performs a robust check for GPU availability and prints detailed troubleshooting
info if issues are detected, specific to Bazzite/VS Code environments. info if issues are detected, specific to Bazzite/VS Code environments.
""" """
print("Checking GPU health...") tracker.logger.info("Checking GPU health...")
# 1. Check if the OS/Driver sees the GPU # 1. Check if the OS/Driver sees the GPU
nvidia_smi_ok = False nvidia_smi_ok = False
try: try:
subprocess.run(["nvidia-smi"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) in_flatpak = os.path.exists("/.flatpak-info")
cmd = ["flatpak-spawn", "--host", "nvidia-smi"] if in_flatpak else ["nvidia-smi"]
subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
nvidia_smi_ok = True nvidia_smi_ok = True
except (subprocess.CalledProcessError, FileNotFoundError): except (subprocess.CalledProcessError, FileNotFoundError):
nvidia_smi_ok = False nvidia_smi_ok = False
@@ -23,36 +26,36 @@ def check_gpu_health():
torch_cuda_ok = torch.cuda.is_available() torch_cuda_ok = torch.cuda.is_available()
if torch_cuda_ok: if torch_cuda_ok:
print(f"✅ GPU is accessible: {torch.cuda.get_device_name(0)}") tracker.logger.info(f"✅ GPU is accessible: {torch.cuda.get_device_name(0)}")
print(f" CUDA Version: {torch.version.cuda}") tracker.logger.info(f" CUDA Version: {torch.version.cuda}")
return True return True
# --- Troubleshooting Block --- # --- Troubleshooting Block ---
print("\n⚠️ WARNING: GPU not detected by PyTorch. Falling back to CPU.") tracker.logger.warning("\n⚠️ WARNING: GPU not detected by PyTorch. Falling back to CPU.")
print(" Transcription will be significantly slower.\n") tracker.logger.warning(" Transcription will be significantly slower.\n")
print("--- Diagnostic Report ---") tracker.logger.info("--- Diagnostic Report ---")
if nvidia_smi_ok: if nvidia_smi_ok:
print("1. [OK] 'nvidia-smi' command works. The system driver is installed and visible.") tracker.logger.info("1. [OK] 'nvidia-smi' command works. The system driver is installed and visible.")
print("2. [FAIL] PyTorch cannot see the GPU.") tracker.logger.info("2. [FAIL] PyTorch cannot see the GPU.")
print(" -> Likely Cause: You might have installed the CPU-only version of PyTorch.") tracker.logger.info(" -> Likely Cause: You might have installed the CPU-only version of PyTorch.")
print(" -> Solution: Reinstall PyTorch with CUDA support:") tracker.logger.info(" -> Solution: Reinstall PyTorch with CUDA support:")
print(" pip uninstall torch torchvision torchaudio") tracker.logger.info(" pip uninstall torch torchvision torchaudio")
print(" pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118") tracker.logger.info(" pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118")
else: else:
print("1. [FAIL] 'nvidia-smi' command failed or not found.") tracker.logger.info("1. [FAIL] 'nvidia-smi' command failed or not found.")
print(" -> Likely Cause: Nvidia drivers are missing, or the container/sandbox cannot access the GPU.") tracker.logger.info(" -> Likely Cause: Nvidia drivers are missing, or the container/sandbox cannot access the GPU.")
print("\n --- Bazzite / VS Code / Container Specific Checks ---") tracker.logger.info("\n --- Bazzite / VS Code / Container Specific Checks ---")
print(" a. If you are running inside a dev container (DevBox/Distrobox/Toolbox):") tracker.logger.info(" a. If you are running inside a dev container (DevBox/Distrobox/Toolbox):")
print(" Ensure the container was created with nvidia support.") tracker.logger.info(" Ensure the container was created with nvidia support.")
print(" (Bazzite usually handles this for 'distrobox', but check your config).") tracker.logger.info(" (Bazzite usually handles this for 'distrobox', but check your config).")
print(" b. If you are using VS Code Flatpak:") tracker.logger.info(" b. If you are using VS Code Flatpak:")
print(" Flatpak might be restricting access. Check Flatseal permissions for VS Code.") tracker.logger.info(" Flatpak might be restricting access. Check Flatseal permissions for VS Code.")
print(" c. Driver Check:") tracker.logger.info(" c. Driver Check:")
print(" Run 'rpm -qa | grep nvidia' in your host terminal to verify drivers are installed.") tracker.logger.info(" Run 'rpm -qa | grep nvidia' in your host terminal to verify drivers are installed.")
print("-------------------------\n") tracker.logger.info("-------------------------\n")
return False return False
def get_vram_gb(): def get_vram_gb():
@@ -80,7 +83,7 @@ def get_optimal_model_size():
if vram == 0: if vram == 0:
return "base" return "base"
print(f"Detected GPU with {vram:.2f} GB VRAM.") tracker.logger.info(f"Detected GPU with {vram:.2f} GB VRAM.")
if vram >= 11: if vram >= 11:
return "large" return "large"
@@ -113,7 +116,7 @@ def save_as_srt(result, output_path):
f.write(f"{i}\n") f.write(f"{i}\n")
f.write(f"{start} --> {end}\n") f.write(f"{start} --> {end}\n")
f.write(f"{text}\n\n") f.write(f"{text}\n\n")
print(f"SRT saved to: {output_path}") tracker.logger.info(f"SRT saved to: {output_path}")
def load_whisper_model(model_size="auto"): def load_whisper_model(model_size="auto"):
""" """
@@ -123,17 +126,17 @@ def load_whisper_model(model_size="auto"):
if model_size == "auto": if model_size == "auto":
model_size = get_optimal_model_size() model_size = get_optimal_model_size()
print(f"Auto-selected model: '{model_size}'") tracker.logger.info(f"Auto-selected model: '{model_size}'")
print(f"Loading Whisper model ('{model_size}')...") tracker.logger.info(f"Loading Whisper model ('{model_size}')...")
device = "cuda" if torch.cuda.is_available() else "cpu" device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}") tracker.logger.info(f"Using device: {device}")
try: try:
model = whisper.load_model(model_size, device=device) model = whisper.load_model(model_size, device=device)
return model return model
except Exception as e: except Exception as e:
print(f"Error loading model: {e}") tracker.logger.error(f"Error loading model: {e}")
sys.exit(1) sys.exit(1)
def transcribe_audio(audio_path, model_size="auto", language=None, loaded_model=None): def transcribe_audio(audio_path, model_size="auto", language=None, loaded_model=None):
@@ -156,12 +159,12 @@ def transcribe_audio(audio_path, model_size="auto", language=None, loaded_model=
if model is None: if model is None:
model = load_whisper_model(model_size) model = load_whisper_model(model_size)
print(f"Transcribing {audio_path}...") tracker.logger.info(f"Transcribing {audio_path}...")
try: try:
# Enable verbose=True to show progress in terminal # Disable verbose to prevent line-by-line output
result = model.transcribe(audio_path, language=language, verbose=True) result = model.transcribe(audio_path, language=language, verbose=False)
print("Transcription complete.") tracker.logger.info("Transcription complete.")
return result return result
except Exception as e: except Exception as e:
print(f"Error during transcription: {e}") tracker.logger.error(f"Error during transcription: {e}")
sys.exit(1) sys.exit(1)
video_transcription/ai_transcriber_v2/translator.py
+6 -6
View File
@@ -25,8 +25,8 @@ def translate_via_ollama(source_srt_content, target_language="English", model="l
subs = pysubs2.SSAFile.from_string(source_srt_content) subs = pysubs2.SSAFile.from_string(source_srt_content)
# Using tqdm for progress bar # Using tqdm for progress bar
# dynamic_ncols=True helps it resize properly. leave=True ensures it stays after completion. # dynamic_ncols=True helps it resize properly.
for line in tqdm(subs, desc=" Ollama Progress", unit="line", dynamic_ncols=True, leave=True): for line in tqdm(subs, desc=" Ollama Progress", unit="line", dynamic_ncols=True, leave=False):
text = line.text.strip() text = line.text.strip()
# Skip empty, numeric-only, or extremely short non-word text # Skip empty, numeric-only, or extremely short non-word text
@@ -81,7 +81,7 @@ def translate_fallback_mymemory(source_srt_content, target_language="en"):
# MyMemory uses ISO 639-1 usually # MyMemory uses ISO 639-1 usually
translator = MyMemoryTranslator(source='auto', target=target_language) translator = MyMemoryTranslator(source='auto', target=target_language)
for line in tqdm(subs, desc=" MyMemory Progress", unit="line"): for line in tqdm(subs, desc=" MyMemory Progress", unit="line", leave=False):
text = line.text.strip() text = line.text.strip()
# Skip empty, numeric-only, or extremely short non-word text # Skip empty, numeric-only, or extremely short non-word text
@@ -121,7 +121,7 @@ def translate_fallback_free(source_srt_content, target_language="en"):
translator = GoogleTranslator(source='auto', target=target_language) translator = GoogleTranslator(source='auto', target=target_language)
# Simple line-by-line translation # Simple line-by-line translation
for line in tqdm(subs, desc=" DeepTranslate Progress", unit="line"): for line in tqdm(subs, desc=" DeepTranslate Progress", unit="line", leave=False):
text = line.text.strip() text = line.text.strip()
# Skip empty, numeric-only, or extremely short non-word text # Skip empty, numeric-only, or extremely short non-word text
@@ -335,8 +335,8 @@ def translate_with_auto_fallback(srt_content, target_language="English", prefer_
# Execute attempts # Execute attempts
for i, method_func in enumerate(attempts): for i, method_func in enumerate(attempts):
if i > 0: # if i > 0:
print(f" Attempt {i} failed. Trying next fallback...") # print(f" Attempt {i} failed. Trying next fallback...")
content, method = method_func() content, method = method_func()
if content: if content: