david/personal_development
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

syncing latest revisions - dk123

Browse Source
This commit is contained in:
david
2026-01-12 15:15:37 -05:00
parent 56494208bd
commit e471d3e1f8
11 changed files with 100 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video_transcription/ai_transcriber_v2/__pycache__/extractor.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/main.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/tracker.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/transcriber.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/translator.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/__pycache__/utils.cpython-313.pyc
BIN
View File
Binary file not shown.
video_transcription/ai_transcriber_v2/extractor.py
+15 -19
View File
@@ -2,6 +2,7 @@ import os
import subprocess
import sys
import json
import tracker
from utils import verify_file_not_empty
def run_ffmpeg(args):
@@ -15,10 +16,10 @@ def run_ffmpeg(args):
subprocess.run(cmd, check=True)
return True
except subprocess.CalledProcessError as e:
print(f"FFmpeg Error: {e}")
tracker.logger.error(f"FFmpeg Error: {e}")
return False
except FileNotFoundError:
print("Error: 'ffmpeg' command not found. Please ensure it is installed on your host system.")
tracker.logger.error("Error: 'ffmpeg' command not found. Please ensure it is installed on your host system.")
return False
def get_video_duration(file_path):
@@ -54,10 +55,10 @@ def extract_audio(video_path, output_path=None):
output_path = f"{base_name}.wav"
if verify_file_not_empty(output_path):
print(f"Audio file already exists: {output_path}")
tracker.logger.info(f"Audio file already exists: {output_path}")
return output_path
print(f"Extracting audio from {video_path}...")
tracker.logger.info(f"Extracting audio from {video_path}...")
args = [
"-i", video_path,
@@ -72,7 +73,7 @@ def extract_audio(video_path, output_path=None):
if run_ffmpeg(args):
if not verify_file_not_empty(output_path):
raise Exception("FFmpeg succeeded but output is empty.")
print(f"Audio extracted to: {output_path}")
tracker.logger.info(f"Audio extracted to: {output_path}")
return output_path
else:
sys.exit(1)
@@ -83,7 +84,7 @@ def embed_subtitles(video_path, srt_path, output_path=None):
Includes strict safety checks (Size & Duration) to prevent replacing videos with corrupted files.
"""
if not os.path.exists(video_path) or not os.path.exists(srt_path):
print("Error: Video or SRT file not found for embedding.")
tracker.logger.error("Error: Video or SRT file not found for embedding.")
return False
# Capture original stats
@@ -91,14 +92,14 @@ def embed_subtitles(video_path, srt_path, output_path=None):
original_duration = get_video_duration(video_path)
if original_size == 0:
print("Error: Source video is 0 bytes.")
tracker.logger.error("Error: Source video is 0 bytes.")
return False
if output_path is None:
base, ext = os.path.splitext(video_path)
output_path = f"{base}.subbed{ext}"
print(f"Embedding subtitles into: {output_path}...")
tracker.logger.info(f"Embedding subtitles into: {output_path}...")
sub_codec = "mov_text" if video_path.lower().endswith(".mp4") else "srt"
@@ -124,7 +125,7 @@ def embed_subtitles(video_path, srt_path, output_path=None):
if run_ffmpeg(args):
# --- Safety Checks ---
if not os.path.exists(output_path):
print("Error: Output file was not created.")
tracker.logger.error("Error: Output file was not created.")
return False
new_size = os.path.getsize(output_path)
@@ -132,29 +133,24 @@ def embed_subtitles(video_path, srt_path, output_path=None):
# 1. Zero Byte Check
if new_size == 0:
print("❌ CRITICAL: Output file is 0 bytes. Deleting corrupted output.")
tracker.logger.error("❌ CRITICAL: Output file is 0 bytes. Deleting corrupted output.")
os.remove(output_path)
raise Exception("Embedding failed: Output is empty.")
# 2. Significant Size Drop Check
if new_size < (original_size * 0.8):
print(f"❌ CRITICAL: Output file is significantly smaller than source!")
print(f" Original: {original_size/1024/1024:.2f} MB")
print(f" New: {new_size/1024/1024:.2f} MB")
tracker.logger.error(f"❌ CRITICAL: Output file is significantly smaller than source!")
os.remove(output_path)
raise Exception("Embedding failed: Suspicious file size reduction.")
# 3. Duration Mismatch Check (New)
if abs(original_duration - new_duration) > 1.0:
print(f"❌ CRITICAL: Duration mismatch detected!")
print(f" Original: {original_duration:.2f}s")
print(f" New: {new_duration:.2f}s")
print(" This indicates truncated video stream. Aborting.")
tracker.logger.error(f"❌ CRITICAL: Duration mismatch detected!")
os.remove(output_path)
raise Exception("Embedding failed: Duration mismatch > 1s.")
print(f"Subtitles embedded successfully: {output_path}")
tracker.logger.info(f"Subtitles embedded successfully: {output_path}")
return True
else:
print("Error: Embedding failed.")
tracker.logger.error("Error: Embedding failed.")
return False
video_transcription/ai_transcriber_v2/main.py
+25 -22
View File
@@ -27,6 +27,8 @@ from diarizer import diarize_audio, merge_diarization_with_transcript
import tracker
from tracker import JobStatus
from tqdm import tqdm
def save_srt_with_speakers(segments, output_path):
"""Helper to save SRT with speaker labels prepended to text."""
def format_timestamp(seconds: float):
@@ -50,7 +52,7 @@ def save_srt_with_speakers(segments, output_path):
f.write(f"{i}\n")
f.write(f"{start} --> {end}\n")
f.write(f"{text}\n\n")
print(f"SRT saved to: {output_path}")
tracker.logger.info(f"SRT saved to: {output_path}")
def process_file(file_path, args, source_lang=None, loaded_model=None, service_status=None):
tracker.logger.info(f"=== Processing: {file_path} ===")
@@ -324,34 +326,35 @@ def main():
loaded_model = load_whisper_model(args.model)
# -----------------------
# Initialize Graceful Exit Handler
killer = GracefulKiller()
# --- Collect All Files ---
all_files = []
video_extensions = ('.mp4', '.mkv', '.mov', '.avi', '.webm', '.flv', '.wmv', '.m4v')
for input_path in valid_inputs:
if os.path.isfile(input_path):
all_files.append(input_path)
elif os.path.isdir(input_path):
for root, dirs, files in os.walk(input_path):
for file in files:
if file.lower().endswith(video_extensions):
all_files.append(os.path.join(root, file))
if not all_files:
print("No video files found to process.")
return
# --- Batch Process with Progress Bar ---
pbar = tqdm(all_files, desc="Batch Progress", unit="file", dynamic_ncols=True, leave=True)
for file_path in pbar:
if killer.kill_now:
break
if os.path.isfile(input_path):
process_file(input_path, args, source_lang, loaded_model=loaded_model, service_status=service_status)
elif os.path.isdir(input_path):
found = False
for root, dirs, files in os.walk(input_path):
if killer.kill_now:
break
# Update progress bar description with current file
filename = os.path.basename(file_path)
pbar.set_description(f"File: {filename[:30]}")
for file in files:
if killer.kill_now:
break
if file.lower().endswith(video_extensions):
found = True
process_file(os.path.join(root, file), args, source_lang, loaded_model=loaded_model, service_status=service_status)
if not found:
print(f"No video files found in {input_path}")
else:
print(f"Error: Invalid input path '{input_path}'")
process_file(file_path, args, source_lang, loaded_model=loaded_model, service_status=service_status)
if killer.kill_now:
print("\n🛑 Process stopped by user. Progress saved in database.")
video_transcription/ai_transcriber_v2/tracker.py
+14 -5
View File
@@ -14,13 +14,22 @@ log_dir = "logs"
os.makedirs(log_dir, exist_ok=True)
log_file = os.path.join(log_dir, f"transcriber_{HOSTNAME}_{datetime.now().strftime('%Y%m%d')}.log")
# Create formatters
log_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
# File Handler (Full Detail)
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(log_formatter)
file_handler.setLevel(logging.INFO)
# Stream Handler (Quiet Detail for Terminal)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(log_formatter)
stream_handler.setLevel(logging.WARNING) # Only warnings/errors to terminal
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(log_file),
logging.StreamHandler()
]
handlers=[file_handler, stream_handler]
)
logger = logging.getLogger(__name__)
video_transcription/ai_transcriber_v2/transcriber.py
+38 -35
View File
@@ -3,18 +3,21 @@ import os
import sys
import subprocess
import torch
import tracker
def check_gpu_health():
"""
Performs a robust check for GPU availability and prints detailed troubleshooting
info if issues are detected, specific to Bazzite/VS Code environments.
"""
print("Checking GPU health...")
tracker.logger.info("Checking GPU health...")
# 1. Check if the OS/Driver sees the GPU
nvidia_smi_ok = False
try:
subprocess.run(["nvidia-smi"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
in_flatpak = os.path.exists("/.flatpak-info")
cmd = ["flatpak-spawn", "--host", "nvidia-smi"] if in_flatpak else ["nvidia-smi"]
subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
nvidia_smi_ok = True
except (subprocess.CalledProcessError, FileNotFoundError):
nvidia_smi_ok = False
@@ -23,36 +26,36 @@ def check_gpu_health():
torch_cuda_ok = torch.cuda.is_available()
if torch_cuda_ok:
print(f"✅ GPU is accessible: {torch.cuda.get_device_name(0)}")
print(f" CUDA Version: {torch.version.cuda}")
tracker.logger.info(f"✅ GPU is accessible: {torch.cuda.get_device_name(0)}")
tracker.logger.info(f" CUDA Version: {torch.version.cuda}")
return True
# --- Troubleshooting Block ---
print("\n⚠️ WARNING: GPU not detected by PyTorch. Falling back to CPU.")
print(" Transcription will be significantly slower.\n")
tracker.logger.warning("\n⚠️ WARNING: GPU not detected by PyTorch. Falling back to CPU.")
tracker.logger.warning(" Transcription will be significantly slower.\n")
print("--- Diagnostic Report ---")
tracker.logger.info("--- Diagnostic Report ---")
if nvidia_smi_ok:
print("1. [OK] 'nvidia-smi' command works. The system driver is installed and visible.")
print("2. [FAIL] PyTorch cannot see the GPU.")
print(" -> Likely Cause: You might have installed the CPU-only version of PyTorch.")
print(" -> Solution: Reinstall PyTorch with CUDA support:")
print(" pip uninstall torch torchvision torchaudio")
print(" pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118")
tracker.logger.info("1. [OK] 'nvidia-smi' command works. The system driver is installed and visible.")
tracker.logger.info("2. [FAIL] PyTorch cannot see the GPU.")
tracker.logger.info(" -> Likely Cause: You might have installed the CPU-only version of PyTorch.")
tracker.logger.info(" -> Solution: Reinstall PyTorch with CUDA support:")
tracker.logger.info(" pip uninstall torch torchvision torchaudio")
tracker.logger.info(" pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118")
else:
print("1. [FAIL] 'nvidia-smi' command failed or not found.")
print(" -> Likely Cause: Nvidia drivers are missing, or the container/sandbox cannot access the GPU.")
tracker.logger.info("1. [FAIL] 'nvidia-smi' command failed or not found.")
tracker.logger.info(" -> Likely Cause: Nvidia drivers are missing, or the container/sandbox cannot access the GPU.")
print("\n --- Bazzite / VS Code / Container Specific Checks ---")
print(" a. If you are running inside a dev container (DevBox/Distrobox/Toolbox):")
print(" Ensure the container was created with nvidia support.")
print(" (Bazzite usually handles this for 'distrobox', but check your config).")
print(" b. If you are using VS Code Flatpak:")
print(" Flatpak might be restricting access. Check Flatseal permissions for VS Code.")
print(" c. Driver Check:")
print(" Run 'rpm -qa | grep nvidia' in your host terminal to verify drivers are installed.")
tracker.logger.info("\n --- Bazzite / VS Code / Container Specific Checks ---")
tracker.logger.info(" a. If you are running inside a dev container (DevBox/Distrobox/Toolbox):")
tracker.logger.info(" Ensure the container was created with nvidia support.")
tracker.logger.info(" (Bazzite usually handles this for 'distrobox', but check your config).")
tracker.logger.info(" b. If you are using VS Code Flatpak:")
tracker.logger.info(" Flatpak might be restricting access. Check Flatseal permissions for VS Code.")
tracker.logger.info(" c. Driver Check:")
tracker.logger.info(" Run 'rpm -qa | grep nvidia' in your host terminal to verify drivers are installed.")
print("-------------------------\n")
tracker.logger.info("-------------------------\n")
return False
def get_vram_gb():
@@ -80,7 +83,7 @@ def get_optimal_model_size():
if vram == 0:
return "base"
print(f"Detected GPU with {vram:.2f} GB VRAM.")
tracker.logger.info(f"Detected GPU with {vram:.2f} GB VRAM.")
if vram >= 11:
return "large"
@@ -113,7 +116,7 @@ def save_as_srt(result, output_path):
f.write(f"{i}\n")
f.write(f"{start} --> {end}\n")
f.write(f"{text}\n\n")
print(f"SRT saved to: {output_path}")
tracker.logger.info(f"SRT saved to: {output_path}")
def load_whisper_model(model_size="auto"):
"""
@@ -123,17 +126,17 @@ def load_whisper_model(model_size="auto"):
if model_size == "auto":
model_size = get_optimal_model_size()
print(f"Auto-selected model: '{model_size}'")
tracker.logger.info(f"Auto-selected model: '{model_size}'")
print(f"Loading Whisper model ('{model_size}')...")
tracker.logger.info(f"Loading Whisper model ('{model_size}')...")
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
tracker.logger.info(f"Using device: {device}")
try:
model = whisper.load_model(model_size, device=device)
return model
except Exception as e:
print(f"Error loading model: {e}")
tracker.logger.error(f"Error loading model: {e}")
sys.exit(1)
def transcribe_audio(audio_path, model_size="auto", language=None, loaded_model=None):
@@ -156,12 +159,12 @@ def transcribe_audio(audio_path, model_size="auto", language=None, loaded_model=
if model is None:
model = load_whisper_model(model_size)
print(f"Transcribing {audio_path}...")
tracker.logger.info(f"Transcribing {audio_path}...")
try:
# Enable verbose=True to show progress in terminal
result = model.transcribe(audio_path, language=language, verbose=True)
print("Transcription complete.")
# Disable verbose to prevent line-by-line output
result = model.transcribe(audio_path, language=language, verbose=False)
tracker.logger.info("Transcription complete.")
return result
except Exception as e:
print(f"Error during transcription: {e}")
tracker.logger.error(f"Error during transcription: {e}")
sys.exit(1)
video_transcription/ai_transcriber_v2/translator.py
+6 -6
View File
@@ -25,8 +25,8 @@ def translate_via_ollama(source_srt_content, target_language="English", model="l
subs = pysubs2.SSAFile.from_string(source_srt_content)
# Using tqdm for progress bar
# dynamic_ncols=True helps it resize properly. leave=True ensures it stays after completion.
for line in tqdm(subs, desc=" Ollama Progress", unit="line", dynamic_ncols=True, leave=True):
# dynamic_ncols=True helps it resize properly.
for line in tqdm(subs, desc=" Ollama Progress", unit="line", dynamic_ncols=True, leave=False):
text = line.text.strip()
# Skip empty, numeric-only, or extremely short non-word text
@@ -81,7 +81,7 @@ def translate_fallback_mymemory(source_srt_content, target_language="en"):
# MyMemory uses ISO 639-1 usually
translator = MyMemoryTranslator(source='auto', target=target_language)
for line in tqdm(subs, desc=" MyMemory Progress", unit="line"):
for line in tqdm(subs, desc=" MyMemory Progress", unit="line", leave=False):
text = line.text.strip()
# Skip empty, numeric-only, or extremely short non-word text
@@ -121,7 +121,7 @@ def translate_fallback_free(source_srt_content, target_language="en"):
translator = GoogleTranslator(source='auto', target=target_language)
# Simple line-by-line translation
for line in tqdm(subs, desc=" DeepTranslate Progress", unit="line"):
for line in tqdm(subs, desc=" DeepTranslate Progress", unit="line", leave=False):
text = line.text.strip()
# Skip empty, numeric-only, or extremely short non-word text
@@ -335,8 +335,8 @@ def translate_with_auto_fallback(srt_content, target_language="English", prefer_
# Execute attempts
for i, method_func in enumerate(attempts):
if i > 0:
print(f" Attempt {i} failed. Trying next fallback...")
# if i > 0:
# print(f" Attempt {i} failed. Trying next fallback...")
content, method = method_func()
if content: