add download service

This commit is contained in:
2026-01-10 11:06:45 +03:00
parent c33c5fd674
commit 266f3768ef
44 changed files with 2652 additions and 4 deletions

View File

@@ -2,7 +2,15 @@
"permissions": {
"allow": [
"Bash(node --version:*)",
"Bash(mkdir:*)"
"Bash(mkdir:*)",
"Bash(cat:*)",
"Bash(python3:*)",
"Bash(docker-compose logs:*)",
"Bash(docker-compose up:*)",
"Bash(curl:*)",
"Bash(docker exec:*)",
"Bash(docker-compose restart:*)",
"Bash(docker-compose exec:*)"
]
}
}

View File

@@ -38,10 +38,30 @@ uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
**Video generation runs in ThreadPoolExecutor (max_workers=1) to avoid blocking the async event loop.**
### Frontend (Vue 3 + Vite)
- `frontend/src/App.vue` - Single component handling all UI, form state, and API calls
- `frontend/src/App.vue` - Main component handling quiz UI, form state, and API calls
- `frontend/src/components/OpeningsDownloader.vue` - Anime openings downloader UI
- `frontend/src/components/DownloadQueue.vue` - Download queue status component
- Uses Vue 3 Composition API (`ref`, `reactive`, `computed`)
- Vite proxies `/api`, `/videos`, `/download` to backend at `http://backend:8000`
### Openings Downloader Module
Located in `backend/app/openings_downloader/`:
- `router.py` - FastAPI router with `/api/downloader/*` endpoints
- `db_models.py` - SQLAlchemy models (Anime, AnimeTheme, DownloadTask)
- `schemas.py` - Pydantic schemas for API requests/responses
- `config.py` - Module settings with `DOWNLOADER_` env prefix
- `services/shikimori.py` - Shikimori GraphQL API client (anime search)
- `services/animethemes.py` - AnimeThemes API client (theme videos)
- `services/downloader.py` - Download queue processing, WebM→MP3 conversion
- `services/storage_tracker.py` - S3 storage usage tracking
**Download flow:**
1. Search anime via Shikimori API
2. Fetch available themes from AnimeThemes API
3. Add themes to download queue
4. Background worker downloads WebM, converts to MP3 via FFmpeg, uploads to S3
5. Creates Opening entity in main table for use in quiz generation
### Media Organization
```
media/
@@ -68,6 +88,7 @@ output/videos/ # Generated MP4 files
## API Endpoints
### Core Endpoints
| Method | Endpoint | Purpose |
|--------|----------|---------|
| GET | `/health` | Health check with FFmpeg status |
@@ -77,10 +98,32 @@ output/videos/ # Generated MP4 files
| DELETE | `/videos/{filename}` | Delete video |
| GET | `/videos-list` | List generated videos |
### Openings Downloader Endpoints (`/api/downloader`)
| Method | Endpoint | Purpose |
|--------|----------|---------|
| GET | `/search` | Search anime via Shikimori (query, year, status params) |
| GET | `/anime/{shikimori_id}` | Get anime details with available themes |
| POST | `/queue/add` | Add specific themes to download queue |
| POST | `/queue/add-all` | Add all themes from anime to queue |
| GET | `/queue` | Get queue status (includes `worker_running` flag) |
| DELETE | `/queue/{task_id}` | Cancel a queued task |
| POST | `/queue/{task_id}/retry` | Retry a failed task |
| DELETE | `/queue/clear` | Clear completed/failed tasks (`include_failed` param) |
| GET | `/storage` | Get S3 storage usage stats |
## Environment Variables
### Core Settings
```
QUIZ_MEDIA_PATH=/app/media
QUIZ_OUTPUT_PATH=/app/output/videos
VITE_API_URL=http://backend:8000
```
### Downloader Settings
```
DOWNLOADER_SHIKIMORI_USER_AGENT=AnimeQuiz/1.0
DOWNLOADER_SHIKIMORI_TOKEN= # Optional OAuth token for higher rate limits
DOWNLOADER_S3_STORAGE_LIMIT_BYTES=107374182400 # 100 GB default
DOWNLOADER_DOWNLOAD_TIMEOUT_SECONDS=300
```

View File

@@ -1,3 +1,3 @@
1) Добавить чекбокс который переключает проигрывание опенинга после отгадывания(продолжает или сначала)
1) Добавить чекбокс который переключает режим проигрывание опенинга после отгадывания(продолжает проигрывать или начинает сначала)
2) Редизайнуть страницу

View File

@@ -20,6 +20,8 @@ from .video_generator import VideoGenerator, check_ffmpeg
from .database import init_db, async_session_maker
from .db_models import Opening
from .routers import openings, backgrounds
from .openings_downloader.router import router as downloader_router
from .openings_downloader import db_models as downloader_db_models # noqa: F401 - import for table creation
from sqlalchemy import select, update
from datetime import datetime, timezone
@@ -55,6 +57,7 @@ app.add_middleware(
# Include routers
app.include_router(openings.router, prefix="/api")
app.include_router(backgrounds.router, prefix="/api")
app.include_router(downloader_router, prefix="/api")
# Mount output directory for serving videos
app.mount("/videos", StaticFiles(directory=str(settings.output_path)), name="videos")

View File

@@ -0,0 +1,2 @@
# Openings Downloader Module
# Search and download anime openings via Shikimori + AnimeThemes APIs

View File

@@ -0,0 +1,24 @@
from pydantic_settings import BaseSettings
class DownloaderSettings(BaseSettings):
"""Settings for the Openings Downloader module."""
# Shikimori API
shikimori_user_agent: str = "AnimeQuiz/1.0"
shikimori_token: str = "" # Optional OAuth token for higher rate limits
# S3 Storage limit (100 GB default)
s3_storage_limit_bytes: int = 107_374_182_400 # 100 GB
# Download settings
download_timeout_seconds: int = 300
default_estimated_size_bytes: int = 6_291_456 # 6 MB default for unknown files
class Config:
env_prefix = "DOWNLOADER_"
env_file = ".env"
extra = "ignore"
downloader_settings = DownloaderSettings()

View File

@@ -0,0 +1,157 @@
from datetime import datetime
from typing import List, Optional, TYPE_CHECKING
from sqlalchemy import String, Integer, ForeignKey, DateTime, BigInteger, Enum as SQLEnum, func, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
import enum
from ..database import Base
if TYPE_CHECKING:
from ..db_models import Opening
class ThemeType(str, enum.Enum):
"""Type of anime theme (opening or ending)."""
OP = "OP"
ED = "ED"
class DownloadStatus(str, enum.Enum):
"""Status of a download task."""
QUEUED = "queued"
DOWNLOADING = "downloading"
CONVERTING = "converting"
UPLOADING = "uploading"
DONE = "done"
FAILED = "failed"
class Anime(Base):
"""Anime entity from Shikimori."""
__tablename__ = "anime"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
shikimori_id: Mapped[int] = mapped_column(Integer, unique=True, index=True, nullable=False)
animethemes_slug: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True)
title_russian: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
title_english: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
title_japanese: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
year: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
poster_url: Mapped[Optional[str]] = mapped_column(String(1024), nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now()
)
# Relationships
themes: Mapped[List["AnimeTheme"]] = relationship(
"AnimeTheme",
back_populates="anime",
cascade="all, delete-orphan"
)
def __repr__(self):
return f"<Anime {self.shikimori_id}: {self.title_russian or self.title_english}>"
class AnimeTheme(Base):
"""Anime opening/ending theme."""
__tablename__ = "anime_themes"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
anime_id: Mapped[int] = mapped_column(
Integer,
ForeignKey("anime.id", ondelete="CASCADE"),
nullable=False
)
theme_type: Mapped[ThemeType] = mapped_column(
SQLEnum(ThemeType, native_enum=False),
nullable=False
)
sequence: Mapped[int] = mapped_column(Integer, nullable=False, default=1) # 1, 2, 3...
song_title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
artist: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
# AnimeThemes video URL (WebM source)
animethemes_video_url: Mapped[Optional[str]] = mapped_column(String(1024), nullable=True)
# Downloaded file info
audio_s3_key: Mapped[Optional[str]] = mapped_column(String(512), nullable=True)
file_size_bytes: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True)
# Link to existing Opening entity (after download)
opening_id: Mapped[Optional[int]] = mapped_column(
Integer,
ForeignKey("openings.id", ondelete="SET NULL"),
nullable=True
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now()
)
# Unique constraint: one anime can have only one OP1, OP2, ED1, etc.
__table_args__ = (
UniqueConstraint('anime_id', 'theme_type', 'sequence', name='uq_anime_theme_sequence'),
)
# Relationships
anime: Mapped["Anime"] = relationship("Anime", back_populates="themes")
opening: Mapped[Optional["Opening"]] = relationship("Opening")
@property
def full_name(self) -> str:
"""Return full theme name like 'OP1' or 'ED2'."""
return f"{self.theme_type.value}{self.sequence}"
@property
def is_downloaded(self) -> bool:
"""Check if theme has been downloaded."""
return self.audio_s3_key is not None
def __repr__(self):
return f"<AnimeTheme {self.full_name}: {self.song_title}>"
class DownloadTask(Base):
"""Download queue task."""
__tablename__ = "download_tasks"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
theme_id: Mapped[int] = mapped_column(
Integer,
ForeignKey("anime_themes.id", ondelete="CASCADE"),
nullable=False,
unique=True # One task per theme
)
status: Mapped[DownloadStatus] = mapped_column(
SQLEnum(DownloadStatus, native_enum=False),
nullable=False,
default=DownloadStatus.QUEUED
)
# Progress tracking
progress_percent: Mapped[int] = mapped_column(Integer, default=0)
error_message: Mapped[Optional[str]] = mapped_column(String(1024), nullable=True)
# Estimated size (6 MB default if unknown)
estimated_size_bytes: Mapped[int] = mapped_column(BigInteger, default=6_291_456) # 6 MB
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now()
)
started_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
# Relationships
theme: Mapped["AnimeTheme"] = relationship("AnimeTheme")
def __repr__(self):
return f"<DownloadTask {self.id}: {self.status.value}>"

View File

@@ -0,0 +1,231 @@
import asyncio
from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from sqlalchemy.orm import selectinload
from typing import Optional
# Lock to prevent multiple workers running simultaneously
_worker_lock = asyncio.Lock()
_worker_running = False
from ..database import get_db
from .schemas import (
SearchResponse,
AnimeDetailResponse,
ThemeInfo,
AddToQueueRequest,
AddAllThemesRequest,
QueueStatusResponse,
StorageStatsResponse,
)
from .db_models import Anime, AnimeTheme, DownloadTask, DownloadStatus
from .services.shikimori import ShikimoriService
from .services.animethemes import AnimeThemesService
from .services.downloader import DownloadService
from .services.storage_tracker import StorageTrackerService
router = APIRouter(prefix="/downloader", tags=["openings-downloader"])
# ============== Search ==============
@router.get("/search", response_model=SearchResponse)
async def search_anime(
query: str = Query(..., min_length=1, description="Search query"),
year: Optional[int] = Query(None, description="Filter by year"),
status: Optional[str] = Query(None, description="Filter by status (ongoing, released, announced)"),
limit: int = Query(20, ge=1, le=50, description="Maximum results"),
db: AsyncSession = Depends(get_db),
):
"""Search anime via Shikimori API."""
service = ShikimoriService()
results = await service.search(query, year=year, status=status, limit=limit)
return SearchResponse(results=results, total=len(results))
# ============== Anime Detail ==============
@router.get("/anime/{shikimori_id}", response_model=AnimeDetailResponse)
async def get_anime_detail(
shikimori_id: int,
db: AsyncSession = Depends(get_db),
):
"""Get anime details with available themes from AnimeThemes."""
shikimori_service = ShikimoriService()
animethemes_service = AnimeThemesService()
# Get or create anime record
anime = await shikimori_service.get_or_create_anime(db, shikimori_id)
# Fetch themes from AnimeThemes API
themes = await animethemes_service.fetch_themes(db, anime)
# Get download status for each theme
theme_ids = [t.id for t in themes]
if theme_ids:
result = await db.execute(
select(DownloadTask)
.where(DownloadTask.theme_id.in_(theme_ids))
)
tasks = {t.theme_id: t for t in result.scalars().all()}
else:
tasks = {}
# Build response
theme_infos = []
for theme in themes:
task = tasks.get(theme.id)
theme_infos.append(ThemeInfo(
id=theme.id,
theme_type=theme.theme_type,
sequence=theme.sequence,
full_name=theme.full_name,
song_title=theme.song_title,
artist=theme.artist,
video_url=theme.animethemes_video_url,
is_downloaded=theme.is_downloaded,
download_status=task.status if task else None,
file_size_bytes=theme.file_size_bytes,
))
return AnimeDetailResponse(
id=anime.id,
shikimori_id=anime.shikimori_id,
title_russian=anime.title_russian,
title_english=anime.title_english,
title_japanese=anime.title_japanese,
year=anime.year,
poster_url=anime.poster_url,
themes=theme_infos,
)
# ============== Download Queue ==============
@router.post("/queue/add", response_model=QueueStatusResponse)
async def add_to_queue(
request: AddToQueueRequest,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
):
"""Add specific themes to download queue."""
storage_service = StorageTrackerService(db)
# Check storage limit
stats = await storage_service.get_stats()
if not stats.can_download:
raise HTTPException(
status_code=400,
detail=f"Storage limit exceeded ({stats.used_bytes}/{stats.limit_bytes} bytes)"
)
download_service = DownloadService(db)
added = await download_service.add_to_queue(request.theme_ids)
if added > 0:
# Trigger worker in background
background_tasks.add_task(_run_download_worker)
return await download_service.get_queue_status(worker_running=_worker_running)
@router.post("/queue/add-all", response_model=QueueStatusResponse)
async def add_all_anime_themes(
request: AddAllThemesRequest,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
):
"""Add all themes from an anime to download queue."""
storage_service = StorageTrackerService(db)
stats = await storage_service.get_stats()
if not stats.can_download:
raise HTTPException(status_code=400, detail="Storage limit exceeded")
download_service = DownloadService(db)
added = await download_service.add_all_anime_themes(request.anime_id)
if added > 0:
background_tasks.add_task(_run_download_worker)
return await download_service.get_queue_status(worker_running=_worker_running)
@router.get("/queue", response_model=QueueStatusResponse)
async def get_queue_status(db: AsyncSession = Depends(get_db)):
"""Get current download queue status."""
download_service = DownloadService(db)
return await download_service.get_queue_status(worker_running=_worker_running)
@router.delete("/queue/{task_id}")
async def cancel_task(task_id: int, db: AsyncSession = Depends(get_db)):
"""Cancel a queued task (not downloading)."""
download_service = DownloadService(db)
success = await download_service.cancel_task(task_id)
if not success:
raise HTTPException(status_code=400, detail="Cannot cancel task (not queued or not found)")
return {"message": "Task cancelled"}
@router.post("/queue/{task_id}/retry")
async def retry_task(
task_id: int,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
):
"""Retry a failed task."""
download_service = DownloadService(db)
success = await download_service.retry_task(task_id)
if not success:
raise HTTPException(status_code=400, detail="Cannot retry task (not failed or not found)")
background_tasks.add_task(_run_download_worker)
return {"message": "Task queued for retry"}
@router.delete("/queue/clear")
async def clear_completed_tasks(
include_failed: bool = Query(False, description="Also clear failed tasks"),
db: AsyncSession = Depends(get_db),
):
"""Clear completed (and optionally failed) tasks from the queue."""
download_service = DownloadService(db)
deleted_count = await download_service.clear_completed_tasks(include_failed=include_failed)
return {"message": f"Cleared {deleted_count} tasks", "deleted_count": deleted_count}
# ============== Storage ==============
@router.get("/storage", response_model=StorageStatsResponse)
async def get_storage_stats(db: AsyncSession = Depends(get_db)):
"""Get S3 storage usage stats (calculated from DB, not scanning S3)."""
storage_service = StorageTrackerService(db)
return await storage_service.get_stats()
# ============== Background Worker ==============
async def _run_download_worker():
"""Background task to process download queue."""
global _worker_running
# Skip if another worker is already running
if _worker_running:
return
async with _worker_lock:
if _worker_running:
return
_worker_running = True
try:
from ..database import async_session_maker
async with async_session_maker() as db:
download_service = DownloadService(db)
await download_service.process_queue()
finally:
_worker_running = False

View File

@@ -0,0 +1,114 @@
from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel, Field
from .db_models import ThemeType, DownloadStatus
# ============== Shikimori Search ==============
class AnimeSearchResult(BaseModel):
"""Single anime search result from Shikimori."""
shikimori_id: int
title_russian: Optional[str] = None
title_english: Optional[str] = None
title_japanese: Optional[str] = None
year: Optional[int] = None
poster_url: Optional[str] = None
class Config:
from_attributes = True
class SearchResponse(BaseModel):
"""Response for anime search."""
results: List[AnimeSearchResult]
total: int
# ============== Anime Themes ==============
class ThemeInfo(BaseModel):
"""Information about a single anime theme (OP/ED)."""
id: int
theme_type: ThemeType
sequence: int
full_name: str # "OP1", "ED2"
song_title: Optional[str] = None
artist: Optional[str] = None
video_url: Optional[str] = None
is_downloaded: bool
download_status: Optional[DownloadStatus] = None
file_size_bytes: Optional[int] = None
class Config:
from_attributes = True
class AnimeDetailResponse(BaseModel):
"""Detailed anime info with themes."""
id: int
shikimori_id: int
title_russian: Optional[str] = None
title_english: Optional[str] = None
title_japanese: Optional[str] = None
year: Optional[int] = None
poster_url: Optional[str] = None
themes: List[ThemeInfo]
class Config:
from_attributes = True
# ============== Download Queue ==============
class AddToQueueRequest(BaseModel):
"""Request to add specific themes to download queue."""
theme_ids: List[int] = Field(..., min_length=1)
class AddAllThemesRequest(BaseModel):
"""Request to add all themes from an anime."""
anime_id: int
class QueueTaskResponse(BaseModel):
"""Single task in the download queue."""
id: int
theme_id: int
anime_title: str
theme_name: str # "OP1", "ED2"
song_title: Optional[str] = None
status: DownloadStatus
progress_percent: int
error_message: Optional[str] = None
estimated_size_bytes: int
created_at: datetime
started_at: Optional[datetime] = None
completed_at: Optional[datetime] = None
class Config:
from_attributes = True
class QueueStatusResponse(BaseModel):
"""Current status of the download queue."""
tasks: List[QueueTaskResponse]
total_queued: int
total_downloading: int
total_done: int
total_failed: int
estimated_queue_size_bytes: int
worker_running: bool = False # Indicates if download worker is currently active
# ============== Storage Stats ==============
class StorageStatsResponse(BaseModel):
"""S3 storage usage statistics."""
used_bytes: int
limit_bytes: int
used_percent: float
available_bytes: int
can_download: bool # False if limit exceeded
openings_count: int

View File

@@ -0,0 +1,12 @@
# Services for Openings Downloader
from .shikimori import ShikimoriService
from .animethemes import AnimeThemesService
from .downloader import DownloadService
from .storage_tracker import StorageTrackerService
__all__ = [
"ShikimoriService",
"AnimeThemesService",
"DownloadService",
"StorageTrackerService",
]

View File

@@ -0,0 +1,187 @@
import httpx
import logging
import re
from typing import List, Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..db_models import Anime, AnimeTheme, ThemeType
logger = logging.getLogger(__name__)
# Shared HTTP client for AnimeThemes API
_animethemes_client: Optional[httpx.AsyncClient] = None
def _get_animethemes_client() -> httpx.AsyncClient:
"""Get or create shared HTTP client for AnimeThemes."""
global _animethemes_client
if _animethemes_client is None or _animethemes_client.is_closed:
_animethemes_client = httpx.AsyncClient(
base_url="https://api.animethemes.moe",
timeout=30.0,
)
return _animethemes_client
class AnimeThemesService:
"""Service for AnimeThemes API (api.animethemes.moe)."""
def __init__(self):
self.client = _get_animethemes_client()
async def _find_anime_slug(self, anime: Anime) -> Optional[str]:
"""Find AnimeThemes slug by searching anime title."""
# Try different title variations
search_terms = [
anime.title_english,
anime.title_russian,
anime.title_japanese,
]
for term in search_terms:
if not term:
continue
try:
response = await self.client.get(
"/anime",
params={
"q": term,
"include": "animethemes.animethemeentries.videos.audio",
},
)
if response.status_code == 200:
data = response.json()
animes = data.get("anime", [])
if animes:
slug = animes[0].get("slug")
logger.info(f"Found AnimeThemes slug '{slug}' for '{term}'")
return slug
except Exception as e:
logger.warning(f"Failed to search AnimeThemes for '{term}': {e}")
continue
return None
async def fetch_themes(self, db: AsyncSession, anime: Anime) -> List[AnimeTheme]:
"""Fetch themes from AnimeThemes API and sync to DB."""
# Always reload anime with themes to avoid lazy loading issues
result = await db.execute(
select(Anime)
.where(Anime.id == anime.id)
.options(selectinload(Anime.themes))
)
anime = result.scalar_one()
current_themes = anime.themes or []
# Find slug if not cached
if not anime.animethemes_slug:
logger.info(f"Searching AnimeThemes slug for: {anime.title_english or anime.title_russian}")
slug = await self._find_anime_slug(anime)
logger.info(f"Found slug: {slug}")
if slug:
anime.animethemes_slug = slug
await db.commit()
if not anime.animethemes_slug:
logger.warning(f"No AnimeThemes slug found for anime {anime.id}: {anime.title_english or anime.title_russian}")
return current_themes
# Fetch themes from AnimeThemes API
try:
response = await self.client.get(
f"/anime/{anime.animethemes_slug}",
params={
"include": "animethemes.animethemeentries.videos.audio,animethemes.song.artists",
},
)
if response.status_code != 200:
logger.warning(f"AnimeThemes API returned {response.status_code} for {anime.animethemes_slug}")
return current_themes
data = response.json()
except Exception as e:
logger.error(f"Failed to fetch themes from AnimeThemes for {anime.animethemes_slug}: {e}")
return current_themes
anime_data = data.get("anime", {})
themes_data = anime_data.get("animethemes", [])
logger.info(f"AnimeThemes API returned {len(themes_data)} themes for {anime.animethemes_slug}")
# Build dict of existing themes
existing_themes = {
(t.theme_type, t.sequence): t
for t in current_themes
}
for theme_data in themes_data:
# Parse theme type and sequence: "OP1", "ED1", etc.
slug = theme_data.get("slug", "") # e.g., "OP1", "ED1"
match = re.match(r"(OP|ED)(\d*)", slug)
if not match:
continue
theme_type = ThemeType.OP if match.group(1) == "OP" else ThemeType.ED
sequence = int(match.group(2)) if match.group(2) else 1
# Get video URL (prioritize audio link, then video link)
video_url = None
entries = theme_data.get("animethemeentries", [])
if entries:
videos = entries[0].get("videos", [])
if videos:
# Try to get audio link first
audio = videos[0].get("audio")
if audio:
video_url = audio.get("link")
# Fallback to video link
if not video_url:
video_url = videos[0].get("link")
# Get song info
song_data = theme_data.get("song", {})
song_title = song_data.get("title")
artist = None
artists = song_data.get("artists", [])
if artists:
artist = artists[0].get("name")
key = (theme_type, sequence)
if key in existing_themes:
# Update existing theme
theme = existing_themes[key]
theme.song_title = song_title
theme.artist = artist
if video_url:
theme.animethemes_video_url = video_url
else:
# Create new theme
theme = AnimeTheme(
anime_id=anime.id,
theme_type=theme_type,
sequence=sequence,
song_title=song_title,
artist=artist,
animethemes_video_url=video_url,
)
db.add(theme)
if anime.themes is None:
anime.themes = []
anime.themes.append(theme)
await db.commit()
# Reload anime with themes to get fresh data
result = await db.execute(
select(Anime)
.where(Anime.id == anime.id)
.options(selectinload(Anime.themes))
)
refreshed_anime = result.scalar_one()
return refreshed_anime.themes

View File

@@ -0,0 +1,326 @@
import asyncio
import tempfile
import re
from pathlib import Path
from typing import List
from datetime import datetime, timezone
import httpx
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..db_models import AnimeTheme, DownloadTask, DownloadStatus, Anime
from ..schemas import QueueStatusResponse, QueueTaskResponse
from ..config import downloader_settings
from ...storage import storage
from ...db_models import Opening
class DownloadService:
"""Service for downloading and converting anime themes."""
def __init__(self, db: AsyncSession):
self.db = db
async def add_to_queue(self, theme_ids: List[int]) -> int:
"""Add themes to download queue (idempotent). Returns number of tasks added."""
added = 0
for theme_id in theme_ids:
# Check if already in queue
existing = await self.db.execute(
select(DownloadTask).where(DownloadTask.theme_id == theme_id)
)
if existing.scalar_one_or_none():
continue
# Check if theme exists and is not already downloaded
result = await self.db.execute(
select(AnimeTheme).where(AnimeTheme.id == theme_id)
)
theme = result.scalar_one_or_none()
if not theme:
continue
# Skip if already downloaded
if theme.audio_s3_key:
continue
# Skip if no video URL available
if not theme.animethemes_video_url:
continue
task = DownloadTask(
theme_id=theme_id,
status=DownloadStatus.QUEUED,
estimated_size_bytes=downloader_settings.default_estimated_size_bytes,
)
self.db.add(task)
added += 1
await self.db.commit()
return added
async def add_all_anime_themes(self, anime_id: int) -> int:
"""Add all themes from anime to queue. Returns number of tasks added."""
result = await self.db.execute(
select(AnimeTheme)
.where(AnimeTheme.anime_id == anime_id)
.where(AnimeTheme.audio_s3_key.is_(None))
.where(AnimeTheme.animethemes_video_url.isnot(None))
)
themes = result.scalars().all()
return await self.add_to_queue([t.id for t in themes])
async def get_queue_status(self, worker_running: bool = False) -> QueueStatusResponse:
"""Get current queue status."""
result = await self.db.execute(
select(DownloadTask)
.options(selectinload(DownloadTask.theme).selectinload(AnimeTheme.anime))
.order_by(DownloadTask.created_at.desc())
)
tasks = result.scalars().all()
task_responses = []
total_queued = 0
total_downloading = 0
total_done = 0
total_failed = 0
estimated_queue_size = 0
for task in tasks:
theme = task.theme
anime = theme.anime
anime_title = anime.title_russian or anime.title_english or "Unknown"
task_responses.append(QueueTaskResponse(
id=task.id,
theme_id=theme.id,
anime_title=anime_title,
theme_name=theme.full_name,
song_title=theme.song_title,
status=task.status,
progress_percent=task.progress_percent,
error_message=task.error_message,
estimated_size_bytes=task.estimated_size_bytes,
created_at=task.created_at,
started_at=task.started_at,
completed_at=task.completed_at,
))
if task.status == DownloadStatus.QUEUED:
total_queued += 1
estimated_queue_size += task.estimated_size_bytes
elif task.status in (DownloadStatus.DOWNLOADING, DownloadStatus.CONVERTING, DownloadStatus.UPLOADING):
total_downloading += 1
estimated_queue_size += task.estimated_size_bytes
elif task.status == DownloadStatus.DONE:
total_done += 1
elif task.status == DownloadStatus.FAILED:
total_failed += 1
return QueueStatusResponse(
tasks=task_responses,
total_queued=total_queued,
total_downloading=total_downloading,
total_done=total_done,
total_failed=total_failed,
estimated_queue_size_bytes=estimated_queue_size,
worker_running=worker_running,
)
async def process_queue(self) -> None:
"""Process download queue (called as background task)."""
while True:
# Get next queued task
result = await self.db.execute(
select(DownloadTask)
.where(DownloadTask.status == DownloadStatus.QUEUED)
.order_by(DownloadTask.created_at)
.limit(1)
)
task = result.scalar_one_or_none()
if not task:
break
await self._process_task(task)
async def _process_task(self, task: DownloadTask) -> None:
"""Process a single download task."""
try:
# Update status to downloading
task.status = DownloadStatus.DOWNLOADING
task.started_at = datetime.now(timezone.utc)
task.progress_percent = 10
await self.db.commit()
# Get theme info with anime
result = await self.db.execute(
select(AnimeTheme)
.options(selectinload(AnimeTheme.anime))
.where(AnimeTheme.id == task.theme_id)
)
theme = result.scalar_one()
anime = theme.anime
if not theme.animethemes_video_url:
raise ValueError("No video URL available")
# Download and convert in temp directory
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
webm_file = tmp_path / "video.webm"
mp3_file = tmp_path / "audio.mp3"
# Stream download WebM file
async with httpx.AsyncClient() as client:
async with client.stream(
"GET",
theme.animethemes_video_url,
timeout=downloader_settings.download_timeout_seconds,
follow_redirects=True,
) as response:
response.raise_for_status()
with open(webm_file, "wb") as f:
async for chunk in response.aiter_bytes(chunk_size=8192):
f.write(chunk)
task.progress_percent = 40
task.status = DownloadStatus.CONVERTING
await self.db.commit()
# Convert to MP3 with FFmpeg
process = await asyncio.create_subprocess_exec(
"ffmpeg", "-i", str(webm_file),
"-vn", "-acodec", "libmp3lame", "-q:a", "2",
str(mp3_file),
"-y",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await process.communicate()
if process.returncode != 0:
raise RuntimeError(f"FFmpeg error: {stderr.decode()[:500]}")
if not mp3_file.exists():
raise RuntimeError("FFmpeg did not create output file")
task.progress_percent = 70
task.status = DownloadStatus.UPLOADING
await self.db.commit()
# Generate safe S3 key
anime_name = self._sanitize_filename(
anime.title_english or anime.title_russian or f"anime_{anime.shikimori_id}"
)
theme_name = theme.full_name
song_part = f"_{self._sanitize_filename(theme.song_title)}" if theme.song_title else ""
s3_key = f"audio/{anime_name}_{theme_name}{song_part}.mp3"
# Read file and upload to S3
file_data = mp3_file.read_bytes()
file_size = len(file_data)
success = storage.upload_file(s3_key, file_data, "audio/mpeg")
if not success:
raise RuntimeError("Failed to upload to S3")
# Update theme with file info
theme.audio_s3_key = s3_key
theme.file_size_bytes = file_size
# Create Opening entity in main table
opening = Opening(
anime_name=anime.title_russian or anime.title_english or f"Anime {anime.shikimori_id}",
op_number=theme_name,
song_name=theme.song_title,
audio_file=s3_key.replace("audio/", ""),
)
self.db.add(opening)
await self.db.flush()
theme.opening_id = opening.id
# Mark task as done
task.status = DownloadStatus.DONE
task.progress_percent = 100
task.completed_at = datetime.now(timezone.utc)
task.estimated_size_bytes = file_size
await self.db.commit()
except Exception as e:
task.status = DownloadStatus.FAILED
task.error_message = str(e)[:1000]
task.progress_percent = 0
await self.db.commit()
def _sanitize_filename(self, name: str) -> str:
"""Sanitize string for use in filename."""
if not name:
return "unknown"
# Remove or replace problematic characters
sanitized = re.sub(r'[<>:"/\\|?*]', '', name)
sanitized = sanitized.replace(' ', '_')
# Limit length
return sanitized[:100]
async def cancel_task(self, task_id: int) -> bool:
"""Cancel a queued task. Returns True if cancelled."""
result = await self.db.execute(
select(DownloadTask).where(DownloadTask.id == task_id)
)
task = result.scalar_one_or_none()
if not task or task.status != DownloadStatus.QUEUED:
return False
await self.db.delete(task)
await self.db.commit()
return True
async def retry_task(self, task_id: int) -> bool:
"""Retry a failed task. Returns True if requeued."""
result = await self.db.execute(
update(DownloadTask)
.where(DownloadTask.id == task_id)
.where(DownloadTask.status == DownloadStatus.FAILED)
.values(
status=DownloadStatus.QUEUED,
error_message=None,
progress_percent=0,
started_at=None,
completed_at=None,
)
.returning(DownloadTask.id)
)
updated = result.scalar_one_or_none()
await self.db.commit()
return updated is not None
async def clear_completed_tasks(self, include_failed: bool = False) -> int:
"""Clear completed (and optionally failed) tasks. Returns number of deleted tasks."""
from sqlalchemy import delete
statuses = [DownloadStatus.DONE]
if include_failed:
statuses.append(DownloadStatus.FAILED)
result = await self.db.execute(
delete(DownloadTask)
.where(DownloadTask.status.in_(statuses))
.returning(DownloadTask.id)
)
deleted_ids = result.scalars().all()
await self.db.commit()
return len(deleted_ids)

View File

@@ -0,0 +1,145 @@
import httpx
from typing import List, Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..db_models import Anime
from ..schemas import AnimeSearchResult
from ..config import downloader_settings
# Shared HTTP client for Shikimori API
_shikimori_client: Optional[httpx.AsyncClient] = None
def _get_shikimori_client() -> httpx.AsyncClient:
"""Get or create shared HTTP client for Shikimori."""
global _shikimori_client
if _shikimori_client is None or _shikimori_client.is_closed:
headers = {
"User-Agent": downloader_settings.shikimori_user_agent,
}
if downloader_settings.shikimori_token:
headers["Authorization"] = f"Bearer {downloader_settings.shikimori_token}"
_shikimori_client = httpx.AsyncClient(
headers=headers,
timeout=30.0,
)
return _shikimori_client
class ShikimoriService:
"""Service for Shikimori GraphQL API."""
GRAPHQL_URL = "https://shikimori.one/api/graphql"
def __init__(self):
self.client = _get_shikimori_client()
async def search(
self,
query: str,
year: Optional[int] = None,
status: Optional[str] = None,
limit: int = 20,
) -> List[AnimeSearchResult]:
"""Search anime by query using Shikimori GraphQL API."""
graphql_query = """
query($search: String, $limit: Int, $season: SeasonString, $status: AnimeStatusString) {
animes(search: $search, limit: $limit, season: $season, status: $status) {
id
russian
english
japanese
airedOn { year }
poster { originalUrl }
}
}
"""
variables = {
"search": query,
"limit": limit,
}
if year:
variables["season"] = str(year)
if status:
variables["status"] = status
response = await self.client.post(
self.GRAPHQL_URL,
json={"query": graphql_query, "variables": variables},
)
response.raise_for_status()
data = response.json()
results = []
for anime in data.get("data", {}).get("animes", []):
results.append(AnimeSearchResult(
shikimori_id=int(anime["id"]),
title_russian=anime.get("russian"),
title_english=anime.get("english"),
title_japanese=anime.get("japanese"),
year=anime.get("airedOn", {}).get("year") if anime.get("airedOn") else None,
poster_url=anime.get("poster", {}).get("originalUrl") if anime.get("poster") else None,
))
return results
async def get_or_create_anime(self, db: AsyncSession, shikimori_id: int) -> Anime:
"""Get anime from DB or fetch from Shikimori and create."""
# Check if exists (with themes eagerly loaded)
query = (
select(Anime)
.where(Anime.shikimori_id == shikimori_id)
.options(selectinload(Anime.themes))
)
result = await db.execute(query)
anime = result.scalar_one_or_none()
if anime:
return anime
# Fetch from Shikimori
graphql_query = """
query($ids: String!) {
animes(ids: $ids, limit: 1) {
id
russian
english
japanese
airedOn { year }
poster { originalUrl }
}
}
"""
response = await self.client.post(
self.GRAPHQL_URL,
json={"query": graphql_query, "variables": {"ids": str(shikimori_id)}},
)
response.raise_for_status()
data = response.json()
animes = data.get("data", {}).get("animes", [])
if not animes:
raise ValueError(f"Anime with ID {shikimori_id} not found on Shikimori")
anime_data = animes[0]
anime = Anime(
shikimori_id=shikimori_id,
title_russian=anime_data.get("russian"),
title_english=anime_data.get("english"),
title_japanese=anime_data.get("japanese"),
year=anime_data.get("airedOn", {}).get("year") if anime_data.get("airedOn") else None,
poster_url=anime_data.get("poster", {}).get("originalUrl") if anime_data.get("poster") else None,
)
db.add(anime)
await db.commit()
await db.refresh(anime)
return anime

View File

@@ -0,0 +1,58 @@
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from ..db_models import AnimeTheme
from ..schemas import StorageStatsResponse
from ..config import downloader_settings
from ...db_models import Opening
class StorageTrackerService:
"""Service for tracking S3 storage usage from DB (without scanning S3)."""
def __init__(self, db: AsyncSession):
self.db = db
async def get_stats(self) -> StorageStatsResponse:
"""Calculate storage stats from database."""
# Sum file sizes from downloaded themes
result = await self.db.execute(
select(func.coalesce(func.sum(AnimeTheme.file_size_bytes), 0))
.where(AnimeTheme.file_size_bytes.isnot(None))
)
used_bytes = result.scalar() or 0
# Count openings in the main Opening table
result = await self.db.execute(
select(func.count(Opening.id))
)
openings_count = result.scalar() or 0
limit_bytes = downloader_settings.s3_storage_limit_bytes
available_bytes = max(0, limit_bytes - used_bytes)
used_percent = (used_bytes / limit_bytes * 100) if limit_bytes > 0 else 0
return StorageStatsResponse(
used_bytes=used_bytes,
limit_bytes=limit_bytes,
used_percent=round(used_percent, 2),
available_bytes=available_bytes,
can_download=used_bytes < limit_bytes,
openings_count=openings_count,
)
async def get_estimated_queue_size(self) -> int:
"""Get estimated size of pending downloads in queue."""
from ..db_models import DownloadTask, DownloadStatus
result = await self.db.execute(
select(func.coalesce(func.sum(DownloadTask.estimated_size_bytes), 0))
.where(DownloadTask.status.in_([
DownloadStatus.QUEUED,
DownloadStatus.DOWNLOADING,
DownloadStatus.CONVERTING,
DownloadStatus.UPLOADING,
]))
)
return result.scalar() or 0

View File

@@ -8,3 +8,4 @@ boto3==1.35.0
sqlalchemy[asyncio]==2.0.36
asyncpg==0.30.0
greenlet==3.1.1
httpx==0.27.0

View File

@@ -23,6 +23,13 @@
>
Upload Files
</button>
<button
class="nav-btn"
:class="{ active: currentPage === 'downloader' }"
@click="currentPage = 'downloader'"
>
Openings Downloader
</button>
</nav>
<!-- Quiz Generator Page -->
@@ -306,6 +313,9 @@
<!-- Admin Page (Upload Files) -->
<AdminPage v-if="currentPage === 'admin'" />
<!-- Openings Downloader -->
<OpeningsDownloader v-if="currentPage === 'downloader'" />
</div>
</template>
@@ -313,6 +323,7 @@
import { ref, reactive, computed, onMounted, watch } from 'vue'
import AdminPage from './components/AdminPage.vue'
import MediaManager from './components/MediaManager.vue'
import OpeningsDownloader from './components/OpeningsDownloader.vue'
const STORAGE_KEY = 'animeQuizSettings'
@@ -320,7 +331,8 @@ export default {
name: 'App',
components: {
AdminPage,
MediaManager
MediaManager,
OpeningsDownloader
},
setup() {
// Navigation

View File

@@ -0,0 +1,510 @@
<template>
<div class="queue-panel">
<div class="queue-header">
<div class="queue-title">
<h3>Download Queue</h3>
<span v-if="queue.worker_running" class="worker-indicator active" title="Worker is processing">
<span class="pulse"></span> Active
</span>
<span v-else class="worker-indicator idle" title="Worker is idle">Idle</span>
</div>
<div class="queue-actions">
<button
v-if="queue.total_done > 0 || queue.total_failed > 0"
@click="$emit('clear')"
class="btn-clear"
title="Clear completed tasks"
>
Clear
</button>
<button @click="$emit('refresh')" class="btn-refresh" title="Refresh">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<path d="M21.5 2v6h-6M2.5 22v-6h6M2 11.5a10 10 0 0 1 18.8-4.3M22 12.5a10 10 0 0 1-18.8 4.3"/>
</svg>
</button>
</div>
</div>
<div class="queue-stats">
<div class="stat">
<span class="stat-value queued">{{ queue.total_queued }}</span>
<span class="stat-label">Queued</span>
</div>
<div class="stat">
<span class="stat-value downloading">{{ queue.total_downloading }}</span>
<span class="stat-label">Active</span>
</div>
<div class="stat">
<span class="stat-value done">{{ queue.total_done }}</span>
<span class="stat-label">Done</span>
</div>
<div class="stat">
<span class="stat-value failed">{{ queue.total_failed }}</span>
<span class="stat-label">Failed</span>
</div>
</div>
<div v-if="queue.estimated_queue_size_bytes > 0" class="queue-size">
Est. queue size: {{ formatBytes(queue.estimated_queue_size_bytes) }}
</div>
<div class="queue-list" v-if="queue.tasks?.length">
<div
v-for="task in sortedTasks"
:key="task.id"
class="queue-item"
:class="task.status"
>
<div class="task-main">
<div class="task-info">
<div class="task-anime">{{ task.anime_title }}</div>
<div class="task-theme">
<span class="theme-badge">{{ task.theme_name }}</span>
<span v-if="task.song_title" class="song-name">{{ task.song_title }}</span>
</div>
</div>
<div class="task-status">
<span :class="'status-badge ' + task.status">{{ formatStatus(task.status) }}</span>
</div>
</div>
<div v-if="isActive(task.status)" class="task-progress">
<div class="progress-bar">
<div class="progress-fill" :style="{ width: task.progress_percent + '%' }"></div>
</div>
<span class="progress-text">{{ task.progress_percent }}%</span>
</div>
<div v-if="task.error_message" class="task-error">
{{ task.error_message }}
</div>
<div class="task-actions">
<button
v-if="task.status === 'queued'"
@click="$emit('cancel', task.id)"
class="btn-action btn-cancel"
title="Cancel"
>
Cancel
</button>
<button
v-if="task.status === 'failed'"
@click="$emit('retry', task.id)"
class="btn-action btn-retry"
title="Retry"
>
Retry
</button>
<span v-if="task.status === 'done'" class="task-size">
{{ formatBytes(task.estimated_size_bytes) }}
</span>
</div>
</div>
</div>
<div v-else class="queue-empty">
No tasks in queue
</div>
</div>
</template>
<script setup>
import { computed } from 'vue'
const props = defineProps({
queue: {
type: Object,
required: true,
},
})
defineEmits(['cancel', 'retry', 'refresh', 'clear'])
const sortedTasks = computed(() => {
if (!props.queue.tasks) return []
// Sort: active first, then queued, then done/failed
const order = {
downloading: 0,
converting: 0,
uploading: 0,
queued: 1,
failed: 2,
done: 3,
}
return [...props.queue.tasks].sort((a, b) => {
const orderDiff = (order[a.status] || 4) - (order[b.status] || 4)
if (orderDiff !== 0) return orderDiff
// Within same status, sort by created_at desc
return new Date(b.created_at) - new Date(a.created_at)
})
})
function formatBytes(bytes) {
if (!bytes) return '0 B'
const units = ['B', 'KB', 'MB', 'GB']
let i = 0
let value = bytes
while (value >= 1024 && i < units.length - 1) {
value /= 1024
i++
}
return `${value.toFixed(1)} ${units[i]}`
}
function formatStatus(status) {
const labels = {
queued: 'Queued',
downloading: 'Downloading',
converting: 'Converting',
uploading: 'Uploading',
done: 'Done',
failed: 'Failed',
}
return labels[status] || status
}
function isActive(status) {
return ['downloading', 'converting', 'uploading'].includes(status)
}
</script>
<style scoped>
.queue-panel {
background: rgba(255, 255, 255, 0.03);
border-radius: 8px;
padding: 1rem;
max-height: 500px;
display: flex;
flex-direction: column;
}
.queue-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1rem;
}
.queue-title {
display: flex;
align-items: center;
gap: 0.75rem;
}
.queue-title h3 {
margin: 0;
color: #fff;
}
.queue-actions {
display: flex;
align-items: center;
gap: 0.5rem;
}
.worker-indicator {
font-size: 0.75rem;
padding: 0.25rem 0.5rem;
border-radius: 4px;
display: flex;
align-items: center;
gap: 0.25rem;
}
.worker-indicator.active {
background: rgba(0, 255, 136, 0.15);
color: #00ff88;
}
.worker-indicator.idle {
background: rgba(136, 136, 136, 0.15);
color: #888;
}
.pulse {
width: 8px;
height: 8px;
background: #00ff88;
border-radius: 50%;
animation: pulse 1.5s ease-in-out infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; transform: scale(1); }
50% { opacity: 0.5; transform: scale(0.8); }
}
.btn-clear {
background: rgba(255, 170, 0, 0.2);
border: 1px solid rgba(255, 170, 0, 0.3);
border-radius: 4px;
padding: 0.4rem 0.75rem;
color: #ffaa00;
font-size: 0.8rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.btn-clear:hover {
background: rgba(255, 170, 0, 0.3);
border-color: #ffaa00;
}
.btn-refresh {
background: transparent;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
padding: 0.5rem;
color: #888;
cursor: pointer;
transition: color 0.2s, border-color 0.2s;
}
.btn-refresh:hover {
color: #00d4ff;
border-color: #00d4ff;
}
/* Stats */
.queue-stats {
display: flex;
gap: 1rem;
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.stat {
display: flex;
flex-direction: column;
align-items: center;
flex: 1;
}
.stat-value {
font-size: 1.5rem;
font-weight: bold;
}
.stat-value.queued {
color: #ffaa00;
}
.stat-value.downloading {
color: #00d4ff;
}
.stat-value.done {
color: #00ff88;
}
.stat-value.failed {
color: #ff4444;
}
.stat-label {
font-size: 0.75rem;
color: #888;
text-transform: uppercase;
}
.queue-size {
text-align: center;
color: #888;
font-size: 0.85rem;
margin-bottom: 1rem;
}
/* Queue List */
.queue-list {
flex: 1;
overflow-y: auto;
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.queue-item {
background: rgba(0, 0, 0, 0.2);
border-radius: 8px;
padding: 0.75rem;
border-left: 3px solid transparent;
}
.queue-item.queued {
border-left-color: #ffaa00;
}
.queue-item.downloading,
.queue-item.converting,
.queue-item.uploading {
border-left-color: #00d4ff;
}
.queue-item.done {
border-left-color: #00ff88;
}
.queue-item.failed {
border-left-color: #ff4444;
}
.task-main {
display: flex;
justify-content: space-between;
align-items: flex-start;
gap: 1rem;
}
.task-info {
flex: 1;
min-width: 0;
}
.task-anime {
font-weight: 500;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.task-theme {
display: flex;
align-items: center;
gap: 0.5rem;
margin-top: 0.25rem;
}
.theme-badge {
font-size: 0.75rem;
padding: 0.125rem 0.5rem;
background: rgba(0, 212, 255, 0.2);
color: #00d4ff;
border-radius: 4px;
}
.song-name {
font-size: 0.85rem;
color: #888;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.task-status {
flex-shrink: 0;
}
.status-badge {
padding: 0.25rem 0.5rem;
border-radius: 4px;
font-size: 0.75rem;
font-weight: 500;
}
.status-badge.queued {
background: rgba(255, 170, 0, 0.2);
color: #ffaa00;
}
.status-badge.downloading,
.status-badge.converting,
.status-badge.uploading {
background: rgba(0, 212, 255, 0.2);
color: #00d4ff;
}
.status-badge.done {
background: rgba(0, 255, 136, 0.2);
color: #00ff88;
}
.status-badge.failed {
background: rgba(255, 68, 68, 0.2);
color: #ff4444;
}
/* Progress */
.task-progress {
display: flex;
align-items: center;
gap: 0.5rem;
margin-top: 0.5rem;
}
.task-progress .progress-bar {
flex: 1;
height: 4px;
background: rgba(255, 255, 255, 0.1);
border-radius: 2px;
overflow: hidden;
}
.task-progress .progress-fill {
height: 100%;
background: linear-gradient(90deg, #00d4ff, #00ff88);
transition: width 0.3s ease;
}
.progress-text {
font-size: 0.75rem;
color: #00d4ff;
min-width: 36px;
text-align: right;
}
/* Error */
.task-error {
margin-top: 0.5rem;
padding: 0.5rem;
background: rgba(255, 68, 68, 0.1);
border-radius: 4px;
color: #ff4444;
font-size: 0.8rem;
word-break: break-word;
}
/* Actions */
.task-actions {
display: flex;
justify-content: flex-end;
gap: 0.5rem;
margin-top: 0.5rem;
}
.btn-action {
padding: 0.25rem 0.75rem;
border: none;
border-radius: 4px;
font-size: 0.8rem;
cursor: pointer;
transition: opacity 0.2s;
}
.btn-action:hover {
opacity: 0.8;
}
.btn-cancel {
background: rgba(255, 68, 68, 0.2);
color: #ff4444;
}
.btn-retry {
background: rgba(0, 212, 255, 0.2);
color: #00d4ff;
}
.task-size {
font-size: 0.8rem;
color: #00ff88;
}
/* Empty */
.queue-empty {
text-align: center;
padding: 2rem;
color: #666;
}
</style>

View File

@@ -0,0 +1,799 @@
<template>
<div class="downloader-page">
<h1 class="page-title">Openings Downloader</h1>
<!-- Storage Stats Bar -->
<div class="storage-bar">
<div class="storage-info">
<span class="storage-label">Storage:</span>
<span class="storage-values">{{ formatBytes(storageStats.used_bytes) }} / {{ formatBytes(storageStats.limit_bytes) }}</span>
<span class="storage-percent" :class="{ warning: storageStats.used_percent > 80, danger: storageStats.used_percent > 95 }">
({{ storageStats.used_percent.toFixed(1) }}%)
</span>
<span class="openings-count">{{ storageStats.openings_count }} openings</span>
</div>
<div class="progress-bar">
<div
class="progress-fill"
:class="{ warning: storageStats.used_percent > 80, danger: storageStats.used_percent > 95 }"
:style="{ width: Math.min(storageStats.used_percent, 100) + '%' }"
></div>
</div>
<div v-if="!storageStats.can_download" class="storage-warning">
Storage limit exceeded! Cannot download new files.
</div>
</div>
<div class="main-content">
<!-- Left Panel: Search & Results -->
<div class="search-panel">
<!-- Search Section -->
<div class="search-section">
<div class="search-input-group">
<input
v-model="searchQuery"
type="text"
placeholder="Search anime..."
@keyup.enter="searchAnime"
class="search-input"
/>
<button @click="searchAnime" class="btn btn-primary" :disabled="searching || !searchQuery.trim()">
{{ searching ? 'Searching...' : 'Search' }}
</button>
</div>
<div class="search-filters">
<input v-model.number="yearFilter" type="number" placeholder="Year" class="filter-input" />
<select v-model="statusFilter" class="filter-select">
<option value="">Any status</option>
<option value="ongoing">Ongoing</option>
<option value="released">Released</option>
<option value="announced">Announced</option>
</select>
</div>
</div>
<!-- Search Results -->
<div class="search-results" v-if="searchResults.length">
<h3>Search Results ({{ searchResults.length }})</h3>
<div class="results-grid">
<div
v-for="anime in searchResults"
:key="anime.shikimori_id"
class="anime-card"
:class="{ selected: selectedAnime?.shikimori_id === anime.shikimori_id }"
@click="selectAnime(anime.shikimori_id)"
>
<img
v-if="anime.poster_url"
:src="anime.poster_url"
:alt="anime.title_russian || anime.title_english"
class="anime-poster"
/>
<div v-else class="anime-poster placeholder">No Image</div>
<div class="anime-info">
<div class="anime-title">{{ anime.title_russian || anime.title_english }}</div>
<div class="anime-year" v-if="anime.year">{{ anime.year }}</div>
</div>
</div>
</div>
</div>
<div v-else-if="searched && !searching" class="no-results">
No anime found for "{{ lastSearchQuery }}"
</div>
</div>
<!-- Right Panel: Selected Anime & Queue -->
<div class="detail-panel">
<!-- Selected Anime Detail -->
<div class="anime-detail" v-if="selectedAnime">
<div class="detail-header">
<img
v-if="selectedAnime.poster_url"
:src="selectedAnime.poster_url"
class="detail-poster"
/>
<div class="detail-info">
<h2>{{ selectedAnime.title_russian || selectedAnime.title_english }}</h2>
<p v-if="selectedAnime.title_japanese" class="japanese-title">{{ selectedAnime.title_japanese }}</p>
<p v-if="selectedAnime.year" class="year-info">Year: {{ selectedAnime.year }}</p>
</div>
</div>
<div v-if="loadingDetail" class="loading">Loading themes...</div>
<div v-else-if="selectedAnime.themes?.length" class="themes-section">
<div class="themes-header">
<h3>Available Themes ({{ selectedAnime.themes.length }})</h3>
<button
@click="addAllToQueue"
class="btn btn-success"
:disabled="!canAddAll || !storageStats.can_download"
>
Add All to Queue
</button>
</div>
<div class="themes-list">
<div
v-for="theme in selectedAnime.themes"
:key="theme.id"
class="theme-item"
:class="{ downloaded: theme.is_downloaded, 'in-queue': theme.download_status }"
>
<div class="theme-info">
<span class="theme-name" :class="theme.theme_type.toLowerCase()">{{ theme.full_name }}</span>
<span class="song-title" v-if="theme.song_title">{{ theme.song_title }}</span>
<span class="artist" v-if="theme.artist">by {{ theme.artist }}</span>
</div>
<div class="theme-status">
<span v-if="theme.is_downloaded" class="status-badge done">
Downloaded ({{ formatBytes(theme.file_size_bytes) }})
</span>
<span v-else-if="theme.download_status" :class="'status-badge ' + theme.download_status">
{{ theme.download_status }}
</span>
<button
v-else-if="theme.video_url"
@click="addToQueue([theme.id])"
class="btn btn-small btn-primary"
:disabled="!storageStats.can_download"
>
Add
</button>
<span v-else class="status-badge unavailable">No source</span>
</div>
</div>
</div>
</div>
<div v-else class="no-themes">
No themes found for this anime on AnimeThemes
</div>
</div>
<!-- Download Queue -->
<DownloadQueue
:queue="queueStatus"
@cancel="cancelTask"
@retry="retryTask"
@refresh="refreshQueue"
@clear="clearQueue"
/>
</div>
</div>
</div>
</template>
<script setup>
import { ref, reactive, computed, onMounted, onUnmounted } from 'vue'
import DownloadQueue from './DownloadQueue.vue'
// State
const searchQuery = ref('')
const lastSearchQuery = ref('')
const yearFilter = ref(null)
const statusFilter = ref('')
const searching = ref(false)
const searched = ref(false)
const searchResults = ref([])
const selectedAnime = ref(null)
const loadingDetail = ref(false)
const storageStats = reactive({
used_bytes: 0,
limit_bytes: 107374182400, // 100 GB default
used_percent: 0,
available_bytes: 107374182400,
can_download: true,
openings_count: 0,
})
const queueStatus = reactive({
tasks: [],
total_queued: 0,
total_downloading: 0,
total_done: 0,
total_failed: 0,
estimated_queue_size_bytes: 0,
worker_running: false,
})
let refreshInterval = null
// Computed
const canAddAll = computed(() => {
if (!selectedAnime.value?.themes) return false
return selectedAnime.value.themes.some(
t => !t.is_downloaded && !t.download_status && t.video_url
)
})
// Methods
function formatBytes(bytes) {
if (!bytes) return '0 B'
const units = ['B', 'KB', 'MB', 'GB', 'TB']
let i = 0
let value = bytes
while (value >= 1024 && i < units.length - 1) {
value /= 1024
i++
}
return `${value.toFixed(1)} ${units[i]}`
}
async function searchAnime() {
if (!searchQuery.value.trim()) return
searching.value = true
searched.value = false
lastSearchQuery.value = searchQuery.value
searchResults.value = []
selectedAnime.value = null
try {
const params = new URLSearchParams({ query: searchQuery.value, limit: '30' })
if (yearFilter.value) params.append('year', yearFilter.value)
if (statusFilter.value) params.append('status', statusFilter.value)
const response = await fetch(`/api/downloader/search?${params}`)
const data = await response.json()
searchResults.value = data.results || []
} catch (error) {
console.error('Search error:', error)
} finally {
searching.value = false
searched.value = true
}
}
async function selectAnime(shikimoriId) {
loadingDetail.value = true
selectedAnime.value = null
try {
const response = await fetch(`/api/downloader/anime/${shikimoriId}`)
const data = await response.json()
selectedAnime.value = data
} catch (error) {
console.error('Error loading anime detail:', error)
} finally {
loadingDetail.value = false
}
}
async function addToQueue(themeIds) {
try {
const response = await fetch('/api/downloader/queue/add', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ theme_ids: themeIds }),
})
const data = await response.json()
if (response.ok) {
Object.assign(queueStatus, data)
// Refresh selected anime to update statuses
if (selectedAnime.value) {
await selectAnime(selectedAnime.value.shikimori_id)
}
} else {
alert(data.detail || 'Failed to add to queue')
}
} catch (error) {
console.error('Error adding to queue:', error)
}
}
async function addAllToQueue() {
if (!selectedAnime.value?.id) return
try {
const response = await fetch('/api/downloader/queue/add-all', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ anime_id: selectedAnime.value.id }),
})
const data = await response.json()
if (response.ok) {
Object.assign(queueStatus, data)
await selectAnime(selectedAnime.value.shikimori_id)
} else {
alert(data.detail || 'Failed to add all to queue')
}
} catch (error) {
console.error('Error adding all to queue:', error)
}
}
async function cancelTask(taskId) {
try {
const response = await fetch(`/api/downloader/queue/${taskId}`, { method: 'DELETE' })
if (response.ok) {
await refreshQueue()
}
} catch (error) {
console.error('Error cancelling task:', error)
}
}
async function retryTask(taskId) {
try {
const response = await fetch(`/api/downloader/queue/${taskId}/retry`, { method: 'POST' })
if (response.ok) {
await refreshQueue()
}
} catch (error) {
console.error('Error retrying task:', error)
}
}
async function refreshQueue() {
try {
const response = await fetch('/api/downloader/queue')
const data = await response.json()
Object.assign(queueStatus, data)
} catch (error) {
console.error('Error refreshing queue:', error)
}
}
async function clearQueue() {
try {
const response = await fetch('/api/downloader/queue/clear?include_failed=true', { method: 'DELETE' })
if (response.ok) {
await refreshQueue()
}
} catch (error) {
console.error('Error clearing queue:', error)
}
}
async function refreshStorage() {
try {
const response = await fetch('/api/downloader/storage')
const data = await response.json()
Object.assign(storageStats, data)
} catch (error) {
console.error('Error refreshing storage:', error)
}
}
async function refreshAll() {
await Promise.all([refreshQueue(), refreshStorage()])
// Also refresh selected anime if any
if (selectedAnime.value?.shikimori_id) {
const response = await fetch(`/api/downloader/anime/${selectedAnime.value.shikimori_id}`)
if (response.ok) {
selectedAnime.value = await response.json()
}
}
}
// Lifecycle
onMounted(async () => {
await refreshAll()
// Auto-refresh every 3 seconds
refreshInterval = setInterval(refreshAll, 3000)
})
onUnmounted(() => {
if (refreshInterval) {
clearInterval(refreshInterval)
}
})
</script>
<style scoped>
.downloader-page {
padding: 1rem;
max-width: 1600px;
margin: 0 auto;
}
.page-title {
background: linear-gradient(90deg, #00d4ff, #7b2cbf);
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
background-clip: text;
margin-bottom: 1rem;
}
/* Storage Bar */
.storage-bar {
background: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1rem;
margin-bottom: 1.5rem;
}
.storage-info {
display: flex;
align-items: center;
gap: 1rem;
margin-bottom: 0.5rem;
flex-wrap: wrap;
}
.storage-label {
color: #888;
}
.storage-values {
font-weight: bold;
color: #fff;
}
.storage-percent {
color: #00ff88;
}
.storage-percent.warning {
color: #ffaa00;
}
.storage-percent.danger {
color: #ff4444;
}
.openings-count {
color: #00d4ff;
margin-left: auto;
}
.progress-bar {
height: 8px;
background: rgba(255, 255, 255, 0.1);
border-radius: 4px;
overflow: hidden;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #00d4ff, #00ff88);
transition: width 0.3s ease;
}
.progress-fill.warning {
background: linear-gradient(90deg, #ffaa00, #ff8800);
}
.progress-fill.danger {
background: linear-gradient(90deg, #ff4444, #ff0000);
}
.storage-warning {
color: #ff4444;
font-weight: bold;
margin-top: 0.5rem;
padding: 0.5rem;
background: rgba(255, 68, 68, 0.1);
border-radius: 4px;
}
/* Main Content */
.main-content {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 1.5rem;
}
@media (max-width: 1200px) {
.main-content {
grid-template-columns: 1fr;
}
}
/* Search Panel */
.search-panel {
background: rgba(255, 255, 255, 0.03);
border-radius: 8px;
padding: 1rem;
}
.search-section {
margin-bottom: 1rem;
}
.search-input-group {
display: flex;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.search-input {
flex: 1;
padding: 0.75rem 1rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 8px;
background: rgba(0, 0, 0, 0.3);
color: #fff;
font-size: 1rem;
}
.search-input:focus {
outline: none;
border-color: #00d4ff;
}
.search-filters {
display: flex;
gap: 0.5rem;
}
.filter-input,
.filter-select {
padding: 0.5rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
background: rgba(0, 0, 0, 0.3);
color: #fff;
font-size: 0.9rem;
}
.filter-input {
width: 100px;
}
/* Results Grid */
.results-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
gap: 1rem;
max-height: 600px;
overflow-y: auto;
}
.anime-card {
background: rgba(255, 255, 255, 0.05);
border-radius: 8px;
overflow: hidden;
cursor: pointer;
transition: transform 0.2s, box-shadow 0.2s;
}
.anime-card:hover {
transform: translateY(-2px);
box-shadow: 0 4px 12px rgba(0, 212, 255, 0.2);
}
.anime-card.selected {
border: 2px solid #00d4ff;
}
.anime-poster {
width: 100%;
aspect-ratio: 2/3;
object-fit: cover;
}
.anime-poster.placeholder {
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 255, 255, 0.05);
color: #666;
}
.anime-info {
padding: 0.5rem;
}
.anime-title {
font-size: 0.85rem;
font-weight: 500;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.anime-year {
font-size: 0.75rem;
color: #888;
}
.no-results {
text-align: center;
padding: 2rem;
color: #888;
}
/* Detail Panel */
.detail-panel {
display: flex;
flex-direction: column;
gap: 1rem;
}
.anime-detail {
background: rgba(255, 255, 255, 0.03);
border-radius: 8px;
padding: 1rem;
}
.detail-header {
display: flex;
gap: 1rem;
margin-bottom: 1rem;
}
.detail-poster {
width: 120px;
height: 180px;
object-fit: cover;
border-radius: 8px;
}
.detail-info h2 {
margin: 0 0 0.5rem 0;
font-size: 1.2rem;
}
.japanese-title {
color: #888;
font-size: 0.9rem;
}
.year-info {
color: #00d4ff;
}
/* Themes Section */
.themes-section {
margin-top: 1rem;
}
.themes-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1rem;
}
.themes-header h3 {
margin: 0;
}
.themes-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
max-height: 400px;
overflow-y: auto;
}
.theme-item {
display: flex;
justify-content: space-between;
align-items: center;
padding: 0.75rem;
background: rgba(0, 0, 0, 0.2);
border-radius: 8px;
}
.theme-item.downloaded {
border-left: 3px solid #00ff88;
}
.theme-item.in-queue {
border-left: 3px solid #00d4ff;
}
.theme-info {
display: flex;
flex-direction: column;
gap: 0.25rem;
}
.theme-name {
font-weight: bold;
padding: 0.125rem 0.5rem;
border-radius: 4px;
display: inline-block;
width: fit-content;
}
.theme-name.op {
background: rgba(0, 212, 255, 0.2);
color: #00d4ff;
}
.theme-name.ed {
background: rgba(123, 44, 191, 0.2);
color: #b57edc;
}
.song-title {
color: #fff;
font-size: 0.9rem;
}
.artist {
color: #888;
font-size: 0.8rem;
}
.theme-status {
display: flex;
align-items: center;
}
.status-badge {
padding: 0.25rem 0.75rem;
border-radius: 12px;
font-size: 0.8rem;
font-weight: 500;
}
.status-badge.done {
background: rgba(0, 255, 136, 0.2);
color: #00ff88;
}
.status-badge.queued {
background: rgba(255, 170, 0, 0.2);
color: #ffaa00;
}
.status-badge.downloading,
.status-badge.converting,
.status-badge.uploading {
background: rgba(0, 212, 255, 0.2);
color: #00d4ff;
}
.status-badge.failed {
background: rgba(255, 68, 68, 0.2);
color: #ff4444;
}
.status-badge.unavailable {
background: rgba(136, 136, 136, 0.2);
color: #888;
}
.no-themes {
text-align: center;
padding: 2rem;
color: #888;
}
.loading {
text-align: center;
padding: 2rem;
color: #00d4ff;
}
/* Buttons */
.btn {
padding: 0.75rem 1.5rem;
border: none;
border-radius: 8px;
font-weight: 500;
cursor: pointer;
transition: opacity 0.2s, transform 0.2s;
}
.btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.btn:not(:disabled):hover {
transform: translateY(-1px);
}
.btn-primary {
background: linear-gradient(90deg, #00d4ff, #0099cc);
color: #fff;
}
.btn-success {
background: linear-gradient(90deg, #00ff88, #00cc6a);
color: #000;
}
.btn-small {
padding: 0.375rem 0.75rem;
font-size: 0.85rem;
}
</style>

View File

@@ -7,6 +7,22 @@ export default defineConfig({
host: '0.0.0.0',
port: 5173,
proxy: {
// Downloader API - already has /api prefix on backend
'/api/downloader': {
target: 'http://backend:8000',
changeOrigin: true
},
// Openings API - already has /api prefix on backend
'/api/openings': {
target: 'http://backend:8000',
changeOrigin: true
},
// Backgrounds API - already has /api prefix on backend
'/api/backgrounds': {
target: 'http://backend:8000',
changeOrigin: true
},
// Core API endpoints - strip /api prefix
'/api': {
target: 'http://backend:8000',
changeOrigin: true,