From cc11f0b773f2d431af393450ec1abdc09dbb87c3 Mon Sep 17 00:00:00 2001 From: "mamonov.ep" Date: Mon, 12 Jan 2026 11:22:46 +0300 Subject: [PATCH] Remove Shikimori API, use AnimeThemes only, switch to WebM format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove ShikimoriService, use AnimeThemes API for search - Replace shikimori_id with animethemes_slug as primary identifier - Remove FFmpeg MP3 conversion, download WebM directly - Add .webm support in storage and upload endpoints - Update frontend to use animethemes_slug 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .env.example | 2 - backend/app/main.py | 3 +- backend/app/openings_downloader/config.py | 4 - backend/app/openings_downloader/db_models.py | 7 +- backend/app/openings_downloader/router.py | 22 ++- backend/app/openings_downloader/schemas.py | 12 +- .../openings_downloader/services/__init__.py | 2 - .../services/animethemes.py | 131 +++++++++++----- .../services/downloader.py | 36 +---- .../openings_downloader/services/shikimori.py | 145 ------------------ backend/app/storage.py | 12 +- .../src/components/OpeningsDownloader.vue | 25 ++- 12 files changed, 138 insertions(+), 263 deletions(-) delete mode 100644 backend/app/openings_downloader/services/shikimori.py diff --git a/.env.example b/.env.example index e3392bc..1c88f62 100644 --- a/.env.example +++ b/.env.example @@ -45,8 +45,6 @@ AUDIO_FADE_DURATION=0.7 # =========================================== # Openings Downloader Settings # =========================================== -DOWNLOADER_SHIKIMORI_USER_AGENT=AnimeQuiz/1.0 -DOWNLOADER_SHIKIMORI_TOKEN=your_shikimori_oauth_token_here DOWNLOADER_S3_STORAGE_LIMIT_BYTES=107374182400 DOWNLOADER_DOWNLOAD_TIMEOUT_SECONDS=300 DOWNLOADER_DEFAULT_ESTIMATED_SIZE_BYTES=6291456 diff --git a/backend/app/main.py b/backend/app/main.py index 6bef926..ce5522a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -126,6 +126,7 @@ async def get_audio(filename: str): ".wav": "audio/wav", ".ogg": "audio/ogg", ".m4a": "audio/mp4", + ".webm": "video/webm", } media_type = media_types.get(suffix, "audio/mpeg") @@ -284,7 +285,7 @@ async def upload_audio(files: List[UploadFile] = File(...)): """Upload audio files to S3.""" results = [] for file in files: - if not file.filename.lower().endswith((".mp3", ".wav", ".ogg", ".m4a")): + if not file.filename.lower().endswith((".mp3", ".wav", ".ogg", ".m4a", ".webm")): results.append({"filename": file.filename, "success": False, "error": "Invalid file type"}) continue diff --git a/backend/app/openings_downloader/config.py b/backend/app/openings_downloader/config.py index 18a7562..1c2b960 100644 --- a/backend/app/openings_downloader/config.py +++ b/backend/app/openings_downloader/config.py @@ -4,10 +4,6 @@ from pydantic_settings import BaseSettings class DownloaderSettings(BaseSettings): """Settings for the Openings Downloader module.""" - # Shikimori API - shikimori_user_agent: str = "AnimeQuiz/1.0" - shikimori_token: str = "" # Optional OAuth token for higher rate limits - # S3 Storage limit (100 GB default) s3_storage_limit_bytes: int = 107_374_182_400 # 100 GB diff --git a/backend/app/openings_downloader/db_models.py b/backend/app/openings_downloader/db_models.py index 1525596..fb7e250 100644 --- a/backend/app/openings_downloader/db_models.py +++ b/backend/app/openings_downloader/db_models.py @@ -27,12 +27,11 @@ class DownloadStatus(str, enum.Enum): class Anime(Base): - """Anime entity from Shikimori.""" + """Anime entity from AnimeThemes.""" __tablename__ = "anime" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - shikimori_id: Mapped[int] = mapped_column(Integer, unique=True, index=True, nullable=False) - animethemes_slug: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True) + animethemes_slug: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False) title_russian: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) title_english: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) @@ -54,7 +53,7 @@ class Anime(Base): ) def __repr__(self): - return f"" + return f"" class AnimeTheme(Base): diff --git a/backend/app/openings_downloader/router.py b/backend/app/openings_downloader/router.py index 8e4f124..485ea92 100644 --- a/backend/app/openings_downloader/router.py +++ b/backend/app/openings_downloader/router.py @@ -21,7 +21,6 @@ from .schemas import ( StorageStatsResponse, ) from .db_models import Anime, AnimeTheme, DownloadTask, DownloadStatus -from .services.shikimori import ShikimoriService from .services.animethemes import AnimeThemesService from .services.downloader import DownloadService from .services.storage_tracker import StorageTrackerService @@ -34,30 +33,29 @@ router = APIRouter(prefix="/downloader", tags=["openings-downloader"]) @router.get("/search", response_model=SearchResponse) async def search_anime( query: str = Query(..., min_length=1, description="Search query"), - year: Optional[int] = Query(None, description="Filter by year"), - status: Optional[str] = Query(None, description="Filter by status (ongoing, released, announced)"), limit: int = Query(20, ge=1, le=50, description="Maximum results"), db: AsyncSession = Depends(get_db), ): - """Search anime via Shikimori API.""" - service = ShikimoriService() - results = await service.search(query, year=year, status=status, limit=limit) + """Search anime via AnimeThemes API.""" + service = AnimeThemesService() + results = await service.search(query, limit=limit) return SearchResponse(results=results, total=len(results)) # ============== Anime Detail ============== -@router.get("/anime/{shikimori_id}", response_model=AnimeDetailResponse) +@router.get("/anime/{slug:path}", response_model=AnimeDetailResponse) async def get_anime_detail( - shikimori_id: int, + slug: str, db: AsyncSession = Depends(get_db), ): """Get anime details with available themes from AnimeThemes.""" - shikimori_service = ShikimoriService() animethemes_service = AnimeThemesService() # Get or create anime record - anime = await shikimori_service.get_or_create_anime(db, shikimori_id) + anime = await animethemes_service.get_or_create_anime(db, slug) + if not anime: + raise HTTPException(status_code=404, detail=f"Anime not found: {slug}") # Fetch themes from AnimeThemes API themes = await animethemes_service.fetch_themes(db, anime) @@ -92,10 +90,8 @@ async def get_anime_detail( return AnimeDetailResponse( id=anime.id, - shikimori_id=anime.shikimori_id, - title_russian=anime.title_russian, + animethemes_slug=anime.animethemes_slug, title_english=anime.title_english, - title_japanese=anime.title_japanese, year=anime.year, poster_url=anime.poster_url, themes=theme_infos, diff --git a/backend/app/openings_downloader/schemas.py b/backend/app/openings_downloader/schemas.py index 9ff7e8b..67bdd42 100644 --- a/backend/app/openings_downloader/schemas.py +++ b/backend/app/openings_downloader/schemas.py @@ -5,14 +5,12 @@ from pydantic import BaseModel, Field from .db_models import ThemeType, DownloadStatus -# ============== Shikimori Search ============== +# ============== AnimeThemes Search ============== class AnimeSearchResult(BaseModel): - """Single anime search result from Shikimori.""" - shikimori_id: int - title_russian: Optional[str] = None + """Single anime search result from AnimeThemes.""" + animethemes_slug: str title_english: Optional[str] = None - title_japanese: Optional[str] = None year: Optional[int] = None poster_url: Optional[str] = None @@ -48,10 +46,8 @@ class ThemeInfo(BaseModel): class AnimeDetailResponse(BaseModel): """Detailed anime info with themes.""" id: int - shikimori_id: int - title_russian: Optional[str] = None + animethemes_slug: str title_english: Optional[str] = None - title_japanese: Optional[str] = None year: Optional[int] = None poster_url: Optional[str] = None themes: List[ThemeInfo] diff --git a/backend/app/openings_downloader/services/__init__.py b/backend/app/openings_downloader/services/__init__.py index a89da10..8db3349 100644 --- a/backend/app/openings_downloader/services/__init__.py +++ b/backend/app/openings_downloader/services/__init__.py @@ -1,11 +1,9 @@ # Services for Openings Downloader -from .shikimori import ShikimoriService from .animethemes import AnimeThemesService from .downloader import DownloadService from .storage_tracker import StorageTrackerService __all__ = [ - "ShikimoriService", "AnimeThemesService", "DownloadService", "StorageTrackerService", diff --git a/backend/app/openings_downloader/services/animethemes.py b/backend/app/openings_downloader/services/animethemes.py index cf13c5a..3ec3578 100644 --- a/backend/app/openings_downloader/services/animethemes.py +++ b/backend/app/openings_downloader/services/animethemes.py @@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from ..db_models import Anime, AnimeTheme, ThemeType +from ..schemas import AnimeSearchResult logger = logging.getLogger(__name__) @@ -31,41 +32,100 @@ class AnimeThemesService: def __init__(self): self.client = _get_animethemes_client() - async def _find_anime_slug(self, anime: Anime) -> Optional[str]: - """Find AnimeThemes slug by searching anime title.""" + async def search(self, query: str, limit: int = 20) -> List[AnimeSearchResult]: + """Search anime by query using AnimeThemes API.""" + try: + response = await self.client.get( + "/anime", + params={ + "q": query, + "include": "images", + "page[size]": limit, + }, + ) + response.raise_for_status() + data = response.json() - # Try different title variations - search_terms = [ - anime.title_english, - anime.title_russian, - anime.title_japanese, - ] + results = [] + for anime in data.get("anime", []): + # Get poster URL from images + poster_url = None + images = anime.get("images", []) + if images: + # Prefer large_cover or first available + for img in images: + if img.get("facet") == "Large Cover": + poster_url = img.get("link") + break + if not poster_url and images: + poster_url = images[0].get("link") - for term in search_terms: - if not term: - continue + results.append(AnimeSearchResult( + animethemes_slug=anime.get("slug"), + title_english=anime.get("name"), + year=anime.get("year"), + poster_url=poster_url, + )) - try: - response = await self.client.get( - "/anime", - params={ - "q": term, - "include": "animethemes.animethemeentries.videos.audio", - }, - ) + return results + except Exception as e: + logger.error(f"Failed to search AnimeThemes: {e}") + return [] - if response.status_code == 200: - data = response.json() - animes = data.get("anime", []) - if animes: - slug = animes[0].get("slug") - logger.info(f"Found AnimeThemes slug '{slug}' for '{term}'") - return slug - except Exception as e: - logger.warning(f"Failed to search AnimeThemes for '{term}': {e}") - continue + async def get_or_create_anime(self, db: AsyncSession, slug: str) -> Optional[Anime]: + """Get anime from DB or fetch from AnimeThemes and create.""" - return None + # Check if exists (with themes eagerly loaded) + query = ( + select(Anime) + .where(Anime.animethemes_slug == slug) + .options(selectinload(Anime.themes)) + ) + result = await db.execute(query) + anime = result.scalar_one_or_none() + + if anime: + return anime + + # Fetch from AnimeThemes + try: + response = await self.client.get( + f"/anime/{slug}", + params={"include": "images"}, + ) + response.raise_for_status() + data = response.json() + except Exception as e: + logger.error(f"Failed to fetch anime from AnimeThemes: {e}") + return None + + anime_data = data.get("anime", {}) + if not anime_data: + return None + + # Get poster URL + poster_url = None + images = anime_data.get("images", []) + if images: + for img in images: + if img.get("facet") == "Large Cover": + poster_url = img.get("link") + break + if not poster_url and images: + poster_url = images[0].get("link") + + anime = Anime( + animethemes_slug=slug, + title_english=anime_data.get("name"), + year=anime_data.get("year"), + poster_url=poster_url, + ) + + db.add(anime) + await db.commit() + await db.refresh(anime) + + return anime async def fetch_themes(self, db: AsyncSession, anime: Anime) -> List[AnimeTheme]: """Fetch themes from AnimeThemes API and sync to DB.""" @@ -79,17 +139,8 @@ class AnimeThemesService: anime = result.scalar_one() current_themes = anime.themes or [] - # Find slug if not cached if not anime.animethemes_slug: - logger.info(f"Searching AnimeThemes slug for: {anime.title_english or anime.title_russian}") - slug = await self._find_anime_slug(anime) - logger.info(f"Found slug: {slug}") - if slug: - anime.animethemes_slug = slug - await db.commit() - - if not anime.animethemes_slug: - logger.warning(f"No AnimeThemes slug found for anime {anime.id}: {anime.title_english or anime.title_russian}") + logger.warning(f"No AnimeThemes slug for anime {anime.id}") return current_themes # Fetch themes from AnimeThemes API diff --git a/backend/app/openings_downloader/services/downloader.py b/backend/app/openings_downloader/services/downloader.py index 75f65fe..de8f3cc 100644 --- a/backend/app/openings_downloader/services/downloader.py +++ b/backend/app/openings_downloader/services/downloader.py @@ -173,11 +173,10 @@ class DownloadService: if not theme.animethemes_video_url: raise ValueError("No video URL available") - # Download and convert in temp directory + # Download WebM file directly (no conversion needed) with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = Path(tmp_dir) - webm_file = tmp_path / "video.webm" - mp3_file = tmp_path / "audio.mp3" + webm_file = tmp_path / "audio.webm" # Stream download WebM file async with httpx.AsyncClient() as client: @@ -192,44 +191,23 @@ class DownloadService: async for chunk in response.aiter_bytes(chunk_size=8192): f.write(chunk) - task.progress_percent = 40 - task.status = DownloadStatus.CONVERTING - await self.db.commit() - - # Convert to MP3 with FFmpeg - process = await asyncio.create_subprocess_exec( - "ffmpeg", "-i", str(webm_file), - "-vn", "-acodec", "libmp3lame", "-q:a", "2", - str(mp3_file), - "-y", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await process.communicate() - - if process.returncode != 0: - raise RuntimeError(f"FFmpeg error: {stderr.decode()[:500]}") - - if not mp3_file.exists(): - raise RuntimeError("FFmpeg did not create output file") - task.progress_percent = 70 task.status = DownloadStatus.UPLOADING await self.db.commit() # Generate safe S3 key anime_name = self._sanitize_filename( - anime.title_english or anime.title_russian or f"anime_{anime.shikimori_id}" + anime.title_english or f"anime_{anime.animethemes_slug}" ) theme_name = theme.full_name song_part = f"_{self._sanitize_filename(theme.song_title)}" if theme.song_title else "" - s3_key = f"audio/{anime_name}_{theme_name}{song_part}.mp3" + s3_key = f"audio/{anime_name}_{theme_name}{song_part}.webm" # Read file and upload to S3 - file_data = mp3_file.read_bytes() + file_data = webm_file.read_bytes() file_size = len(file_data) - success = storage.upload_file(s3_key, file_data, "audio/mpeg") + success = storage.upload_file(s3_key, file_data, "video/webm") if not success: raise RuntimeError("Failed to upload to S3") @@ -239,7 +217,7 @@ class DownloadService: # Create Opening entity in main table opening = Opening( - anime_name=anime.title_russian or anime.title_english or f"Anime {anime.shikimori_id}", + anime_name=anime.title_english or f"Anime {anime.animethemes_slug}", op_number=theme_name, song_name=theme.song_title, audio_file=s3_key.replace("audio/", ""), diff --git a/backend/app/openings_downloader/services/shikimori.py b/backend/app/openings_downloader/services/shikimori.py deleted file mode 100644 index 8d147da..0000000 --- a/backend/app/openings_downloader/services/shikimori.py +++ /dev/null @@ -1,145 +0,0 @@ -import httpx -from typing import List, Optional -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload - -from ..db_models import Anime -from ..schemas import AnimeSearchResult -from ..config import downloader_settings - -# Shared HTTP client for Shikimori API -_shikimori_client: Optional[httpx.AsyncClient] = None - - -def _get_shikimori_client() -> httpx.AsyncClient: - """Get or create shared HTTP client for Shikimori.""" - global _shikimori_client - if _shikimori_client is None or _shikimori_client.is_closed: - headers = { - "User-Agent": downloader_settings.shikimori_user_agent, - } - if downloader_settings.shikimori_token: - headers["Authorization"] = f"Bearer {downloader_settings.shikimori_token}" - _shikimori_client = httpx.AsyncClient( - headers=headers, - timeout=30.0, - ) - return _shikimori_client - - -class ShikimoriService: - """Service for Shikimori GraphQL API.""" - - GRAPHQL_URL = "https://shikimori.one/api/graphql" - - def __init__(self): - self.client = _get_shikimori_client() - - async def search( - self, - query: str, - year: Optional[int] = None, - status: Optional[str] = None, - limit: int = 20, - ) -> List[AnimeSearchResult]: - """Search anime by query using Shikimori GraphQL API.""" - - graphql_query = """ - query($search: String, $limit: Int, $season: SeasonString, $status: AnimeStatusString) { - animes(search: $search, limit: $limit, season: $season, status: $status) { - id - russian - english - japanese - airedOn { year } - poster { originalUrl } - } - } - """ - - variables = { - "search": query, - "limit": limit, - } - if year: - variables["season"] = str(year) - if status: - variables["status"] = status - - response = await self.client.post( - self.GRAPHQL_URL, - json={"query": graphql_query, "variables": variables}, - ) - response.raise_for_status() - data = response.json() - - results = [] - for anime in data.get("data", {}).get("animes", []): - results.append(AnimeSearchResult( - shikimori_id=int(anime["id"]), - title_russian=anime.get("russian"), - title_english=anime.get("english"), - title_japanese=anime.get("japanese"), - year=anime.get("airedOn", {}).get("year") if anime.get("airedOn") else None, - poster_url=anime.get("poster", {}).get("originalUrl") if anime.get("poster") else None, - )) - - return results - - async def get_or_create_anime(self, db: AsyncSession, shikimori_id: int) -> Anime: - """Get anime from DB or fetch from Shikimori and create.""" - - # Check if exists (with themes eagerly loaded) - query = ( - select(Anime) - .where(Anime.shikimori_id == shikimori_id) - .options(selectinload(Anime.themes)) - ) - result = await db.execute(query) - anime = result.scalar_one_or_none() - - if anime: - return anime - - # Fetch from Shikimori - graphql_query = """ - query($ids: String!) { - animes(ids: $ids, limit: 1) { - id - russian - english - japanese - airedOn { year } - poster { originalUrl } - } - } - """ - - response = await self.client.post( - self.GRAPHQL_URL, - json={"query": graphql_query, "variables": {"ids": str(shikimori_id)}}, - ) - response.raise_for_status() - data = response.json() - - animes = data.get("data", {}).get("animes", []) - if not animes: - raise ValueError(f"Anime with ID {shikimori_id} not found on Shikimori") - - anime_data = animes[0] - - anime = Anime( - shikimori_id=shikimori_id, - title_russian=anime_data.get("russian"), - title_english=anime_data.get("english"), - title_japanese=anime_data.get("japanese"), - year=anime_data.get("airedOn", {}).get("year") if anime_data.get("airedOn") else None, - poster_url=anime_data.get("poster", {}).get("originalUrl") if anime_data.get("poster") else None, - ) - - db.add(anime) - await db.commit() - await db.refresh(anime) - - return anime diff --git a/backend/app/storage.py b/backend/app/storage.py index 8010e4a..b41dcf3 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -117,7 +117,7 @@ class S3Storage: def list_audio_files(self) -> list[str]: """List available audio files.""" - return self.list_files("audio/", [".mp3", ".wav", ".ogg", ".m4a"]) + return self.list_files("audio/", [".mp3", ".wav", ".ogg", ".m4a", ".webm"]) def list_background_videos(self) -> list[str]: """List available background videos.""" @@ -167,7 +167,15 @@ class S3Storage: def upload_audio(self, filename: str, file_data: bytes) -> bool: """Upload audio file to S3.""" - content_type = "audio/mpeg" if filename.lower().endswith(".mp3") else "audio/wav" + ext = filename.lower().split(".")[-1] + content_types = { + "mp3": "audio/mpeg", + "wav": "audio/wav", + "ogg": "audio/ogg", + "m4a": "audio/mp4", + "webm": "video/webm", + } + content_type = content_types.get(ext, "audio/mpeg") return self.upload_file(f"audio/{filename}", file_data, content_type) def upload_background(self, filename: str, file_data: bytes) -> bool: diff --git a/frontend/src/components/OpeningsDownloader.vue b/frontend/src/components/OpeningsDownloader.vue index 3d73b7a..72cd6de 100644 --- a/frontend/src/components/OpeningsDownloader.vue +++ b/frontend/src/components/OpeningsDownloader.vue @@ -58,20 +58,20 @@
No Image
-
{{ anime.title_russian || anime.title_english }}
+
{{ anime.title_english }}
{{ anime.year }}
@@ -94,8 +94,7 @@ class="detail-poster" />
-

{{ selectedAnime.title_russian || selectedAnime.title_english }}

-

{{ selectedAnime.title_japanese }}

+

{{ selectedAnime.title_english }}

Year: {{ selectedAnime.year }}

@@ -247,12 +246,12 @@ async function searchAnime() { } } -async function selectAnime(shikimoriId) { +async function selectAnime(slug) { loadingDetail.value = true selectedAnime.value = null try { - const response = await fetch(`/api/downloader/anime/${shikimoriId}`) + const response = await fetch(`/api/downloader/anime/${slug}`) const data = await response.json() selectedAnime.value = data } catch (error) { @@ -274,7 +273,7 @@ async function addToQueue(themeIds) { Object.assign(queueStatus, data) // Refresh selected anime to update statuses if (selectedAnime.value) { - await selectAnime(selectedAnime.value.shikimori_id) + await selectAnime(selectedAnime.value.animethemes_slug) } } else { alert(data.detail || 'Failed to add to queue') @@ -296,7 +295,7 @@ async function addAllToQueue() { const data = await response.json() if (response.ok) { Object.assign(queueStatus, data) - await selectAnime(selectedAnime.value.shikimori_id) + await selectAnime(selectedAnime.value.animethemes_slug) } else { alert(data.detail || 'Failed to add all to queue') } @@ -361,8 +360,8 @@ async function refreshStorage() { async function refreshAll() { await Promise.all([refreshQueue(), refreshStorage()]) // Also refresh selected anime if any - if (selectedAnime.value?.shikimori_id) { - const response = await fetch(`/api/downloader/anime/${selectedAnime.value.shikimori_id}`) + if (selectedAnime.value?.animethemes_slug) { + const response = await fetch(`/api/downloader/anime/${selectedAnime.value.animethemes_slug}`) if (response.ok) { selectedAnime.value = await response.json() }