'commit'

DouYin/dy-downloader/storage/__init__.py

@@ -0,0 +1,5 @@
+from .database import Database
+from .file_manager import FileManager
+from .metadata_handler import MetadataHandler
+
+__all__ = ['Database', 'FileManager', 'MetadataHandler']

DouYin/dy-downloader/storage/database.py

@@ -0,0 +1,115 @@
+import aiosqlite
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+
+class Database:
+    def __init__(self, db_path: str = 'dy_downloader.db'):
+        self.db_path = db_path
+        self._initialized = False
+
+    async def initialize(self):
+        if self._initialized:
+            return
+
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute('''
+                CREATE TABLE IF NOT EXISTS aweme (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    aweme_id TEXT UNIQUE NOT NULL,
+                    aweme_type TEXT NOT NULL,
+                    title TEXT,
+                    author_id TEXT,
+                    author_name TEXT,
+                    create_time INTEGER,
+                    download_time INTEGER,
+                    file_path TEXT,
+                    metadata TEXT
+                )
+            ''')
+
+            await db.execute('''
+                CREATE TABLE IF NOT EXISTS download_history (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    url TEXT NOT NULL,
+                    url_type TEXT NOT NULL,
+                    download_time INTEGER,
+                    total_count INTEGER,
+                    success_count INTEGER,
+                    config TEXT
+                )
+            ''')
+
+            await db.execute('CREATE INDEX IF NOT EXISTS idx_aweme_id ON aweme(aweme_id)')
+            await db.execute('CREATE INDEX IF NOT EXISTS idx_author_id ON aweme(author_id)')
+            await db.execute('CREATE INDEX IF NOT EXISTS idx_download_time ON aweme(download_time)')
+
+            await db.commit()
+
+        self._initialized = True
+
+    async def is_downloaded(self, aweme_id: str) -> bool:
+        async with aiosqlite.connect(self.db_path) as db:
+            cursor = await db.execute(
+                'SELECT id FROM aweme WHERE aweme_id = ?',
+                (aweme_id,)
+            )
+            result = await cursor.fetchone()
+            return result is not None
+
+    async def add_aweme(self, aweme_data: Dict[str, Any]):
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute('''
+                INSERT OR REPLACE INTO aweme
+                (aweme_id, aweme_type, title, author_id, author_name, create_time, download_time, file_path, metadata)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ''', (
+                aweme_data.get('aweme_id'),
+                aweme_data.get('aweme_type'),
+                aweme_data.get('title'),
+                aweme_data.get('author_id'),
+                aweme_data.get('author_name'),
+                aweme_data.get('create_time'),
+                int(datetime.now().timestamp()),
+                aweme_data.get('file_path'),
+                aweme_data.get('metadata'),
+            ))
+            await db.commit()
+
+    async def get_latest_aweme_time(self, author_id: str) -> Optional[int]:
+        async with aiosqlite.connect(self.db_path) as db:
+            cursor = await db.execute(
+                'SELECT MAX(create_time) FROM aweme WHERE author_id = ?',
+                (author_id,)
+            )
+            result = await cursor.fetchone()
+            return result[0] if result and result[0] else None
+
+    async def add_history(self, history_data: Dict[str, Any]):
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute('''
+                INSERT INTO download_history
+                (url, url_type, download_time, total_count, success_count, config)
+                VALUES (?, ?, ?, ?, ?, ?)
+            ''', (
+                history_data.get('url'),
+                history_data.get('url_type'),
+                int(datetime.now().timestamp()),
+                history_data.get('total_count'),
+                history_data.get('success_count'),
+                history_data.get('config'),
+            ))
+            await db.commit()
+
+    async def get_aweme_count_by_author(self, author_id: str) -> int:
+        async with aiosqlite.connect(self.db_path) as db:
+            cursor = await db.execute(
+                'SELECT COUNT(*) FROM aweme WHERE author_id = ?',
+                (author_id,)
+            )
+            result = await cursor.fetchone()
+            return result[0] if result else 0
+
+    async def close(self):
+        pass

DouYin/dy-downloader/storage/file_manager.py

@@ -0,0 +1,75 @@
+import aiofiles
+import aiohttp
+from pathlib import Path
+from typing import Dict, Optional
+from utils.validators import sanitize_filename
+from utils.logger import setup_logger
+
+logger = setup_logger('FileManager')
+
+
+class FileManager:
+    def __init__(self, base_path: str = './Downloaded'):
+        self.base_path = Path(base_path)
+        self.base_path.mkdir(parents=True, exist_ok=True)
+
+    def get_save_path(self, author_name: str, mode: str = None, aweme_title: str = None,
+                     aweme_id: str = None, folderstyle: bool = True) -> Path:
+        safe_author = sanitize_filename(author_name)
+
+        if mode:
+            save_dir = self.base_path / safe_author / mode
+        else:
+            save_dir = self.base_path / safe_author
+
+        if folderstyle and aweme_title and aweme_id:
+            safe_title = sanitize_filename(aweme_title)
+            save_dir = save_dir / f"{safe_title}_{aweme_id}"
+
+        save_dir.mkdir(parents=True, exist_ok=True)
+        return save_dir
+
+    async def download_file(
+        self,
+        url: str,
+        save_path: Path,
+        session: aiohttp.ClientSession = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> bool:
+        should_close = False
+        if session is None:
+            default_headers = headers or {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+                               'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+                'Referer': 'https://www.douyin.com/',
+                'Accept': '*/*',
+            }
+            session = aiohttp.ClientSession(headers=default_headers)
+            should_close = True
+
+        try:
+            async with session.get(
+                url,
+                timeout=aiohttp.ClientTimeout(total=300),
+                headers=headers,
+            ) as response:
+                if response.status == 200:
+                    async with aiofiles.open(save_path, 'wb') as f:
+                        async for chunk in response.content.iter_chunked(8192):
+                            await f.write(chunk)
+                    return True
+                else:
+                    logger.error(f"Download failed: {url}, status: {response.status}")
+                    return False
+        except Exception as e:
+            logger.error(f"Download error: {url}, error: {e}")
+            return False
+        finally:
+            if should_close:
+                await session.close()
+
+    def file_exists(self, file_path: Path) -> bool:
+        return file_path.exists() and file_path.stat().st_size > 0
+
+    def get_file_size(self, file_path: Path) -> int:
+        return file_path.stat().st_size if self.file_exists(file_path) else 0

DouYin/dy-downloader/storage/metadata_handler.py

@@ -0,0 +1,27 @@
+import json
+import aiofiles
+from pathlib import Path
+from typing import Dict, Any
+from utils.logger import setup_logger
+
+logger = setup_logger('MetadataHandler')
+
+
+class MetadataHandler:
+    @staticmethod
+    async def save_metadata(data: Dict[str, Any], save_path: Path):
+        try:
+            async with aiofiles.open(save_path, 'w', encoding='utf-8') as f:
+                await f.write(json.dumps(data, ensure_ascii=False, indent=2))
+        except Exception as e:
+            logger.error(f"Failed to save metadata: {save_path}, error: {e}")
+
+    @staticmethod
+    async def load_metadata(file_path: Path) -> Dict[str, Any]:
+        try:
+            async with aiofiles.open(file_path, 'r', encoding='utf-8') as f:
+                content = await f.read()
+                return json.loads(content)
+        except Exception as e:
+            logger.error(f"Failed to load metadata: {file_path}, error: {e}")
+            return {}