This commit is contained in:
HuangHai
2026-01-20 14:05:19 +08:00
parent 646f3acd6d
commit d89388ab51
89 changed files with 12526 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
from .api_client import DouyinAPIClient
from .url_parser import URLParser
from .downloader_factory import DownloaderFactory
__all__ = ['DouyinAPIClient', 'URLParser', 'DownloaderFactory']

View File

@@ -0,0 +1,169 @@
from __future__ import annotations
import aiohttp
from typing import Any, Dict, Optional, Tuple
from urllib.parse import urlencode
from utils.logger import setup_logger
from utils.xbogus import XBogus
logger = setup_logger('APIClient')
class DouyinAPIClient:
BASE_URL = 'https://www.douyin.com'
def __init__(self, cookies: Dict[str, str]):
self.cookies = cookies or {}
self._session: Optional[aiohttp.ClientSession] = None
self.headers = {
'User-Agent': (
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
),
'Referer': 'https://www.douyin.com/',
'Accept': 'application/json',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'Connection': 'keep-alive',
}
self._signer = XBogus(self.headers['User-Agent'])
async def __aenter__(self) -> 'DouyinAPIClient':
await self._ensure_session()
return self
async def __aexit__(self, exc_type, exc, tb):
await self.close()
async def _ensure_session(self):
if self._session is None or self._session.closed:
self._session = aiohttp.ClientSession(
headers=self.headers,
cookies=self.cookies,
timeout=aiohttp.ClientTimeout(total=30),
raise_for_status=False,
)
async def close(self):
if self._session and not self._session.closed:
await self._session.close()
async def get_session(self) -> aiohttp.ClientSession:
await self._ensure_session()
assert self._session is not None
return self._session
def _default_query(self) -> Dict[str, Any]:
return {
'device_platform': 'webapp',
'aid': '6383',
'channel': 'channel_pc_web',
'pc_client_type': '1',
'version_code': '170400',
'version_name': '17.4.0',
'cookie_enabled': 'true',
'screen_width': '1920',
'screen_height': '1080',
'browser_language': 'zh-CN',
'browser_platform': 'Win32',
'browser_name': 'Chrome',
'browser_version': '123.0.0.0',
'browser_online': 'true',
'engine_name': 'Blink',
'engine_version': '123.0.0.0',
'os_name': 'Windows',
'os_version': '10',
'cpu_core_num': '8',
'device_memory': '8',
'platform': 'PC',
'downlink': '10',
'effective_type': '4g',
'round_trip_time': '50',
'msToken': self.cookies.get('msToken', ''),
}
def sign_url(self, url: str) -> Tuple[str, str]:
signed_url, _xbogus, ua = self._signer.build(url)
return signed_url, ua
def build_signed_path(self, path: str, params: Dict[str, Any]) -> Tuple[str, str]:
query = urlencode(params)
url = f"{self.BASE_URL}{path}?{query}"
return self.sign_url(url)
async def get_video_detail(self, aweme_id: str) -> Optional[Dict[str, Any]]:
params = self._default_query()
params.update({
'aweme_id': aweme_id,
'aid': '1128',
})
await self._ensure_session()
signed_url, ua = self.build_signed_path('/aweme/v1/web/aweme/detail/', params)
try:
async with self._session.get(signed_url, headers={**self.headers, 'User-Agent': ua}) as response:
if response.status == 200:
data = await response.json(content_type=None)
return data.get('aweme_detail')
logger.error(f"Video detail request failed: {aweme_id}, status={response.status}")
except Exception as e:
logger.error(f"Failed to get video detail: {aweme_id}, error: {e}")
return None
async def get_user_post(self, sec_uid: str, max_cursor: int = 0, count: int = 20) -> Dict[str, Any]:
params = self._default_query()
params.update({
'sec_user_id': sec_uid,
'max_cursor': max_cursor,
'count': count,
'locate_query': 'false',
'show_live_replay_strategy': '1',
'need_time_list': '1',
'time_list_query': '0',
'whale_cut_token': '',
'cut_version': '1',
'publish_video_strategy_type': '2',
})
await self._ensure_session()
signed_url, ua = self.build_signed_path('/aweme/v1/web/aweme/post/', params)
try:
async with self._session.get(signed_url, headers={**self.headers, 'User-Agent': ua}) as response:
if response.status == 200:
return await response.json(content_type=None)
logger.error(f"User post request failed: {sec_uid}, status={response.status}")
except Exception as e:
logger.error(f"Failed to get user post: {sec_uid}, error: {e}")
return {}
async def get_user_info(self, sec_uid: str) -> Optional[Dict[str, Any]]:
params = self._default_query()
params.update({'sec_user_id': sec_uid})
await self._ensure_session()
signed_url, ua = self.build_signed_path('/aweme/v1/web/user/profile/other/', params)
try:
async with self._session.get(signed_url, headers={**self.headers, 'User-Agent': ua}) as response:
if response.status == 200:
data = await response.json(content_type=None)
return data.get('user')
logger.error(f"User info request failed: {sec_uid}, status={response.status}")
except Exception as e:
logger.error(f"Failed to get user info: {sec_uid}, error: {e}")
return None
async def resolve_short_url(self, short_url: str) -> Optional[str]:
try:
await self._ensure_session()
async with self._session.get(short_url, allow_redirects=True) as response:
return str(response.url)
except Exception as e:
logger.error(f"Failed to resolve short URL: {short_url}, error: {e}")
return None

View File

@@ -0,0 +1,309 @@
import json
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import urlparse
from config import ConfigLoader
from storage import Database, FileManager, MetadataHandler
from auth import CookieManager
from control import QueueManager, RateLimiter, RetryHandler
from core.api_client import DouyinAPIClient
from utils.logger import setup_logger
from utils.validators import sanitize_filename
logger = setup_logger('BaseDownloader')
class DownloadResult:
def __init__(self):
self.total = 0
self.success = 0
self.failed = 0
self.skipped = 0
def __str__(self):
return f"Total: {self.total}, Success: {self.success}, Failed: {self.failed}, Skipped: {self.skipped}"
class BaseDownloader(ABC):
def __init__(
self,
config: ConfigLoader,
api_client: DouyinAPIClient,
file_manager: FileManager,
cookie_manager: CookieManager,
database: Optional[Database] = None,
rate_limiter: Optional[RateLimiter] = None,
retry_handler: Optional[RetryHandler] = None,
queue_manager: Optional[QueueManager] = None,
):
self.config = config
self.api_client = api_client
self.file_manager = file_manager
self.cookie_manager = cookie_manager
self.database = database
self.rate_limiter = rate_limiter or RateLimiter()
self.retry_handler = retry_handler or RetryHandler()
thread_count = int(self.config.get('thread', 5) or 5)
self.queue_manager = queue_manager or QueueManager(max_workers=thread_count)
self.metadata_handler = MetadataHandler()
def _download_headers(self, user_agent: Optional[str] = None) -> Dict[str, str]:
headers = {
'Referer': f'{self.api_client.BASE_URL}/',
'Origin': self.api_client.BASE_URL,
'Accept': '*/*',
}
headers['User-Agent'] = user_agent or self.api_client.headers.get('User-Agent', '')
return headers
@abstractmethod
async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult:
pass
async def _should_download(self, aweme_id: str) -> bool:
if self.database:
return not await self.database.is_downloaded(aweme_id)
return True
def _filter_by_time(self, aweme_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
start_time = self.config.get('start_time')
end_time = self.config.get('end_time')
if not start_time and not end_time:
return aweme_list
filtered: List[Dict[str, Any]] = []
for aweme in aweme_list:
create_time = aweme.get('create_time', 0)
if start_time:
from datetime import datetime
start_ts = int(datetime.strptime(start_time, '%Y-%m-%d').timestamp())
if create_time < start_ts:
continue
if end_time:
from datetime import datetime
end_ts = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp())
if create_time > end_ts:
continue
filtered.append(aweme)
return filtered
def _limit_count(self, aweme_list: List[Dict[str, Any]], mode: str) -> List[Dict[str, Any]]:
number_config = self.config.get('number', {})
limit = number_config.get(mode, 0)
if limit > 0:
return aweme_list[:limit]
return aweme_list
async def _download_aweme_assets(
self,
aweme_data: Dict[str, Any],
author_name: str,
mode: Optional[str] = None,
) -> bool:
aweme_id = aweme_data.get('aweme_id')
if not aweme_id:
logger.error('Missing aweme_id in aweme data')
return False
desc = aweme_data.get('desc', 'no_title')
safe_title = sanitize_filename(desc)
save_dir = self.file_manager.get_save_path(
author_name=author_name,
mode=mode,
aweme_title=desc,
aweme_id=aweme_id,
folderstyle=self.config.get('folderstyle', True)
)
session = await self.api_client.get_session()
media_type = self._detect_media_type(aweme_data)
if media_type == 'video':
video_info = self._build_no_watermark_url(aweme_data)
if not video_info:
logger.error(f'No playable video URL found for aweme {aweme_id}')
return False
video_url, video_headers = video_info
video_path = save_dir / f"{safe_title}_{aweme_id}.mp4"
if not await self._download_with_retry(video_url, video_path, session, headers=video_headers):
return False
if self.config.get('cover'):
cover_url = self._extract_first_url(aweme_data.get('video', {}).get('cover'))
if cover_url:
cover_path = save_dir / f"{safe_title}_{aweme_id}_cover.jpg"
await self._download_with_retry(
cover_url,
cover_path,
session,
headers=self._download_headers(),
optional=True,
)
if self.config.get('music'):
music_url = self._extract_first_url(aweme_data.get('music', {}).get('play_url'))
if music_url:
music_path = save_dir / f"{safe_title}_{aweme_id}_music.mp3"
await self._download_with_retry(
music_url,
music_path,
session,
headers=self._download_headers(),
optional=True,
)
elif media_type == 'gallery':
image_urls = self._collect_image_urls(aweme_data)
if not image_urls:
logger.error(f'No images found for aweme {aweme_id}')
return False
for index, image_url in enumerate(image_urls, start=1):
suffix = Path(urlparse(image_url).path).suffix or '.jpg'
image_path = save_dir / f"{safe_title}_{aweme_id}_{index}{suffix}"
success = await self._download_with_retry(
image_url,
image_path,
session,
headers=self._download_headers(),
)
if not success:
logger.error(f'Failed downloading image {index} for aweme {aweme_id}')
return False
else:
logger.error(f"Unsupported media type for aweme {aweme_id}: {media_type}")
return False
if self.config.get('avatar'):
author = aweme_data.get('author', {})
avatar_url = self._extract_first_url(author.get('avatar_larger'))
if avatar_url:
avatar_path = save_dir / 'avatar.jpg'
await self._download_with_retry(
avatar_url,
avatar_path,
session,
headers=self._download_headers(),
optional=True,
)
if self.config.get('json'):
json_path = save_dir / f"{safe_title}_{aweme_id}_data.json"
await self.metadata_handler.save_metadata(aweme_data, json_path)
if self.database:
author = aweme_data.get('author', {})
metadata_json = json.dumps(aweme_data, ensure_ascii=False)
await self.database.add_aweme({
'aweme_id': aweme_id,
'aweme_type': media_type,
'title': desc,
'author_id': author.get('uid'),
'author_name': author.get('nickname', author_name),
'create_time': aweme_data.get('create_time'),
'file_path': str(save_dir),
'metadata': metadata_json,
})
logger.info(f"Downloaded {media_type}: {desc} ({aweme_id})")
return True
async def _download_with_retry(
self,
url: str,
save_path: Path,
session,
*,
headers: Optional[Dict[str, str]] = None,
optional: bool = False,
) -> bool:
async def _task():
success = await self.file_manager.download_file(url, save_path, session, headers=headers)
if not success:
raise RuntimeError(f'Download failed for {url}')
return True
try:
await self.retry_handler.execute_with_retry(_task)
return True
except Exception as error:
log_fn = logger.warning if optional else logger.error
log_fn(f"Download error for {save_path.name}: {error}")
return False
def _detect_media_type(self, aweme_data: Dict[str, Any]) -> str:
if aweme_data.get('image_post_info') or aweme_data.get('images'):
return 'gallery'
return 'video'
def _build_no_watermark_url(self, aweme_data: Dict[str, Any]) -> Optional[Tuple[str, Dict[str, str]]]:
video = aweme_data.get('video', {})
play_addr = video.get('play_addr', {})
url_candidates = [c for c in (play_addr.get('url_list') or []) if c]
url_candidates.sort(key=lambda u: 0 if 'watermark=0' in u else 1)
fallback_candidate: Optional[Tuple[str, Dict[str, str]]] = None
for candidate in url_candidates:
parsed = urlparse(candidate)
headers = self._download_headers()
if parsed.netloc.endswith('douyin.com'):
if 'X-Bogus=' not in candidate:
signed_url, ua = self.api_client.sign_url(candidate)
headers = self._download_headers(user_agent=ua)
return signed_url, headers
return candidate, headers
fallback_candidate = (candidate, headers)
if fallback_candidate:
return fallback_candidate
uri = play_addr.get('uri') or video.get('vid') or video.get('download_addr', {}).get('uri')
if uri:
params = {
'video_id': uri,
'ratio': '1080p',
'line': '0',
'is_play_url': '1',
'watermark': '0',
'source': 'PackSourceEnum_PUBLISH',
}
signed_url, ua = self.api_client.build_signed_path('/aweme/v1/play/', params)
return signed_url, self._download_headers(user_agent=ua)
return None
def _collect_image_urls(self, aweme_data: Dict[str, Any]) -> List[str]:
image_urls: List[str] = []
image_post = aweme_data.get('image_post_info', {})
images = image_post.get('images') or aweme_data.get('images') or []
for item in images:
url_list = item.get('url_list') if isinstance(item, dict) else None
if url_list:
image_urls.append(url_list[0])
return image_urls
@staticmethod
def _extract_first_url(source: Any) -> Optional[str]:
if isinstance(source, dict):
url_list = source.get('url_list')
if isinstance(url_list, list) and url_list:
return url_list[0]
elif isinstance(source, list) and source:
return source[0]
elif isinstance(source, str):
return source
return None

View File

@@ -0,0 +1,48 @@
from typing import Dict, Any, Optional
from core.downloader_base import BaseDownloader
from core.video_downloader import VideoDownloader
from core.user_downloader import UserDownloader
from config import ConfigLoader
from storage import Database, FileManager
from auth import CookieManager
from control import QueueManager, RateLimiter, RetryHandler
from core.api_client import DouyinAPIClient
from utils.logger import setup_logger
logger = setup_logger('DownloaderFactory')
class DownloaderFactory:
@staticmethod
def create(
url_type: str,
config: ConfigLoader,
api_client: DouyinAPIClient,
file_manager: FileManager,
cookie_manager: CookieManager,
database: Optional[Database] = None,
rate_limiter: Optional[RateLimiter] = None,
retry_handler: Optional[RetryHandler] = None,
queue_manager: Optional[QueueManager] = None,
) -> Optional[BaseDownloader]:
common_args = {
'config': config,
'api_client': api_client,
'file_manager': file_manager,
'cookie_manager': cookie_manager,
'database': database,
'rate_limiter': rate_limiter,
'retry_handler': retry_handler,
'queue_manager': queue_manager,
}
if url_type == 'video':
return VideoDownloader(**common_args)
elif url_type == 'user':
return UserDownloader(**common_args)
elif url_type == 'gallery':
return VideoDownloader(**common_args)
else:
logger.error(f"Unsupported URL type: {url_type}")
return None

View File

@@ -0,0 +1,79 @@
import re
from typing import Optional, Dict, Any
from urllib.parse import urlparse, parse_qs
from utils.validators import parse_url_type
from utils.logger import setup_logger
logger = setup_logger('URLParser')
class URLParser:
@staticmethod
def parse(url: str) -> Optional[Dict[str, Any]]:
url_type = parse_url_type(url)
if not url_type:
logger.error(f"Unsupported URL type: {url}")
return None
result = {
'original_url': url,
'type': url_type,
}
if url_type == 'video':
aweme_id = URLParser._extract_video_id(url)
if aweme_id:
result['aweme_id'] = aweme_id
elif url_type == 'user':
sec_uid = URLParser._extract_user_id(url)
if sec_uid:
result['sec_uid'] = sec_uid
elif url_type == 'collection':
mix_id = URLParser._extract_mix_id(url)
if mix_id:
result['mix_id'] = mix_id
elif url_type == 'gallery':
note_id = URLParser._extract_note_id(url)
if note_id:
result['note_id'] = note_id
result['aweme_id'] = note_id
return result
@staticmethod
def _extract_video_id(url: str) -> Optional[str]:
match = re.search(r'/video/(\d+)', url)
if match:
return match.group(1)
match = re.search(r'modal_id=(\d+)', url)
if match:
return match.group(1)
return None
@staticmethod
def _extract_user_id(url: str) -> Optional[str]:
match = re.search(r'/user/([A-Za-z0-9_-]+)', url)
if match:
return match.group(1)
return None
@staticmethod
def _extract_mix_id(url: str) -> Optional[str]:
match = re.search(r'/collection/(\d+)', url)
if not match:
match = re.search(r'/mix/(\d+)', url)
if match:
return match.group(1)
return None
@staticmethod
def _extract_note_id(url: str) -> Optional[str]:
match = re.search(r'/note/(\d+)', url)
if match:
return match.group(1)
return None

View File

@@ -0,0 +1,105 @@
from typing import Any, Dict
from core.downloader_base import BaseDownloader, DownloadResult
from utils.logger import setup_logger
logger = setup_logger('UserDownloader')
class UserDownloader(BaseDownloader):
async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult:
result = DownloadResult()
sec_uid = parsed_url.get('sec_uid')
if not sec_uid:
logger.error("No sec_uid found in parsed URL")
return result
user_info = await self.api_client.get_user_info(sec_uid)
if not user_info:
logger.error(f"Failed to get user info: {sec_uid}")
return result
modes = self.config.get('mode', ['post'])
for mode in modes:
if mode == 'post':
mode_result = await self._download_user_post(sec_uid, user_info)
result.total += mode_result.total
result.success += mode_result.success
result.failed += mode_result.failed
result.skipped += mode_result.skipped
return result
async def _download_user_post(self, sec_uid: str, user_info: Dict[str, Any]) -> DownloadResult:
result = DownloadResult()
aweme_list = []
max_cursor = 0
has_more = True
increase_enabled = self.config.get('increase', {}).get('post', False)
latest_time = None
if increase_enabled and self.database:
latest_time = await self.database.get_latest_aweme_time(user_info.get('uid'))
while has_more:
await self.rate_limiter.acquire()
data = await self.api_client.get_user_post(sec_uid, max_cursor)
if not data:
break
aweme_items = data.get('aweme_list', [])
if not aweme_items:
break
if increase_enabled and latest_time:
new_items = [a for a in aweme_items if a.get('create_time', 0) > latest_time]
aweme_list.extend(new_items)
if len(new_items) < len(aweme_items):
break
else:
aweme_list.extend(aweme_items)
has_more = data.get('has_more', False)
max_cursor = data.get('max_cursor', 0)
number_limit = self.config.get('number', {}).get('post', 0)
if number_limit > 0 and len(aweme_list) >= number_limit:
aweme_list = aweme_list[:number_limit]
break
aweme_list = self._filter_by_time(aweme_list)
aweme_list = self._limit_count(aweme_list, 'post')
result.total = len(aweme_list)
author_name = user_info.get('nickname', 'unknown')
async def _process_aweme(item: Dict[str, Any]):
aweme_id = item.get('aweme_id')
if not await self._should_download(aweme_id):
return {'status': 'skipped', 'aweme_id': aweme_id}
success = await self._download_aweme_assets(item, author_name, mode='post')
return {
'status': 'success' if success else 'failed',
'aweme_id': aweme_id,
}
download_results = await self.queue_manager.download_batch(_process_aweme, aweme_list)
for entry in download_results:
status = entry.get('status') if isinstance(entry, dict) else None
if status == 'success':
result.success += 1
elif status == 'failed':
result.failed += 1
elif status == 'skipped':
result.skipped += 1
else:
result.failed += 1
return result

View File

@@ -0,0 +1,44 @@
from typing import Any, Dict
from core.downloader_base import BaseDownloader, DownloadResult
from utils.logger import setup_logger
logger = setup_logger('VideoDownloader')
class VideoDownloader(BaseDownloader):
async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult:
result = DownloadResult()
aweme_id = parsed_url.get('aweme_id')
if not aweme_id:
logger.error("No aweme_id found in parsed URL")
return result
result.total = 1
if not await self._should_download(aweme_id):
logger.info(f"Video {aweme_id} already downloaded, skipping")
result.skipped += 1
return result
await self.rate_limiter.acquire()
aweme_data = await self.api_client.get_video_detail(aweme_id)
if not aweme_data:
logger.error(f"Failed to get video detail: {aweme_id}")
result.failed += 1
return result
success = await self._download_aweme(aweme_data)
if success:
result.success += 1
else:
result.failed += 1
return result
async def _download_aweme(self, aweme_data: Dict[str, Any]) -> bool:
author = aweme_data.get('author', {})
author_name = author.get('nickname', 'unknown')
return await self._download_aweme_assets(aweme_data, author_name)