'commit'
This commit is contained in:
5
DouYin/dy-downloader/core/__init__.py
Normal file
5
DouYin/dy-downloader/core/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
from .api_client import DouyinAPIClient
|
||||
from .url_parser import URLParser
|
||||
from .downloader_factory import DownloaderFactory
|
||||
|
||||
__all__ = ['DouyinAPIClient', 'URLParser', 'DownloaderFactory']
|
||||
169
DouYin/dy-downloader/core/api_client.py
Normal file
169
DouYin/dy-downloader/core/api_client.py
Normal file
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import aiohttp
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from utils.logger import setup_logger
|
||||
from utils.xbogus import XBogus
|
||||
|
||||
logger = setup_logger('APIClient')
|
||||
|
||||
|
||||
class DouyinAPIClient:
|
||||
BASE_URL = 'https://www.douyin.com'
|
||||
|
||||
def __init__(self, cookies: Dict[str, str]):
|
||||
self.cookies = cookies or {}
|
||||
self._session: Optional[aiohttp.ClientSession] = None
|
||||
self.headers = {
|
||||
'User-Agent': (
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
|
||||
),
|
||||
'Referer': 'https://www.douyin.com/',
|
||||
'Accept': 'application/json',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'Connection': 'keep-alive',
|
||||
}
|
||||
self._signer = XBogus(self.headers['User-Agent'])
|
||||
|
||||
async def __aenter__(self) -> 'DouyinAPIClient':
|
||||
await self._ensure_session()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
await self.close()
|
||||
|
||||
async def _ensure_session(self):
|
||||
if self._session is None or self._session.closed:
|
||||
self._session = aiohttp.ClientSession(
|
||||
headers=self.headers,
|
||||
cookies=self.cookies,
|
||||
timeout=aiohttp.ClientTimeout(total=30),
|
||||
raise_for_status=False,
|
||||
)
|
||||
|
||||
async def close(self):
|
||||
if self._session and not self._session.closed:
|
||||
await self._session.close()
|
||||
|
||||
async def get_session(self) -> aiohttp.ClientSession:
|
||||
await self._ensure_session()
|
||||
assert self._session is not None
|
||||
return self._session
|
||||
|
||||
def _default_query(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'device_platform': 'webapp',
|
||||
'aid': '6383',
|
||||
'channel': 'channel_pc_web',
|
||||
'pc_client_type': '1',
|
||||
'version_code': '170400',
|
||||
'version_name': '17.4.0',
|
||||
'cookie_enabled': 'true',
|
||||
'screen_width': '1920',
|
||||
'screen_height': '1080',
|
||||
'browser_language': 'zh-CN',
|
||||
'browser_platform': 'Win32',
|
||||
'browser_name': 'Chrome',
|
||||
'browser_version': '123.0.0.0',
|
||||
'browser_online': 'true',
|
||||
'engine_name': 'Blink',
|
||||
'engine_version': '123.0.0.0',
|
||||
'os_name': 'Windows',
|
||||
'os_version': '10',
|
||||
'cpu_core_num': '8',
|
||||
'device_memory': '8',
|
||||
'platform': 'PC',
|
||||
'downlink': '10',
|
||||
'effective_type': '4g',
|
||||
'round_trip_time': '50',
|
||||
'msToken': self.cookies.get('msToken', ''),
|
||||
}
|
||||
|
||||
def sign_url(self, url: str) -> Tuple[str, str]:
|
||||
signed_url, _xbogus, ua = self._signer.build(url)
|
||||
return signed_url, ua
|
||||
|
||||
def build_signed_path(self, path: str, params: Dict[str, Any]) -> Tuple[str, str]:
|
||||
query = urlencode(params)
|
||||
url = f"{self.BASE_URL}{path}?{query}"
|
||||
return self.sign_url(url)
|
||||
|
||||
async def get_video_detail(self, aweme_id: str) -> Optional[Dict[str, Any]]:
|
||||
params = self._default_query()
|
||||
params.update({
|
||||
'aweme_id': aweme_id,
|
||||
'aid': '1128',
|
||||
})
|
||||
|
||||
await self._ensure_session()
|
||||
signed_url, ua = self.build_signed_path('/aweme/v1/web/aweme/detail/', params)
|
||||
|
||||
try:
|
||||
async with self._session.get(signed_url, headers={**self.headers, 'User-Agent': ua}) as response:
|
||||
if response.status == 200:
|
||||
data = await response.json(content_type=None)
|
||||
return data.get('aweme_detail')
|
||||
logger.error(f"Video detail request failed: {aweme_id}, status={response.status}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get video detail: {aweme_id}, error: {e}")
|
||||
|
||||
return None
|
||||
|
||||
async def get_user_post(self, sec_uid: str, max_cursor: int = 0, count: int = 20) -> Dict[str, Any]:
|
||||
params = self._default_query()
|
||||
params.update({
|
||||
'sec_user_id': sec_uid,
|
||||
'max_cursor': max_cursor,
|
||||
'count': count,
|
||||
'locate_query': 'false',
|
||||
'show_live_replay_strategy': '1',
|
||||
'need_time_list': '1',
|
||||
'time_list_query': '0',
|
||||
'whale_cut_token': '',
|
||||
'cut_version': '1',
|
||||
'publish_video_strategy_type': '2',
|
||||
})
|
||||
|
||||
await self._ensure_session()
|
||||
signed_url, ua = self.build_signed_path('/aweme/v1/web/aweme/post/', params)
|
||||
|
||||
try:
|
||||
async with self._session.get(signed_url, headers={**self.headers, 'User-Agent': ua}) as response:
|
||||
if response.status == 200:
|
||||
return await response.json(content_type=None)
|
||||
logger.error(f"User post request failed: {sec_uid}, status={response.status}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get user post: {sec_uid}, error: {e}")
|
||||
|
||||
return {}
|
||||
|
||||
async def get_user_info(self, sec_uid: str) -> Optional[Dict[str, Any]]:
|
||||
params = self._default_query()
|
||||
params.update({'sec_user_id': sec_uid})
|
||||
|
||||
await self._ensure_session()
|
||||
signed_url, ua = self.build_signed_path('/aweme/v1/web/user/profile/other/', params)
|
||||
|
||||
try:
|
||||
async with self._session.get(signed_url, headers={**self.headers, 'User-Agent': ua}) as response:
|
||||
if response.status == 200:
|
||||
data = await response.json(content_type=None)
|
||||
return data.get('user')
|
||||
logger.error(f"User info request failed: {sec_uid}, status={response.status}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get user info: {sec_uid}, error: {e}")
|
||||
|
||||
return None
|
||||
|
||||
async def resolve_short_url(self, short_url: str) -> Optional[str]:
|
||||
try:
|
||||
await self._ensure_session()
|
||||
async with self._session.get(short_url, allow_redirects=True) as response:
|
||||
return str(response.url)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to resolve short URL: {short_url}, error: {e}")
|
||||
return None
|
||||
309
DouYin/dy-downloader/core/downloader_base.py
Normal file
309
DouYin/dy-downloader/core/downloader_base.py
Normal file
@@ -0,0 +1,309 @@
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from config import ConfigLoader
|
||||
from storage import Database, FileManager, MetadataHandler
|
||||
from auth import CookieManager
|
||||
from control import QueueManager, RateLimiter, RetryHandler
|
||||
from core.api_client import DouyinAPIClient
|
||||
from utils.logger import setup_logger
|
||||
from utils.validators import sanitize_filename
|
||||
|
||||
logger = setup_logger('BaseDownloader')
|
||||
|
||||
|
||||
class DownloadResult:
|
||||
def __init__(self):
|
||||
self.total = 0
|
||||
self.success = 0
|
||||
self.failed = 0
|
||||
self.skipped = 0
|
||||
|
||||
def __str__(self):
|
||||
return f"Total: {self.total}, Success: {self.success}, Failed: {self.failed}, Skipped: {self.skipped}"
|
||||
|
||||
|
||||
class BaseDownloader(ABC):
|
||||
def __init__(
|
||||
self,
|
||||
config: ConfigLoader,
|
||||
api_client: DouyinAPIClient,
|
||||
file_manager: FileManager,
|
||||
cookie_manager: CookieManager,
|
||||
database: Optional[Database] = None,
|
||||
rate_limiter: Optional[RateLimiter] = None,
|
||||
retry_handler: Optional[RetryHandler] = None,
|
||||
queue_manager: Optional[QueueManager] = None,
|
||||
):
|
||||
self.config = config
|
||||
self.api_client = api_client
|
||||
self.file_manager = file_manager
|
||||
self.cookie_manager = cookie_manager
|
||||
self.database = database
|
||||
self.rate_limiter = rate_limiter or RateLimiter()
|
||||
self.retry_handler = retry_handler or RetryHandler()
|
||||
thread_count = int(self.config.get('thread', 5) or 5)
|
||||
self.queue_manager = queue_manager or QueueManager(max_workers=thread_count)
|
||||
self.metadata_handler = MetadataHandler()
|
||||
|
||||
def _download_headers(self, user_agent: Optional[str] = None) -> Dict[str, str]:
|
||||
headers = {
|
||||
'Referer': f'{self.api_client.BASE_URL}/',
|
||||
'Origin': self.api_client.BASE_URL,
|
||||
'Accept': '*/*',
|
||||
}
|
||||
|
||||
headers['User-Agent'] = user_agent or self.api_client.headers.get('User-Agent', '')
|
||||
return headers
|
||||
|
||||
@abstractmethod
|
||||
async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult:
|
||||
pass
|
||||
|
||||
async def _should_download(self, aweme_id: str) -> bool:
|
||||
if self.database:
|
||||
return not await self.database.is_downloaded(aweme_id)
|
||||
return True
|
||||
|
||||
def _filter_by_time(self, aweme_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
start_time = self.config.get('start_time')
|
||||
end_time = self.config.get('end_time')
|
||||
|
||||
if not start_time and not end_time:
|
||||
return aweme_list
|
||||
|
||||
filtered: List[Dict[str, Any]] = []
|
||||
for aweme in aweme_list:
|
||||
create_time = aweme.get('create_time', 0)
|
||||
|
||||
if start_time:
|
||||
from datetime import datetime
|
||||
start_ts = int(datetime.strptime(start_time, '%Y-%m-%d').timestamp())
|
||||
if create_time < start_ts:
|
||||
continue
|
||||
|
||||
if end_time:
|
||||
from datetime import datetime
|
||||
end_ts = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp())
|
||||
if create_time > end_ts:
|
||||
continue
|
||||
|
||||
filtered.append(aweme)
|
||||
|
||||
return filtered
|
||||
|
||||
def _limit_count(self, aweme_list: List[Dict[str, Any]], mode: str) -> List[Dict[str, Any]]:
|
||||
number_config = self.config.get('number', {})
|
||||
limit = number_config.get(mode, 0)
|
||||
|
||||
if limit > 0:
|
||||
return aweme_list[:limit]
|
||||
return aweme_list
|
||||
|
||||
async def _download_aweme_assets(
|
||||
self,
|
||||
aweme_data: Dict[str, Any],
|
||||
author_name: str,
|
||||
mode: Optional[str] = None,
|
||||
) -> bool:
|
||||
aweme_id = aweme_data.get('aweme_id')
|
||||
if not aweme_id:
|
||||
logger.error('Missing aweme_id in aweme data')
|
||||
return False
|
||||
|
||||
desc = aweme_data.get('desc', 'no_title')
|
||||
safe_title = sanitize_filename(desc)
|
||||
|
||||
save_dir = self.file_manager.get_save_path(
|
||||
author_name=author_name,
|
||||
mode=mode,
|
||||
aweme_title=desc,
|
||||
aweme_id=aweme_id,
|
||||
folderstyle=self.config.get('folderstyle', True)
|
||||
)
|
||||
|
||||
session = await self.api_client.get_session()
|
||||
|
||||
media_type = self._detect_media_type(aweme_data)
|
||||
if media_type == 'video':
|
||||
video_info = self._build_no_watermark_url(aweme_data)
|
||||
if not video_info:
|
||||
logger.error(f'No playable video URL found for aweme {aweme_id}')
|
||||
return False
|
||||
|
||||
video_url, video_headers = video_info
|
||||
video_path = save_dir / f"{safe_title}_{aweme_id}.mp4"
|
||||
if not await self._download_with_retry(video_url, video_path, session, headers=video_headers):
|
||||
return False
|
||||
|
||||
if self.config.get('cover'):
|
||||
cover_url = self._extract_first_url(aweme_data.get('video', {}).get('cover'))
|
||||
if cover_url:
|
||||
cover_path = save_dir / f"{safe_title}_{aweme_id}_cover.jpg"
|
||||
await self._download_with_retry(
|
||||
cover_url,
|
||||
cover_path,
|
||||
session,
|
||||
headers=self._download_headers(),
|
||||
optional=True,
|
||||
)
|
||||
|
||||
if self.config.get('music'):
|
||||
music_url = self._extract_first_url(aweme_data.get('music', {}).get('play_url'))
|
||||
if music_url:
|
||||
music_path = save_dir / f"{safe_title}_{aweme_id}_music.mp3"
|
||||
await self._download_with_retry(
|
||||
music_url,
|
||||
music_path,
|
||||
session,
|
||||
headers=self._download_headers(),
|
||||
optional=True,
|
||||
)
|
||||
|
||||
elif media_type == 'gallery':
|
||||
image_urls = self._collect_image_urls(aweme_data)
|
||||
if not image_urls:
|
||||
logger.error(f'No images found for aweme {aweme_id}')
|
||||
return False
|
||||
|
||||
for index, image_url in enumerate(image_urls, start=1):
|
||||
suffix = Path(urlparse(image_url).path).suffix or '.jpg'
|
||||
image_path = save_dir / f"{safe_title}_{aweme_id}_{index}{suffix}"
|
||||
success = await self._download_with_retry(
|
||||
image_url,
|
||||
image_path,
|
||||
session,
|
||||
headers=self._download_headers(),
|
||||
)
|
||||
if not success:
|
||||
logger.error(f'Failed downloading image {index} for aweme {aweme_id}')
|
||||
return False
|
||||
else:
|
||||
logger.error(f"Unsupported media type for aweme {aweme_id}: {media_type}")
|
||||
return False
|
||||
|
||||
if self.config.get('avatar'):
|
||||
author = aweme_data.get('author', {})
|
||||
avatar_url = self._extract_first_url(author.get('avatar_larger'))
|
||||
if avatar_url:
|
||||
avatar_path = save_dir / 'avatar.jpg'
|
||||
await self._download_with_retry(
|
||||
avatar_url,
|
||||
avatar_path,
|
||||
session,
|
||||
headers=self._download_headers(),
|
||||
optional=True,
|
||||
)
|
||||
|
||||
if self.config.get('json'):
|
||||
json_path = save_dir / f"{safe_title}_{aweme_id}_data.json"
|
||||
await self.metadata_handler.save_metadata(aweme_data, json_path)
|
||||
|
||||
if self.database:
|
||||
author = aweme_data.get('author', {})
|
||||
metadata_json = json.dumps(aweme_data, ensure_ascii=False)
|
||||
await self.database.add_aweme({
|
||||
'aweme_id': aweme_id,
|
||||
'aweme_type': media_type,
|
||||
'title': desc,
|
||||
'author_id': author.get('uid'),
|
||||
'author_name': author.get('nickname', author_name),
|
||||
'create_time': aweme_data.get('create_time'),
|
||||
'file_path': str(save_dir),
|
||||
'metadata': metadata_json,
|
||||
})
|
||||
|
||||
logger.info(f"Downloaded {media_type}: {desc} ({aweme_id})")
|
||||
return True
|
||||
|
||||
async def _download_with_retry(
|
||||
self,
|
||||
url: str,
|
||||
save_path: Path,
|
||||
session,
|
||||
*,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
optional: bool = False,
|
||||
) -> bool:
|
||||
async def _task():
|
||||
success = await self.file_manager.download_file(url, save_path, session, headers=headers)
|
||||
if not success:
|
||||
raise RuntimeError(f'Download failed for {url}')
|
||||
return True
|
||||
|
||||
try:
|
||||
await self.retry_handler.execute_with_retry(_task)
|
||||
return True
|
||||
except Exception as error:
|
||||
log_fn = logger.warning if optional else logger.error
|
||||
log_fn(f"Download error for {save_path.name}: {error}")
|
||||
return False
|
||||
|
||||
def _detect_media_type(self, aweme_data: Dict[str, Any]) -> str:
|
||||
if aweme_data.get('image_post_info') or aweme_data.get('images'):
|
||||
return 'gallery'
|
||||
return 'video'
|
||||
|
||||
def _build_no_watermark_url(self, aweme_data: Dict[str, Any]) -> Optional[Tuple[str, Dict[str, str]]]:
|
||||
video = aweme_data.get('video', {})
|
||||
play_addr = video.get('play_addr', {})
|
||||
url_candidates = [c for c in (play_addr.get('url_list') or []) if c]
|
||||
url_candidates.sort(key=lambda u: 0 if 'watermark=0' in u else 1)
|
||||
|
||||
fallback_candidate: Optional[Tuple[str, Dict[str, str]]] = None
|
||||
|
||||
for candidate in url_candidates:
|
||||
parsed = urlparse(candidate)
|
||||
headers = self._download_headers()
|
||||
|
||||
if parsed.netloc.endswith('douyin.com'):
|
||||
if 'X-Bogus=' not in candidate:
|
||||
signed_url, ua = self.api_client.sign_url(candidate)
|
||||
headers = self._download_headers(user_agent=ua)
|
||||
return signed_url, headers
|
||||
return candidate, headers
|
||||
|
||||
fallback_candidate = (candidate, headers)
|
||||
|
||||
if fallback_candidate:
|
||||
return fallback_candidate
|
||||
|
||||
uri = play_addr.get('uri') or video.get('vid') or video.get('download_addr', {}).get('uri')
|
||||
if uri:
|
||||
params = {
|
||||
'video_id': uri,
|
||||
'ratio': '1080p',
|
||||
'line': '0',
|
||||
'is_play_url': '1',
|
||||
'watermark': '0',
|
||||
'source': 'PackSourceEnum_PUBLISH',
|
||||
}
|
||||
signed_url, ua = self.api_client.build_signed_path('/aweme/v1/play/', params)
|
||||
return signed_url, self._download_headers(user_agent=ua)
|
||||
|
||||
return None
|
||||
|
||||
def _collect_image_urls(self, aweme_data: Dict[str, Any]) -> List[str]:
|
||||
image_urls: List[str] = []
|
||||
image_post = aweme_data.get('image_post_info', {})
|
||||
images = image_post.get('images') or aweme_data.get('images') or []
|
||||
for item in images:
|
||||
url_list = item.get('url_list') if isinstance(item, dict) else None
|
||||
if url_list:
|
||||
image_urls.append(url_list[0])
|
||||
return image_urls
|
||||
|
||||
@staticmethod
|
||||
def _extract_first_url(source: Any) -> Optional[str]:
|
||||
if isinstance(source, dict):
|
||||
url_list = source.get('url_list')
|
||||
if isinstance(url_list, list) and url_list:
|
||||
return url_list[0]
|
||||
elif isinstance(source, list) and source:
|
||||
return source[0]
|
||||
elif isinstance(source, str):
|
||||
return source
|
||||
return None
|
||||
48
DouYin/dy-downloader/core/downloader_factory.py
Normal file
48
DouYin/dy-downloader/core/downloader_factory.py
Normal file
@@ -0,0 +1,48 @@
|
||||
from typing import Dict, Any, Optional
|
||||
from core.downloader_base import BaseDownloader
|
||||
from core.video_downloader import VideoDownloader
|
||||
from core.user_downloader import UserDownloader
|
||||
from config import ConfigLoader
|
||||
from storage import Database, FileManager
|
||||
from auth import CookieManager
|
||||
from control import QueueManager, RateLimiter, RetryHandler
|
||||
from core.api_client import DouyinAPIClient
|
||||
from utils.logger import setup_logger
|
||||
|
||||
logger = setup_logger('DownloaderFactory')
|
||||
|
||||
|
||||
class DownloaderFactory:
|
||||
@staticmethod
|
||||
def create(
|
||||
url_type: str,
|
||||
config: ConfigLoader,
|
||||
api_client: DouyinAPIClient,
|
||||
file_manager: FileManager,
|
||||
cookie_manager: CookieManager,
|
||||
database: Optional[Database] = None,
|
||||
rate_limiter: Optional[RateLimiter] = None,
|
||||
retry_handler: Optional[RetryHandler] = None,
|
||||
queue_manager: Optional[QueueManager] = None,
|
||||
) -> Optional[BaseDownloader]:
|
||||
|
||||
common_args = {
|
||||
'config': config,
|
||||
'api_client': api_client,
|
||||
'file_manager': file_manager,
|
||||
'cookie_manager': cookie_manager,
|
||||
'database': database,
|
||||
'rate_limiter': rate_limiter,
|
||||
'retry_handler': retry_handler,
|
||||
'queue_manager': queue_manager,
|
||||
}
|
||||
|
||||
if url_type == 'video':
|
||||
return VideoDownloader(**common_args)
|
||||
elif url_type == 'user':
|
||||
return UserDownloader(**common_args)
|
||||
elif url_type == 'gallery':
|
||||
return VideoDownloader(**common_args)
|
||||
else:
|
||||
logger.error(f"Unsupported URL type: {url_type}")
|
||||
return None
|
||||
79
DouYin/dy-downloader/core/url_parser.py
Normal file
79
DouYin/dy-downloader/core/url_parser.py
Normal file
@@ -0,0 +1,79 @@
|
||||
import re
|
||||
from typing import Optional, Dict, Any
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from utils.validators import parse_url_type
|
||||
from utils.logger import setup_logger
|
||||
|
||||
logger = setup_logger('URLParser')
|
||||
|
||||
|
||||
class URLParser:
|
||||
@staticmethod
|
||||
def parse(url: str) -> Optional[Dict[str, Any]]:
|
||||
url_type = parse_url_type(url)
|
||||
if not url_type:
|
||||
logger.error(f"Unsupported URL type: {url}")
|
||||
return None
|
||||
|
||||
result = {
|
||||
'original_url': url,
|
||||
'type': url_type,
|
||||
}
|
||||
|
||||
if url_type == 'video':
|
||||
aweme_id = URLParser._extract_video_id(url)
|
||||
if aweme_id:
|
||||
result['aweme_id'] = aweme_id
|
||||
|
||||
elif url_type == 'user':
|
||||
sec_uid = URLParser._extract_user_id(url)
|
||||
if sec_uid:
|
||||
result['sec_uid'] = sec_uid
|
||||
|
||||
elif url_type == 'collection':
|
||||
mix_id = URLParser._extract_mix_id(url)
|
||||
if mix_id:
|
||||
result['mix_id'] = mix_id
|
||||
|
||||
elif url_type == 'gallery':
|
||||
note_id = URLParser._extract_note_id(url)
|
||||
if note_id:
|
||||
result['note_id'] = note_id
|
||||
result['aweme_id'] = note_id
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _extract_video_id(url: str) -> Optional[str]:
|
||||
match = re.search(r'/video/(\d+)', url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
match = re.search(r'modal_id=(\d+)', url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_user_id(url: str) -> Optional[str]:
|
||||
match = re.search(r'/user/([A-Za-z0-9_-]+)', url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_mix_id(url: str) -> Optional[str]:
|
||||
match = re.search(r'/collection/(\d+)', url)
|
||||
if not match:
|
||||
match = re.search(r'/mix/(\d+)', url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_note_id(url: str) -> Optional[str]:
|
||||
match = re.search(r'/note/(\d+)', url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
105
DouYin/dy-downloader/core/user_downloader.py
Normal file
105
DouYin/dy-downloader/core/user_downloader.py
Normal file
@@ -0,0 +1,105 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
from core.downloader_base import BaseDownloader, DownloadResult
|
||||
from utils.logger import setup_logger
|
||||
|
||||
logger = setup_logger('UserDownloader')
|
||||
|
||||
|
||||
class UserDownloader(BaseDownloader):
|
||||
async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult:
|
||||
result = DownloadResult()
|
||||
|
||||
sec_uid = parsed_url.get('sec_uid')
|
||||
if not sec_uid:
|
||||
logger.error("No sec_uid found in parsed URL")
|
||||
return result
|
||||
|
||||
user_info = await self.api_client.get_user_info(sec_uid)
|
||||
if not user_info:
|
||||
logger.error(f"Failed to get user info: {sec_uid}")
|
||||
return result
|
||||
|
||||
modes = self.config.get('mode', ['post'])
|
||||
|
||||
for mode in modes:
|
||||
if mode == 'post':
|
||||
mode_result = await self._download_user_post(sec_uid, user_info)
|
||||
result.total += mode_result.total
|
||||
result.success += mode_result.success
|
||||
result.failed += mode_result.failed
|
||||
result.skipped += mode_result.skipped
|
||||
|
||||
return result
|
||||
|
||||
async def _download_user_post(self, sec_uid: str, user_info: Dict[str, Any]) -> DownloadResult:
|
||||
result = DownloadResult()
|
||||
aweme_list = []
|
||||
max_cursor = 0
|
||||
has_more = True
|
||||
|
||||
increase_enabled = self.config.get('increase', {}).get('post', False)
|
||||
latest_time = None
|
||||
|
||||
if increase_enabled and self.database:
|
||||
latest_time = await self.database.get_latest_aweme_time(user_info.get('uid'))
|
||||
|
||||
while has_more:
|
||||
await self.rate_limiter.acquire()
|
||||
|
||||
data = await self.api_client.get_user_post(sec_uid, max_cursor)
|
||||
if not data:
|
||||
break
|
||||
|
||||
aweme_items = data.get('aweme_list', [])
|
||||
if not aweme_items:
|
||||
break
|
||||
|
||||
if increase_enabled and latest_time:
|
||||
new_items = [a for a in aweme_items if a.get('create_time', 0) > latest_time]
|
||||
aweme_list.extend(new_items)
|
||||
if len(new_items) < len(aweme_items):
|
||||
break
|
||||
else:
|
||||
aweme_list.extend(aweme_items)
|
||||
|
||||
has_more = data.get('has_more', False)
|
||||
max_cursor = data.get('max_cursor', 0)
|
||||
|
||||
number_limit = self.config.get('number', {}).get('post', 0)
|
||||
if number_limit > 0 and len(aweme_list) >= number_limit:
|
||||
aweme_list = aweme_list[:number_limit]
|
||||
break
|
||||
|
||||
aweme_list = self._filter_by_time(aweme_list)
|
||||
aweme_list = self._limit_count(aweme_list, 'post')
|
||||
|
||||
result.total = len(aweme_list)
|
||||
|
||||
author_name = user_info.get('nickname', 'unknown')
|
||||
|
||||
async def _process_aweme(item: Dict[str, Any]):
|
||||
aweme_id = item.get('aweme_id')
|
||||
if not await self._should_download(aweme_id):
|
||||
return {'status': 'skipped', 'aweme_id': aweme_id}
|
||||
|
||||
success = await self._download_aweme_assets(item, author_name, mode='post')
|
||||
return {
|
||||
'status': 'success' if success else 'failed',
|
||||
'aweme_id': aweme_id,
|
||||
}
|
||||
|
||||
download_results = await self.queue_manager.download_batch(_process_aweme, aweme_list)
|
||||
|
||||
for entry in download_results:
|
||||
status = entry.get('status') if isinstance(entry, dict) else None
|
||||
if status == 'success':
|
||||
result.success += 1
|
||||
elif status == 'failed':
|
||||
result.failed += 1
|
||||
elif status == 'skipped':
|
||||
result.skipped += 1
|
||||
else:
|
||||
result.failed += 1
|
||||
|
||||
return result
|
||||
44
DouYin/dy-downloader/core/video_downloader.py
Normal file
44
DouYin/dy-downloader/core/video_downloader.py
Normal file
@@ -0,0 +1,44 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
from core.downloader_base import BaseDownloader, DownloadResult
|
||||
from utils.logger import setup_logger
|
||||
|
||||
logger = setup_logger('VideoDownloader')
|
||||
|
||||
|
||||
class VideoDownloader(BaseDownloader):
|
||||
async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult:
|
||||
result = DownloadResult()
|
||||
|
||||
aweme_id = parsed_url.get('aweme_id')
|
||||
if not aweme_id:
|
||||
logger.error("No aweme_id found in parsed URL")
|
||||
return result
|
||||
|
||||
result.total = 1
|
||||
|
||||
if not await self._should_download(aweme_id):
|
||||
logger.info(f"Video {aweme_id} already downloaded, skipping")
|
||||
result.skipped += 1
|
||||
return result
|
||||
|
||||
await self.rate_limiter.acquire()
|
||||
|
||||
aweme_data = await self.api_client.get_video_detail(aweme_id)
|
||||
if not aweme_data:
|
||||
logger.error(f"Failed to get video detail: {aweme_id}")
|
||||
result.failed += 1
|
||||
return result
|
||||
|
||||
success = await self._download_aweme(aweme_data)
|
||||
if success:
|
||||
result.success += 1
|
||||
else:
|
||||
result.failed += 1
|
||||
|
||||
return result
|
||||
|
||||
async def _download_aweme(self, aweme_data: Dict[str, Any]) -> bool:
|
||||
author = aweme_data.get('author', {})
|
||||
author_name = author.get('nickname', 'unknown')
|
||||
return await self._download_aweme_assets(aweme_data, author_name)
|
||||
Reference in New Issue
Block a user