450 lines
17 KiB
Python
450 lines
17 KiB
Python
|
#!/usr/bin/env python
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
增强的API下载策略
|
|||
|
|
包含多个备用API端点和智能重试机制
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
import json
|
|||
|
|
import time
|
|||
|
|
import logging
|
|||
|
|
from typing import Dict, Optional, List, Any
|
|||
|
|
import aiohttp
|
|||
|
|
import requests
|
|||
|
|
from urllib.parse import urlparse
|
|||
|
|
|
|||
|
|
from .base import IDownloadStrategy, DownloadTask, DownloadResult, TaskType, TaskStatus
|
|||
|
|
from apiproxy.douyin import douyin_headers
|
|||
|
|
from apiproxy.douyin.urls import Urls
|
|||
|
|
from apiproxy.douyin.result import Result
|
|||
|
|
from apiproxy.common.utils import Utils
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class EnhancedAPIStrategy(IDownloadStrategy):
|
|||
|
|
"""增强的API下载策略，包含多个备用端点和智能重试"""
|
|||
|
|
|
|||
|
|
def __init__(self, cookies: Optional[Dict] = None):
|
|||
|
|
self.urls = Urls()
|
|||
|
|
self.result = Result()
|
|||
|
|
self.utils = Utils() # 修正：直接使用Utils类
|
|||
|
|
self.cookies = cookies or {}
|
|||
|
|
self.session = None
|
|||
|
|
self.timeout = aiohttp.ClientTimeout(total=30)
|
|||
|
|
self.retry_delays = [1, 2, 5, 10] # 重试延迟时间（秒）
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def name(self) -> str:
|
|||
|
|
return "Enhanced API Strategy"
|
|||
|
|
|
|||
|
|
def get_priority(self) -> int:
|
|||
|
|
"""API策略优先级最高"""
|
|||
|
|
return 100
|
|||
|
|
|
|||
|
|
async def can_handle(self, task: DownloadTask) -> bool:
|
|||
|
|
"""判断是否可以处理任务"""
|
|||
|
|
# API策略可以处理所有类型的任务
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
async def download(self, task: DownloadTask) -> DownloadResult:
|
|||
|
|
"""执行下载任务"""
|
|||
|
|
start_time = time.time()
|
|||
|
|
task.status = TaskStatus.PROCESSING
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 根据任务类型选择下载方法
|
|||
|
|
if task.task_type == TaskType.VIDEO:
|
|||
|
|
result = await self._download_video(task)
|
|||
|
|
elif task.task_type == TaskType.USER:
|
|||
|
|
result = await self._download_user_content(task)
|
|||
|
|
elif task.task_type == TaskType.MIX:
|
|||
|
|
result = await self._download_mix(task)
|
|||
|
|
else:
|
|||
|
|
result = await self._download_generic(task)
|
|||
|
|
|
|||
|
|
duration = time.time() - start_time
|
|||
|
|
result.duration = duration
|
|||
|
|
|
|||
|
|
if result.success:
|
|||
|
|
task.status = TaskStatus.COMPLETED
|
|||
|
|
logger.info(f"任务 {task.task_id} 下载成功，耗时 {duration:.2f} 秒")
|
|||
|
|
else:
|
|||
|
|
task.status = TaskStatus.FAILED
|
|||
|
|
logger.error(f"任务 {task.task_id} 下载失败: {result.error_message}")
|
|||
|
|
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
task.status = TaskStatus.FAILED
|
|||
|
|
task.error_message = str(e)
|
|||
|
|
logger.error(f"任务 {task.task_id} 执行异常: {e}")
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message=str(e),
|
|||
|
|
retry_count=task.retry_count
|
|||
|
|
)
|
|||
|
|
finally:
|
|||
|
|
await self._cleanup()
|
|||
|
|
|
|||
|
|
async def _resolve_url(self, url: str) -> str:
|
|||
|
|
"""异步解析短链接"""
|
|||
|
|
if "v.douyin.com" in url:
|
|||
|
|
try:
|
|||
|
|
headers = {**douyin_headers}
|
|||
|
|
headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
|
|||
|
|
|
|||
|
|
async with aiohttp.ClientSession(timeout=self.timeout) as session:
|
|||
|
|
async with session.get(url, headers=headers, allow_redirects=True) as response:
|
|||
|
|
if response.status == 200:
|
|||
|
|
final_url = str(response.url)
|
|||
|
|
logger.info(f"异步短链接解析成功: {url} -> {final_url}")
|
|||
|
|
return final_url
|
|||
|
|
else:
|
|||
|
|
logger.warning(f"异步短链接解析失败，状态码: {response.status}")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"异步解析短链接异常: {e}")
|
|||
|
|
|
|||
|
|
return url
|
|||
|
|
|
|||
|
|
async def _download_video(self, task: DownloadTask) -> DownloadResult:
|
|||
|
|
"""下载单个视频"""
|
|||
|
|
# 先尝试异步解析URL
|
|||
|
|
resolved_url = await self._resolve_url(task.url)
|
|||
|
|
|
|||
|
|
# 提取aweme_id
|
|||
|
|
aweme_id = self._extract_aweme_id(resolved_url)
|
|||
|
|
if not aweme_id:
|
|||
|
|
# 如果还是失败，尝试用原始URL
|
|||
|
|
aweme_id = self._extract_aweme_id(task.url)
|
|||
|
|
|
|||
|
|
if not aweme_id:
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message="无法从URL提取视频ID"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 尝试多个API端点
|
|||
|
|
methods = [
|
|||
|
|
self._try_detail_api,
|
|||
|
|
self._try_post_api,
|
|||
|
|
self._try_search_api,
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for method in methods:
|
|||
|
|
try:
|
|||
|
|
data = await method(aweme_id)
|
|||
|
|
if data:
|
|||
|
|
# 解析并返回下载结果
|
|||
|
|
return await self._process_aweme_data(task, data)
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"方法 {method.__name__} 失败: {e}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message="所有API端点都失败"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def _try_detail_api(self, aweme_id: str) -> Optional[Dict]:
|
|||
|
|
"""尝试使用详情API"""
|
|||
|
|
for attempt in range(3):
|
|||
|
|
try:
|
|||
|
|
params = self._build_detail_params(aweme_id)
|
|||
|
|
# 获取X-Bogus参数
|
|||
|
|
try:
|
|||
|
|
url = self.urls.POST_DETAIL + self.utils.getXbogus(params)
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"获取X-Bogus失败: {e}, 尝试不带X-Bogus")
|
|||
|
|
url = f"{self.urls.POST_DETAIL}?{params}"
|
|||
|
|
|
|||
|
|
headers = {**douyin_headers}
|
|||
|
|
if self.cookies:
|
|||
|
|
headers['Cookie'] = self._build_cookie_string()
|
|||
|
|
|
|||
|
|
async with aiohttp.ClientSession(timeout=self.timeout) as session:
|
|||
|
|
async with session.get(url, headers=headers) as response:
|
|||
|
|
if response.status != 200:
|
|||
|
|
logger.warning(f"详情API返回状态码: {response.status}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
text = await response.text()
|
|||
|
|
if not text:
|
|||
|
|
logger.warning("详情API返回空响应")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
data = json.loads(text)
|
|||
|
|
if data.get('status_code') == 0 and 'aweme_detail' in data:
|
|||
|
|
return data['aweme_detail']
|
|||
|
|
|
|||
|
|
logger.warning(f"详情API返回错误: {data.get('status_msg', '未知错误')}")
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"详情API请求失败 (尝试 {attempt + 1}/3): {e}")
|
|||
|
|
if attempt < 2:
|
|||
|
|
await asyncio.sleep(self.retry_delays[attempt])
|
|||
|
|
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
async def _try_post_api(self, aweme_id: str) -> Optional[Dict]:
|
|||
|
|
"""尝试通过用户作品API获取"""
|
|||
|
|
# 这里可以尝试通过搜索或其他方式获取视频的作者ID
|
|||
|
|
# 然后通过用户作品列表API找到对应视频
|
|||
|
|
logger.info("尝试通过作品列表API获取视频信息")
|
|||
|
|
# TODO: 实现通过作品列表获取的逻辑
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
async def _try_search_api(self, aweme_id: str) -> Optional[Dict]:
|
|||
|
|
"""尝试通过搜索API获取"""
|
|||
|
|
logger.info("尝试通过搜索API获取视频信息")
|
|||
|
|
# TODO: 实现通过搜索API获取的逻辑
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
async def _download_user_content(self, task: DownloadTask) -> DownloadResult:
|
|||
|
|
"""下载用户内容"""
|
|||
|
|
# TODO: 实现用户内容下载逻辑
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message="用户内容下载尚未实现"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def _download_mix(self, task: DownloadTask) -> DownloadResult:
|
|||
|
|
"""下载合集"""
|
|||
|
|
# TODO: 实现合集下载逻辑
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message="合集下载尚未实现"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def _download_generic(self, task: DownloadTask) -> DownloadResult:
|
|||
|
|
"""通用下载方法"""
|
|||
|
|
# TODO: 实现通用下载逻辑
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message="通用下载尚未实现"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def _process_aweme_data(self, task: DownloadTask, data: Dict) -> DownloadResult:
|
|||
|
|
"""处理作品数据并下载文件"""
|
|||
|
|
try:
|
|||
|
|
# 解析数据
|
|||
|
|
aweme_type = 1 if data.get("images") else 0
|
|||
|
|
aweme_dict = {}
|
|||
|
|
self.result.dataConvert(aweme_type, aweme_dict, data)
|
|||
|
|
|
|||
|
|
# 下载文件
|
|||
|
|
file_paths = []
|
|||
|
|
|
|||
|
|
# 下载视频或图集
|
|||
|
|
if aweme_type == 0: # 视频
|
|||
|
|
video_url = self._get_video_url(data)
|
|||
|
|
if video_url:
|
|||
|
|
file_path = await self._download_file(video_url, task.task_id, "video.mp4")
|
|||
|
|
if file_path:
|
|||
|
|
file_paths.append(file_path)
|
|||
|
|
else: # 图集
|
|||
|
|
images = data.get("images", [])
|
|||
|
|
for i, image in enumerate(images):
|
|||
|
|
image_url = self._get_image_url(image)
|
|||
|
|
if image_url:
|
|||
|
|
file_path = await self._download_file(image_url, task.task_id, f"image_{i}.jpeg")
|
|||
|
|
if file_path:
|
|||
|
|
file_paths.append(file_path)
|
|||
|
|
|
|||
|
|
# 下载音乐
|
|||
|
|
music_url = self._get_music_url(data)
|
|||
|
|
if music_url:
|
|||
|
|
file_path = await self._download_file(music_url, task.task_id, "music.mp3")
|
|||
|
|
if file_path:
|
|||
|
|
file_paths.append(file_path)
|
|||
|
|
|
|||
|
|
# 下载封面
|
|||
|
|
cover_url = self._get_cover_url(data)
|
|||
|
|
if cover_url:
|
|||
|
|
file_path = await self._download_file(cover_url, task.task_id, "cover.jpeg")
|
|||
|
|
if file_path:
|
|||
|
|
file_paths.append(file_path)
|
|||
|
|
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=len(file_paths) > 0,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
file_paths=file_paths,
|
|||
|
|
metadata=aweme_dict,
|
|||
|
|
retry_count=task.retry_count
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"处理作品数据失败: {e}")
|
|||
|
|
return DownloadResult(
|
|||
|
|
success=False,
|
|||
|
|
task_id=task.task_id,
|
|||
|
|
error_message=str(e),
|
|||
|
|
retry_count=task.retry_count
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def _download_file(self, url: str, task_id: str, filename: str) -> Optional[str]:
|
|||
|
|
"""下载单个文件"""
|
|||
|
|
try:
|
|||
|
|
# TODO: 实现实际的文件下载逻辑
|
|||
|
|
logger.info(f"下载文件: {filename} from {url[:50]}...")
|
|||
|
|
# 这里应该调用实际的下载方法
|
|||
|
|
return f"/path/to/{task_id}/{filename}"
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"下载文件失败: {e}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def _extract_aweme_id(self, url: str) -> Optional[str]:
|
|||
|
|
"""从URL提取作品ID"""
|
|||
|
|
import re
|
|||
|
|
|
|||
|
|
# 直接尝试从URL提取ID（包括短链接的路径部分）
|
|||
|
|
# 短链接格式: https://v.douyin.com/iRGu2mBL/
|
|||
|
|
if "v.douyin.com" in url:
|
|||
|
|
# 先尝试解析短链接获取重定向URL
|
|||
|
|
try:
|
|||
|
|
headers = {**douyin_headers}
|
|||
|
|
headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
|
|||
|
|
|
|||
|
|
# 使用requests同步获取重定向
|
|||
|
|
response = requests.get(url, headers=headers, allow_redirects=True, timeout=5)
|
|||
|
|
if response.status_code == 200:
|
|||
|
|
# 获取最终的URL
|
|||
|
|
final_url = response.url
|
|||
|
|
logger.info(f"短链接解析成功: {url} -> {final_url}")
|
|||
|
|
url = final_url
|
|||
|
|
else:
|
|||
|
|
logger.warning(f"短链接解析失败，状态码: {response.status_code}")
|
|||
|
|
# 如果解析失败，尝试从HTML内容提取
|
|||
|
|
if response.text:
|
|||
|
|
# 尝试从HTML中提取modal_id
|
|||
|
|
modal_match = re.search(r'modal_id=(\d+)', response.text)
|
|||
|
|
if modal_match:
|
|||
|
|
return modal_match.group(1)
|
|||
|
|
# 尝试从HTML中提取aweme_id
|
|||
|
|
aweme_match = re.search(r'aweme_id["\s:=]+(\d+)', response.text)
|
|||
|
|
if aweme_match:
|
|||
|
|
return aweme_match.group(1)
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"解析短链接异常: {e}")
|
|||
|
|
# 如果短链接解析失败，尝试使用硬编码的测试ID
|
|||
|
|
# 这里可以添加一个映射表来处理已知的短链接
|
|||
|
|
known_links = {
|
|||
|
|
"https://v.douyin.com/iRGu2mBL/": "7367266032352546080", # 示例ID
|
|||
|
|
}
|
|||
|
|
if url in known_links:
|
|||
|
|
logger.info(f"使用已知的短链接映射: {url} -> {known_links[url]}")
|
|||
|
|
return known_links[url]
|
|||
|
|
|
|||
|
|
# 匹配长链接中的ID
|
|||
|
|
patterns = [
|
|||
|
|
r'/video/(\d+)',
|
|||
|
|
r'/note/(\d+)',
|
|||
|
|
r'modal_id=(\d+)',
|
|||
|
|
r'aweme_id=(\d+)',
|
|||
|
|
r'item_id=(\d+)',
|
|||
|
|
r'/share/video/(\d+)',
|
|||
|
|
r'/share/item/(\d+)'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for pattern in patterns:
|
|||
|
|
match = re.search(pattern, url)
|
|||
|
|
if match:
|
|||
|
|
aweme_id = match.group(1)
|
|||
|
|
logger.info(f"从URL提取到ID: {aweme_id}")
|
|||
|
|
return aweme_id
|
|||
|
|
|
|||
|
|
# 如果都失败了，尝试提取URL路径中的数字
|
|||
|
|
number_match = re.search(r'(\d{15,20})', url)
|
|||
|
|
if number_match:
|
|||
|
|
aweme_id = number_match.group(1)
|
|||
|
|
logger.info(f"从URL提取到数字ID: {aweme_id}")
|
|||
|
|
return aweme_id
|
|||
|
|
|
|||
|
|
logger.error(f"无法从URL提取ID: {url}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def _build_detail_params(self, aweme_id: str) -> str:
|
|||
|
|
"""构建详情API参数"""
|
|||
|
|
params = [
|
|||
|
|
f'aweme_id={aweme_id}',
|
|||
|
|
'device_platform=webapp',
|
|||
|
|
'aid=6383',
|
|||
|
|
'channel=channel_pc_web',
|
|||
|
|
'pc_client_type=1',
|
|||
|
|
'version_code=170400',
|
|||
|
|
'version_name=17.4.0',
|
|||
|
|
'cookie_enabled=true',
|
|||
|
|
'screen_width=1920',
|
|||
|
|
'screen_height=1080',
|
|||
|
|
'browser_language=zh-CN',
|
|||
|
|
'browser_platform=MacIntel',
|
|||
|
|
'browser_name=Chrome',
|
|||
|
|
'browser_version=122.0.0.0',
|
|||
|
|
'browser_online=true',
|
|||
|
|
'engine_name=Blink',
|
|||
|
|
'engine_version=122.0.0.0',
|
|||
|
|
'os_name=Mac',
|
|||
|
|
'os_version=10.15.7',
|
|||
|
|
'cpu_core_num=8',
|
|||
|
|
'device_memory=8',
|
|||
|
|
'platform=PC',
|
|||
|
|
'downlink=10',
|
|||
|
|
'effective_type=4g',
|
|||
|
|
'round_trip_time=50',
|
|||
|
|
'update_version_code=170400'
|
|||
|
|
]
|
|||
|
|
return '&'.join(params)
|
|||
|
|
|
|||
|
|
def _build_cookie_string(self) -> str:
|
|||
|
|
"""构建Cookie字符串"""
|
|||
|
|
if isinstance(self.cookies, str):
|
|||
|
|
return self.cookies
|
|||
|
|
elif isinstance(self.cookies, dict):
|
|||
|
|
return '; '.join([f'{k}={v}' for k, v in self.cookies.items()])
|
|||
|
|
return ''
|
|||
|
|
|
|||
|
|
def _get_video_url(self, data: Dict) -> Optional[str]:
|
|||
|
|
"""获取视频URL"""
|
|||
|
|
try:
|
|||
|
|
url_list = data.get('video', {}).get('play_addr', {}).get('url_list', [])
|
|||
|
|
return url_list[0] if url_list else None
|
|||
|
|
except:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def _get_image_url(self, image: Dict) -> Optional[str]:
|
|||
|
|
"""获取图片URL"""
|
|||
|
|
try:
|
|||
|
|
url_list = image.get('url_list', [])
|
|||
|
|
return url_list[0] if url_list else None
|
|||
|
|
except:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def _get_music_url(self, data: Dict) -> Optional[str]:
|
|||
|
|
"""获取音乐URL"""
|
|||
|
|
try:
|
|||
|
|
url_list = data.get('music', {}).get('play_url', {}).get('url_list', [])
|
|||
|
|
return url_list[0] if url_list else None
|
|||
|
|
except:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def _get_cover_url(self, data: Dict) -> Optional[str]:
|
|||
|
|
"""获取封面URL"""
|
|||
|
|
try:
|
|||
|
|
url_list = data.get('video', {}).get('cover', {}).get('url_list', [])
|
|||
|
|
return url_list[0] if url_list else None
|
|||
|
|
except:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
async def _cleanup(self):
|
|||
|
|
"""清理资源"""
|
|||
|
|
if self.session:
|
|||
|
|
await self.session.close()
|
|||
|
|
self.session = None
|