'commit'
This commit is contained in:
HuangHai
64
Tools/T3_GetCookie.py
Normal file
64
Tools/T3_GetCookie.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# pip install selenium
|
||||
# 将项目根目录添加到系统路径，确保可以导入 Util 模块
|
||||
# 注意：PaChongGaoDeKit 位于 Util 目录下
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.append(os.path.join(current_dir, '..'))
|
||||
|
||||
from Util.PaChongGaoDeKit import PaChongGaoDeKit
|
||||
|
||||
def run_login_tool(force=False):
|
||||
"""
|
||||
运行高德地图 Cookie 获取工具。
|
||||
:param force: 如果为 True，则跳过 'Cookie 已存在' 的询问，直接开始流程。
|
||||
:return: Boolean, True 表示获取成功, False 表示失败或取消。
|
||||
"""
|
||||
logger.info("正在启动高德地图 Cookie 获取工具,使用手机号18686619970+验证码登录即可！")
|
||||
|
||||
# 默认将 Cookie 保存到项目根目录下的 Json/amap_cookies.json
|
||||
cookie_path = os.path.join(current_dir, '../Json/amap_cookies.json')
|
||||
|
||||
# 检查 Cookie 文件是否已存在
|
||||
if os.path.exists(cookie_path) and not force:
|
||||
logger.info(f"提示：检测到已存在 Cookie 文件: {os.path.abspath(cookie_path)}")
|
||||
logger.info("如果您本地的 Cookie 文件没有过期且可以正常使用，则不需要重新生成。")
|
||||
|
||||
user_choice = input("是否仍要继续运行并重新获取 Cookie？(y/n): ").strip().lower()
|
||||
if user_choice != 'y':
|
||||
logger.info("用户取消操作，程序退出。")
|
||||
return False
|
||||
|
||||
logger.info("\n" + "-"*60)
|
||||
logger.info("即将开始获取 Cookie，请按照以下步骤操作：")
|
||||
logger.info("1. 程序将自动打开一个 Chrome 浏览器窗口。")
|
||||
logger.info("2. 请在该窗口中完成高德地图账号登录（扫码或账号密码）。")
|
||||
logger.info("3. 登录成功后，保持页面处于控制台或个人中心状态。")
|
||||
logger.info("4. 回到此终端窗口，按下回车键，程序将保存 Cookie 并自动关闭浏览器。")
|
||||
logger.info("-"*60 + "\n")
|
||||
|
||||
# 初始化工具类，强制开启非无头模式（显示浏览器窗口）以便手动登录
|
||||
# 确保 Json 目录存在
|
||||
os.makedirs(os.path.dirname(cookie_path), exist_ok=True)
|
||||
|
||||
kit = PaChongGaoDeKit(headless=False, cookie_file=cookie_path)
|
||||
|
||||
try:
|
||||
# 执行登录并保存流程
|
||||
return kit.login_and_save_cookies()
|
||||
except Exception as e:
|
||||
logger.error(f"执行过程中发生意外错误: {e}")
|
||||
return False
|
||||
finally:
|
||||
# 关闭浏览器资源
|
||||
kit.close()
|
||||
logger.info("工具运行结束。")
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_login_tool()
|
||||
145
Tools/T4_Coord.py
Normal file
145
Tools/T4_Coord.py
Normal file
@@ -0,0 +1,145 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from sqlalchemy import text
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# python -m pip install webdriver-manager selenium
|
||||
# https://googlechromelabs.github.io/chrome-for-testing/
|
||||
# https://storage.googleapis.com/chrome-for-testing-public/143.0.7499.192/win64/chromedriver-win64.zip
|
||||
# update t_station_profile_scd set coord_x=null,coord_y=null where id='95783932-1d32-48e4-9a38-6a6696a6a423'
|
||||
# 将项目根目录添加到系统路径，确保可以导入 Util, DbKit 和 Config 模块
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from Util.PaChongGaoDeKit import PaChongGaoDeKit
|
||||
from DbKit.Db import Db
|
||||
from Config.Config import DB_URL
|
||||
|
||||
class CookieInvalidError(Exception):
|
||||
"""自定义异常：表示 Cookie 无效或过期"""
|
||||
pass
|
||||
|
||||
async def main(check_cookie_first=True):
|
||||
"""
|
||||
坐标回填任务：自动扫描数据库中缺失坐标的场站地址，并通过高德地图补全。
|
||||
:param check_cookie_first: 是否在开始前验证 Cookie 有效性
|
||||
"""
|
||||
logger.info("正在启动 T4_Coord: 高德地图坐标回填任务...")
|
||||
|
||||
# 初始化数据库连接
|
||||
db = Db(db_url=DB_URL)
|
||||
await db.init_db()
|
||||
|
||||
# 初始化高德地图工具类（使用无头模式）
|
||||
# 显式指定 Cookie 路径，确保跨目录运行时能找到文件
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
cookie_path = os.path.join(project_root, 'Json', 'amap_cookies.json')
|
||||
|
||||
if not os.path.exists(cookie_path):
|
||||
logger.error(f"错误：未找到 Cookie 文件：{cookie_path}")
|
||||
# 如果要求检查且文件不存在，直接抛出异常以便上层处理
|
||||
if check_cookie_first:
|
||||
await db.close()
|
||||
raise CookieInvalidError("Cookie file not found")
|
||||
logger.error("请先运行 Tools/T3_GetAmapCookie.py 生成有效 Cookie。")
|
||||
return
|
||||
|
||||
kit = PaChongGaoDeKit(cookie_file=cookie_path, headless=False)
|
||||
|
||||
try:
|
||||
# 0. Cookie 有效性探针
|
||||
if check_cookie_first:
|
||||
logger.info("正在验证高德地图 Cookie 有效性...")
|
||||
loop = asyncio.get_event_loop()
|
||||
# 使用一个肯定存在的地址进行测试
|
||||
test_addr = "北京市天安门"
|
||||
t_lng, t_lat = await loop.run_in_executor(None, kit.get_coordinate, test_addr)
|
||||
|
||||
if not t_lng:
|
||||
logger.warning(f"Cookie 验证失败：无法获取测试地址 '{test_addr}' 的坐标。")
|
||||
logger.warning("判定 Cookie 已失效或过期。")
|
||||
raise CookieInvalidError("Cookie invalid or expired")
|
||||
else:
|
||||
logger.info(f"Cookie 验证通过 (测试坐标: {t_lng}, {t_lat})。")
|
||||
|
||||
async with await db.get_session() as session:
|
||||
# 1. 查询所有地址不为空但缺失坐标的唯一地址
|
||||
sql_missing = """
|
||||
SELECT DISTINCT address
|
||||
FROM t_station_profile_scd
|
||||
WHERE address IS NOT NULL
|
||||
AND address != ''
|
||||
AND (coord_x IS NULL OR coord_y IS NULL)
|
||||
"""
|
||||
result = await session.execute(text(sql_missing))
|
||||
missing_addresses = [row[0] for row in result.fetchall()]
|
||||
|
||||
logger.info(f"共发现 {len(missing_addresses)} 个缺失坐标的唯一地址。")
|
||||
|
||||
for address in missing_addresses:
|
||||
logger.info(f"\n正在处理地址: {address}")
|
||||
|
||||
# 2. 检查数据库中其他记录是否已经存有该地址的坐标（去重复查询）
|
||||
# 这样可以最大程度减少对高德地图接口的调用
|
||||
sql_check = """
|
||||
SELECT coord_x, coord_y
|
||||
FROM t_station_profile_scd
|
||||
WHERE address = :addr
|
||||
AND coord_x IS NOT NULL
|
||||
AND coord_y IS NOT NULL
|
||||
LIMIT 1
|
||||
"""
|
||||
existing = await session.execute(text(sql_check), {"addr": address})
|
||||
row = existing.fetchone()
|
||||
|
||||
lng, lat = None, None
|
||||
|
||||
if row:
|
||||
lng, lat = row[0], row[1]
|
||||
logger.info(f" -> 在数据库中找到现有坐标: {lng}, {lat}")
|
||||
else:
|
||||
# 3. 数据库中没有，调用高德地图进行查询
|
||||
logger.info(" -> 数据库中无记录，正在查询高德地图...")
|
||||
# 在线程池中运行同步的 Selenium 调用，避免阻塞异步循环
|
||||
loop = asyncio.get_event_loop()
|
||||
lng, lat = await loop.run_in_executor(None, kit.get_coordinate, address)
|
||||
|
||||
if lng and lat:
|
||||
logger.info(f" -> 高德地图返回坐标: {lng}, {lat}")
|
||||
else:
|
||||
logger.info(" -> 高德地图未返回有效坐标。")
|
||||
|
||||
# 4. 如果获取到了坐标，更新数据库中所有匹配该地址的记录
|
||||
if lng and lat:
|
||||
sql_update = """
|
||||
UPDATE t_station_profile_scd
|
||||
SET coord_x = :lng, coord_y = :lat
|
||||
WHERE address = :addr
|
||||
"""
|
||||
await session.execute(text(sql_update), {
|
||||
"lng": lng,
|
||||
"lat": lat,
|
||||
"addr": address
|
||||
})
|
||||
await session.commit()
|
||||
logger.info(f" -> 已成功更新数据库中地址为 '{address}' 的记录。")
|
||||
else:
|
||||
logger.info(f" -> 跳过更新 (未找到有效坐标)。")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"执行过程中出错: {e}")
|
||||
finally:
|
||||
# 释放资源
|
||||
kit.close()
|
||||
await db.close()
|
||||
logger.info("\n任务执行结束。")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 针对 Windows 平台的异步策略设置
|
||||
if sys.platform == 'win32':
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user