huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-15 07:40:53 +08:00
parent 3e9fc3737e
commit a61e18bf26
6 changed files with 43 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
Apps/AiTeJiYiChong/ClearHistory.py
View File

@@ -1,85 +0,0 @@
import asyncio
import os
import sys
import logging
# 确保项目根目录在 sys.path 中
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
from DbKit.Db import Db
from Util.RedisKit import RedisKit
from sqlalchemy.sql import text
from Config.Config import DB_URL
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger("ClearAiTeJiYiChong")
async def main():
"""
1. 删除数据库中所有 operator='艾特吉易充' 的记录
2. 删除 Redis 中所有关于艾特吉易充的缓存信息
"""
operator = '艾特吉易充'
# 1. 数据库清理
logger.info(f"开始清理数据库中 operator='{operator}' 的数据...")
db = Db(db_url=DB_URL)
await db.init_db()
try:
async with db.AsyncSessionLocal() as session:
async with session.begin():
# 先删除从表记录（通过 station_hash 关联）
# 1. t_station_status_scd
sql_status = """
DELETE FROM t_station_status_scd
WHERE station_hash IN (
SELECT station_hash FROM t_station_profile_scd WHERE operator = :operator
)
"""
logger.info("正在清理 t_station_status_scd...")
result_status = await session.execute(text(sql_status), {"operator": operator})
logger.info(f"t_station_status_scd 已删除 {result_status.rowcount} 行记录。")
# 2. t_station_price_schedule_scd
sql_price = """
DELETE FROM t_station_price_schedule_scd
WHERE station_hash IN (
SELECT station_hash FROM t_station_profile_scd WHERE operator = :operator
)
"""
logger.info("正在清理 t_station_price_schedule_scd...")
result_price = await session.execute(text(sql_price), {"operator": operator})
logger.info(f"t_station_price_schedule_scd 已删除 {result_price.rowcount} 行记录。")
# 3. 最后删除主表 t_station_profile_scd
sql_profile = "DELETE FROM t_station_profile_scd WHERE operator = :operator"
logger.info("正在清理 t_station_profile_scd...")
result_profile = await session.execute(text(sql_profile), {"operator": operator})
logger.info(f"t_station_profile_scd 已删除 {result_profile.rowcount} 行记录。")
logger.info("数据库记录清理完成。")
except Exception as e:
logger.error(f"数据库清理失败: {e}")
# 2. Redis 清理
logger.info("开始清理 Redis 中的缓存数据...")
redis_kit = RedisKit()
# 根据 Crawler.py 中的逻辑，key 的模式为 crawled:aite:*
pattern = "crawled:aite:*"
try:
keys = await redis_kit.keys(pattern)
if keys:
logger.info(f"匹配到 {len(keys)} 个键，正在删除...")
await redis_kit.delete(*keys)
logger.info("Redis 缓存清理完成。")
else:
logger.info(f"未匹配到模式为 '{pattern}' 的键。")
except Exception as e:
logger.error(f"Redis 清理失败: {e}")
if __name__ == "__main__":
asyncio.run(main())

85
Apps/XinDianTu/ClearHistory.py
View File

@@ -1,85 +0,0 @@
import asyncio
import os
import sys
import logging
# 确保项目根目录在 sys.path 中
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
from DbKit.Db import Db
from Util.RedisKit import RedisKit
from sqlalchemy.sql import text
from Config.Config import DB_URL
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger("ClearXinDianTu")
async def main():
"""
1. 删除数据库中所有 operator='新电途' 的记录
2. 删除 Redis 中所有关于新电途的缓存信息
"""
operator = '新电途'
# 1. 数据库清理
logger.info(f"开始清理数据库中 operator='{operator}' 的数据...")
db = Db(db_url=DB_URL)
await db.init_db()
try:
async with db.AsyncSessionLocal() as session:
async with session.begin():
# 先删除从表记录（通过 station_hash 关联）
# 1. t_station_status_scd
sql_status = """
DELETE FROM t_station_status_scd
WHERE station_hash IN (
SELECT station_hash FROM t_station_profile_scd WHERE operator = :operator
)
"""
logger.info("正在清理 t_station_status_scd...")
result_status = await session.execute(text(sql_status), {"operator": operator})
logger.info(f"t_station_status_scd 已删除 {result_status.rowcount} 行记录。")
# 2. t_station_price_schedule_scd
sql_price = """
DELETE FROM t_station_price_schedule_scd
WHERE station_hash IN (
SELECT station_hash FROM t_station_profile_scd WHERE operator = :operator
)
"""
logger.info("正在清理 t_station_price_schedule_scd...")
result_price = await session.execute(text(sql_price), {"operator": operator})
logger.info(f"t_station_price_schedule_scd 已删除 {result_price.rowcount} 行记录。")
# 3. 最后删除主表 t_station_profile_scd
sql_profile = "DELETE FROM t_station_profile_scd WHERE operator = :operator"
logger.info("正在清理 t_station_profile_scd...")
result_profile = await session.execute(text(sql_profile), {"operator": operator})
logger.info(f"t_station_profile_scd 已删除 {result_profile.rowcount} 行记录。")
logger.info("数据库记录清理完成。")
except Exception as e:
logger.error(f"数据库清理失败: {e}")
# 2. Redis 清理
logger.info("开始清理 Redis 中的缓存数据...")
redis_kit = RedisKit()
# 根据 Crawler.py 中的逻辑，key 的模式为 crawled:xdt:*
pattern = "crawled:xdt:*"
try:
keys = await redis_kit.keys(pattern)
if keys:
logger.info(f"匹配到 {len(keys)} 个键，正在删除...")
await redis_kit.delete(*keys)
logger.info("Redis 缓存清理完成。")
else:
logger.info(f"未匹配到模式为 '{pattern}' 的键。")
except Exception as e:
logger.error(f"Redis 清理失败: {e}")
if __name__ == "__main__":
asyncio.run(main())

88
Apps/YeLiTe/ClearHistory.py
View File

@@ -1,88 +0,0 @@
import asyncio
import os
import sys
import logging
from Util import Win32Patch
# 确保项目根目录在 sys.path 中
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
from DbKit.Db import Db
from Util.RedisKit import RedisKit
from sqlalchemy.sql import text
from Config.Config import DB_URL
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger("ClearYeLiTe")
Win32Patch.patch()
async def main():
"""
根据注释完成代码：
1. 删除数据库中所有 operator='驿来特' 的记录
2. 删除 Redis 中所有关于驿来特的缓存信息
"""
operator = '驿来特'
# 1. 数据库清理
logger.info(f"开始清理数据库中 operator='{operator}' 的数据...")
db = Db(db_url=DB_URL)
await db.init_db()
try:
async with db.AsyncSessionLocal() as session:
async with session.begin():
# 先删除从表记录（通过 station_hash 关联）
# 1. t_station_status_scd
sql_status = """
DELETE FROM t_station_status_scd
WHERE station_hash IN (
SELECT station_hash FROM t_station_profile_scd WHERE operator = :operator
)
"""
logger.info("正在清理 t_station_status_scd...")
result_status = await session.execute(text(sql_status), {"operator": operator})
logger.info(f"t_station_status_scd 已删除 {result_status.rowcount} 行记录。")
# 2. t_station_price_schedule_scd
sql_price = """
DELETE FROM t_station_price_schedule_scd
WHERE station_hash IN (
SELECT station_hash FROM t_station_profile_scd WHERE operator = :operator
)
"""
logger.info("正在清理 t_station_price_schedule_scd...")
result_price = await session.execute(text(sql_price), {"operator": operator})
logger.info(f"t_station_price_schedule_scd 已删除 {result_price.rowcount} 行记录。")
# 3. 最后删除主表 t_station_profile_scd
sql_profile = "DELETE FROM t_station_profile_scd WHERE operator = :operator"
logger.info("正在清理 t_station_profile_scd...")
result_profile = await session.execute(text(sql_profile), {"operator": operator})
logger.info(f"t_station_profile_scd 已删除 {result_profile.rowcount} 行记录。")
logger.info("数据库记录清理完成。")
except Exception as e:
logger.error(f"数据库清理失败: {e}")
# 2. Redis 清理
logger.info("开始清理 Redis 中的缓存数据...")
redis_kit = RedisKit()
# 根据 Crawler.py 中的逻辑，key 的模式为 crawled:ylt:*
pattern = "crawled:ylt:*"
try:
keys = await redis_kit.keys(pattern)
if keys:
logger.info(f"匹配到 {len(keys)} 个键，正在删除...")
await redis_kit.delete(*keys)
logger.info("Redis 缓存清理完成。")
else:
logger.info(f"未匹配到模式为 '{pattern}' 的键。")
except Exception as e:
logger.error(f"Redis 清理失败: {e}")
if __name__ == "__main__":
asyncio.run(main())

27
Apps/TeLaiDian/ClearHistory.py → Tools/T5_ClearHistory.py
View File

@@ -3,15 +3,27 @@ import os
import sys import sys
import logging import logging
# 确保项目根目录在 sys.path 中 # 确保项目根目录在 sys.path 中，并且优先于外部同名模块
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path: if project_root not in sys.path:
sys.path.append(project_root) sys.path.insert(0, project_root)
from Util import Win32Patch
from DbKit.Db import Db from DbKit.Db import Db
from Util.RedisKit import RedisKit from Util.RedisKit import RedisKit
from sqlalchemy.sql import text from sqlalchemy.sql import text
from Config.Config import DB_URL try:
from Config.Config import DB_URL
except ModuleNotFoundError:
import importlib.util
_root_dir_cfg = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
_config_path_cfg = os.path.join(_root_dir_cfg, "Config", "Config.py")
_spec_cfg = importlib.util.spec_from_file_location("project_config_fallback", _config_path_cfg)
_cfg_mod = importlib.util.module_from_spec(_spec_cfg)
assert _spec_cfg.loader is not None
_spec_cfg.loader.exec_module(_cfg_mod)
DB_URL = _cfg_mod.DB_URL
# 配置日志 # 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -64,7 +76,11 @@ async def main():
logger.info("数据库记录清理完成。") logger.info("数据库记录清理完成。")
except Exception as e: except Exception as e:
logger.error(f"数据库清理失败: {e}") logger.error(f"数据库清理失败: {e}")
finally:
# 3. 资源清理
logger.info("正在关闭数据库连接...")
await db.close()
# 2. Redis 清理 # 2. Redis 清理
logger.info("开始清理 Redis 中的缓存数据...") logger.info("开始清理 Redis 中的缓存数据...")
redis_kit = RedisKit() redis_kit = RedisKit()
@@ -80,6 +96,9 @@ async def main():
logger.info(f"未匹配到模式为 '{pattern}' 的键。") logger.info(f"未匹配到模式为 '{pattern}' 的键。")
except Exception as e: except Exception as e:
logger.error(f"Redis 清理失败: {e}") logger.error(f"Redis 清理失败: {e}")
# RedisKit 目前是单例且内部使用线程池，通常不需要显式关闭，
# 但如果未来改为异步驱动，则需要在此处添加清理逻辑
if __name__ == "__main__": if __name__ == "__main__":
Win32Patch.patch()
asyncio.run(main()) asyncio.run(main())

24
Util/RedisKit.py
View File

@@ -4,10 +4,26 @@ import uuid
import json import json
import redis import redis
from Config.Config import ( try:
REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD, from Config.Config import (
REDIS_DECODE_RESPONSES, REDIS_MAX_CONNECTIONS REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD,
) REDIS_DECODE_RESPONSES, REDIS_MAX_CONNECTIONS
)
except ModuleNotFoundError:
import importlib.util
import os
_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_config_path = os.path.join(_root_dir, "Config", "Config.py")
_spec = importlib.util.spec_from_file_location("project_config_fallback", _config_path)
_cfg = importlib.util.module_from_spec(_spec)
assert _spec.loader is not None
_spec.loader.exec_module(_cfg)
REDIS_HOST = _cfg.REDIS_HOST
REDIS_PORT = _cfg.REDIS_PORT
REDIS_DB = _cfg.REDIS_DB
REDIS_PASSWORD = getattr(_cfg, "REDIS_PASSWORD", None)
REDIS_DECODE_RESPONSES = _cfg.REDIS_DECODE_RESPONSES
REDIS_MAX_CONNECTIONS = _cfg.REDIS_MAX_CONNECTIONS
# 创建logger实例 # 创建logger实例
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

BIN
Util/__pycache__/RedisKit.cpython-310.pyc
View File

Binary file not shown.