huhao/drl_2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
user9994793890
2026-05-29 10:28:07 +08:00
commit ee860ce0ae
87 changed files with 12595 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
xuexiao/etl/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""课时核对表 Excel ETL 包"""

80
xuexiao/etl/create_sample_workbooks.py Normal file
View File

@@ -0,0 +1,80 @@
"""生成示例课时核对表（3月+4月，用于 ETL 冒烟测试）"""
from pathlib import Path
from openpyxl import Workbook
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / 'keshibiao'
def _write_month(path: Path, year: int, month: int, prev_label: str, rows: list):
wb = Workbook()
ws = wb.active
ws.title = 'code学龄后'
ws.append([f'达尔琳{year}年{month}月份code学员课时明细'])
headers = [
'序号', '学员姓名', '课程级别',
f'{prev_label}剩余课时', '赠送课时', f'{prev_label}剩余总课时',
'单价', f'{prev_label}末剩余金额',
'新签金额', '新签课时', '新签赠送课时',
'续费金额', '续费课时', '续费赠送课时',
f'{month}月已上课时', f'{month}月已上金额',
'退费课时', '账号费', '退赠送课时', '退费金额',
f'{month}月剩余课时', f'{month}月剩余总课时', f'{month}月末剩余金额',
'', '', '', '备注',
]
ws.append(headers)
for row in rows:
ws.append(row)
path.parent.mkdir(parents=True, exist_ok=True)
wb.save(path)
print('wrote', path)
def main():
# 3月：杨霈淞示例
_write_month(
OUT / '课时核对表_2026_3月份.xlsx', 2026, 3, '2月',
[[
1, '杨霈淞', 'scratch',
68, 8, 76, 96.875, 6703.7,
None, None, None,
None, None, None,
6, 581.25,
None, None, None, None,
62, 70, 6122.45,
'3.8', '3.15', '3.29', '',
]],
)
# 4月：链式衔接
_write_month(
OUT / '课时核对表_2026_4月份.xlsx', 2026, 4, '3月',
[[
1, '杨霈淞', 'scratch',
62, 8, 70, 96.875, 6122.45,
None, None, None,
None, None, None,
6, 581.25,
None, None, None, None,
56, 64, 5541.2,
'4.5', '4.12', '4.19', '',
]],
)
# 新签学员示例
_write_month(
OUT / '课时核对表_2026_5月份.xlsx', 2026, 5, '4月',
[[
None, '柳昱泽（三月）', 'python',
None, None, None, None, None,
9600, 48, 4,
None, None, None,
4, 800,
None, None, None, None,
44, 48, 8800,
'5.10', '5.17', '', '5月新签',
]],
)
if __name__ == '__main__':
main()

578
xuexiao/etl/import_keshibiao.py Normal file
View File

@@ -0,0 +1,578 @@
"""
课时核对表 Excel → 数据库 ETL（批量校验 + 批量写入）
用法:
python -m etl.import_keshibiao --file keshibiao/课时核对表（2026.1月份）.xlsx --update
python -m etl.import_keshibiao --dir keshibiao --update --month 1
"""
from __future__ import annotations
import argparse
import logging
import sys
import time
from collections import defaultdict
from decimal import Decimal
from pathlib import Path
from typing import Optional
ROOT = Path(__file__).resolve().parent.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from etl.keshi_parser import (
ParsedWorkbook,
ParsedRow,
parse_workbook,
batch_validate_rows,
is_real_workbook_file,
)
logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')
logger = logging.getLogger('keshi_etl')
DEFAULT_DIR = ROOT / 'keshibiao'
BULK_CHUNK = 500
COURSE_DISPLAY = {
'scratch': 'Scratch',
'python': 'Python',
'c++': 'C++',
'wedo': 'WeDo',
'大班': '大班',
'中班': '中班',
'小班': '小班',
'lab': 'Lab',
'stem': 'STEM',
'free': '免费课',
'trial': '试学',
}
def _d(val) -> Optional[Decimal]:
if val is None:
return None
return Decimal(str(round(float(val), 4)))
def _graduated_from_notes(notes: str) -> bool:
if not notes:
return False
return any(k in notes for k in ('结课', '毕业', '退费结课', '不再上课'))
def _fill_sparse_snapshot(data: dict, row: ParsedRow) -> dict:
"""试学缴费等仅含金额的行：补全数值字段，避免 Doris strict 模式过滤 INSERT"""
if row.course_code != 'trial' and data.get('end_balance') is not None:
return data
if row.course_code == 'trial' or (
data.get('new_signup_amount') is not None
and data.get('prev_lessons') is None
and data.get('end_lessons') is None
):
amt = data.get('new_signup_amount') or Decimal(0)
for key in (
'prev_lessons', 'prev_gift_lessons', 'prev_total_lessons',
'end_lessons', 'end_gift_lessons', 'end_total_lessons',
'consumed_lessons', 'consumed_amount',
'refund_lessons', 'refund_gift_lessons', 'refund_amount', 'account_fee',
):
data.setdefault(key, Decimal(0))
data.setdefault('prev_balance', Decimal(0))
data.setdefault('end_balance', amt)
return data
def build_snapshot_fields(row: ParsedRow, year: int, month: int, source_file: str) -> dict:
f = row.fields
end_lessons = f.get('end_lessons')
end_total = f.get('end_total_lessons')
end_gift = None
if end_lessons is not None and end_total is not None:
end_gift = end_total - end_lessons
elif f.get('prev_gift_lessons') is not None:
end_gift = f.get('prev_gift_lessons')
return {
'year': year,
'month': month,
'seq_no': row.seq_no,
'prev_lessons': _d(f.get('prev_lessons')),
'prev_gift_lessons': _d(f.get('prev_gift_lessons')),
'prev_total_lessons': _d(f.get('prev_total_lessons')),
'prev_balance': _d(f.get('prev_balance')),
'new_signup_amount': _d(f.get('new_signup_amount')),
'new_signup_lessons': _d(f.get('new_signup_lessons')),
'new_signup_gift_lessons': _d(f.get('new_signup_gift_lessons')),
'renewal_amount': _d(f.get('renewal_amount')),
'renewal_lessons': _d(f.get('renewal_lessons')),
'renewal_gift_lessons': _d(f.get('renewal_gift_lessons')),
'consumed_lessons': _d(f.get('consumed_lessons')),
'consumed_amount': _d(f.get('consumed_amount')),
'refund_lessons': _d(f.get('refund_lessons')),
'account_fee': _d(f.get('account_fee')),
'refund_gift_lessons': _d(f.get('refund_gift_lessons')),
'refund_amount': _d(f.get('refund_amount')),
'end_lessons': _d(end_lessons),
'end_gift_lessons': _d(end_gift),
'end_total_lessons': _d(end_total),
'end_balance': _d(f.get('end_balance')),
'unit_price': _d(f.get('unit_price')),
'notes': row.notes or None,
'source_file': source_file,
}
class ImportCaches:
"""预加载学员/课程/账户/当月快照，减少逐行查询"""
def __init__(self, db, Student, Course, StudentAccount, MonthlySnapshot, year, month):
self.db = db
self.Student = Student
self.Course = Course
self.StudentAccount = StudentAccount
self.MonthlySnapshot = MonthlySnapshot
self.year = year
self.month = month
self.courses_by_code: dict[str, Course] = {}
for c in Course.query.all():
for key in (c.course_code, c.level):
if key:
self.courses_by_code[str(key).lower()] = c
self.students_by_name: dict[str, Student] = {
s.name: s for s in Student.query.all()
}
self.accounts: dict[tuple[int, int], StudentAccount] = {}
for a in StudentAccount.query.all():
self.accounts[(a.student_id, a.course_id)] = a
self.snapshots: dict[int, MonthlySnapshot] = {
s.account_id: s
for s in MonthlySnapshot.query.filter_by(year=year, month=month).all()
}
self._pending_students: list = []
self._pending_courses: list = []
self._pending_accounts: list = []
def _course_key(self, code: str) -> str:
return (code or 'unknown').lower()
def ensure_courses(self, codes: set[str]) -> None:
for code in codes:
key = self._course_key(code)
if key in self.courses_by_code:
continue
name = COURSE_DISPLAY.get(key, key.upper())
course = self.Course(
name=name, course_code=key, level=key,
type='编程', status=1,
)
self._pending_courses.append(course)
self.courses_by_code[key] = course
def ensure_students(self, rows: list[ParsedRow]) -> None:
seen = set(self.students_by_name)
for row in rows:
if row.student_name in seen:
continue
seen.add(row.student_name)
student = self.Student(
name=row.student_name,
display_name=row.display_name or row.student_name,
nickname=row.nickname or None,
status=1,
source='课时表导入',
)
self._pending_students.append(student)
self.students_by_name[row.student_name] = student
def flush_pending(self) -> None:
need_flush = False
if self._pending_courses:
self.db.session.add_all(self._pending_courses)
self._pending_courses.clear()
need_flush = True
if self._pending_students:
self.db.session.add_all(self._pending_students)
self._pending_students.clear()
need_flush = True
if need_flush:
self.db.session.flush()
if self._pending_accounts:
self.db.session.add_all(self._pending_accounts)
self._pending_accounts.clear()
self.db.session.flush()
def ensure_accounts(self, pairs: set[tuple[str, str]]) -> None:
for student_name, course_code in pairs:
student = self.students_by_name.get(student_name)
course = self.courses_by_code.get(self._course_key(course_code))
if not student or not course:
continue
key = (student.id, course.id)
if key in self.accounts:
continue
orig = float(course.price_per_hour or 0) if course else 0
acc = self.StudentAccount(
student_id=student.id,
course_id=course.id,
normal_hours=0,
gifted_hours=0,
account_status='active',
original_price_per_lesson=orig if orig > 0 else None,
)
self._pending_accounts.append(acc)
self.accounts[key] = acc
def get_student(self, row: ParsedRow):
s = self.students_by_name[row.student_name]
if row.display_name and not s.display_name:
s.display_name = row.display_name
if row.nickname and not s.nickname:
s.nickname = row.nickname
return s
def get_course(self, course_code: str):
return self.courses_by_code[self._course_key(course_code)]
def get_account(self, student, course, unit_price=None):
acc = self.accounts[(student.id, course.id)]
if unit_price is not None:
acc.unit_price = _d(unit_price) or acc.unit_price
return acc
def get_snapshot(self, account_id: int):
return self.snapshots.get(account_id)
def _chunked(items: list, size: int):
for i in range(0, len(items), size):
yield items[i:i + size]
def import_workbook(
app,
path: Path,
*,
update: bool = False,
default_lessons: float = 2.0,
verbose: bool = False,
repair_chains: bool = True,
) -> dict:
from models import db, Student, Course, StudentAccount, MonthlySnapshot, ClassRecord
import business_rules as rules
t0 = time.perf_counter()
stats = {
'file': path.name,
'rows': 0,
'parsed_rows': 0,
'snapshots_new': 0,
'snapshots_updated': 0,
'snapshots_skipped': 0,
'class_records': 0,
'warnings': [],
'validation_warning_count': 0,
'sheets_parsed': [],
'sheets_skipped': [],
'elapsed_sec': 0,
'account_ids_touched': [],
}
parsed: ParsedWorkbook = parse_workbook(path, default_lessons=default_lessons)
stats['year'] = parsed.year
stats['month'] = parsed.month
stats['sheets_parsed'] = parsed.sheets_parsed
stats['sheets_skipped'] = parsed.sheets_skipped
stats['parsed_rows'] = len(parsed.rows)
valid_rows: list[ParsedRow] = []
for row in parsed.rows:
if not row.student_name or not row.course_code:
stats['warnings'].append(f'跳过缺姓名/课程行: {row.student_name}')
continue
valid_rows.append(row)
val_warnings = batch_validate_rows(valid_rows)
stats['validation_warning_count'] = len(val_warnings)
if val_warnings:
stats['warnings'].extend(val_warnings[:30])
logger.info(
'批量校验完成: %s 条余额/课时警告（已合并，不逐条刷屏；--verbose 可看样例）',
len(val_warnings),
)
if verbose:
for w in val_warnings[:20]:
logger.warning(w)
with app.app_context():
from db_bootstrap import migrate_keshi_ledger_columns
migrate_keshi_ledger_columns()
caches = ImportCaches(
db, Student, Course, StudentAccount, MonthlySnapshot,
parsed.year, parsed.month,
)
course_codes = {r.course_code for r in valid_rows}
account_pairs = {(r.student_name, r.course_code) for r in valid_rows}
caches.ensure_courses(course_codes)
caches.ensure_students(valid_rows)
caches.flush_pending()
caches.ensure_accounts(account_pairs)
caches.flush_pending()
new_snapshots: list = []
class_mappings: list[dict] = []
account_ids_touched: set[int] = set()
for row in valid_rows:
stats['rows'] += 1
course = caches.get_course(row.course_code)
student = caches.get_student(row)
unit_price = row.fields.get('unit_price')
account = caches.get_account(student, course, unit_price)
account_ids_touched.add(account.id)
snap_data = _fill_sparse_snapshot(
build_snapshot_fields(row, parsed.year, parsed.month, path.name),
row,
)
snap = caches.get_snapshot(account.id)
if snap:
if not update:
stats['snapshots_skipped'] += 1
else:
for k, v in snap_data.items():
setattr(snap, k, v)
stats['snapshots_updated'] += 1
else:
new_snapshots.append(
MonthlySnapshot(account_id=account.id, **snap_data)
)
stats['snapshots_new'] += 1
caches.snapshots[account.id] = new_snapshots[-1]
if snap_data.get('end_lessons') is not None:
account.normal_hours = snap_data['end_lessons']
if snap_data.get('end_gift_lessons') is not None:
account.gifted_hours = snap_data['end_gift_lessons']
elif snap_data.get('end_total_lessons') is not None and snap_data.get('end_lessons') is not None:
account.gifted_hours = snap_data['end_total_lessons'] - snap_data['end_lessons']
if snap_data.get('unit_price') is not None:
account.unit_price = snap_data['unit_price']
if _graduated_from_notes(row.notes):
account.account_status = 'graduated'
student.status = 0
for class_date, lessons, raw in row.class_sessions:
class_mappings.append({
'account_id': account.id,
'year': int(parsed.year),
'month': int(parsed.month),
'class_date': class_date,
'lessons_consumed': float(lessons) if lessons is not None else 2.0,
'raw_text': (raw or '')[:50] or None,
})
if update and account_ids_touched:
ClassRecord.query.filter(
ClassRecord.account_id.in_(account_ids_touched),
ClassRecord.year == parsed.year,
ClassRecord.month == parsed.month,
).delete(synchronize_session=False)
if new_snapshots:
db.session.add_all(new_snapshots)
if class_mappings:
# Doris 对 executemany + Decimal/datetime 不友好，用 add_all 分批提交
for chunk in _chunked(class_mappings, BULK_CHUNK):
db.session.add_all([ClassRecord(**m) for m in chunk])
stats['class_records'] = len(class_mappings)
db.session.commit()
stats['account_ids_touched'] = list(account_ids_touched)
if repair_chains and account_ids_touched:
repaired = rules.repair_snapshot_chains_for_accounts(account_ids_touched)
stats['chain_repaired'] = repaired
logger.info('快照链修复: %s 个衔接点（仅本次涉及 %s 个账户）', repaired, len(account_ids_touched))
stats['elapsed_sec'] = round(time.perf_counter() - t0, 2)
return stats
def validate_month_chain(app, tolerance: float = 0.05) -> list[str]:
from models import MonthlySnapshot
warnings = []
with app.app_context():
snaps = MonthlySnapshot.query.order_by(
MonthlySnapshot.account_id,
MonthlySnapshot.year,
MonthlySnapshot.month,
).all()
by_account: dict[int, list] = defaultdict(list)
for s in snaps:
by_account[s.account_id].append(s)
for account_id, items in by_account.items():
items.sort(key=lambda x: (x.year, x.month))
for i in range(len(items) - 1):
cur, nxt = items[i], items[i + 1]
expected_next = cur.month + 1
expected_year = cur.year
if expected_next > 12:
expected_next = 1
expected_year += 1
if (nxt.year, nxt.month) != (expected_year, expected_next):
continue
pairs = [
('正课', cur.end_lessons, nxt.prev_lessons),
('赠课', cur.end_gift_lessons, nxt.prev_gift_lessons),
('余额', cur.end_balance, nxt.prev_balance),
]
for label, end_v, prev_v in pairs:
if end_v is None or prev_v is None:
continue
if abs(float(end_v) - float(prev_v)) > tolerance:
warnings.append(
f'账户{account_id} {cur.period_label}→{nxt.period_label} '
f'{label}不衔接: 月末{float(end_v)} vs 下月初{float(prev_v)}'
)
return warnings
def _filter_files(files: list[Path], month: Optional[int]) -> list[Path]:
if month is None:
return files
out = []
for f in files:
name = f.name
if (
f'_{month}月' in name
or f'.{month}月' in name
or f'.{month}月份' in name
or f'（{month}月' in name
or f'({month}月' in name
):
out.append(f)
return out
def import_directory(
app,
directory: Path,
*,
update: bool = False,
default_lessons: float = 2.0,
month: Optional[int] = None,
verbose: bool = False,
repair_chains: bool = True,
validate_chain: bool = False,
) -> list[dict]:
directory = Path(directory)
if not directory.is_dir():
raise FileNotFoundError(f'目录不存在: {directory}')
files = sorted(directory.glob('*.xlsx'))
files = [
f for f in files
if not f.name.startswith('~$') and is_real_workbook_file(f)
]
files = _filter_files(files, month)
if not files:
logger.warning('目录内无匹配 xlsx: %s month=%s', directory, month)
return []
results = []
all_touched: set[int] = set()
t0 = time.perf_counter()
for path in files:
logger.info('导入 %s', path.name)
try:
stats = import_workbook(
app, path, update=update, default_lessons=default_lessons,
verbose=verbose, repair_chains=False,
)
all_touched.update(stats.get('account_ids_touched') or [])
results.append(stats)
logger.info(
' 完成 [%ss]: 解析=%s 入库=%s 表=%s 新增=%s 更新=%s 上课=%s 校验警告=%s',
stats['elapsed_sec'],
stats.get('parsed_rows'), stats['rows'],
stats.get('sheets_parsed'),
stats['snapshots_new'],
stats['snapshots_updated'],
stats['class_records'],
stats.get('validation_warning_count', 0),
)
except Exception as e:
logger.exception('导入失败 %s: %s', path.name, e)
results.append({'file': path.name, 'error': str(e)})
if repair_chains and all_touched:
import business_rules as rules
with app.app_context():
n = rules.repair_snapshot_chains_for_accounts(all_touched)
logger.info('目录导入后快照链修复: %s 点 / %s 账户', n, len(all_touched))
if validate_chain:
chain_warnings = validate_month_chain(app)
logger.info('全库链式校验: %s 条', len(chain_warnings))
if results:
results[-1]['chain_warnings'] = chain_warnings
logger.info('目录导入总耗时 %.1fs，文件数 %s', time.perf_counter() - t0, len(files))
return results
def main():
parser = argparse.ArgumentParser(description='课时核对表 Excel 导入（批量模式）')
parser.add_argument('--dir', type=str, default=str(DEFAULT_DIR), help='xlsx 目录')
parser.add_argument('--file', type=str, help='单个 xlsx 文件')
parser.add_argument('--month', type=int, help='仅导入文件名含该月份的表（与 --dir 联用）')
parser.add_argument('--update', action='store_true', help='已存在快照则更新')
parser.add_argument('--lessons-per-class', type=float, default=2.0, help='默认每次上课课时')
parser.add_argument('--verbose', action='store_true', help='打印校验警告样例')
parser.add_argument('--no-repair', action='store_true', help='跳过快照链修复（最快）')
parser.add_argument('--validate-chain', action='store_true', help='导入后全库链式校验（慢）')
args = parser.parse_args()
from app import create_app
app = create_app()
repair = not args.no_repair
if args.file:
stats = import_workbook(
app, Path(args.file),
update=args.update,
default_lessons=args.lessons_per_class,
verbose=args.verbose,
repair_chains=repair,
)
print(stats)
else:
results = import_directory(
app, Path(args.dir),
update=args.update,
default_lessons=args.lessons_per_class,
month=args.month,
verbose=args.verbose,
repair_chains=repair,
validate_chain=args.validate_chain,
)
print(f'共处理 {len(results)} 个文件')
for r in results:
print(r)
if __name__ == '__main__':
main()

857
xuexiao/etl/keshi_parser.py Normal file
View File

@@ -0,0 +1,857 @@
"""课时核对表 Excel 解析：多工作表、多表头区块、列映射、上课日期解析"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path
from typing import Any, Optional
from openpyxl import load_workbook
# 文件名：课时核对表_YYYY_N月份.xlsx / 达尔琳2026年1月份
FILE_MONTH_PATTERNS = [
re.compile(r'课时核对表[_\s]*(\d{4})[_\s]*(\d{1,2})\s*月', re.IGNORECASE),
re.compile(r'课时核对表[（(]?(\d{4})[.\s年]*(\d{1,2})\s*月', re.IGNORECASE),
re.compile(r'(\d{4})[.\s年]*(\d{1,2})\s*月', re.IGNORECASE),
]
# 上课情况：3.8 / 1.13（补2次）/ 1.24(补2节)
CLASS_CELL_RE = re.compile(
r'^(\d{1,2})\.(\d{1,2})(?:[（(]([^）)]+)[）)]|\(([^)]+)\))?$',
re.IGNORECASE,
)
NAME_RE = re.compile(r'^(.+?)[（(](.+?)[）)]$')
PAYMENT_ROW_RE = re.compile(r'^\d{4}-\d{2}-\d{2}\s+\d{2}:')
LONG_ID_RE = re.compile(r'^\d{15,}$')
COURSE_ALIASES = {
'scratch': 'scratch',
'python': 'python',
'c++': 'c++',
'cpp': 'c++',
'wedo': 'wedo',
'lab': 'lab',
'stem': 'stem',
'code': 'code',
'大班': '大班',
'中班': '中班',
'小班': '小班',
'编程': 'python',
'python编程': 'python',
'scratch编程': 'scratch',
'乐高': 'wedo',
'wedo乐高': 'wedo',
'待定': 'lab',
'免费': 'free',
'trial': 'trial',
'试学': 'trial',
}
# 仅跳过「转账/缴费流水」类工作表（不跳过「试学明细」等含学员数据的子表）
# 列标题 → 逻辑字段（靠前者优先匹配）
HEADER_RULES: list[tuple[str, str]] = [
('12月份剩余课时', 'prev_lessons'),
('12月剩余总课时', 'prev_total_lessons'),
('12月末剩余金额', 'prev_balance'),
('12月末累计已上总课时', 'prev_cumulative_consumed'),
('12月末剩余课时', 'prev_lessons'),
('1月份剩余课时', 'end_lessons'),
('1月剩余总课时', 'end_total_lessons'),
('1月末剩余金额', 'end_balance'),
('1月末累计已上总课时', 'end_cumulative_consumed'),
('1月份已上金额', 'consumed_amount'),
('1月份已上课时', 'consumed_lessons'),
('1月已上课时', 'consumed_lessons'),
('1月份课时登记', '_class_register'),
('1月份上课情况', '_class_register'),
('1月已上课时登记', '_class_register'),
('序号', 'seq_no'),
('学员姓名', 'student_name'),
('学员名字', 'student_name'),
('课程级别', 'course_level'),
('课程', 'course_level'),
('单价/节', 'unit_price'),
('退费课时', 'refund_lessons'),
('退赠送课时', 'refund_gift_lessons'),
('退费金额', 'refund_amount'),
('账号费', 'account_fee'),
('退费', 'refund_amount'),
('赠送课时', 'prev_gift_lessons'),
('上月剩余总课时', 'prev_total_lessons'),
('剩余总课时', 'prev_total_lessons'),
('上月剩余课时', 'prev_lessons'),
('月份剩余课时', 'prev_lessons'),
('剩余课时', 'prev_lessons'),
('单价', 'unit_price'),
('上月末剩余金额', 'prev_balance'),
('月末剩余金额', 'prev_balance'),
('剩余金额', 'prev_balance'),
('新签金额', 'new_signup_amount'),
('新签课时', 'new_signup_lessons'),
('新签赠送', 'new_signup_gift_lessons'),
('续费金额', 'renewal_amount'),
('续费课时', 'renewal_lessons'),
('续费赠送', 'renewal_gift_lessons'),
('本月已上课时', 'consumed_lessons'),
('已上课时', 'consumed_lessons'),
('本月已上金额', 'consumed_amount'),
('已上金额', 'consumed_amount'),
('本月剩余总课时', 'end_total_lessons'),
('本月剩余课时', 'end_lessons'),
('本月末剩余金额', 'end_balance'),
('总课时', 'total_hours'),
('备注', 'notes'),
('缴费金额', 'new_signup_amount'),
('交费金额', 'new_signup_amount'),
('缴费时间', '_pay_time'),
]
def parse_filename_month(path: Path) -> Optional[tuple[int, int]]:
stem = path.stem
for pat in FILE_MONTH_PATTERNS:
m = pat.search(stem)
if m:
return int(m.group(1)), int(m.group(2))
return None
def parse_title_month(title: str) -> Optional[tuple[int, int]]:
if not title:
return None
m = re.search(r'(\d{4})\s*年\s*(\d{1,2})\s*月', str(title))
if m:
return int(m.group(1)), int(m.group(2))
return None
def split_student_name(full: str) -> tuple[str, str, str]:
"""返回 (全名, 正式名, 小名)"""
full = (full or '').strip().replace('\n', '')
if not full:
return '', '', ''
m = NAME_RE.match(full)
if m:
display = m.group(1).strip()
nick = m.group(2).strip()
return full, display, nick
return full, full, ''
def normalize_course_code(raw: str) -> str:
s = (raw or '').strip().lower().replace('\n', '')
s = s.replace('＋', '+').replace(' ', '')
if not s or s in ('-', '—', '/', '无', 'nan'):
return ''
return COURSE_ALIASES.get(s, s)
def infer_course_from_schedule(text: str) -> str:
t = (text or '').lower()
if 'python' in t:
return 'python'
if 'scratch' in t:
return 'scratch'
if 'wedo' in t:
return 'wedo'
if 'c++' in t or 'cpp' in t:
return 'c++'
if 'lab' in t:
return 'lab'
return ''
def sheet_default_course(sheet_name: str) -> str:
sn = (sheet_name or '').lower()
if 'lab' in sn:
return 'lab'
return ''
def should_skip_sheet(sheet_name: str) -> bool:
"""跳过纯转账流水表；保留 code/stem/lab/免费/试学明细等业务子表"""
sn = (sheet_name or '').replace(' ', '')
if '缴费流水' in sn:
return True
if '转账' in sn and ('明细' in sn or '月转账' in sn or sn.endswith('转账')):
return True
if re.search(r'\d+月转账', sn):
return True
return False
def is_real_workbook_file(path: Path, min_size: int = 50_000) -> bool:
"""排除体积过小的示例/测试表，只导入完整月度核对表"""
return path.suffix.lower() == '.xlsx' and path.stat().st_size >= min_size
def _safe_float(val: Any) -> Optional[float]:
if val is None or val == '':
return None
if isinstance(val, (int, float)):
return float(val)
s = str(val).strip().replace(',', '').replace('\n', '')
if not s or s in ('-', '—', '/', '无', '待定', 'nan'):
return None
try:
return float(s)
except ValueError:
return None
def _row_text(row: tuple) -> str:
return ''.join(str(c or '') for c in (row or ()))
def _has_student_col_header(text: str) -> bool:
"""标准表为「学员姓名」，免费课表为「学员名字」（字可能为不同 Unicode 码位）"""
if '学员' not in text:
return False
return '姓名' in text or '名字' in text or bool(
re.search(r'学员.{0,2}名', text)
)
def _find_title_in_row(row: tuple) -> str:
for cell in row or ():
if cell is None:
continue
s = str(cell).strip()
if s and ('课时明细' in s or '课时核对' in s or re.search(r'\d{4}\s*年', s)):
return s
return ''
def is_standard_header_row(row: tuple) -> bool:
text = _row_text(row)
return _has_student_col_header(text) and ('课程' in text or '课程级别' in text)
def is_free_class_header_row(row: tuple) -> bool:
text = _row_text(row)
return _has_student_col_header(text) and ('班级' in text or '总课时' in text)
def is_tracking_header_row(row: tuple) -> bool:
text = _row_text(row)
return _has_student_col_header(text) and '累计已上' in text and '总课时' in text
def is_trial_payment_header_row(row: tuple) -> bool:
text = _row_text(row)
return _has_student_col_header(text) and ('缴费' in text or '交费' in text)
def is_summary_row(row: tuple) -> bool:
text = _row_text(row)
return '合计' in text or text.strip().startswith('总计')
def is_payment_data_row(row: tuple) -> bool:
if not row:
return False
text = _row_text(row)
if '微信支付' in text or '已交款' in text or '网络银行' in text:
return True
first = str(row[0] or '').strip()
if PAYMENT_ROW_RE.match(first):
second = str(row[1] if len(row) > 1 else '').strip()
if LONG_ID_RE.match(second) or second == '':
return True
name_cell = ''
for cell in row[1:4]:
if cell and str(cell).strip():
name_cell = str(cell).strip()
break
if LONG_ID_RE.match(name_cell):
return True
if name_cell in ('DEL', 'XF'):
return True
return False
def _header_month_num(h: str) -> Optional[int]:
"""表头中的月份数字，如 4月份剩余课时 → 4"""
m = re.search(r'(\d{1,2})\s*月', h)
return int(m.group(1)) if m else None
def _prev_month_num(book_month: int) -> int:
return 12 if book_month == 1 else book_month - 1
def _map_header_by_month(h: str, book_month: int) -> Optional[str]:
"""按表头月份与核对表月份对齐：上月→prev_*，当月→end_* / consumed_*"""
hm = _header_month_num(h)
if hm is None:
return None
prev_m = _prev_month_num(book_month)
if hm == book_month:
if '剩余总课时' in h or ('剩余总' in h and '课时' in h):
return 'end_total_lessons'
if '月末' in h and ('剩余金额' in h or '余额' in h):
return 'end_balance'
if '剩余课时' in h:
return 'end_lessons'
if '已上课时' in h:
return 'consumed_lessons'
if '已上金额' in h:
return 'consumed_amount'
if '累计已上' in h:
return 'end_cumulative_consumed'
if hm == prev_m:
if '剩余总课时' in h or ('剩余总' in h and '课时' in h):
return 'prev_total_lessons'
if '月末' in h and ('剩余金额' in h or '余额' in h):
return 'prev_balance'
if '剩余课时' in h:
return 'prev_lessons'
if '累计已上' in h:
return 'prev_cumulative_consumed'
return None
def _map_header(cell_val: Any, book_year: Optional[int] = None, book_month: Optional[int] = None) -> Optional[str]:
if cell_val is None:
return None
h = str(cell_val).strip().replace('\n', '').replace(' ', '')
if not h:
return None
if book_month is not None:
by_month = _map_header_by_month(h, book_month)
if by_month:
return by_month
for keyword, field_key in HEADER_RULES:
if keyword not in str(cell_val):
continue
if field_key.startswith('_') or field_key == '_pay_time':
return field_key
if field_key in ('prev_lessons', 'prev_total_lessons', 'prev_balance', 'prev_gift_lessons'):
if '本月' in h or re.search(r'^1月|^1月份', h):
continue
if field_key == 'prev_balance' and re.search(r'^1月', h):
continue
if field_key in ('end_lessons', 'end_total_lessons', 'end_balance'):
if '12月' in h or '上月' in h:
continue
if '剩余' in h and '本月' not in h and not re.search(r'^1月|^1月份', h):
if field_key != 'end_balance' or '月末' not in h:
continue
if field_key == 'prev_balance' and re.search(r'^1月', h):
continue
if field_key == 'end_balance' and '12月' in h:
continue
if field_key == 'prev_lessons' and re.search(r'^1月', h) and '12' not in h[:3]:
if '剩余' in h:
continue
return field_key
return None
def _lessons_from_class_suffix(suffix: str, default_lessons: float) -> float:
if not suffix:
return default_lessons
m = re.search(r'(\d+(?:\.\d+)?)\s*课时', suffix)
if m:
return float(m.group(1))
m = re.search(r'补?(\d+)\s*次', suffix)
if m:
return float(m.group(1)) * default_lessons
m = re.search(r'(\d+)\s*节', suffix)
if m:
return float(m.group(1)) * default_lessons
return default_lessons
def parse_class_cell(text: str, year: int, month: int, default_lessons: float = 2.0):
"""解析上课情况单元格 → (class_date, lessons, raw_text)"""
if not text:
return None
raw = str(text).strip()
if not raw:
return None
compact = raw.replace(' ', '')
m = CLASS_CELL_RE.match(compact)
if not m:
return None
d_month = int(m.group(1))
d_day = int(m.group(2))
suffix = m.group(3) or m.group(4) or ''
lessons = _lessons_from_class_suffix(suffix, default_lessons)
use_month = month
if 1 <= d_month <= 12:
use_month = d_month
try:
class_date = date(year, use_month, d_day)
except ValueError:
return None
return class_date, lessons, raw
def _build_col_map(
header_row: tuple,
book_year: Optional[int] = None,
book_month: Optional[int] = None,
) -> tuple[dict[int, str], list[int]]:
col_map: dict[int, str] = {}
class_cols: list[int] = []
for idx, cell in enumerate(header_row):
field_key = _map_header(cell, book_year, book_month)
if field_key == '_class_register':
class_cols.append(idx)
elif field_key:
col_map[idx] = field_key
else:
h = str(cell or '').strip().replace(' ', '')
if (
not h
or '上课' in str(cell or '')
or '情况' in str(cell or '')
or '登记' in str(cell or '')
or CLASS_CELL_RE.match(h)
):
class_cols.append(idx)
notes_idx = next((i for i, k in col_map.items() if k == 'notes'), None)
end_idx = next(
(i for i, k in col_map.items() if k in ('end_balance', 'end_total_lessons', 'end_lessons')),
None,
)
scan_start = (end_idx + 1) if end_idx is not None else 0
scan_end = notes_idx if notes_idx is not None else len(header_row)
for idx in range(scan_start, scan_end):
if idx not in col_map and idx not in class_cols:
class_cols.append(idx)
if not class_cols:
mapped = set(col_map.keys())
start = (notes_idx + 1) if notes_idx is not None else (max(mapped) + 1 if mapped else 0)
for idx in range(start, len(header_row)):
if idx not in col_map:
class_cols.append(idx)
return col_map, class_cols
def _parse_standard_row(
data_row: tuple,
col_map: dict[int, str],
class_cols: list[int],
year: int,
month: int,
default_lessons: float,
section_course: str,
sheet_name: str,
) -> Optional[ParsedRow]:
name_idx = next((i for i, k in col_map.items() if k == 'student_name'), None)
if name_idx is None:
return None
name_val = data_row[name_idx] if name_idx < len(data_row) else None
if not name_val or not str(name_val).strip():
return None
name_str = str(name_val).strip().replace('\n', '')
if LONG_ID_RE.match(name_str) or name_str in ('DEL', 'XF'):
return None
full, display, nick = split_student_name(str(name_val))
course_idx = next((i for i, k in col_map.items() if k == 'course_level'), None)
course_raw = ''
if course_idx is not None and course_idx < len(data_row):
course_raw = str(data_row[course_idx] or '')
course_code = normalize_course_code(course_raw)
if not course_code:
course_code = section_course or sheet_default_course(sheet_name)
fields: dict[str, Optional[float]] = {}
notes = ''
for idx, key in col_map.items():
if idx >= len(data_row):
continue
val = data_row[idx]
if key in ('student_name', 'course_level', 'seq_no', '_pay_time'):
if key == '_pay_time':
pay_t = str(val or '').strip()
if pay_t:
notes = f'缴费时间:{pay_t}' + (f'；{notes}' if notes else '')
continue
if key == 'notes':
notes = str(val or '').strip()
continue
if key == 'total_hours':
th = _safe_float(val)
if th is not None:
fields['total_hours'] = th
continue
fv = _safe_float(val)
if fv is not None:
fields[key] = fv
# lab 累计课消表：用累计差推算本月消耗（仅当未提供 consumed_lessons）
if 'prev_cumulative_consumed' in fields or 'end_cumulative_consumed' in fields:
prev_c = fields.get('prev_cumulative_consumed')
end_c = fields.get('end_cumulative_consumed')
if fields.get('consumed_lessons') is None and prev_c is not None and end_c is not None:
fields['consumed_lessons'] = end_c - prev_c
seq_no = None
seq_idx = next((i for i, k in col_map.items() if k == 'seq_no'), None)
if seq_idx is not None and seq_idx < len(data_row):
try:
if data_row[seq_idx] not in (None, ''):
seq_no = int(float(data_row[seq_idx]))
except (TypeError, ValueError):
seq_no = None
sessions = []
for cidx in class_cols:
if cidx >= len(data_row):
continue
cell = data_row[cidx]
if cell is None or str(cell).strip() == '':
continue
parsed_cell = parse_class_cell(str(cell), year, month, default_lessons)
if parsed_cell:
sessions.append(parsed_cell)
return ParsedRow(
seq_no=seq_no,
student_name=full,
display_name=display,
nickname=nick,
course_code=course_code,
fields=fields,
class_sessions=sessions,
notes=notes,
source_sheet=sheet_name,
)
def _parse_free_class_row(
data_row: tuple,
class_cols: list[int],
year: int,
month: int,
default_lessons: float,
section_course: str,
sheet_name: str,
) -> tuple[Optional[ParsedRow], str]:
"""免费课表：班级|姓名|总课时|12月末剩余|1月已上|1月剩余|登记…"""
schedule = str(data_row[0] or '').strip() if len(data_row) > 0 else ''
new_section = infer_course_from_schedule(schedule) if schedule else ''
if new_section:
section_course = new_section
name_val = data_row[1] if len(data_row) > 1 else None
if not name_val or not str(name_val).strip():
return None, section_course
name_str = str(name_val).strip()
if '姓名' in name_str or LONG_ID_RE.match(name_str):
return None, section_course
full, display, nick = split_student_name(name_str)
course_code = section_course or 'free'
fields: dict[str, Optional[float]] = {}
if len(data_row) > 2:
v = _safe_float(data_row[2])
if v is not None:
fields['total_hours'] = v
if len(data_row) > 3:
v = _safe_float(data_row[3])
if v is not None:
fields['prev_lessons'] = v
if len(data_row) > 4:
v = _safe_float(data_row[4])
if v is not None:
fields['consumed_lessons'] = v
if len(data_row) > 5:
v = _safe_float(data_row[5])
if v is not None:
fields['end_lessons'] = v
notes = str(data_row[-1] or '').strip() if data_row else ''
if notes in ('免费', '备注'):
notes = ''
sessions = []
for cidx in class_cols:
if cidx >= len(data_row):
continue
cell = data_row[cidx]
if cell is None or str(cell).strip() == '':
continue
parsed_cell = parse_class_cell(str(cell), year, month, default_lessons)
if parsed_cell:
sessions.append(parsed_cell)
row = ParsedRow(
student_name=full,
display_name=display,
nickname=nick,
course_code=course_code,
fields=fields,
class_sessions=sessions,
notes=notes or '免费课',
source_sheet=sheet_name,
)
return row, section_course
def _parse_trial_payment_row(
data_row: tuple,
col_map: dict[int, str],
year: int,
month: int,
sheet_name: str,
) -> Optional[ParsedRow]:
name_idx = next((i for i, k in col_map.items() if k == 'student_name'), None)
if name_idx is None:
return None
name_val = data_row[name_idx] if name_idx < len(data_row) else None
if not name_val or not str(name_val).strip():
return None
name_str = str(name_val).strip()
if '姓名' in name_str or '合计' in name_str:
return None
full, display, nick = split_student_name(name_str)
fields: dict[str, Optional[float]] = {}
notes = '试学缴费'
for idx, key in col_map.items():
if idx >= len(data_row):
continue
val = data_row[idx]
if key == 'student_name':
continue
if key == '_pay_time':
pay_t = str(val or '').strip()
if pay_t:
notes = f'缴费时间:{pay_t}'
continue
if key == 'notes':
extra = str(val or '').strip()
if extra:
notes = f'{notes}；{extra}' if notes else extra
continue
if key == 'seq_no':
continue
fv = _safe_float(val)
if fv is not None:
fields[key] = fv
if not fields.get('new_signup_amount'):
return None
if name_str.isdigit() or LONG_ID_RE.match(name_str.replace(' ', '')):
return None
return ParsedRow(
student_name=full,
display_name=display,
nickname=nick,
course_code='trial',
fields=fields,
notes=notes,
source_sheet=sheet_name,
)
def parse_sheet_rows(
rows: list[tuple],
year: int,
month: int,
sheet_name: str,
default_lessons: float = 2.0,
) -> list[ParsedRow]:
parsed_rows: list[ParsedRow] = []
col_map: Optional[dict[int, str]] = None
class_cols: list[int] = []
layout = 'standard' # standard | free | tracking | trial
section_course = sheet_default_course(sheet_name)
for row in rows:
if not row or all(c is None or str(c).strip() == '' for c in row):
continue
if is_summary_row(row):
continue
if is_payment_data_row(row):
continue
if is_standard_header_row(row):
col_map, class_cols = _build_col_map(row, year, month)
layout = 'tracking' if is_tracking_header_row(row) else 'standard'
continue
if is_free_class_header_row(row) and not is_standard_header_row(row):
col_map, class_cols = _build_col_map(row, year, month)
if not class_cols:
class_cols = list(range(6, len(row)))
layout = 'free'
continue
if is_trial_payment_header_row(row):
col_map, class_cols = _build_col_map(row, year, month)
layout = 'trial'
continue
if layout == 'trial':
parsed = _parse_trial_payment_row(
row, col_map or {}, year, month, sheet_name,
)
if parsed and parsed.student_name:
parsed_rows.append(parsed)
continue
if layout == 'free':
result, section_course = _parse_free_class_row(
row, class_cols, year, month, default_lessons, section_course, sheet_name,
)
if result and result.student_name and result.course_code:
parsed_rows.append(result)
continue
if not col_map:
continue
parsed = _parse_standard_row(
row, col_map, class_cols, year, month, default_lessons, section_course, sheet_name,
)
if parsed and parsed.student_name and parsed.course_code:
parsed_rows.append(parsed)
return parsed_rows
@dataclass
class ParsedRow:
seq_no: Optional[int] = None
student_name: str = ''
display_name: str = ''
nickname: str = ''
course_code: str = ''
fields: dict[str, Optional[float]] = field(default_factory=dict)
class_sessions: list[tuple[date, float, str]] = field(default_factory=list)
notes: str = ''
source_sheet: str = ''
@dataclass
class ParsedWorkbook:
path: Path
year: int
month: int
sheet_name: str
title: str
rows: list[ParsedRow] = field(default_factory=list)
sheets_parsed: list[str] = field(default_factory=list)
sheets_skipped: list[str] = field(default_factory=list)
def parse_workbook(path: Path, default_lessons: float = 2.0) -> ParsedWorkbook:
path = Path(path)
ym = parse_filename_month(path)
wb = load_workbook(path, read_only=True, data_only=True)
all_rows: list[ParsedRow] = []
sheets_parsed: list[str] = []
sheets_skipped: list[str] = []
title = ''
for sheet_name in wb.sheetnames:
if should_skip_sheet(sheet_name):
sheets_skipped.append(sheet_name)
continue
ws = wb[sheet_name]
rows = list(ws.iter_rows(values_only=True))
if not rows:
continue
sheet_title = _find_title_in_row(rows[0])
if not sheet_title and len(rows) > 1:
sheet_title = _find_title_in_row(rows[1])
if sheet_title and not title:
title = sheet_title
if not ym and sheet_title:
ym = parse_title_month(sheet_title)
if not ym:
continue
year, month = ym
sheet_rows = parse_sheet_rows(rows, year, month, sheet_name, default_lessons)
if sheet_rows:
sheets_parsed.append(sheet_name)
all_rows.extend(sheet_rows)
wb.close()
if not ym:
raise ValueError(f'无法从文件名或标题解析年月: {path.name}')
year, month = ym
return ParsedWorkbook(
path=path,
year=year,
month=month,
sheet_name=','.join(sheets_parsed) if sheets_parsed else '',
title=title,
rows=all_rows,
sheets_parsed=sheets_parsed,
sheets_skipped=sheets_skipped,
)
def validate_balance(row: ParsedRow, tolerance: float = 1.0) -> list[str]:
"""校验月末余额公式，返回 warning 消息列表"""
f = row.fields
warnings = []
prev_bal = f.get('prev_balance')
end_bal = f.get('end_balance')
if prev_bal is None or end_bal is None:
return warnings
income = sum(f.get(k) or 0 for k in (
'new_signup_amount', 'renewal_amount',
))
outgo = sum(f.get(k) or 0 for k in (
'consumed_amount', 'refund_amount', 'account_fee',
))
expected = prev_bal + income - outgo
diff = abs(expected - end_bal)
if diff > tolerance:
warnings.append(
f'{row.student_name}/{row.course_code}: 余额不平 '
f'期望≈{expected:.2f} 实际={end_bal:.2f} 差={diff:.2f}'
)
return warnings
def batch_validate_rows(
rows: list[ParsedRow],
*,
balance_tolerance: float = 1.0,
lessons_tolerance: float = 0.05,
) -> list[str]:
"""批量校验所有行，一次性返回全部警告（不打印日志）"""
warnings: list[str] = []
for row in rows:
warnings.extend(validate_balance(row, tolerance=balance_tolerance))
warnings.extend(validate_lessons(row, tolerance=lessons_tolerance))
return warnings
def validate_lessons(row: ParsedRow, tolerance: float = 0.05) -> list[str]:
f = row.fields
warnings = []
prev = f.get('prev_lessons')
end = f.get('end_lessons')
if prev is None or end is None:
return warnings
add = sum(f.get(k) or 0 for k in ('new_signup_lessons', 'renewal_lessons'))
sub = sum(f.get(k) or 0 for k in ('consumed_lessons', 'refund_lessons'))
consumed = f.get('consumed_lessons') or 0
if consumed < 0:
sub = abs(consumed)
add = 0
expected = prev + add - sub
if abs(expected - end) > tolerance:
warnings.append(
f'{row.student_name}/{row.course_code}: 课时不平 '
f'期望≈{expected:.2f} 实际={end:.2f}'
)
return warnings