#!/bin/bash # 环境变量补丁 export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin [ -f /etc/profile ] && . /etc/profile [ -f ~/.bash_profile ] && . ~/.bash_profile [ -f ~/.profile ] && . ~/.profile # 检查参数 TARGET=$1 if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then echo "使用方法: $0 [doris|mysql]" exit 1 fi # 配置文件 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" CONF_DIR="$SCRIPT_DIR/json" CONFIG_FILE="$SCRIPT_DIR/db_config.sh" # 加载数据库配置 if [ -f "$CONFIG_FILE" ]; then source "$CONFIG_FILE" else echo "错误: 找不到配置文件 $CONFIG_FILE" exit 1 fi DATAX_HOME="/usr/local/datax" BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" # 明确指定要执行的15个文件 JOBS=( "t_equipment_charge_order.json" "t_equipment_charge_order_detail.json" "t_account_recharge.json" "t_account_water.json" "t_car.json" "t_company.json" "t_company_info_value.json" "t_connector.json" "t_equipment.json" "t_station.json" "t_ext_hurry_quit.json" "t_time_day.json" "t_user.json" "t_user_account.json" "t_user_upload_fault.json" ) # 日志配置 LOG_DIR="$SCRIPT_DIR/logs" if [ ! -d "$LOG_DIR" ]; then mkdir -p "$LOG_DIR" fi TIMESTAMP=$(date +%Y%m%d_%H%M%S) LOG_FILE="$LOG_DIR/run_full_${TARGET}_${TIMESTAMP}.log" echo "日志将同时记录到: $LOG_FILE" # 根据目标设置参数 if [ "$TARGET" == "doris" ]; then DEST_PARAMS="$DORIS_DEST_PARAMS" TITLE="MySQL -> Doris Full Sync" else DEST_PARAMS="$MYSQL_DEST_PARAMS" TITLE="MySQL -> MySQL Full Sync" fi # 在脚本最开始记录开始时间 SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) echo "DataX全量同步脚本 ($TITLE)" echo "==========================" # 切换到配置文件目录 cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR" exit 1 } SUCCESS_COUNT=0 FAIL_COUNT=0 TOTAL=${#JOBS[@]} declare -a TASK_RESULTS=() declare -a TASK_DURATIONS=() declare -a TASK_STATUSES=() for JOB in "${JOBS[@]}"; do TABLE_NAME="${JOB%.json}" echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" # 动态构造全量 SQL:SELECT * FROM table QUERY_SQL="SELECT * FROM $TABLE_NAME" # 动态构造清空 SQL:TRUNCATE TABLE table PRE_SQL="TRUNCATE TABLE $TABLE_NAME" # 注意:这里需要仔细处理参数引用,将变量作为整体传递 PARAMS="$SRC_BASE_PARAMS $DEST_PARAMS -Dsrc_query_sql=\"$QUERY_SQL\" -Ddest_pre_sql=\"$PRE_SQL\"" # 打印最终的执行命令(去除敏感信息后),用于调试 # echo "执行命令: python $DATAX_PY -p \"$PARAMS\" $JOB" START_TIME=$(date +%s) # 使用 eval 来正确处理带引号的参数,或者直接修改调用方式 # 由于PARAMS中包含空格和引号,直接引用可能会导致参数解析错误 # 最稳妥的方式是将 -p 后面的参数用单引号包起来,但内部又有双引号,比较麻烦 # 我们可以尝试直接将所有参数拼接好,让python去解析 /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" EXIT_CODE=${PIPESTATUS[0]} END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) TASK_RESULTS+=("$JOB") TASK_DURATIONS+=("$DURATION") if [ $EXIT_CODE -eq 0 ]; then echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) TASK_STATUSES+=("成功") else echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" FAIL_COUNT=$((FAIL_COUNT + 1)) TASK_STATUSES+=("失败") fi done # ... (汇总逻辑与之前相同) echo "==========================" echo "执行结果汇总 (Full Sync to $TARGET)" echo "==========================" echo "总任务数: $TOTAL" echo "成功: $SUCCESS_COUNT" echo "失败: $FAIL_COUNT" echo "=========================="