#!/bin/bash
export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
[ -f /etc/profile ] && . /etc/profile
[ -f ~/.bash_profile ] && . ~/.bash_profile
[ -f ~/.profile ] && . ~/.profile

TARGET=$1

if [ -z "$TARGET" ]; then
    echo "用法: $0 <target>"
    echo "请指定同步目标:"
    echo "  doris       - 全量同步到 Doris (默认端口 9030)"
    echo "  mysql       - 全量同步到 MySQL (CSV Load 模式，高性能)"
    # echo "  mysql_jdbc  - 全量同步到 MySQL (JDBC Batch 模式，速度较慢)"
    exit 1
fi

DATAX_HOME="/usr/local/datax"
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
CONF_DIR="$SCRIPT_DIR/json"  # 更新为统一目录
BIN_DIR="$DATAX_HOME/bin"
DATAX_PY="$BIN_DIR/datax.py"
TOOL_DIR="$SCRIPT_DIR/tool"

JOBS=(
    "t_equipment_charge_order.json"
    "t_equipment_charge_order_detail.json"
    "t_account_recharge.json"
    "t_account_water.json"
    "t_car.json"
    "t_company.json"
    "t_company_info_value.json"
    "t_connector.json"
    "t_equipment.json"
    "t_station.json"
    "t_ext_hurry_quit.json"
    "t_time_day.json"
    "t_user.json"
    "t_user_account.json"
    "t_user_upload_fault.json"
)

# 公共源端参数
SRC_PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"

if [ "$TARGET" == "doris" ]; then
    echo "模式: Doris 全量同步"
    DEST_PARAMS="-Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge?useSSL=false"
    PARAMS="$SRC_PARAMS $DEST_PARAMS"
elif [ "$TARGET" == "mysql" ]; then
    echo "模式: MySQL 全量同步 (CSV Load)"
    # 注意: CSV Load 模式下 DataX 负责写文件，Python 负责 Load。JDBC 参数用于 Python 连接。
    DEST_PARAMS="-Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowLoadLocalInfile=true"
    PARAMS="$SRC_PARAMS $DEST_PARAMS"
# elif [ "$TARGET" == "mysql_jdbc" ]; then
#     echo "模式: MySQL 全量同步 (JDBC Batch)"
#     DEST_PARAMS="-Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false"
#     PARAMS="$SRC_PARAMS $DEST_PARAMS"
else
    echo "错误: 未知目标 '$TARGET'。请使用 'doris' 或 'mysql'。"
    exit 1
fi

SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S')
SCRIPT_START_TIMESTAMP=$(date +%s)

echo "====================================="
echo "DataX 全量同步脚本 (Target: $TARGET)"
echo "====================================="
echo "配置文件目录: $CONF_DIR"
echo "任务数量: ${#JOBS[@]}"
echo "脚本开始时间: $SCRIPT_START_TIME"
echo "====================================="

cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR"; exit 1; }

SUCCESS_COUNT=0
FAIL_COUNT=0
TOTAL=${#JOBS[@]}
CURRENT=0

# 清理旧的 CSV 临时目录
if [ "$TARGET" == "mysql" ]; then
    rm -rf "$SCRIPT_DIR/datax_tmp_csv"
fi

for JOB in "${JOBS[@]}"; do
    CURRENT=$((CURRENT + 1))
    echo "任务 [$CURRENT/$TOTAL] $JOB"
    echo "----------------------------------------"

    if [ ! -f "$JOB" ]; then
        echo "✗ 错误: 文件不存在 - $JOB"
        FAIL_COUNT=$((FAIL_COUNT + 1))
        continue
    fi

    START_TIME=$(date +%s)
    JOB_FILE="$JOB"
    
    # 变量用于存储 mysql_load 模式下的元数据
    TABLE_NAME=""
    COLUMNS_JSON=""
    CSV_DIR=""

    # ---------------------------------------------------------
    # 配置转换逻辑
    # ---------------------------------------------------------
    
    TMP_FILE="/tmp/datax_optimized_${TARGET}_${JOB}"
    
    # 传递目标类型和 CSV 输出目录给 Python
    export DATAX_TARGET="$TARGET"
    if [ "$TARGET" == "mysql" ]; then
        CSV_DIR="$SCRIPT_DIR/datax_tmp_csv/${JOB%.json}"
        export DATAX_CSV_DIR="$CSV_DIR"
    fi

    # 执行 Python 转换脚本并捕获输出
    TRANSFORM_OUTPUT=$(/usr/bin/python - <<'PY' "$JOB" "$TMP_FILE"
import json, sys, re, os

src_path = sys.argv[1]
dst_path = sys.argv[2]
target = os.environ.get('DATAX_TARGET', 'doris')
csv_dir = os.environ.get('DATAX_CSV_DIR', '')

with open(src_path, 'r', encoding='utf-8') as f:
    data = json.load(f)

# 1. 基础性能优化 (设置并发和内存限制)
if 'setting' not in data['job']:
    data['job']['setting'] = {}
data['job']['setting']['speed'] = {
    "channel": 8,  # 提高并发到 8，加速 Reader 读取
}
data['job']['setting']['errorLimit'] = {
    "record": 0
}

unit = data['job']['content'][0]
reader = unit.get('reader', {})
writer = unit.get('writer', {})
rp = reader.get('parameter', {})
wp = writer.get('parameter', {})

# 2. Reader 优化 (FetchSize)
rp['fetchSize'] = 10000  # 极大增加读取缓存，减少网络往返

# 获取表名
table_name = None
try:
    # 优先尝试从 writer.parameter.connection[0].table[0] 获取
    table_name = wp.get('connection', [{}])[0].get('table', [None])[0]
except Exception:
    pass
if not table_name:
    try:
        # 其次尝试从 reader.parameter.connection[0].table[0] 获取
        table_name = rp.get('connection', [{}])[0].get('table', [None])[0]
    except Exception:
        pass
if not table_name:
    # 最后尝试从 querySql 解析
    q = rp.get('connection', [{}])[0].get('querySql', [None])[0]
    if q:
        m = re.search(r'FROM\s+`?(\w+)`?', q, re.IGNORECASE)
        if m:
            table_name = m.group(1)

if not table_name:
    # 如果还是没找到，使用文件名去掉 .json
    table_name = os.path.basename(src_path).replace('.json', '')

# 获取列
columns = wp.get('column') or rp.get('column')
if not columns:
    # 默认使用 * (虽然 DataX 不推荐，但作为兜底)
    columns = ["*"]

# 3. 根据目标转换 Writer
if target == 'mysql':
    if not os.path.exists(csv_dir):
        os.makedirs(csv_dir)
        
    new_writer = {
        "name": "txtfilewriter",
        "parameter": {
            "path": csv_dir,
            "fileName": table_name,
            "writeMode": "truncate",
            "fileFormat": "csv",
            "separator": ",",
            "quoteChar": "\"",
            "escapeChar": "\\",
            "nullFormat": "\\N",
            "header": [],
            "column": columns
        }
    }
    unit['writer'] = new_writer

# 输出元数据供 Shell 使用
print(table_name)
print(json.dumps(columns))

with open(dst_path, 'w', encoding='utf-8') as f:
    json.dump(data, f, ensure_ascii=False, indent=4)
PY
    )
        
    if [ $? -ne 0 ]; then
        echo "✗ 错误: 生成配置文件失败 - $JOB"
        echo "$TRANSFORM_OUTPUT"
        FAIL_COUNT=$((FAIL_COUNT + 1))
        continue
    fi
    
    JOB_FILE="$TMP_FILE"
    
    # 解析元数据
    TABLE_NAME=$(echo "$TRANSFORM_OUTPUT" | sed -n '1p' | tr -d '\r')
    COLUMNS_JSON=$(echo "$TRANSFORM_OUTPUT" | sed -n '2p' | tr -d '\r')

    # ---------------------------------------------------------
    # 执行 DataX
    # ---------------------------------------------------------
    
    # 构造 Java 命令
    CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:."
    # 优化 JVM 参数: 增大内存，使用 G1 回收器提高吞吐，增加 Metaspace 空间
    JVM_OPTS="-server -Xms2g -Xmx2g -XX:MaxMetaspaceSize=256m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError"
    
    java $JVM_OPTS -classpath "$CLASS_PATH" \
        -Dfile.encoding=UTF-8 \
        -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \
        -Djava.security.egd=file:///dev/urandom \
        -Ddatax.home="$DATAX_HOME" \
        -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \
        $PARAMS \
        com.alibaba.datax.core.Engine \
        -mode standalone \
        -jobid -1 \
        -job "$JOB_FILE" | sed "s/^/[$TABLE_NAME] /"
    
    EXIT_CODE=$?

    # ---------------------------------------------------------
    # 后置处理: mysql 模式下执行导入
    # ---------------------------------------------------------
    
    if [ $EXIT_CODE -eq 0 ] && [ "$TARGET" == "mysql" ]; then
        echo "DataX 导出完成，开始执行 MySQL Load Data..."
        
        # 提取连接参数
        # 注意: 这里简化处理，直接写死或从 PARAMS 中解析有点麻烦
        # 我们直接复用脚本顶部的变量，但要注意这些变量包含 -D 前缀
        # 所以最好直接传递硬编码的值或者重新定义变量
        
        # 解析参数 (简单暴力去除 -Ddest_mysql_... 前缀)
        # 这里为了稳健，我们直接使用 Python 脚本解析 JDBC URL
        
        # 定义 DB 连接信息 (与 DEST_PARAMS 保持一致)
        DB_USER="ylt"
        DB_PWD="Ycharge666"
        DB_JDBC="jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowLoadLocalInfile=true"
        
        python "$TOOL_DIR/LoadCsvToMysql.py" \
            "$DB_JDBC" \
            "$DB_USER" \
            "$DB_PWD" \
            "$TABLE_NAME" \
            "$CSV_DIR" \
            "$COLUMNS_JSON"
            
        EXIT_CODE=$?
        
        # 清理临时 CSV
        rm -rf "$CSV_DIR"
    fi

    END_TIME=$(date +%s)
    DURATION=$((END_TIME - START_TIME))

    if [ $EXIT_CODE -eq 0 ]; then
        echo "✓ 成功: $JOB 用时 ${DURATION}s"
        SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
    else
        echo "✗ 失败: $JOB 用时 ${DURATION}s (exit=$EXIT_CODE)"
        FAIL_COUNT=$((FAIL_COUNT + 1))
    fi
    echo "----------------------------------------"
done

TOTAL_TIME=$(( $(date +%s) - SCRIPT_START_TIMESTAMP ))
echo "完成: 成功 $SUCCESS_COUNT, 失败 $FAIL_COUNT, 总耗时 ${TOTAL_TIME}s"
exit $([ $FAIL_COUNT -eq 0 ] && echo 0 || echo 1)