diff --git a/DataX/run_jobs_full.sh b/DataX/run_jobs_full_doris.sh similarity index 88% rename from DataX/run_jobs_full.sh rename to DataX/run_jobs_full_doris.sh index ae8d830..74338ab 100644 --- a/DataX/run_jobs_full.sh +++ b/DataX/run_jobs_full_doris.sh @@ -91,8 +91,25 @@ for JOB in "${JOBS[@]}"; do # 记录开始时间(用于计算耗时) START_TIME=$(date +%s) - # 执行任务 (添加 -p 参数传递变量) - /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" + # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) + CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." + + # 构造 JVM 参数 (可根据服务器配置调整堆内存) + JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" + + # 执行 DataX + java $JVM_OPTS -classpath "$CLASS_PATH" \ + -Dfile.encoding=UTF-8 \ + -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ + -Djava.security.egd=file:///dev/urandom \ + -Ddatax.home="$DATAX_HOME" \ + -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ + $PARAMS \ + com.alibaba.datax.core.Engine \ + -mode standalone \ + -jobid -1 \ + -job "$JOB" + EXIT_CODE=$? # 计算耗时 @@ -191,3 +208,4 @@ if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then fi echo "==========================" + diff --git a/DataX/run_jobs_full_mysql.sh b/DataX/run_jobs_full_mysql.sh new file mode 100644 index 0000000..2dc54fe --- /dev/null +++ b/DataX/run_jobs_full_mysql.sh @@ -0,0 +1,180 @@ +#!/bin/bash +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +DATAX_HOME="/usr/local/datax" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +CONF_DIR="$SCRIPT_DIR/json_full" +BIN_DIR="$DATAX_HOME/bin" +DATAX_PY="$BIN_DIR/datax.py" + +JOBS=( + "t_equipment_charge_order.json" + "t_equipment_charge_order_detail.json" + "t_account_recharge.json" + "t_account_water.json" + "t_car.json" + "t_company.json" + "t_company_info_value.json" + "t_connector.json" + "t_equipment.json" + "t_station.json" + "t_ext_hurry_quit.json" + "t_time_day.json" + "t_user.json" + "t_user_account.json" + "t_user_upload_fault.json" +) + +# 源端保持与现有 DataX 配置一致 +# 目标为 MySQL:10.10.14.210:22066 数据库 ylt 用户 ylt 密码 Ycharge666 +# 同时启用 rewriteBatchedStatements 以获得真正的批量写入性能 +PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false" + +SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_START_TIMESTAMP=$(date +%s) + +echo "DataX MySQL全量同步脚本 (MySQL -> MySQL)" +echo "=====================================" +echo "DataX路径: $DATAX_PY" +echo "配置文件目录: $CONF_DIR" +echo "任务数量: ${#JOBS[@]}" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "策略: 每张表 TRUNCATE 后以 INSERT 批量写入" +echo "=====================================" + +cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR"; exit 1; } + +SUCCESS_COUNT=0 +FAIL_COUNT=0 +TOTAL=${#JOBS[@]} +CURRENT=0 + +for JOB in "${JOBS[@]}"; do + CURRENT=$((CURRENT + 1)) + echo "任务 [$CURRENT/$TOTAL] $JOB" + echo "----------------------------------------" + + if [ ! -f "$JOB" ]; then + echo "✗ 错误: 文件不存在 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "----------------------------------------" + continue + fi + + START_TIME=$(date +%s) + + TMP_FILE="/tmp/datax_mysql_full_${JOB}" + /usr/bin/python - <<'PY' "$JOB" "$TMP_FILE" +import json, sys, re +src_path = sys.argv[1] +dst_path = sys.argv[2] + +with open(src_path, 'r', encoding='utf-8') as f: + data = json.load(f) + +unit = data['job']['content'][0] +reader = unit.get('reader', {}) +writer = unit.get('writer', {}) +rp = reader.get('parameter', {}) +wp = writer.get('parameter', {}) + +# 获取表名:优先从原 writer.connection.table 获取;否则从 reader.connection.table;最后从 querySql 解析 +table_name = None +try: + table_name = wp.get('connection', [{}])[0].get('table', [None])[0] +except Exception: + table_name = None +if not table_name: + try: + table_name = rp.get('connection', [{}])[0].get('table', [None])[0] + except Exception: + table_name = None +if not table_name: + q = rp.get('connection', [{}])[0].get('querySql', [None])[0] + if q: + m = re.search(r'FROM\\s+`?(\\w+)`?', q, re.IGNORECASE) + if m: + table_name = m.group(1) + +if not table_name: + raise RuntimeError("无法确定目标表名,请检查配置: %s" % src_path) + +# 列:从原 writer.column 优先获取,否则回退 reader.column +columns = wp.get('column') or rp.get('column') +if not columns: + raise RuntimeError("无法确定列列表,请检查配置: %s" % src_path) + +new_writer = { + "name": "mysqlwriter", + "parameter": { + "connection": [ + { + "jdbcUrl": "${dest_mysql_jdbc}", + "table": [table_name] + } + ], + "username": "${dest_mysql_user}", + "password": "${dest_mysql_pwd}", + "column": columns, + "writeMode": "insert", + "batchSize": 2000, + "preSql": [f"TRUNCATE TABLE `{table_name}`"], + "postSql": [] + } +} + +unit['writer'] = new_writer + +with open(dst_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=4) +PY + + if [ $? -ne 0 ]; then + echo "✗ 错误: 生成 MySQL Writer 配置失败 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "----------------------------------------" + continue + fi + + # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) + CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." + + # 构造 JVM 参数 (可根据服务器配置调整堆内存) + JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" + + # 执行 DataX + java $JVM_OPTS -classpath "$CLASS_PATH" \ + -Dfile.encoding=UTF-8 \ + -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ + -Djava.security.egd=file:///dev/urandom \ + -Ddatax.home="$DATAX_HOME" \ + -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ + $PARAMS \ + com.alibaba.datax.core.Engine \ + -mode standalone \ + -jobid -1 \ + -job "$TMP_FILE" + + EXIT_CODE=$? + + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + if [ $EXIT_CODE -eq 0 ]; then + echo "✓ 成功: $JOB 用时 ${DURATION}s" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + else + echo "✗ 失败: $JOB 用时 ${DURATION}s (exit=$EXIT_CODE)" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi + + echo "----------------------------------------" +done + +TOTAL_TIME=$(( $(date +%s) - SCRIPT_START_TIMESTAMP )) +echo "完成: 成功 $SUCCESS_COUNT, 失败 $FAIL_COUNT, 总耗时 ${TOTAL_TIME}s" +exit $([ $FAIL_COUNT -eq 0 ] && echo 0 || echo 1) + diff --git a/DataX/run_jobs_inc.sh b/DataX/run_jobs_inc_doris.sh similarity index 88% rename from DataX/run_jobs_inc.sh rename to DataX/run_jobs_inc_doris.sh index 0080f5f..2ea19cc 100644 --- a/DataX/run_jobs_inc.sh +++ b/DataX/run_jobs_inc_doris.sh @@ -93,8 +93,25 @@ for JOB in "${JOBS[@]}"; do # 记录开始时间(用于计算耗时) START_TIME=$(date +%s) - # 执行任务 (添加 -p 参数传递变量) - /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" + # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) + CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." + + # 构造 JVM 参数 (可根据服务器配置调整堆内存) + JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" + + # 执行 DataX + java $JVM_OPTS -classpath "$CLASS_PATH" \ + -Dfile.encoding=UTF-8 \ + -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ + -Djava.security.egd=file:///dev/urandom \ + -Ddatax.home="$DATAX_HOME" \ + -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ + $PARAMS \ + com.alibaba.datax.core.Engine \ + -mode standalone \ + -jobid -1 \ + -job "$JOB" + EXIT_CODE=$? # 计算耗时 @@ -182,3 +199,4 @@ if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then fi echo "==========================" + diff --git a/DataX/run_jobs_inc_mysql.sh b/DataX/run_jobs_inc_mysql.sh new file mode 100644 index 0000000..096c384 --- /dev/null +++ b/DataX/run_jobs_inc_mysql.sh @@ -0,0 +1,178 @@ +#!/bin/bash +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +DATAX_HOME="/usr/local/datax" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +CONF_DIR="$SCRIPT_DIR/json_inc" +BIN_DIR="$DATAX_HOME/bin" +DATAX_PY="$BIN_DIR/datax.py" + +JOBS=( + "t_equipment_charge_order.json" + "t_equipment_charge_order_detail.json" + "t_account_recharge.json" + "t_account_water.json" + "t_car.json" + "t_company.json" + "t_company_info_value.json" + "t_connector.json" + "t_equipment.json" + "t_station.json" + "t_ext_hurry_quit.json" + "t_time_day.json" + "t_user.json" + "t_user_account.json" + "t_user_upload_fault.json" +) + +# 源端保持与现有 DataX 配置一致 +# 目标为 MySQL:10.10.14.210:22066 数据库 ylt 用户 ylt 密码 Ycharge666 +# 增量同步使用 REPLACE 实现覆盖更新 +PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false" + +SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_START_TIMESTAMP=$(date +%s) + +echo "DataX MySQL增量同步脚本 (MySQL -> MySQL)" +echo "=====================================" +echo "DataX路径: $DATAX_PY" +echo "配置文件目录: $CONF_DIR" +echo "任务数量: ${#JOBS[@]}" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "策略: 每次同步最近10000条(由 reader 配置决定),目标端使用 REPLACE 实现覆盖更新" +echo "=====================================" + +cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR"; exit 1; } + +SUCCESS_COUNT=0 +FAIL_COUNT=0 +TOTAL=${#JOBS[@]} +CURRENT=0 + +for JOB in "${JOBS[@]}"; do + CURRENT=$((CURRENT + 1)) + echo "任务 [$CURRENT/$TOTAL] $JOB" + echo "----------------------------------------" + + if [ ! -f "$JOB" ]; then + echo "✗ 错误: 文件不存在 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "----------------------------------------" + continue + fi + + START_TIME=$(date +%s) + + TMP_FILE="/tmp/datax_mysql_inc_${JOB}" + /usr/bin/python - <<'PY' "$JOB" "$TMP_FILE" +import json, sys, re +src_path = sys.argv[1] +dst_path = sys.argv[2] + +with open(src_path, 'r', encoding='utf-8') as f: + data = json.load(f) + +unit = data['job']['content'][0] +reader = unit.get('reader', {}) +writer = unit.get('writer', {}) +rp = reader.get('parameter', {}) +wp = writer.get('parameter', {}) + +table_name = None +try: + table_name = wp.get('connection', [{}])[0].get('table', [None])[0] +except Exception: + table_name = None +if not table_name: + try: + table_name = rp.get('connection', [{}])[0].get('table', [None])[0] + except Exception: + table_name = None +if not table_name: + q = rp.get('connection', [{}])[0].get('querySql', [None])[0] + if q: + m = re.search(r'FROM\\s+`?(\\w+)`?', q, re.IGNORECASE) + if m: + table_name = m.group(1) + +if not table_name: + raise RuntimeError("无法确定目标表名,请检查配置: %s" % src_path) + +columns = wp.get('column') or rp.get('column') +if not columns: + raise RuntimeError("无法确定列列表,请检查配置: %s" % src_path) + +new_writer = { + "name": "mysqlwriter", + "parameter": { + "connection": [ + { + "jdbcUrl": "${dest_mysql_jdbc}", + "table": [table_name] + } + ], + "username": "${dest_mysql_user}", + "password": "${dest_mysql_pwd}", + "column": columns, + "writeMode": "replace", + "batchSize": 2000, + "preSql": [], + "postSql": [] + } +} + +unit['writer'] = new_writer + +with open(dst_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=4) +PY + + if [ $? -ne 0 ]; then + echo "✗ 错误: 生成 MySQL Writer 配置失败 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "----------------------------------------" + continue + fi + + # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) + CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." + + # 构造 JVM 参数 (可根据服务器配置调整堆内存) + JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" + + # 执行 DataX + java $JVM_OPTS -classpath "$CLASS_PATH" \ + -Dfile.encoding=UTF-8 \ + -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ + -Djava.security.egd=file:///dev/urandom \ + -Ddatax.home="$DATAX_HOME" \ + -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ + $PARAMS \ + com.alibaba.datax.core.Engine \ + -mode standalone \ + -jobid -1 \ + -job "$TMP_FILE" + + EXIT_CODE=$? + + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + if [ $EXIT_CODE -eq 0 ]; then + echo "✓ 成功: $JOB 用时 ${DURATION}s" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + else + echo "✗ 失败: $JOB 用时 ${DURATION}s (exit=$EXIT_CODE)" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi + + echo "----------------------------------------" +done + +TOTAL_TIME=$(( $(date +%s) - SCRIPT_START_TIMESTAMP )) +echo "完成: 成功 $SUCCESS_COUNT, 失败 $FAIL_COUNT, 总耗时 ${TOTAL_TIME}s" +exit $([ $FAIL_COUNT -eq 0 ] && echo 0 || echo 1) + diff --git a/DataX/start_cron.sh b/DataX/start_cron.sh deleted file mode 100644 index d51ce3b..0000000 --- a/DataX/start_cron.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -# ========================================== -# DataX 增量同步定时任务配置脚本 -# 适用系统: Rocky Linux 9 / CentOS / RHEL -# 功能: 添加每10分钟执行一次的 crontab 任务 -# ========================================== - -# 1. 基础配置 -# 假设您将整个文件夹上传到了 /usr/local/datax -WORK_DIR="/usr/local/datax" -SCRIPT_NAME="run_jobs_inc.sh" -SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" -# 日志存放位置,建议单独放在 logs 目录 -LOG_FILE="$WORK_DIR/logs/datax_inc_cron.log" - -# 2. 检查环境 -echo "正在检查环境..." - -# 检查工作目录 -if [ ! -d "$WORK_DIR" ]; then - echo "错误: 工作目录 $WORK_DIR 不存在" - echo "请先将 DataX 配置相关文件上传至服务器的 $WORK_DIR" - exit 1 -fi - -# 检查脚本是否存在 -if [ ! -f "$SCRIPT_PATH" ]; then - echo "错误: 执行脚本 $SCRIPT_PATH 不存在" - echo "请确保 $SCRIPT_NAME 位于 $WORK_DIR 目录下" - exit 1 -fi - -# 创建日志目录 -if [ ! -d "$(dirname "$LOG_FILE")" ]; then - mkdir -p "$(dirname "$LOG_FILE")" - echo "创建日志目录: $(dirname "$LOG_FILE")" -fi - -# 3. 赋予执行权限 -echo "正在赋予脚本执行权限..." -chmod +x "$SCRIPT_PATH" - -# 4. 配置 Crontab -# 任务定义: 每10分钟执行一次 (*/10) -# >> $LOG_FILE 2>&1 : 将标准输出和错误输出都追加写入日志文件 -CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" - -# 检查任务是否已存在 (避免重复添加) -# 使用 grep 查找包含脚本路径的行 -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") - -if [ -n "$EXISTING_JOB" ]; then - echo "警告: 检测到该脚本的定时任务已存在:" - echo " $EXISTING_JOB" - echo "----------------------------------------" - read -p "是否先删除旧任务再重新添加? (y/n): " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "已取消操作,未修改 Crontab。" - exit 0 - fi - - # 移除旧任务 (通过 grep -v 反向过滤掉包含脚本路径的行) - # 注意: 这会保留其他不相关的定时任务 - crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - - echo "旧任务已移除" -fi - -# 添加新任务 -# (crontab -l; echo "new job") | crontab - 这种写法可以保留原有任务并追加新任务 -(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - - -# 5. 验证与提示 -echo "----------------------------------------" -echo "✅ 定时任务设置成功!" -echo "----------------------------------------" -echo "当前用户的 Crontab 列表:" -crontab -l -echo "----------------------------------------" -echo "执行策略: 每10分钟执行一次" -echo "日志文件: $LOG_FILE" -echo "----------------------------------------" -echo "提示: 请确保 crond 服务正在运行" -echo "检查命令: systemctl status crond" -echo "启动命令: systemctl start crond" diff --git a/DataX/start_cron_doris_inc.sh b/DataX/start_cron_doris_inc.sh new file mode 100644 index 0000000..c2d4d04 --- /dev/null +++ b/DataX/start_cron_doris_inc.sh @@ -0,0 +1,33 @@ +#!/bin/bash +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc_doris.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" +LOG_FILE="$WORK_DIR/logs/datax_inc_doris_cron.log" + +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +echo "正在检查环境..." +if [ ! -d "$WORK_DIR" ]; then + echo "错误: 工作目录 $WORK_DIR 不存在"; exit 1 +fi +if [ ! -f "$SCRIPT_PATH" ]; then + echo "错误: 执行脚本 $SCRIPT_PATH 不存在"; exit 1 +fi +if [ ! -d "$(dirname "$LOG_FILE")" ]; then + mkdir -p "$(dirname "$LOG_FILE")" +fi + +chmod +x "$SCRIPT_PATH" +CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" + +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +if [ -n "$EXISTING_JOB" ]; then + crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - +fi +(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - + +echo "✅ Doris 增量定时任务设置成功" +crontab -l diff --git a/DataX/start_cron_mysql_inc.sh b/DataX/start_cron_mysql_inc.sh new file mode 100644 index 0000000..e25c252 --- /dev/null +++ b/DataX/start_cron_mysql_inc.sh @@ -0,0 +1,33 @@ +#!/bin/bash +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc_mysql.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" +LOG_FILE="$WORK_DIR/logs/datax_inc_mysql_cron.log" + +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +echo "正在检查环境..." +if [ ! -d "$WORK_DIR" ]; then + echo "错误: 工作目录 $WORK_DIR 不存在"; exit 1 +fi +if [ ! -f "$SCRIPT_PATH" ]; then + echo "错误: 执行脚本 $SCRIPT_PATH 不存在"; exit 1 +fi +if [ ! -d "$(dirname "$LOG_FILE")" ]; then + mkdir -p "$(dirname "$LOG_FILE")" +fi + +chmod +x "$SCRIPT_PATH" +CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" + +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +if [ -n "$EXISTING_JOB" ]; then + crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - +fi +(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - + +echo "✅ MySQL 增量定时任务设置成功" +crontab -l diff --git a/DataX/stop_cron.sh b/DataX/stop_cron.sh deleted file mode 100644 index ac633eb..0000000 --- a/DataX/stop_cron.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# ========================================== -# DataX 增量同步定时任务停止脚本 -# 适用系统: Rocky Linux 9 / CentOS / RHEL -# 功能: 移除 run_jobs_inc.sh 的 crontab 任务 -# 场景: 全量同步前防止增量任务干扰 -# ========================================== - -# 1. 基础配置 (需与 start_cron.sh 保持一致以确保匹配正确) -WORK_DIR="/usr/local/datax" -SCRIPT_NAME="run_jobs_inc.sh" -SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" - -# 2. 检查任务是否存在 -echo "正在检查定时任务..." - -# 使用 grep 查找包含脚本路径的行 -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") - -if [ -z "$EXISTING_JOB" ]; then - echo "----------------------------------------" - echo "提示: 未发现该脚本的定时任务,无需停止。" - echo "目标脚本: $SCRIPT_PATH" - echo "----------------------------------------" - exit 0 -fi - -echo "发现正在运行的定时任务:" -echo " $EXISTING_JOB" -echo "----------------------------------------" - -# 3. 移除任务 -echo "正在移除任务..." - -# 备份当前的 crontab -BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" -crontab -l > "$BACKUP_FILE" 2>/dev/null -echo "已备份当前 Crontab 到: $BACKUP_FILE" - -# 核心逻辑: 移除包含脚本路径的行 -# grep -v "$SCRIPT_PATH" : 排除包含脚本路径的行 -# crontab - : 将过滤后的内容重新写入 crontab -crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - - -# 4. 验证结果 -REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") - -if [ -z "$REMAINING_JOB" ]; then - echo "----------------------------------------" - echo "✅ 增量同步定时任务已成功停止 (移除)!" - echo "----------------------------------------" - echo "当前用户的 Crontab 列表:" - crontab -l - echo "----------------------------------------" - echo "现在您可以安全地执行全量同步操作了。" -else - echo "❌ 停止失败,任务仍然存在。请尝试手动编辑: crontab -e" - exit 1 -fi diff --git a/DataX/stop_cron_doris_inc.sh b/DataX/stop_cron_doris_inc.sh new file mode 100644 index 0000000..5dba5b2 --- /dev/null +++ b/DataX/stop_cron_doris_inc.sh @@ -0,0 +1,29 @@ +#!/bin/bash +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc_doris.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" + +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +echo "正在检查定时任务..." +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +if [ -z "$EXISTING_JOB" ]; then + echo "提示: 未发现 Doris 增量任务,无需停止。" + exit 0 +fi + +BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" +crontab -l > "$BACKUP_FILE" 2>/dev/null +crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - + +REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +if [ -z "$REMAINING_JOB" ]; then + echo "✅ Doris 增量定时任务已停止" + crontab -l +else + echo "❌ 停止失败,请手动执行: crontab -e" + exit 1 +fi diff --git a/DataX/stop_cron_mysql_inc.sh b/DataX/stop_cron_mysql_inc.sh new file mode 100644 index 0000000..075f2f3 --- /dev/null +++ b/DataX/stop_cron_mysql_inc.sh @@ -0,0 +1,29 @@ +#!/bin/bash +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc_mysql.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" + +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +echo "正在检查定时任务..." +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +if [ -z "$EXISTING_JOB" ]; then + echo "提示: 未发现 MySQL 增量任务,无需停止。" + exit 0 +fi + +BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" +crontab -l > "$BACKUP_FILE" 2>/dev/null +crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - + +REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +if [ -z "$REMAINING_JOB" ]; then + echo "✅ MySQL 增量定时任务已停止" + crontab -l +else + echo "❌ 停止失败,请手动执行: crontab -e" + exit 1 +fi