From 3a6c19e850ce9190977f47f466255ea7fb8c437c Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Fri, 6 Feb 2026 07:27:23 +0800 Subject: [PATCH] 'commit' --- .../t_account_recharge.json | 0 .../{json_full => json}/t_account_water.json | 0 DataX/{json_full => json}/t_car.json | 0 DataX/{json_full => json}/t_company.json | 0 .../t_company_info_value.json | 0 DataX/{json_full => json}/t_connector.json | 0 DataX/{json_full => json}/t_equipment.json | 0 .../t_equipment_charge_order.json | 0 .../t_equipment_charge_order_detail.json | 0 .../{json_full => json}/t_ext_hurry_quit.json | 0 DataX/{json_full => json}/t_station.json | 0 DataX/{json_full => json}/t_time_day.json | 0 DataX/{json_full => json}/t_user.json | 0 DataX/{json_full => json}/t_user_account.json | 0 .../t_user_upload_fault.json | 0 DataX/json_inc/t_account_recharge.json | 124 -------- DataX/json_inc/t_account_water.json | 102 ------- DataX/json_inc/t_car.json | 96 ------ DataX/json_inc/t_company.json | 130 -------- DataX/json_inc/t_company_info_value.json | 74 ----- DataX/json_inc/t_connector.json | 110 ------- DataX/json_inc/t_equipment.json | 120 -------- DataX/json_inc/t_equipment_charge_order.json | 282 ------------------ .../t_equipment_charge_order_detail.json | 106 ------- DataX/json_inc/t_ext_hurry_quit.json | 118 -------- DataX/json_inc/t_station.json | 224 -------------- DataX/json_inc/t_time_day.json | 70 ----- DataX/json_inc/t_user.json | 150 ---------- DataX/json_inc/t_user_account.json | 94 ------ DataX/json_inc/t_user_upload_fault.json | 74 ----- ...un_jobs_full_mysql.sh => run_jobs_full.sh} | 71 +++-- DataX/run_jobs_full_doris.sh | 211 ------------- DataX/run_jobs_inc.sh | 227 ++++++++++++++ DataX/run_jobs_inc_doris.sh | 202 ------------- DataX/run_jobs_inc_mysql.sh | 178 ----------- DataX/start_cron_doris_inc.sh | 33 -- DataX/start_cron_inc.sh | 45 +++ DataX/start_cron_mysql_inc.sh | 33 -- DataX/stop_cron_doris_inc.sh | 29 -- DataX/stop_cron_inc.sh | 39 +++ DataX/stop_cron_mysql_inc.sh | 29 -- 41 files changed, 351 insertions(+), 2620 deletions(-) rename DataX/{json_full => json}/t_account_recharge.json (100%) rename DataX/{json_full => json}/t_account_water.json (100%) rename DataX/{json_full => json}/t_car.json (100%) rename DataX/{json_full => json}/t_company.json (100%) rename DataX/{json_full => json}/t_company_info_value.json (100%) rename DataX/{json_full => json}/t_connector.json (100%) rename DataX/{json_full => json}/t_equipment.json (100%) rename DataX/{json_full => json}/t_equipment_charge_order.json (100%) rename DataX/{json_full => json}/t_equipment_charge_order_detail.json (100%) rename DataX/{json_full => json}/t_ext_hurry_quit.json (100%) rename DataX/{json_full => json}/t_station.json (100%) rename DataX/{json_full => json}/t_time_day.json (100%) rename DataX/{json_full => json}/t_user.json (100%) rename DataX/{json_full => json}/t_user_account.json (100%) rename DataX/{json_full => json}/t_user_upload_fault.json (100%) delete mode 100644 DataX/json_inc/t_account_recharge.json delete mode 100644 DataX/json_inc/t_account_water.json delete mode 100644 DataX/json_inc/t_car.json delete mode 100644 DataX/json_inc/t_company.json delete mode 100644 DataX/json_inc/t_company_info_value.json delete mode 100644 DataX/json_inc/t_connector.json delete mode 100644 DataX/json_inc/t_equipment.json delete mode 100644 DataX/json_inc/t_equipment_charge_order.json delete mode 100644 DataX/json_inc/t_equipment_charge_order_detail.json delete mode 100644 DataX/json_inc/t_ext_hurry_quit.json delete mode 100644 DataX/json_inc/t_station.json delete mode 100644 DataX/json_inc/t_time_day.json delete mode 100644 DataX/json_inc/t_user.json delete mode 100644 DataX/json_inc/t_user_account.json delete mode 100644 DataX/json_inc/t_user_upload_fault.json rename DataX/{run_jobs_full_mysql.sh => run_jobs_full.sh} (69%) delete mode 100644 DataX/run_jobs_full_doris.sh create mode 100644 DataX/run_jobs_inc.sh delete mode 100644 DataX/run_jobs_inc_doris.sh delete mode 100644 DataX/run_jobs_inc_mysql.sh delete mode 100644 DataX/start_cron_doris_inc.sh create mode 100644 DataX/start_cron_inc.sh delete mode 100644 DataX/start_cron_mysql_inc.sh delete mode 100644 DataX/stop_cron_doris_inc.sh create mode 100644 DataX/stop_cron_inc.sh delete mode 100644 DataX/stop_cron_mysql_inc.sh diff --git a/DataX/json_full/t_account_recharge.json b/DataX/json/t_account_recharge.json similarity index 100% rename from DataX/json_full/t_account_recharge.json rename to DataX/json/t_account_recharge.json diff --git a/DataX/json_full/t_account_water.json b/DataX/json/t_account_water.json similarity index 100% rename from DataX/json_full/t_account_water.json rename to DataX/json/t_account_water.json diff --git a/DataX/json_full/t_car.json b/DataX/json/t_car.json similarity index 100% rename from DataX/json_full/t_car.json rename to DataX/json/t_car.json diff --git a/DataX/json_full/t_company.json b/DataX/json/t_company.json similarity index 100% rename from DataX/json_full/t_company.json rename to DataX/json/t_company.json diff --git a/DataX/json_full/t_company_info_value.json b/DataX/json/t_company_info_value.json similarity index 100% rename from DataX/json_full/t_company_info_value.json rename to DataX/json/t_company_info_value.json diff --git a/DataX/json_full/t_connector.json b/DataX/json/t_connector.json similarity index 100% rename from DataX/json_full/t_connector.json rename to DataX/json/t_connector.json diff --git a/DataX/json_full/t_equipment.json b/DataX/json/t_equipment.json similarity index 100% rename from DataX/json_full/t_equipment.json rename to DataX/json/t_equipment.json diff --git a/DataX/json_full/t_equipment_charge_order.json b/DataX/json/t_equipment_charge_order.json similarity index 100% rename from DataX/json_full/t_equipment_charge_order.json rename to DataX/json/t_equipment_charge_order.json diff --git a/DataX/json_full/t_equipment_charge_order_detail.json b/DataX/json/t_equipment_charge_order_detail.json similarity index 100% rename from DataX/json_full/t_equipment_charge_order_detail.json rename to DataX/json/t_equipment_charge_order_detail.json diff --git a/DataX/json_full/t_ext_hurry_quit.json b/DataX/json/t_ext_hurry_quit.json similarity index 100% rename from DataX/json_full/t_ext_hurry_quit.json rename to DataX/json/t_ext_hurry_quit.json diff --git a/DataX/json_full/t_station.json b/DataX/json/t_station.json similarity index 100% rename from DataX/json_full/t_station.json rename to DataX/json/t_station.json diff --git a/DataX/json_full/t_time_day.json b/DataX/json/t_time_day.json similarity index 100% rename from DataX/json_full/t_time_day.json rename to DataX/json/t_time_day.json diff --git a/DataX/json_full/t_user.json b/DataX/json/t_user.json similarity index 100% rename from DataX/json_full/t_user.json rename to DataX/json/t_user.json diff --git a/DataX/json_full/t_user_account.json b/DataX/json/t_user_account.json similarity index 100% rename from DataX/json_full/t_user_account.json rename to DataX/json/t_user_account.json diff --git a/DataX/json_full/t_user_upload_fault.json b/DataX/json/t_user_upload_fault.json similarity index 100% rename from DataX/json_full/t_user_upload_fault.json rename to DataX/json/t_user_upload_fault.json diff --git a/DataX/json_inc/t_account_recharge.json b/DataX/json_inc/t_account_recharge.json deleted file mode 100644 index 35d2510..0000000 --- a/DataX/json_inc/t_account_recharge.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "prid", - "order_id", - "outtradeno", - "pay_time", - "pay_money", - "refund_time", - "refund_money", - "refund_state", - "order_time", - "pay_xml", - "refund_xml", - "check_xml", - "pay_state", - "pay_type", - "phone", - "check_time", - "user_id", - "create_time", - "update_time", - "donate_money", - "memo", - "refund_memo", - "refund_outtradno", - "refund_image", - "refund_operer", - "rechage_operer", - "refund_reason", - "vip", - "meal_id", - "refund_apply_time", - "pay_method" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `prid`, `order_id`, `outtradeno`, `pay_time`, `pay_money`, `refund_time`, `refund_money`, `refund_state`, `order_time`, `pay_xml`, `refund_xml`, `check_xml`, `pay_state`, `pay_type`, `phone`, `check_time`, `user_id`, `create_time`, `update_time`, `donate_money`, `memo`, `refund_memo`, `refund_outtradno`, `refund_image`, `refund_operer`, `rechage_operer`, `refund_reason`, `vip`, `meal_id`, `refund_apply_time`, `pay_method` FROM `t_account_recharge` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "prid", - "order_id", - "outtradeno", - "pay_time", - "pay_money", - "refund_time", - "refund_money", - "refund_state", - "order_time", - "pay_xml", - "refund_xml", - "check_xml", - "pay_state", - "pay_type", - "phone", - "check_time", - "user_id", - "create_time", - "update_time", - "donate_money", - "memo", - "refund_memo", - "refund_outtradno", - "refund_image", - "refund_operer", - "rechage_operer", - "refund_reason", - "vip", - "meal_id", - "refund_apply_time", - "pay_method" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_account_recharge" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_account_water.json b/DataX/json_inc/t_account_water.json deleted file mode 100644 index 85f6fcb..0000000 --- a/DataX/json_inc/t_account_water.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "cash", - "phone", - "txt_desc", - "sign_msg", - "reason", - "sign", - "type", - "order_id", - "treatment", - "user_id", - "pay_status", - "pay_method", - "update_time", - "create_time", - "pre_balance", - "bef_balance", - "memo", - "donate_money", - "ua_balance", - "to_user_id" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `cash`, `phone`, `txt_desc`, `sign_msg`, `reason`, `sign`, `type`, `order_id`, `treatment`, `user_id`, `pay_status`, `pay_method`, `update_time`, `create_time`, `pre_balance`, `bef_balance`, `memo`, `donate_money`, `ua_balance`, `to_user_id` FROM `t_account_water` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "cash", - "phone", - "txt_desc", - "sign_msg", - "reason", - "sign", - "type", - "order_id", - "treatment", - "user_id", - "pay_status", - "pay_method", - "update_time", - "create_time", - "pre_balance", - "bef_balance", - "memo", - "donate_money", - "ua_balance", - "to_user_id" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_account_water" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_car.json b/DataX/json_inc/t_car.json deleted file mode 100644 index 4a1ad53..0000000 --- a/DataX/json_inc/t_car.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "car_plate_no", - "car_sn", - "car_brand_name", - "car_model_name", - "car_vin", - "car_engine", - "car_type", - "car_owner_company_id", - "car_business", - "car_bus_path", - "is_auto_charge", - "user_id", - "user_card_no", - "is_disable", - "create_time", - "update_time", - "operator_id" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `car_plate_no`, `car_sn`, `car_brand_name`, `car_model_name`, `car_vin`, `car_engine`, `car_type`, `car_owner_company_id`, `car_business`, `car_bus_path`, `is_auto_charge`, `user_id`, `user_card_no`, `is_disable`, `create_time`, `update_time`, `operator_id` FROM `t_car` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "car_plate_no", - "car_sn", - "car_brand_name", - "car_model_name", - "car_vin", - "car_engine", - "car_type", - "car_owner_company_id", - "car_business", - "car_bus_path", - "is_auto_charge", - "user_id", - "user_card_no", - "is_disable", - "create_time", - "update_time", - "operator_id" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_car" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_company.json b/DataX/json_inc/t_company.json deleted file mode 100644 index 8d66cff..0000000 --- a/DataX/json_inc/t_company.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "company_no", - "company_type", - "company_name", - "company_state", - "company_logo", - "company_admin_phone", - "company_admin_name", - "balance_remind", - "balance_remind_phone", - "agent_id", - "station_set_type", - "specified_range", - "specified_operator_ids", - "is_company_discount", - "is_overlay_coupons", - "station_ids", - "no_discount_stationids", - "remark", - "invoice_eligible", - "refuse_invoice_tip", - "is_share_master", - "settlement_method", - "charge_discount_type", - "charge_discount", - "settle_status", - "charge_unit_service", - "charge_unit_electric", - "is_custom_price", - "company_order_permission", - "open_station_ids", - "create_time", - "update_time", - "minimum_recharge_amount", - "inner_person" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `company_no`, `company_type`, `company_name`, `company_state`, `company_logo`, `company_admin_phone`, `company_admin_name`, `balance_remind`, `balance_remind_phone`, `agent_id`, `station_set_type`, `specified_range`, `specified_operator_ids`, `is_company_discount`, `is_overlay_coupons`, `station_ids`, `no_discount_stationids`, `remark`, `invoice_eligible`, `refuse_invoice_tip`, `is_share_master`, `settlement_method`, `charge_discount_type`, `charge_discount`, `settle_status`, `charge_unit_service`, `charge_unit_electric`, `is_custom_price`, `company_order_permission`, `open_station_ids`, `create_time`, `update_time`, `minimum_recharge_amount`, `inner_person` FROM `t_company` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "company_no", - "company_type", - "company_name", - "company_state", - "company_logo", - "company_admin_phone", - "company_admin_name", - "balance_remind", - "balance_remind_phone", - "agent_id", - "station_set_type", - "specified_range", - "specified_operator_ids", - "is_company_discount", - "is_overlay_coupons", - "station_ids", - "no_discount_stationids", - "remark", - "invoice_eligible", - "refuse_invoice_tip", - "is_share_master", - "settlement_method", - "charge_discount_type", - "charge_discount", - "settle_status", - "charge_unit_service", - "charge_unit_electric", - "is_custom_price", - "company_order_permission", - "open_station_ids", - "create_time", - "update_time", - "minimum_recharge_amount", - "inner_person" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_company" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_company_info_value.json b/DataX/json_inc/t_company_info_value.json deleted file mode 100644 index e8fbaeb..0000000 --- a/DataX/json_inc/t_company_info_value.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "info_value", - "type_id", - "create_time", - "update_time", - "remark", - "renzheng_flag" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `info_value`, `type_id`, `create_time`, `update_time`, `remark`, `renzheng_flag` FROM `t_company_info_value` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "info_value", - "type_id", - "create_time", - "update_time", - "remark", - "renzheng_flag" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_company_info_value" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_connector.json b/DataX/json_inc/t_connector.json deleted file mode 100644 index a18460f..0000000 --- a/DataX/json_inc/t_connector.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "equipment_id", - "connector_sn", - "connector_short_sn", - "connector_no", - "connector_name", - "connector_type", - "voltage_upper_limits", - "voltage_lower_limits", - "current", - "power", - "parkno", - "appointment_id", - "appointment_user_id", - "open_lock_user_id", - "detection_type", - "is_ground_lock", - "is_ground_lock_msg", - "is_geomagnetic", - "is_geomagnetic_msg", - "is_appointment", - "is_appointment_msg", - "auto_open_lock_time", - "create_time", - "update_time" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `equipment_id`, `connector_sn`, `connector_short_sn`, `connector_no`, `connector_name`, `connector_type`, `voltage_upper_limits`, `voltage_lower_limits`, `current`, `power`, `parkno`, `appointment_id`, `appointment_user_id`, `open_lock_user_id`, `detection_type`, `is_ground_lock`, `is_ground_lock_msg`, `is_geomagnetic`, `is_geomagnetic_msg`, `is_appointment`, `is_appointment_msg`, `auto_open_lock_time`, `create_time`, `update_time` FROM `t_connector` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "equipment_id", - "connector_sn", - "connector_short_sn", - "connector_no", - "connector_name", - "connector_type", - "voltage_upper_limits", - "voltage_lower_limits", - "current", - "power", - "parkno", - "appointment_id", - "appointment_user_id", - "open_lock_user_id", - "detection_type", - "is_ground_lock", - "is_ground_lock_msg", - "is_geomagnetic", - "is_geomagnetic_msg", - "is_appointment", - "is_appointment_msg", - "auto_open_lock_time", - "create_time", - "update_time" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_connector" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment.json b/DataX/json_inc/t_equipment.json deleted file mode 100644 index ce1b5ec..0000000 --- a/DataX/json_inc/t_equipment.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "station_id", - "agreement_id", - "equipment_name", - "equipment_sn", - "equipment_short_sn", - "equipment_no", - "manufacturer_id", - "manufacturer_sn", - "manufacturer_name", - "equipment_model", - "production_date", - "equipment_type", - "equipment_type_msg", - "equipment_lng", - "equipment_lat", - "connector_count", - "power", - "national_standard", - "national_standard_msg", - "status", - "equipment_business_type", - "equipment_business_type_msg", - "is_use_card", - "is_use_card_msg", - "charge_host_esn", - "create_time", - "update_time", - "aux_power", - "app_show" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `station_id`, `agreement_id`, `equipment_name`, `equipment_sn`, `equipment_short_sn`, `equipment_no`, `manufacturer_id`, `manufacturer_sn`, `manufacturer_name`, `equipment_model`, `production_date`, `equipment_type`, `equipment_type_msg`, `equipment_lng`, `equipment_lat`, `connector_count`, `power`, `national_standard`, `national_standard_msg`, `status`, `equipment_business_type`, `equipment_business_type_msg`, `is_use_card`, `is_use_card_msg`, `charge_host_esn`, `create_time`, `update_time`, `aux_power`, `app_show` FROM `t_equipment` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "station_id", - "agreement_id", - "equipment_name", - "equipment_sn", - "equipment_short_sn", - "equipment_no", - "manufacturer_id", - "manufacturer_sn", - "manufacturer_name", - "equipment_model", - "production_date", - "equipment_type", - "equipment_type_msg", - "equipment_lng", - "equipment_lat", - "connector_count", - "power", - "national_standard", - "national_standard_msg", - "status", - "equipment_business_type", - "equipment_business_type_msg", - "is_use_card", - "is_use_card_msg", - "charge_host_esn", - "create_time", - "update_time", - "aux_power", - "app_show" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment_charge_order.json b/DataX/json_inc/t_equipment_charge_order.json deleted file mode 100644 index 817438c..0000000 --- a/DataX/json_inc/t_equipment_charge_order.json +++ /dev/null @@ -1,282 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_id", - "company_id", - "station_id", - "equipment_id", - "connector_id", - "user_card_no", - "state", - "operate_state", - "order_no", - "platform_order_no", - "platform_sale_type", - "platform_driver_id", - "platform_phone", - "charging_amt", - "order_type", - "order_time", - "charge_begin_time", - "charge_end_time", - "charge_begin_degree", - "charge_end_degree", - "charge_degree", - "charge_ah", - "charge_times_degree", - "charge_begin_soc", - "charge_end_soc", - "charge_cur_soc", - "charge_duration", - "charge_settle_type", - "charge_unit_price", - "charge_unit_cost", - "charge_unit_service_fee", - "receivable_electric_fee", - "receivable_service_fee", - "receivable_total_fee", - "charge_elecfee_amount", - "charge_elecfee_cost_amount", - "charge_service_amount", - "actual_pay_amount", - "pay_order_no", - "pay_amount", - "pay_method", - "pay_status", - "pay_time", - "user_coupon_id", - "coupon_name", - "coupon_type", - "coupon_amount", - "elecfee_coupon_amount", - "servicefee_coupon_amount", - "invoice_id", - "invoice_fee", - "ticket_status", - "activity_electric_fee", - "activity_service_fee", - "charge_plate_no", - "charge_vin", - "charge_strategy", - "charge_strategy_param", - "operator_id", - "operator_income", - "plat_service_fee", - "charge_discount_type", - "settle_electric_fee", - "settle_service_fee", - "settle_fee", - "settle_method", - "settle_coupon_amount", - "settle_coupon_electric_fee", - "settle_coupon_service_fee", - "settle_status", - "cancel_type", - "cancel_code", - "cancel_msg", - "cancel_time", - "finish_type", - "finish_code", - "finish_msg", - "finish_time", - "boot_mode", - "settlement_type", - "is_parking_coupon", - "control_source", - "exception_flag", - "exception_msg", - "report_time", - "com_service_fee", - "com_electric_fee", - "com_total_fee", - "com_charge_discount_type", - "com_charge_discount", - "has_shallow_report", - "has_deep_report", - "is_exception", - "shallow_report_is_received", - "create_time", - "update_time", - "charge_market_id", - "coupon_total_amount", - "coupon_service_total_amount", - "coupon_elecfee_total_amount", - "activity_name", - "activity_total_fee", - "company_activity_id", - "company_activity_name", - "company_activity_total_fee", - "company_activity_service_fee", - "company_activity_electric_fee", - "subsidy_degree", - "subsidy_fee", - "special_channel_order_source" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `user_id`, `company_id`, `station_id`, `equipment_id`, `connector_id`, `user_card_no`, `state`, `operate_state`, `order_no`, `platform_order_no`, `platform_sale_type`, `platform_driver_id`, `platform_phone`, `charging_amt`, `order_type`, `order_time`, `charge_begin_time`, `charge_end_time`, `charge_begin_degree`, `charge_end_degree`, `charge_degree`, `charge_ah`, `charge_times_degree`, `charge_begin_soc`, `charge_end_soc`, `charge_cur_soc`, `charge_duration`, `charge_settle_type`, `charge_unit_price`, `charge_unit_cost`, `charge_unit_service_fee`, `receivable_electric_fee`, `receivable_service_fee`, `receivable_total_fee`, `charge_elecfee_amount`, `charge_elecfee_cost_amount`, `charge_service_amount`, `actual_pay_amount`, `pay_order_no`, `pay_amount`, `pay_method`, `pay_status`, `pay_time`, `user_coupon_id`, `coupon_name`, `coupon_type`, `coupon_amount`, `elecfee_coupon_amount`, `servicefee_coupon_amount`, `invoice_id`, `invoice_fee`, `ticket_status`, `activity_electric_fee`, `activity_service_fee`, `charge_plate_no`, `charge_vin`, `charge_strategy`, `charge_strategy_param`, `operator_id`, `operator_income`, `plat_service_fee`, `charge_discount_type`, `settle_electric_fee`, `settle_service_fee`, `settle_fee`, `settle_method`, `settle_coupon_amount`, `settle_coupon_electric_fee`, `settle_coupon_service_fee`, `settle_status`, `cancel_type`, `cancel_code`, `cancel_msg`, `cancel_time`, `finish_type`, `finish_code`, `finish_msg`, `finish_time`, `boot_mode`, `settlement_type`, `is_parking_coupon`, `control_source`, `exception_flag`, `exception_msg`, `report_time`, `com_service_fee`, `com_electric_fee`, `com_total_fee`, `com_charge_discount_type`, `com_charge_discount`, `has_shallow_report`, `has_deep_report`, `is_exception`, `shallow_report_is_received`, `create_time`, `update_time`, `charge_market_id`, `coupon_total_amount`, `coupon_service_total_amount`, `coupon_elecfee_total_amount`, `activity_name`, `activity_total_fee`, `company_activity_id`, `company_activity_name`, `company_activity_total_fee`, `company_activity_service_fee`, `company_activity_electric_fee`, `subsidy_degree`, `subsidy_fee`, `special_channel_order_source` FROM `t_equipment_charge_order` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_id", - "company_id", - "station_id", - "equipment_id", - "connector_id", - "user_card_no", - "state", - "operate_state", - "order_no", - "platform_order_no", - "platform_sale_type", - "platform_driver_id", - "platform_phone", - "charging_amt", - "order_type", - "order_time", - "charge_begin_time", - "charge_end_time", - "charge_begin_degree", - "charge_end_degree", - "charge_degree", - "charge_ah", - "charge_times_degree", - "charge_begin_soc", - "charge_end_soc", - "charge_cur_soc", - "charge_duration", - "charge_settle_type", - "charge_unit_price", - "charge_unit_cost", - "charge_unit_service_fee", - "receivable_electric_fee", - "receivable_service_fee", - "receivable_total_fee", - "charge_elecfee_amount", - "charge_elecfee_cost_amount", - "charge_service_amount", - "actual_pay_amount", - "pay_order_no", - "pay_amount", - "pay_method", - "pay_status", - "pay_time", - "user_coupon_id", - "coupon_name", - "coupon_type", - "coupon_amount", - "elecfee_coupon_amount", - "servicefee_coupon_amount", - "invoice_id", - "invoice_fee", - "ticket_status", - "activity_electric_fee", - "activity_service_fee", - "charge_plate_no", - "charge_vin", - "charge_strategy", - "charge_strategy_param", - "operator_id", - "operator_income", - "plat_service_fee", - "charge_discount_type", - "settle_electric_fee", - "settle_service_fee", - "settle_fee", - "settle_method", - "settle_coupon_amount", - "settle_coupon_electric_fee", - "settle_coupon_service_fee", - "settle_status", - "cancel_type", - "cancel_code", - "cancel_msg", - "cancel_time", - "finish_type", - "finish_code", - "finish_msg", - "finish_time", - "boot_mode", - "settlement_type", - "is_parking_coupon", - "control_source", - "exception_flag", - "exception_msg", - "report_time", - "com_service_fee", - "com_electric_fee", - "com_total_fee", - "com_charge_discount_type", - "com_charge_discount", - "has_shallow_report", - "has_deep_report", - "is_exception", - "shallow_report_is_received", - "create_time", - "update_time", - "charge_market_id", - "coupon_total_amount", - "coupon_service_total_amount", - "coupon_elecfee_total_amount", - "activity_name", - "activity_total_fee", - "company_activity_id", - "company_activity_name", - "company_activity_total_fee", - "company_activity_service_fee", - "company_activity_electric_fee", - "subsidy_degree", - "subsidy_fee", - "special_channel_order_source" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment_charge_order" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment_charge_order_detail.json b/DataX/json_inc/t_equipment_charge_order_detail.json deleted file mode 100644 index c76e67e..0000000 --- a/DataX/json_inc/t_equipment_charge_order_detail.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "order_no", - "charge_degree", - "service_fee", - "electric_fee", - "cost_fee", - "service_fee_unit_price", - "electric_fee_unit_price", - "cost_fee_unit_price", - "activity_service_fee", - "activity_electric_fee", - "charge_startr_time", - "charge_end_time", - "charge_meal_electric_fee", - "charge_meal_service_fee", - "charge_meal_total_fee", - "electric_card_use_dgree", - "check_electric_unit_price", - "check_service_unit_price", - "time_flag", - "create_time", - "update_time", - "detail_more" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `order_no`, `charge_degree`, `service_fee`, `electric_fee`, `cost_fee`, `service_fee_unit_price`, `electric_fee_unit_price`, `cost_fee_unit_price`, `activity_service_fee`, `activity_electric_fee`, `charge_startr_time`, `charge_end_time`, `charge_meal_electric_fee`, `charge_meal_service_fee`, `charge_meal_total_fee`, `electric_card_use_dgree`, `check_electric_unit_price`, `check_service_unit_price`, `time_flag`, `create_time`, `update_time`, `detail_more` FROM `t_equipment_charge_order_detail` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "order_no", - "charge_degree", - "service_fee", - "electric_fee", - "cost_fee", - "service_fee_unit_price", - "electric_fee_unit_price", - "cost_fee_unit_price", - "activity_service_fee", - "activity_electric_fee", - "charge_startr_time", - "charge_end_time", - "charge_meal_electric_fee", - "charge_meal_service_fee", - "charge_meal_total_fee", - "electric_card_use_dgree", - "check_electric_unit_price", - "check_service_unit_price", - "time_flag", - "create_time", - "update_time", - "detail_more" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment_charge_order_detail" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_ext_hurry_quit.json b/DataX/json_inc/t_ext_hurry_quit.json deleted file mode 100644 index fd3b1f5..0000000 --- a/DataX/json_inc/t_ext_hurry_quit.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "order_no", - "telephone", - "user_id", - "message1", - "message2", - "create_time", - "station_id", - "equipment_id", - "connector_id", - "flag", - "close_time", - "finish_time", - "minutesDifference", - "reduce_money", - "beforeBalance", - "afterBalance", - "fee_station", - "manual_count", - "last_manual_time", - "last_manual_message", - "person_id", - "refund_status", - "refund_time", - "refund_person_id", - "refund_person_name", - "refund_amount", - "refund_type", - "unplug_time" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `order_no`, `telephone`, `user_id`, `message1`, `message2`, `create_time`, `station_id`, `equipment_id`, `connector_id`, `flag`, `close_time`, `finish_time`, `minutesDifference`, `reduce_money`, `beforeBalance`, `afterBalance`, `fee_station`, `manual_count`, `last_manual_time`, `last_manual_message`, `person_id`, `refund_status`, `refund_time`, `refund_person_id`, `refund_person_name`, `refund_amount`, `refund_type`, `unplug_time` FROM `t_ext_hurry_quit` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "order_no", - "telephone", - "user_id", - "message1", - "message2", - "create_time", - "station_id", - "equipment_id", - "connector_id", - "flag", - "close_time", - "finish_time", - "minutesDifference", - "reduce_money", - "beforeBalance", - "afterBalance", - "fee_station", - "manual_count", - "last_manual_time", - "last_manual_message", - "person_id", - "refund_status", - "refund_time", - "refund_person_id", - "refund_person_name", - "refund_amount", - "refund_type", - "unplug_time" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_ext_hurry_quit" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_station.json b/DataX/json_inc/t_station.json deleted file mode 100644 index 11dc396..0000000 --- a/DataX/json_inc/t_station.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "station_sn", - "operator_id", - "equioment_owner_id", - "station_name", - "country_code", - "region_id", - "region_code", - "regional_name", - "regional_code", - "address", - "station_tel", - "service_tel", - "station_type", - "station_type_msg", - "station_business_type", - "station_business_type_msg", - "station_status", - "station_status_msg", - "park_nums", - "station_lng", - "station_lat", - "site_guide", - "construction", - "construction_msg", - "pictures", - "match_cars", - "park_info", - "business_start_time", - "business_end_time", - "electricity_fee", - "service_fee", - "is_park_fee", - "park_fee", - "payment", - "payment_msg", - "support_order", - "support_order_msg", - "service_item", - "service_item_image_url", - "service_item_msg", - "station_location_type", - "station_location_type_msg", - "station_charge_type", - "station_charge_type_msg", - "station_power", - "station_voltage", - "station_current", - "is_open", - "is_open_msg", - "is_display", - "is_display_msg", - "sharing_code", - "sharing_applet_picture", - "theme_picture", - "is_barrier_flag", - "is_barrier_flag_msg", - "charge_settle_type", - "charge_settle_type_msg", - "fast_connector_nums", - "slow_connector_nums", - "transformer_capacity", - "installed_capacity", - "remark", - "record_unique_no", - "equipment_owner_name", - "supply_type", - "resident_no", - "watt_hour_meter_no", - "forward_power", - "operation_date", - "charge_replace_type", - "is_hlht", - "is_subsidy", - "create_time", - "update_time", - "area_code_countryside", - "business_expand_type", - "capacity", - "video_monitor", - "electricity_type", - "receive_shop" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `station_sn`, `operator_id`, `equioment_owner_id`, `station_name`, `country_code`, `region_id`, `region_code`, `regional_name`, `regional_code`, `address`, `station_tel`, `service_tel`, `station_type`, `station_type_msg`, `station_business_type`, `station_business_type_msg`, `station_status`, `station_status_msg`, `park_nums`, `station_lng`, `station_lat`, `site_guide`, `construction`, `construction_msg`, `pictures`, `match_cars`, `park_info`, `business_start_time`, `business_end_time`, `electricity_fee`, `service_fee`, `is_park_fee`, `park_fee`, `payment`, `payment_msg`, `support_order`, `support_order_msg`, `service_item`, `service_item_image_url`, `service_item_msg`, `station_location_type`, `station_location_type_msg`, `station_charge_type`, `station_charge_type_msg`, `station_power`, `station_voltage`, `station_current`, `is_open`, `is_open_msg`, `is_display`, `is_display_msg`, `sharing_code`, `sharing_applet_picture`, `theme_picture`, `is_barrier_flag`, `is_barrier_flag_msg`, `charge_settle_type`, `charge_settle_type_msg`, `fast_connector_nums`, `slow_connector_nums`, `transformer_capacity`, `installed_capacity`, `remark`, `record_unique_no`, `equipment_owner_name`, `supply_type`, `resident_no`, `watt_hour_meter_no`, `forward_power`, `operation_date`, `charge_replace_type`, `is_hlht`, `is_subsidy`, `create_time`, `update_time`, `area_code_countryside`, `business_expand_type`, `capacity`, `video_monitor`, `electricity_type`, `receive_shop` FROM `t_station` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "station_sn", - "operator_id", - "equioment_owner_id", - "station_name", - "country_code", - "region_id", - "region_code", - "regional_name", - "regional_code", - "address", - "station_tel", - "service_tel", - "station_type", - "station_type_msg", - "station_business_type", - "station_business_type_msg", - "station_status", - "station_status_msg", - "park_nums", - "station_lng", - "station_lat", - "site_guide", - "construction", - "construction_msg", - "pictures", - "match_cars", - "park_info", - "business_start_time", - "business_end_time", - "electricity_fee", - "service_fee", - "is_park_fee", - "park_fee", - "payment", - "payment_msg", - "support_order", - "support_order_msg", - "service_item", - "service_item_image_url", - "service_item_msg", - "station_location_type", - "station_location_type_msg", - "station_charge_type", - "station_charge_type_msg", - "station_power", - "station_voltage", - "station_current", - "is_open", - "is_open_msg", - "is_display", - "is_display_msg", - "sharing_code", - "sharing_applet_picture", - "theme_picture", - "is_barrier_flag", - "is_barrier_flag_msg", - "charge_settle_type", - "charge_settle_type_msg", - "fast_connector_nums", - "slow_connector_nums", - "transformer_capacity", - "installed_capacity", - "remark", - "record_unique_no", - "equipment_owner_name", - "supply_type", - "resident_no", - "watt_hour_meter_no", - "forward_power", - "operation_date", - "charge_replace_type", - "is_hlht", - "is_subsidy", - "create_time", - "update_time", - "area_code_countryside", - "business_expand_type", - "capacity", - "video_monitor", - "electricity_type", - "receive_shop" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_station" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_time_day.json b/DataX/json_inc/t_time_day.json deleted file mode 100644 index 6c17810..0000000 --- a/DataX/json_inc/t_time_day.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "create_time", - "update_time", - "time_fq", - "teme_ve" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `create_time`, `update_time`, `time_fq`, `teme_ve` FROM `t_time_day` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "create_time", - "update_time", - "time_fq", - "teme_ve" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_time_day" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_user.json b/DataX/json_inc/t_user.json deleted file mode 100644 index aed8105..0000000 --- a/DataX/json_inc/t_user.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_name", - "nicker_name", - "sex", - "phone", - "qk_openid", - "open_id", - "ali_user_id", - "birthday", - "avatar", - "app_type", - "password", - "upper_user", - "master_flag", - "user_type", - "multi_charge", - "relation_phone", - "user_owner_id", - "union_id", - "state", - "reg_channel", - "reg_time", - "city_label_code", - "city_label", - "reg_source", - "create_time", - "update_time", - "operator", - "invite_code", - "identity_id", - "operator_id", - "address", - "vip_level", - "province_code", - "province", - "work_company", - "is_bind_wxinfo", - "company_min_charge_balance", - "enable_online_recharge", - "integral", - "shared_user", - "enable_online_refund", - "recommenderId", - "lx_wx_user_id", - "phone_area" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `user_name`, `nicker_name`, `sex`, `phone`, `qk_openid`, `open_id`, `ali_user_id`, `birthday`, `avatar`, `app_type`, `password`, `upper_user`, `master_flag`, `user_type`, `multi_charge`, `relation_phone`, `user_owner_id`, `union_id`, `state`, `reg_channel`, `reg_time`, `city_label_code`, `city_label`, `reg_source`, `create_time`, `update_time`, `operator`, `invite_code`, `identity_id`, `operator_id`, `address`, `vip_level`, `province_code`, `province`, `work_company`, `is_bind_wxinfo`, `company_min_charge_balance`, `enable_online_recharge`, `integral`, `shared_user`, `enable_online_refund`, `recommenderId`, `lx_wx_user_id`, `phone_area` FROM `t_user` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_name", - "nicker_name", - "sex", - "phone", - "qk_openid", - "open_id", - "ali_user_id", - "birthday", - "avatar", - "app_type", - "password", - "upper_user", - "master_flag", - "user_type", - "multi_charge", - "relation_phone", - "user_owner_id", - "union_id", - "state", - "reg_channel", - "reg_time", - "city_label_code", - "city_label", - "reg_source", - "create_time", - "update_time", - "operator", - "invite_code", - "identity_id", - "operator_id", - "address", - "vip_level", - "province_code", - "province", - "work_company", - "is_bind_wxinfo", - "company_min_charge_balance", - "enable_online_recharge", - "integral", - "shared_user", - "enable_online_refund", - "recommenderId", - "lx_wx_user_id", - "phone_area" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_user" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_user_account.json b/DataX/json_inc/t_user_account.json deleted file mode 100644 index d44036e..0000000 --- a/DataX/json_inc/t_user_account.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_id", - "ua_balance", - "ua_deposit", - "ua_balance_time", - "ua_password", - "ua_state", - "ua_real_balance", - "ua_real_degree", - "create_time", - "update_time", - "donate_money", - "donate_able_money", - "ua_able_balance", - "refund_money", - "ua_consume_money", - "ua_clear_zero_money" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `user_id`, `ua_balance`, `ua_deposit`, `ua_balance_time`, `ua_password`, `ua_state`, `ua_real_balance`, `ua_real_degree`, `create_time`, `update_time`, `donate_money`, `donate_able_money`, `ua_able_balance`, `refund_money`, `ua_consume_money`, `ua_clear_zero_money` FROM `t_user_account` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_id", - "ua_balance", - "ua_deposit", - "ua_balance_time", - "ua_password", - "ua_state", - "ua_real_balance", - "ua_real_degree", - "create_time", - "update_time", - "donate_money", - "donate_able_money", - "ua_able_balance", - "refund_money", - "ua_consume_money", - "ua_clear_zero_money" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_user_account" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_user_upload_fault.json b/DataX/json_inc/t_user_upload_fault.json deleted file mode 100644 index e382b7f..0000000 --- a/DataX/json_inc/t_user_upload_fault.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "equipment_id", - "user_id", - "fault_config_id", - "remark", - "create_time", - "update_time" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `equipment_id`, `user_id`, `fault_config_id`, `remark`, `create_time`, `update_time` FROM `t_user_upload_fault` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "equipment_id", - "user_id", - "fault_config_id", - "remark", - "create_time", - "update_time" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_user_upload_fault" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/run_jobs_full_mysql.sh b/DataX/run_jobs_full.sh similarity index 69% rename from DataX/run_jobs_full_mysql.sh rename to DataX/run_jobs_full.sh index 2dc54fe..c5b11b5 100644 --- a/DataX/run_jobs_full_mysql.sh +++ b/DataX/run_jobs_full.sh @@ -4,9 +4,11 @@ export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin [ -f ~/.bash_profile ] && . ~/.bash_profile [ -f ~/.profile ] && . ~/.profile +TARGET=${1:-doris} # 默认为 doris + DATAX_HOME="/usr/local/datax" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -CONF_DIR="$SCRIPT_DIR/json_full" +CONF_DIR="$SCRIPT_DIR/json" # 更新为统一目录 BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" @@ -28,21 +30,31 @@ JOBS=( "t_user_upload_fault.json" ) -# 源端保持与现有 DataX 配置一致 -# 目标为 MySQL：10.10.14.210:22066 数据库 ylt 用户 ylt 密码 Ycharge666 -# 同时启用 rewriteBatchedStatements 以获得真正的批量写入性能 -PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false" +# 公共源端参数 +SRC_PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge" + +if [ "$TARGET" == "doris" ]; then + echo "模式: Doris 全量同步" + DEST_PARAMS="-Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" + PARAMS="$SRC_PARAMS $DEST_PARAMS" +elif [ "$TARGET" == "mysql" ]; then + echo "模式: MySQL 全量同步 (10.10.14.210:22066)" + DEST_PARAMS="-Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false" + PARAMS="$SRC_PARAMS $DEST_PARAMS" +else + echo "错误: 未知目标 '$TARGET'。请使用 'doris' 或 'mysql'。" + exit 1 +fi SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) -echo "DataX MySQL全量同步脚本 (MySQL -> MySQL)" echo "=====================================" -echo "DataX路径: $DATAX_PY" +echo "DataX 全量同步脚本 (Target: $TARGET)" +echo "=====================================" echo "配置文件目录: $CONF_DIR" echo "任务数量: ${#JOBS[@]}" echo "脚本开始时间: $SCRIPT_START_TIME" -echo "策略: 每张表 TRUNCATE 后以 INSERT 批量写入" echo "=====================================" cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR"; exit 1; } @@ -60,14 +72,16 @@ for JOB in "${JOBS[@]}"; do if [ ! -f "$JOB" ]; then echo "✗ 错误: 文件不存在 - $JOB" FAIL_COUNT=$((FAIL_COUNT + 1)) - echo "----------------------------------------" continue fi START_TIME=$(date +%s) + JOB_FILE="$JOB" - TMP_FILE="/tmp/datax_mysql_full_${JOB}" - /usr/bin/python - <<'PY' "$JOB" "$TMP_FILE" + # 如果是 MySQL，需要生成临时配置文件修改 Writer + if [ "$TARGET" == "mysql" ]; then + TMP_FILE="/tmp/datax_mysql_full_${JOB}" + /usr/bin/python - <<'PY' "$JOB" "$TMP_FILE" import json, sys, re src_path = sys.argv[1] dst_path = sys.argv[2] @@ -81,31 +95,31 @@ writer = unit.get('writer', {}) rp = reader.get('parameter', {}) wp = writer.get('parameter', {}) -# 获取表名：优先从原 writer.connection.table 获取；否则从 reader.connection.table；最后从 querySql 解析 +# 获取表名 table_name = None try: table_name = wp.get('connection', [{}])[0].get('table', [None])[0] except Exception: - table_name = None + pass if not table_name: try: table_name = rp.get('connection', [{}])[0].get('table', [None])[0] except Exception: - table_name = None + pass if not table_name: q = rp.get('connection', [{}])[0].get('querySql', [None])[0] if q: - m = re.search(r'FROM\\s+`?(\\w+)`?', q, re.IGNORECASE) + m = re.search(r'FROM\s+`?(\w+)`?', q, re.IGNORECASE) if m: table_name = m.group(1) if not table_name: - raise RuntimeError("无法确定目标表名，请检查配置: %s" % src_path) + raise RuntimeError("无法确定目标表名: %s" % src_path) -# 列：从原 writer.column 优先获取，否则回退 reader.column +# 获取列 columns = wp.get('column') or rp.get('column') if not columns: - raise RuntimeError("无法确定列列表，请检查配置: %s" % src_path) + raise RuntimeError("无法确定列列表: %s" % src_path) new_writer = { "name": "mysqlwriter", @@ -131,21 +145,18 @@ unit['writer'] = new_writer with open(dst_path, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=4) PY - - if [ $? -ne 0 ]; then - echo "✗ 错误: 生成 MySQL Writer 配置失败 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - echo "----------------------------------------" - continue + if [ $? -ne 0 ]; then + echo "✗ 错误: 生成 MySQL Writer 配置失败 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + continue + fi + JOB_FILE="$TMP_FILE" fi - # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) + # 构造 Java 命令 CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." - - # 构造 JVM 参数 (可根据服务器配置调整堆内存) JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" - # 执行 DataX java $JVM_OPTS -classpath "$CLASS_PATH" \ -Dfile.encoding=UTF-8 \ -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ @@ -156,7 +167,7 @@ PY com.alibaba.datax.core.Engine \ -mode standalone \ -jobid -1 \ - -job "$TMP_FILE" + -job "$JOB_FILE" EXIT_CODE=$? @@ -170,11 +181,9 @@ PY echo "✗ 失败: $JOB 用时 ${DURATION}s (exit=$EXIT_CODE)" FAIL_COUNT=$((FAIL_COUNT + 1)) fi - echo "----------------------------------------" done TOTAL_TIME=$(( $(date +%s) - SCRIPT_START_TIMESTAMP )) echo "完成: 成功 $SUCCESS_COUNT, 失败 $FAIL_COUNT, 总耗时 ${TOTAL_TIME}s" exit $([ $FAIL_COUNT -eq 0 ] && echo 0 || echo 1) - diff --git a/DataX/run_jobs_full_doris.sh b/DataX/run_jobs_full_doris.sh deleted file mode 100644 index 74338ab..0000000 --- a/DataX/run_jobs_full_doris.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash - -# 环境变量补丁 (针对 Crontab 环境下可能找不到 java 的问题) -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -# 配置路径 -DATAX_HOME="/usr/local/datax" -# 获取当前脚本所在目录 -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# JSON配置文件目录指向 json_full -CONF_DIR="$SCRIPT_DIR/json_full" -BIN_DIR="$DATAX_HOME/bin" -DATAX_PY="$BIN_DIR/datax.py" - -# 明确指定要执行的15个文件（全量同步） -JOBS=( - "t_equipment_charge_order.json" - "t_equipment_charge_order_detail.json" - "t_account_recharge.json" - "t_account_water.json" - "t_car.json" - "t_company.json" - "t_company_info_value.json" - "t_connector.json" - "t_equipment.json" - "t_station.json" - "t_ext_hurry_quit.json" - "t_time_day.json" - "t_user.json" - "t_user_account.json" - "t_user_upload_fault.json" -) - -# 定义全局参数 -PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" - - -# 在脚本最开始记录开始时间 -SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_START_TIMESTAMP=$(date +%s) - -echo "DataX全量同步脚本 (Full Sync)" -echo "==========================" -echo "DataX路径: $DATAX_PY" -echo "配置文件目录: $CONF_DIR" -echo "任务数量: ${#JOBS[@]}" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "==========================" - -# 切换到配置文件目录 -cd "$CONF_DIR" || { - echo "错误: 无法进入配置文件目录 $CONF_DIR" - exit 1 -} - -echo "当前目录: $(pwd)" -echo - -# 执行计数器 -SUCCESS_COUNT=0 -FAIL_COUNT=0 -TOTAL=${#JOBS[@]} -CURRENT=0 - -# 数组存储每个任务的执行详情 -declare -a TASK_RESULTS=() -declare -a TASK_DURATIONS=() -declare -a TASK_STATUSES=() - -for JOB in "${JOBS[@]}"; do - CURRENT=$((CURRENT + 1)) - - echo "任务 [$CURRENT/$TOTAL] $JOB" - echo "任务开始时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" - - # 检查文件是否存在 - if [ ! -f "$JOB" ]; then - echo "✗ 错误: 文件不存在 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - TASK_RESULTS+=("$JOB") - TASK_STATUSES+=("文件不存在") - TASK_DURATIONS+=("0") - echo "----------------------------------------" - continue - fi - - # 记录开始时间（用于计算耗时） - START_TIME=$(date +%s) - - # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) - CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." - - # 构造 JVM 参数 (可根据服务器配置调整堆内存) - JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" - - # 执行 DataX - java $JVM_OPTS -classpath "$CLASS_PATH" \ - -Dfile.encoding=UTF-8 \ - -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ - -Djava.security.egd=file:///dev/urandom \ - -Ddatax.home="$DATAX_HOME" \ - -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ - $PARAMS \ - com.alibaba.datax.core.Engine \ - -mode standalone \ - -jobid -1 \ - -job "$JOB" - - EXIT_CODE=$? - - # 计算耗时 - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME)) - - # 保存任务详情 - TASK_RESULTS+=("$JOB") - TASK_DURATIONS+=("$DURATION") - - # 处理执行结果 - if [ $EXIT_CODE -eq 0 ]; then - echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" - SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) - TASK_STATUSES+=("成功") - else - echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" - echo "错误代码: $EXIT_CODE" - FAIL_COUNT=$((FAIL_COUNT + 1)) - TASK_STATUSES+=("失败") - - # 询问是否继续执行后续任务 - # 全量同步失败可能需要人工干预，但这里保持自动询问或直接继续 - # 为方便自动化，这里改为仅提示，不阻塞 (或者保持原样) - # 保持原样: - # echo "----------------------------------------" - # read -p "任务失败，是否继续执行下一个任务？(y/n): " -n 1 -r - # echo - # if [[ ! $REPLY =~ ^[Yy]$ ]]; then - # echo "用户选择停止执行" - # break - # fi - fi - - echo "任务结束时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" - echo - - # 可选：任务间等待时间（秒） - # sleep 1 -done - -# 记录脚本结束时间 -SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_END_TIMESTAMP=$(date +%s) -TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) - -echo "==========================" -echo "执行结果汇总 (Full Sync)" -echo "==========================" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "脚本结束时间: $SCRIPT_END_TIME" -echo "脚本总耗时: ${TOTAL_DURATION}秒" -echo "==========================" -echo "任务执行详情:" -echo "----------------------------------------" - -# 输出每个任务的执行结果 -for i in "${!TASK_RESULTS[@]}"; do - STATUS_ICON="✓" - if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then - STATUS_ICON="✗" - fi - printf "%-40s %-2s %-10s %-8s秒\n" \ - "${TASK_RESULTS[$i]}" \ - "$STATUS_ICON" \ - "${TASK_STATUSES[$i]}" \ - "${TASK_DURATIONS[$i]}" -done - -echo "----------------------------------------" -echo "汇总统计:" -echo "总任务数: $TOTAL" -echo "成功: $SUCCESS_COUNT" -echo "失败: $FAIL_COUNT" -if [ $TOTAL -gt 0 ]; then - SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) - echo "成功率: ${SUCCESS_RATE}%" -fi - -# 计算平均耗时 -if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then - TOTAL_TASK_TIME=0 - VALID_TASKS=0 - for duration in "${TASK_DURATIONS[@]}"; do - if [ "$duration" -gt 0 ]; then - TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) - VALID_TASKS=$((VALID_TASKS + 1)) - fi - done - - if [ $VALID_TASKS -gt 0 ]; then - AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) - echo "平均任务耗时: ${AVG_DURATION}秒" - fi -fi - -echo "==========================" - diff --git a/DataX/run_jobs_inc.sh b/DataX/run_jobs_inc.sh new file mode 100644 index 0000000..bb6834f --- /dev/null +++ b/DataX/run_jobs_inc.sh @@ -0,0 +1,227 @@ +#!/bin/bash +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +TARGET=${1:-doris} # 默认为 doris + +DATAX_HOME="/usr/local/datax" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +CONF_DIR="$SCRIPT_DIR/json" # 更新为统一目录 +BIN_DIR="$DATAX_HOME/bin" +DATAX_PY="$BIN_DIR/datax.py" + +JOBS=( + "t_equipment_charge_order.json" + "t_equipment_charge_order_detail.json" + "t_account_recharge.json" + "t_account_water.json" + "t_car.json" + "t_company.json" + "t_company_info_value.json" + "t_connector.json" + "t_equipment.json" + "t_station.json" + "t_ext_hurry_quit.json" + "t_time_day.json" + "t_user.json" + "t_user_account.json" + "t_user_upload_fault.json" +) + +# 公共源端参数 +SRC_PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge" + +if [ "$TARGET" == "doris" ]; then + echo "模式: Doris 增量同步" + DEST_PARAMS="-Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" + PARAMS="$SRC_PARAMS $DEST_PARAMS" +elif [ "$TARGET" == "mysql" ]; then + echo "模式: MySQL 增量同步 (10.10.14.210:22066)" + DEST_PARAMS="-Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false" + PARAMS="$SRC_PARAMS $DEST_PARAMS" +else + echo "错误: 未知目标 '$TARGET'。请使用 'doris' 或 'mysql'。" + exit 1 +fi + +SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_START_TIMESTAMP=$(date +%s) + +echo "=====================================" +echo "DataX 增量同步脚本 (Target: $TARGET)" +echo "=====================================" +echo "配置文件目录: $CONF_DIR" +echo "任务数量: ${#JOBS[@]}" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "=====================================" + +cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR"; exit 1; } + +SUCCESS_COUNT=0 +FAIL_COUNT=0 +TOTAL=${#JOBS[@]} +CURRENT=0 + +for JOB in "${JOBS[@]}"; do + CURRENT=$((CURRENT + 1)) + echo "任务 [$CURRENT/$TOTAL] $JOB" + echo "----------------------------------------" + + if [ ! -f "$JOB" ]; then + echo "✗ 错误: 文件不存在 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + continue + fi + + START_TIME=$(date +%s) + JOB_FILE="$JOB" + + # 生成临时配置文件: 修改 Reader 为 querySql (增量逻辑) 且根据目标修改 Writer + TMP_FILE="/tmp/datax_inc_${TARGET}_${JOB}" + + # 传递目标类型给 Python 脚本 + export DATAX_TARGET="$TARGET" + + /usr/bin/python - <<'PY' "$JOB" "$TMP_FILE" +import json, sys, os, re + +src_path = sys.argv[1] +dst_path = sys.argv[2] +target = os.environ.get('DATAX_TARGET', 'doris') + +with open(src_path, 'r', encoding='utf-8') as f: + data = json.load(f) + +unit = data['job']['content'][0] +reader = unit.get('reader', {}) +writer = unit.get('writer', {}) +rp = reader.get('parameter', {}) +wp = writer.get('parameter', {}) + +# --------------------------------------------------------- +# 1. Reader 转换: 从 table 模式转换为 querySql 模式 (增量逻辑) +# --------------------------------------------------------- + +# 获取列列表 +columns = rp.get('column') +if not columns: + # 尝试从 writer 获取 + columns = wp.get('column') + +if not columns: + raise RuntimeError("无法确定列列表: %s" % src_path) + +# 获取表名 +table_name = None +try: + table_name = rp.get('connection', [{}])[0].get('table', [None])[0] +except Exception: + pass + +if not table_name: + # 尝试从 querySql 解析 (如果源文件已经是 querySql 模式) + q = rp.get('connection', [{}])[0].get('querySql', [None])[0] + if q: + m = re.search(r'FROM\s+`?(\w+)`?', q, re.IGNORECASE) + if m: + table_name = m.group(1) + +if not table_name: + raise RuntimeError("无法确定表名: %s" % src_path) + +# 构建增量查询 SQL: SELECT col1, col2... FROM table ORDER BY id DESC LIMIT 10000 +# 注意: 这里假设所有表都有 id 列。如果无 id 列，可能需要特殊处理。 +# 安全起见，检查 columns 中是否包含 'id' +has_id = 'id' in columns + +if has_id: + # 构建列字符串，处理反引号 + col_str = ", ".join(["`{}`".format(c) for c in columns]) + inc_sql = "SELECT {} FROM `{}` ORDER BY id DESC LIMIT 10000".format(col_str, table_name) + + # 更新 Reader 配置 + new_reader_conn = [ + { + "jdbcUrl": rp.get('connection', [{}])[0].get('jdbcUrl', ["${src_jdbc}"]), + "querySql": [inc_sql] + } + ] + rp['connection'] = new_reader_conn + # 清除 table 和 where 参数，因为 querySql 模式下这些通常不生效或冲突 + if 'table' in rp: del rp['table'] + if 'where' in rp: del rp['where'] +else: + print("警告: 表 %s 没有 id 列，无法使用 ORDER BY id DESC 策略，保持原样" % table_name) + +# --------------------------------------------------------- +# 2. Writer 转换: 如果目标是 MySQL，转换为 mysqlwriter +# --------------------------------------------------------- + +if target == 'mysql': + new_writer = { + "name": "mysqlwriter", + "parameter": { + "connection": [ + { + "jdbcUrl": "${dest_mysql_jdbc}", + "table": [table_name] + } + ], + "username": "${dest_mysql_user}", + "password": "${dest_mysql_pwd}", + "column": columns, + "writeMode": "replace", # 增量同步使用 replace 模式 + "batchSize": 2000, + "preSql": [], + "postSql": [] + } + } + unit['writer'] = new_writer + +with open(dst_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=4) +PY + + if [ $? -ne 0 ]; then + echo "✗ 错误: 生成增量配置文件失败 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + continue + fi + JOB_FILE="$TMP_FILE" + + # 构造 Java 命令 + CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." + JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" + + java $JVM_OPTS -classpath "$CLASS_PATH" \ + -Dfile.encoding=UTF-8 \ + -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ + -Djava.security.egd=file:///dev/urandom \ + -Ddatax.home="$DATAX_HOME" \ + -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ + $PARAMS \ + com.alibaba.datax.core.Engine \ + -mode standalone \ + -jobid -1 \ + -job "$JOB_FILE" + + EXIT_CODE=$? + + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + if [ $EXIT_CODE -eq 0 ]; then + echo "✓ 成功: $JOB 用时 ${DURATION}s" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + else + echo "✗ 失败: $JOB 用时 ${DURATION}s (exit=$EXIT_CODE)" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi + echo "----------------------------------------" +done + +TOTAL_TIME=$(( $(date +%s) - SCRIPT_START_TIMESTAMP )) +echo "完成: 成功 $SUCCESS_COUNT, 失败 $FAIL_COUNT, 总耗时 ${TOTAL_TIME}s" +exit $([ $FAIL_COUNT -eq 0 ] && echo 0 || echo 1) diff --git a/DataX/run_jobs_inc_doris.sh b/DataX/run_jobs_inc_doris.sh deleted file mode 100644 index 2ea19cc..0000000 --- a/DataX/run_jobs_inc_doris.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/bin/bash - -# 环境变量补丁 (针对 Crontab 环境下可能找不到 java 的问题) -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -# 配置路径 -DATAX_HOME="/usr/local/datax" -# 获取当前脚本所在目录 -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# JSON配置文件目录指向 json_inc -CONF_DIR="$SCRIPT_DIR/json_inc" -BIN_DIR="$DATAX_HOME/bin" -DATAX_PY="$BIN_DIR/datax.py" - -# 明确指定要执行的15个文件（增量同步） -JOBS=( - "t_equipment_charge_order.json" - "t_equipment_charge_order_detail.json" - "t_account_recharge.json" - "t_account_water.json" - "t_car.json" - "t_company.json" - "t_company_info_value.json" - "t_connector.json" - "t_equipment.json" - "t_station.json" - "t_ext_hurry_quit.json" - "t_time_day.json" - "t_user.json" - "t_user_account.json" - "t_user_upload_fault.json" -) - -# 定义全局参数 -PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" - - -# 在脚本最开始记录开始时间 -SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_START_TIMESTAMP=$(date +%s) - -echo "DataX增量同步脚本 (Incremental Sync)" -echo "==========================" -echo "DataX路径: $DATAX_PY" -echo "配置文件目录: $CONF_DIR" -echo "任务数量: ${#JOBS[@]}" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "注意: 增量模式下不执行 Truncate，请确保目标表为 Unique Key 模型以实现 Upsert 效果" -echo "策略: 每次同步最近10000条数据(按ID倒序)，利用Doris主键机制自动覆盖更新" -echo "==========================" - -# 切换到配置文件目录 -cd "$CONF_DIR" || { - echo "错误: 无法进入配置文件目录 $CONF_DIR" - exit 1 -} - -echo "当前目录: $(pwd)" -echo - -# 执行计数器 -SUCCESS_COUNT=0 -FAIL_COUNT=0 -TOTAL=${#JOBS[@]} -CURRENT=0 - -# 数组存储每个任务的执行详情 -declare -a TASK_RESULTS=() -declare -a TASK_DURATIONS=() -declare -a TASK_STATUSES=() - -for JOB in "${JOBS[@]}"; do - CURRENT=$((CURRENT + 1)) - - echo "任务 [$CURRENT/$TOTAL] $JOB" - echo "任务开始时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" - - # 检查文件是否存在 - if [ ! -f "$JOB" ]; then - echo "✗ 错误: 文件不存在 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - TASK_RESULTS+=("$JOB") - TASK_STATUSES+=("文件不存在") - TASK_DURATIONS+=("0") - echo "----------------------------------------" - continue - fi - - # 记录开始时间（用于计算耗时） - START_TIME=$(date +%s) - - # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) - CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." - - # 构造 JVM 参数 (可根据服务器配置调整堆内存) - JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" - - # 执行 DataX - java $JVM_OPTS -classpath "$CLASS_PATH" \ - -Dfile.encoding=UTF-8 \ - -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ - -Djava.security.egd=file:///dev/urandom \ - -Ddatax.home="$DATAX_HOME" \ - -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ - $PARAMS \ - com.alibaba.datax.core.Engine \ - -mode standalone \ - -jobid -1 \ - -job "$JOB" - - EXIT_CODE=$? - - # 计算耗时 - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME)) - - # 保存任务详情 - TASK_RESULTS+=("$JOB") - TASK_DURATIONS+=("$DURATION") - - # 处理执行结果 - if [ $EXIT_CODE -eq 0 ]; then - echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" - SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) - TASK_STATUSES+=("成功") - else - echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" - echo "错误代码: $EXIT_CODE" - FAIL_COUNT=$((FAIL_COUNT + 1)) - TASK_STATUSES+=("失败") - # 增量任务通常无人值守，失败不暂停 - fi - - echo "任务结束时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" - echo - - # 可选：任务间等待时间（秒） - # sleep 1 -done - -# 记录脚本结束时间 -SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_END_TIMESTAMP=$(date +%s) -TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) - -echo "==========================" -echo "执行结果汇总 (Incremental Sync)" -echo "==========================" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "脚本结束时间: $SCRIPT_END_TIME" -echo "脚本总耗时: ${TOTAL_DURATION}秒" -echo "==========================" -echo "任务执行详情:" -echo "----------------------------------------" - -# 输出每个任务的执行结果 -for i in "${!TASK_RESULTS[@]}"; do - STATUS_ICON="✓" - if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then - STATUS_ICON="✗" - fi - printf "%-40s %-2s %-10s %-8s秒\n" \ - "${TASK_RESULTS[$i]}" \ - "$STATUS_ICON" \ - "${TASK_STATUSES[$i]}" \ - "${TASK_DURATIONS[$i]}" -done - -echo "----------------------------------------" -echo "汇总统计:" -echo "总任务数: $TOTAL" -echo "成功: $SUCCESS_COUNT" -echo "失败: $FAIL_COUNT" -if [ $TOTAL -gt 0 ]; then - SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) - echo "成功率: ${SUCCESS_RATE}%" -fi - -# 计算平均耗时 -if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then - TOTAL_TASK_TIME=0 - VALID_TASKS=0 - for duration in "${TASK_DURATIONS[@]}"; do - if [ "$duration" -gt 0 ]; then - TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) - VALID_TASKS=$((VALID_TASKS + 1)) - fi - done - - if [ $VALID_TASKS -gt 0 ]; then - AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) - echo "平均任务耗时: ${AVG_DURATION}秒" - fi -fi - -echo "==========================" - diff --git a/DataX/run_jobs_inc_mysql.sh b/DataX/run_jobs_inc_mysql.sh deleted file mode 100644 index 096c384..0000000 --- a/DataX/run_jobs_inc_mysql.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -DATAX_HOME="/usr/local/datax" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -CONF_DIR="$SCRIPT_DIR/json_inc" -BIN_DIR="$DATAX_HOME/bin" -DATAX_PY="$BIN_DIR/datax.py" - -JOBS=( - "t_equipment_charge_order.json" - "t_equipment_charge_order_detail.json" - "t_account_recharge.json" - "t_account_water.json" - "t_car.json" - "t_company.json" - "t_company_info_value.json" - "t_connector.json" - "t_equipment.json" - "t_station.json" - "t_ext_hurry_quit.json" - "t_time_day.json" - "t_user.json" - "t_user_account.json" - "t_user_upload_fault.json" -) - -# 源端保持与现有 DataX 配置一致 -# 目标为 MySQL：10.10.14.210:22066 数据库 ylt 用户 ylt 密码 Ycharge666 -# 增量同步使用 REPLACE 实现覆盖更新 -PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_mysql_user=ylt -Ddest_mysql_pwd=Ycharge666 -Ddest_mysql_jdbc=jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=UTF-8&useSSL=false&rewriteBatchedStatements=true&autoReconnect=true&failOverReadOnly=false" - -SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_START_TIMESTAMP=$(date +%s) - -echo "DataX MySQL增量同步脚本 (MySQL -> MySQL)" -echo "=====================================" -echo "DataX路径: $DATAX_PY" -echo "配置文件目录: $CONF_DIR" -echo "任务数量: ${#JOBS[@]}" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "策略: 每次同步最近10000条(由 reader 配置决定)，目标端使用 REPLACE 实现覆盖更新" -echo "=====================================" - -cd "$CONF_DIR" || { echo "错误: 无法进入配置文件目录 $CONF_DIR"; exit 1; } - -SUCCESS_COUNT=0 -FAIL_COUNT=0 -TOTAL=${#JOBS[@]} -CURRENT=0 - -for JOB in "${JOBS[@]}"; do - CURRENT=$((CURRENT + 1)) - echo "任务 [$CURRENT/$TOTAL] $JOB" - echo "----------------------------------------" - - if [ ! -f "$JOB" ]; then - echo "✗ 错误: 文件不存在 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - echo "----------------------------------------" - continue - fi - - START_TIME=$(date +%s) - - TMP_FILE="/tmp/datax_mysql_inc_${JOB}" - /usr/bin/python - <<'PY' "$JOB" "$TMP_FILE" -import json, sys, re -src_path = sys.argv[1] -dst_path = sys.argv[2] - -with open(src_path, 'r', encoding='utf-8') as f: - data = json.load(f) - -unit = data['job']['content'][0] -reader = unit.get('reader', {}) -writer = unit.get('writer', {}) -rp = reader.get('parameter', {}) -wp = writer.get('parameter', {}) - -table_name = None -try: - table_name = wp.get('connection', [{}])[0].get('table', [None])[0] -except Exception: - table_name = None -if not table_name: - try: - table_name = rp.get('connection', [{}])[0].get('table', [None])[0] - except Exception: - table_name = None -if not table_name: - q = rp.get('connection', [{}])[0].get('querySql', [None])[0] - if q: - m = re.search(r'FROM\\s+`?(\\w+)`?', q, re.IGNORECASE) - if m: - table_name = m.group(1) - -if not table_name: - raise RuntimeError("无法确定目标表名，请检查配置: %s" % src_path) - -columns = wp.get('column') or rp.get('column') -if not columns: - raise RuntimeError("无法确定列列表，请检查配置: %s" % src_path) - -new_writer = { - "name": "mysqlwriter", - "parameter": { - "connection": [ - { - "jdbcUrl": "${dest_mysql_jdbc}", - "table": [table_name] - } - ], - "username": "${dest_mysql_user}", - "password": "${dest_mysql_pwd}", - "column": columns, - "writeMode": "replace", - "batchSize": 2000, - "preSql": [], - "postSql": [] - } -} - -unit['writer'] = new_writer - -with open(dst_path, 'w', encoding='utf-8') as f: - json.dump(data, f, ensure_ascii=False, indent=4) -PY - - if [ $? -ne 0 ]; then - echo "✗ 错误: 生成 MySQL Writer 配置失败 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - echo "----------------------------------------" - continue - fi - - # 构造 Java 命令 (绕过 datax.py 以避免环境兼容问题) - CLASS_PATH="$DATAX_HOME/lib/*:$DATAX_HOME/conf:." - - # 构造 JVM 参数 (可根据服务器配置调整堆内存) - JVM_OPTS="-server -Xms1g -Xmx1g -XX:+HeapDumpOnOutOfMemoryError" - - # 执行 DataX - java $JVM_OPTS -classpath "$CLASS_PATH" \ - -Dfile.encoding=UTF-8 \ - -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ - -Djava.security.egd=file:///dev/urandom \ - -Ddatax.home="$DATAX_HOME" \ - -Dlogback.configurationFile="$DATAX_HOME/conf/logback.xml" \ - $PARAMS \ - com.alibaba.datax.core.Engine \ - -mode standalone \ - -jobid -1 \ - -job "$TMP_FILE" - - EXIT_CODE=$? - - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME)) - - if [ $EXIT_CODE -eq 0 ]; then - echo "✓ 成功: $JOB 用时 ${DURATION}s" - SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) - else - echo "✗ 失败: $JOB 用时 ${DURATION}s (exit=$EXIT_CODE)" - FAIL_COUNT=$((FAIL_COUNT + 1)) - fi - - echo "----------------------------------------" -done - -TOTAL_TIME=$(( $(date +%s) - SCRIPT_START_TIMESTAMP )) -echo "完成: 成功 $SUCCESS_COUNT, 失败 $FAIL_COUNT, 总耗时 ${TOTAL_TIME}s" -exit $([ $FAIL_COUNT -eq 0 ] && echo 0 || echo 1) - diff --git a/DataX/start_cron_doris_inc.sh b/DataX/start_cron_doris_inc.sh deleted file mode 100644 index c2d4d04..0000000 --- a/DataX/start_cron_doris_inc.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -WORK_DIR="/usr/local/datax" -SCRIPT_NAME="run_jobs_inc_doris.sh" -SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" -LOG_FILE="$WORK_DIR/logs/datax_inc_doris_cron.log" - -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -echo "正在检查环境..." -if [ ! -d "$WORK_DIR" ]; then - echo "错误: 工作目录 $WORK_DIR 不存在"; exit 1 -fi -if [ ! -f "$SCRIPT_PATH" ]; then - echo "错误: 执行脚本 $SCRIPT_PATH 不存在"; exit 1 -fi -if [ ! -d "$(dirname "$LOG_FILE")" ]; then - mkdir -p "$(dirname "$LOG_FILE")" -fi - -chmod +x "$SCRIPT_PATH" -CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" - -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") -if [ -n "$EXISTING_JOB" ]; then - crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - -fi -(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - - -echo "✅ Doris 增量定时任务设置成功" -crontab -l diff --git a/DataX/start_cron_inc.sh b/DataX/start_cron_inc.sh new file mode 100644 index 0000000..59ca3c4 --- /dev/null +++ b/DataX/start_cron_inc.sh @@ -0,0 +1,45 @@ +#!/bin/bash +TARGET=${1:-doris} # 默认为 doris + +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "错误: 请指定目标 (doris 或 mysql)" + exit 1 +fi + +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" +LOG_FILE="$WORK_DIR/logs/datax_inc_${TARGET}_cron.log" + +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +echo "正在检查环境..." +if [ ! -d "$WORK_DIR" ]; then + echo "错误: 工作目录 $WORK_DIR 不存在"; exit 1 +fi +if [ ! -f "$SCRIPT_PATH" ]; then + echo "错误: 执行脚本 $SCRIPT_PATH 不存在"; exit 1 +fi +if [ ! -d "$(dirname "$LOG_FILE")" ]; then + mkdir -p "$(dirname "$LOG_FILE")" +fi + +chmod +x "$SCRIPT_PATH" +# 构造 cron 命令，注意转义和参数传递 +CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH $TARGET >> $LOG_FILE 2>&1" + +# 查找已存在的任务 (匹配脚本路径和目标参数) +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") + +if [ -n "$EXISTING_JOB" ]; then + echo "提示: 该任务已存在，将先删除旧任务再重新添加..." + crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - +fi + +(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - + +echo "✅ $TARGET 增量定时任务设置成功" +crontab -l diff --git a/DataX/start_cron_mysql_inc.sh b/DataX/start_cron_mysql_inc.sh deleted file mode 100644 index e25c252..0000000 --- a/DataX/start_cron_mysql_inc.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -WORK_DIR="/usr/local/datax" -SCRIPT_NAME="run_jobs_inc_mysql.sh" -SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" -LOG_FILE="$WORK_DIR/logs/datax_inc_mysql_cron.log" - -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -echo "正在检查环境..." -if [ ! -d "$WORK_DIR" ]; then - echo "错误: 工作目录 $WORK_DIR 不存在"; exit 1 -fi -if [ ! -f "$SCRIPT_PATH" ]; then - echo "错误: 执行脚本 $SCRIPT_PATH 不存在"; exit 1 -fi -if [ ! -d "$(dirname "$LOG_FILE")" ]; then - mkdir -p "$(dirname "$LOG_FILE")" -fi - -chmod +x "$SCRIPT_PATH" -CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" - -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") -if [ -n "$EXISTING_JOB" ]; then - crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - -fi -(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - - -echo "✅ MySQL 增量定时任务设置成功" -crontab -l diff --git a/DataX/stop_cron_doris_inc.sh b/DataX/stop_cron_doris_inc.sh deleted file mode 100644 index 5dba5b2..0000000 --- a/DataX/stop_cron_doris_inc.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -WORK_DIR="/usr/local/datax" -SCRIPT_NAME="run_jobs_inc_doris.sh" -SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" - -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -echo "正在检查定时任务..." -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") -if [ -z "$EXISTING_JOB" ]; then - echo "提示: 未发现 Doris 增量任务，无需停止。" - exit 0 -fi - -BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" -crontab -l > "$BACKUP_FILE" 2>/dev/null -crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - - -REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") -if [ -z "$REMAINING_JOB" ]; then - echo "✅ Doris 增量定时任务已停止" - crontab -l -else - echo "❌ 停止失败，请手动执行: crontab -e" - exit 1 -fi diff --git a/DataX/stop_cron_inc.sh b/DataX/stop_cron_inc.sh new file mode 100644 index 0000000..eec61d9 --- /dev/null +++ b/DataX/stop_cron_inc.sh @@ -0,0 +1,39 @@ +#!/bin/bash +TARGET=${1:-doris} # 默认为 doris + +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "错误: 请指定目标 (doris 或 mysql)" + exit 1 +fi + +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" + +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +echo "正在检查定时任务..." +# 精确匹配包含脚本路径和参数的行 +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") + +if [ -z "$EXISTING_JOB" ]; then + echo "提示: 未发现 $TARGET 增量任务，无需停止。" + exit 0 +fi + +BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" +crontab -l > "$BACKUP_FILE" 2>/dev/null +# 删除特定任务 +crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - + +REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") +if [ -z "$REMAINING_JOB" ]; then + echo "✅ $TARGET 增量定时任务已停止" + crontab -l +else + echo "❌ 停止失败，请手动执行: crontab -e" + exit 1 +fi diff --git a/DataX/stop_cron_mysql_inc.sh b/DataX/stop_cron_mysql_inc.sh deleted file mode 100644 index 075f2f3..0000000 --- a/DataX/stop_cron_mysql_inc.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -WORK_DIR="/usr/local/datax" -SCRIPT_NAME="run_jobs_inc_mysql.sh" -SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" - -export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -[ -f /etc/profile ] && . /etc/profile -[ -f ~/.bash_profile ] && . ~/.bash_profile -[ -f ~/.profile ] && . ~/.profile - -echo "正在检查定时任务..." -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") -if [ -z "$EXISTING_JOB" ]; then - echo "提示: 未发现 MySQL 增量任务，无需停止。" - exit 0 -fi - -BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" -crontab -l > "$BACKUP_FILE" 2>/dev/null -crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - - -REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") -if [ -z "$REMAINING_JOB" ]; then - echo "✅ MySQL 增量定时任务已停止" - crontab -l -else - echo "❌ 停止失败，请手动执行: crontab -e" - exit 1 -fi