From 4712967fbecb7b44b72bf552385fe569cedb795b Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Thu, 5 Feb 2026 15:55:17 +0800 Subject: [PATCH] 'commit' --- DataX/json_full/t_account_recharge.json | 127 ++++++++ DataX/json_full/t_account_water.json | 105 +++++++ DataX/json_full/t_car.json | 99 ++++++ DataX/json_full/t_company.json | 133 ++++++++ DataX/json_full/t_company_info_value.json | 77 +++++ DataX/json_full/t_connector.json | 113 +++++++ DataX/json_full/t_equipment.json | 123 ++++++++ DataX/json_full/t_equipment_charge_order.json | 285 ++++++++++++++++++ .../t_equipment_charge_order_detail.json | 109 +++++++ DataX/json_full/t_ext_hurry_quit.json | 120 ++++++++ DataX/json_full/t_station.json | 226 ++++++++++++++ DataX/json_full/t_time_day.json | 72 +++++ DataX/json_full/t_user.json | 152 ++++++++++ DataX/json_full/t_user_account.json | 96 ++++++ DataX/json_full/t_user_upload_fault.json | 76 +++++ DataX/json_inc/t_account_recharge.json | 124 ++++++++ DataX/json_inc/t_account_water.json | 102 +++++++ DataX/json_inc/t_car.json | 96 ++++++ DataX/json_inc/t_company.json | 130 ++++++++ DataX/json_inc/t_company_info_value.json | 74 +++++ DataX/json_inc/t_connector.json | 110 +++++++ DataX/json_inc/t_equipment.json | 120 ++++++++ DataX/json_inc/t_equipment_charge_order.json | 282 +++++++++++++++++ .../t_equipment_charge_order_detail.json | 106 +++++++ DataX/json_inc/t_ext_hurry_quit.json | 118 ++++++++ DataX/json_inc/t_station.json | 224 ++++++++++++++ DataX/json_inc/t_time_day.json | 70 +++++ DataX/json_inc/t_user.json | 150 +++++++++ DataX/json_inc/t_user_account.json | 94 ++++++ DataX/json_inc/t_user_upload_fault.json | 74 +++++ DataX/run_jobs_full.sh | 174 +++++++---- DataX/run_jobs_inc.sh | 154 +++++++--- DataX/start_cron.sh | 30 +- DataX/stop_cron.sh | 28 +- DataX/说明.md | 102 +++++++ {DataX => DataX_Bad}/Doc/说明.md | 0 {DataX => DataX_Bad}/db_config.sh | 4 +- .../json/t_account_recharge.json | 19 +- .../json/t_account_water.json | 19 +- {DataX => DataX_Bad}/json/t_car.json | 19 +- {DataX => DataX_Bad}/json/t_company.json | 19 +- .../json/t_company_info_value.json | 19 +- {DataX => DataX_Bad}/json/t_connector.json | 19 +- {DataX => DataX_Bad}/json/t_equipment.json | 19 +- .../json/t_equipment_charge_order.json | 19 +- .../json/t_equipment_charge_order_detail.json | 19 +- .../json/t_ext_hurry_quit.json | 19 +- {DataX => DataX_Bad}/json/t_station.json | 19 +- {DataX => DataX_Bad}/json/t_time_day.json | 19 +- {DataX => DataX_Bad}/json/t_user.json | 19 +- {DataX => DataX_Bad}/json/t_user_account.json | 19 +- .../json/t_user_upload_fault.json | 19 +- DataX_Bad/run_jobs_full.sh | 143 +++++++++ DataX_Bad/run_jobs_inc.sh | 128 ++++++++ {DataX => DataX_Bad}/scheme/README.md | 0 .../scheme/generate_schema.py | 0 .../output/t_account_recharge_doris.sql | 0 .../output/t_account_recharge_mysql.sql | 0 .../scheme/output/t_account_water_doris.sql | 0 .../scheme/output/t_account_water_mysql.sql | 0 .../scheme/output/t_car_doris.sql | 0 .../scheme/output/t_car_mysql.sql | 0 .../scheme/output/t_company_doris.sql | 0 .../output/t_company_info_value_doris.sql | 0 .../output/t_company_info_value_mysql.sql | 0 .../scheme/output/t_company_mysql.sql | 0 .../scheme/output/t_connector_doris.sql | 0 .../scheme/output/t_connector_mysql.sql | 0 .../t_equipment_charge_order_detail_doris.sql | 0 .../t_equipment_charge_order_detail_mysql.sql | 0 .../output/t_equipment_charge_order_doris.sql | 0 .../output/t_equipment_charge_order_mysql.sql | 0 .../scheme/output/t_equipment_doris.sql | 0 .../scheme/output/t_equipment_mysql.sql | 0 .../scheme/output/t_ext_hurry_quit_doris.sql | 0 .../scheme/output/t_ext_hurry_quit_mysql.sql | 0 .../scheme/output/t_station_doris.sql | 0 .../scheme/output/t_station_mysql.sql | 0 .../scheme/output/t_time_day_doris.sql | 0 .../scheme/output/t_time_day_mysql.sql | 0 .../scheme/output/t_user_account_doris.sql | 0 .../scheme/output/t_user_account_mysql.sql | 0 .../scheme/output/t_user_doris.sql | 0 .../scheme/output/t_user_mysql.sql | 0 .../output/t_user_upload_fault_doris.sql | 0 .../output/t_user_upload_fault_mysql.sql | 0 DataX_Bad/start_cron.sh | 88 ++++++ DataX_Bad/stop_cron.sh | 64 ++++ 88 files changed, 4768 insertions(+), 219 deletions(-) create mode 100644 DataX/json_full/t_account_recharge.json create mode 100644 DataX/json_full/t_account_water.json create mode 100644 DataX/json_full/t_car.json create mode 100644 DataX/json_full/t_company.json create mode 100644 DataX/json_full/t_company_info_value.json create mode 100644 DataX/json_full/t_connector.json create mode 100644 DataX/json_full/t_equipment.json create mode 100644 DataX/json_full/t_equipment_charge_order.json create mode 100644 DataX/json_full/t_equipment_charge_order_detail.json create mode 100644 DataX/json_full/t_ext_hurry_quit.json create mode 100644 DataX/json_full/t_station.json create mode 100644 DataX/json_full/t_time_day.json create mode 100644 DataX/json_full/t_user.json create mode 100644 DataX/json_full/t_user_account.json create mode 100644 DataX/json_full/t_user_upload_fault.json create mode 100644 DataX/json_inc/t_account_recharge.json create mode 100644 DataX/json_inc/t_account_water.json create mode 100644 DataX/json_inc/t_car.json create mode 100644 DataX/json_inc/t_company.json create mode 100644 DataX/json_inc/t_company_info_value.json create mode 100644 DataX/json_inc/t_connector.json create mode 100644 DataX/json_inc/t_equipment.json create mode 100644 DataX/json_inc/t_equipment_charge_order.json create mode 100644 DataX/json_inc/t_equipment_charge_order_detail.json create mode 100644 DataX/json_inc/t_ext_hurry_quit.json create mode 100644 DataX/json_inc/t_station.json create mode 100644 DataX/json_inc/t_time_day.json create mode 100644 DataX/json_inc/t_user.json create mode 100644 DataX/json_inc/t_user_account.json create mode 100644 DataX/json_inc/t_user_upload_fault.json create mode 100644 DataX/说明.md rename {DataX => DataX_Bad}/Doc/说明.md (100%) rename {DataX => DataX_Bad}/db_config.sh (84%) rename {DataX => DataX_Bad}/json/t_account_recharge.json (81%) rename {DataX => DataX_Bad}/json/t_account_water.json (78%) rename {DataX => DataX_Bad}/json/t_car.json (78%) rename {DataX => DataX_Bad}/json/t_company.json (82%) rename {DataX => DataX_Bad}/json/t_company_info_value.json (74%) rename {DataX => DataX_Bad}/json/t_connector.json (80%) rename {DataX => DataX_Bad}/json/t_equipment.json (81%) rename {DataX => DataX_Bad}/json/t_equipment_charge_order.json (89%) rename {DataX => DataX_Bad}/json/t_equipment_charge_order_detail.json (80%) rename {DataX => DataX_Bad}/json/t_ext_hurry_quit.json (80%) rename {DataX => DataX_Bad}/json/t_station.json (87%) rename {DataX => DataX_Bad}/json/t_time_day.json (73%) rename {DataX => DataX_Bad}/json/t_user.json (83%) rename {DataX => DataX_Bad}/json/t_user_account.json (77%) rename {DataX => DataX_Bad}/json/t_user_upload_fault.json (74%) create mode 100644 DataX_Bad/run_jobs_full.sh create mode 100644 DataX_Bad/run_jobs_inc.sh rename {DataX => DataX_Bad}/scheme/README.md (100%) rename {DataX => DataX_Bad}/scheme/generate_schema.py (100%) rename {DataX => DataX_Bad}/scheme/output/t_account_recharge_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_account_recharge_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_account_water_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_account_water_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_car_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_car_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_company_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_company_info_value_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_company_info_value_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_company_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_connector_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_connector_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_equipment_charge_order_detail_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_equipment_charge_order_detail_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_equipment_charge_order_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_equipment_charge_order_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_equipment_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_equipment_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_ext_hurry_quit_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_ext_hurry_quit_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_station_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_station_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_time_day_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_time_day_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_user_account_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_user_account_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_user_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_user_mysql.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_user_upload_fault_doris.sql (100%) rename {DataX => DataX_Bad}/scheme/output/t_user_upload_fault_mysql.sql (100%) create mode 100644 DataX_Bad/start_cron.sh create mode 100644 DataX_Bad/stop_cron.sh diff --git a/DataX/json_full/t_account_recharge.json b/DataX/json_full/t_account_recharge.json new file mode 100644 index 0000000..727ce5f --- /dev/null +++ b/DataX/json_full/t_account_recharge.json @@ -0,0 +1,127 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "prid", + "order_id", + "outtradeno", + "pay_time", + "pay_money", + "refund_time", + "refund_money", + "refund_state", + "order_time", + "pay_xml", + "refund_xml", + "check_xml", + "pay_state", + "pay_type", + "phone", + "check_time", + "user_id", + "create_time", + "update_time", + "donate_money", + "memo", + "refund_memo", + "refund_outtradno", + "refund_image", + "refund_operer", + "rechage_operer", + "refund_reason", + "vip", + "meal_id", + "refund_apply_time", + "pay_method" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_account_recharge" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "prid", + "order_id", + "outtradeno", + "pay_time", + "pay_money", + "refund_time", + "refund_money", + "refund_state", + "order_time", + "pay_xml", + "refund_xml", + "check_xml", + "pay_state", + "pay_type", + "phone", + "check_time", + "user_id", + "create_time", + "update_time", + "donate_money", + "memo", + "refund_memo", + "refund_outtradno", + "refund_image", + "refund_operer", + "rechage_operer", + "refund_reason", + "vip", + "meal_id", + "refund_apply_time", + "pay_method" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_account_recharge" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_account_recharge" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_account_water.json b/DataX/json_full/t_account_water.json new file mode 100644 index 0000000..54b63f8 --- /dev/null +++ b/DataX/json_full/t_account_water.json @@ -0,0 +1,105 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "cash", + "phone", + "txt_desc", + "sign_msg", + "reason", + "sign", + "type", + "order_id", + "treatment", + "user_id", + "pay_status", + "pay_method", + "update_time", + "create_time", + "pre_balance", + "bef_balance", + "memo", + "donate_money", + "ua_balance", + "to_user_id" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_account_water" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "cash", + "phone", + "txt_desc", + "sign_msg", + "reason", + "sign", + "type", + "order_id", + "treatment", + "user_id", + "pay_status", + "pay_method", + "update_time", + "create_time", + "pre_balance", + "bef_balance", + "memo", + "donate_money", + "ua_balance", + "to_user_id" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_account_water" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_account_water" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_car.json b/DataX/json_full/t_car.json new file mode 100644 index 0000000..be62364 --- /dev/null +++ b/DataX/json_full/t_car.json @@ -0,0 +1,99 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_car" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_car" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_car" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_company.json b/DataX/json_full/t_company.json new file mode 100644 index 0000000..9c82856 --- /dev/null +++ b/DataX/json_full/t_company.json @@ -0,0 +1,133 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "company_no", + "company_type", + "company_name", + "company_state", + "company_logo", + "company_admin_phone", + "company_admin_name", + "balance_remind", + "balance_remind_phone", + "agent_id", + "station_set_type", + "specified_range", + "specified_operator_ids", + "is_company_discount", + "is_overlay_coupons", + "station_ids", + "no_discount_stationids", + "remark", + "invoice_eligible", + "refuse_invoice_tip", + "is_share_master", + "settlement_method", + "charge_discount_type", + "charge_discount", + "settle_status", + "charge_unit_service", + "charge_unit_electric", + "is_custom_price", + "company_order_permission", + "open_station_ids", + "create_time", + "update_time", + "minimum_recharge_amount", + "inner_person" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_company" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "company_no", + "company_type", + "company_name", + "company_state", + "company_logo", + "company_admin_phone", + "company_admin_name", + "balance_remind", + "balance_remind_phone", + "agent_id", + "station_set_type", + "specified_range", + "specified_operator_ids", + "is_company_discount", + "is_overlay_coupons", + "station_ids", + "no_discount_stationids", + "remark", + "invoice_eligible", + "refuse_invoice_tip", + "is_share_master", + "settlement_method", + "charge_discount_type", + "charge_discount", + "settle_status", + "charge_unit_service", + "charge_unit_electric", + "is_custom_price", + "company_order_permission", + "open_station_ids", + "create_time", + "update_time", + "minimum_recharge_amount", + "inner_person" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_company" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_company" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_company_info_value.json b/DataX/json_full/t_company_info_value.json new file mode 100644 index 0000000..4147538 --- /dev/null +++ b/DataX/json_full/t_company_info_value.json @@ -0,0 +1,77 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "info_value", + "type_id", + "create_time", + "update_time", + "remark", + "renzheng_flag" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_company_info_value" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "info_value", + "type_id", + "create_time", + "update_time", + "remark", + "renzheng_flag" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_company_info_value" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_company_info_value" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_connector.json b/DataX/json_full/t_connector.json new file mode 100644 index 0000000..9e915c1 --- /dev/null +++ b/DataX/json_full/t_connector.json @@ -0,0 +1,113 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "equipment_id", + "connector_sn", + "connector_short_sn", + "connector_no", + "connector_name", + "connector_type", + "voltage_upper_limits", + "voltage_lower_limits", + "current", + "power", + "parkno", + "appointment_id", + "appointment_user_id", + "open_lock_user_id", + "detection_type", + "is_ground_lock", + "is_ground_lock_msg", + "is_geomagnetic", + "is_geomagnetic_msg", + "is_appointment", + "is_appointment_msg", + "auto_open_lock_time", + "create_time", + "update_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_connector" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "equipment_id", + "connector_sn", + "connector_short_sn", + "connector_no", + "connector_name", + "connector_type", + "voltage_upper_limits", + "voltage_lower_limits", + "current", + "power", + "parkno", + "appointment_id", + "appointment_user_id", + "open_lock_user_id", + "detection_type", + "is_ground_lock", + "is_ground_lock_msg", + "is_geomagnetic", + "is_geomagnetic_msg", + "is_appointment", + "is_appointment_msg", + "auto_open_lock_time", + "create_time", + "update_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_connector" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_connector" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_equipment.json b/DataX/json_full/t_equipment.json new file mode 100644 index 0000000..bf81d8b --- /dev/null +++ b/DataX/json_full/t_equipment.json @@ -0,0 +1,123 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "station_id", + "agreement_id", + "equipment_name", + "equipment_sn", + "equipment_short_sn", + "equipment_no", + "manufacturer_id", + "manufacturer_sn", + "manufacturer_name", + "equipment_model", + "production_date", + "equipment_type", + "equipment_type_msg", + "equipment_lng", + "equipment_lat", + "connector_count", + "power", + "national_standard", + "national_standard_msg", + "status", + "equipment_business_type", + "equipment_business_type_msg", + "is_use_card", + "is_use_card_msg", + "charge_host_esn", + "create_time", + "update_time", + "aux_power", + "app_show" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_equipment" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "station_id", + "agreement_id", + "equipment_name", + "equipment_sn", + "equipment_short_sn", + "equipment_no", + "manufacturer_id", + "manufacturer_sn", + "manufacturer_name", + "equipment_model", + "production_date", + "equipment_type", + "equipment_type_msg", + "equipment_lng", + "equipment_lat", + "connector_count", + "power", + "national_standard", + "national_standard_msg", + "status", + "equipment_business_type", + "equipment_business_type_msg", + "is_use_card", + "is_use_card_msg", + "charge_host_esn", + "create_time", + "update_time", + "aux_power", + "app_show" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_equipment" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_equipment_charge_order.json b/DataX/json_full/t_equipment_charge_order.json new file mode 100644 index 0000000..aef7367 --- /dev/null +++ b/DataX/json_full/t_equipment_charge_order.json @@ -0,0 +1,285 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_equipment_charge_order" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_equipment_charge_order" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment_charge_order" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_equipment_charge_order_detail.json b/DataX/json_full/t_equipment_charge_order_detail.json new file mode 100644 index 0000000..d5d1808 --- /dev/null +++ b/DataX/json_full/t_equipment_charge_order_detail.json @@ -0,0 +1,109 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "order_no", + "charge_degree", + "service_fee", + "electric_fee", + "cost_fee", + "service_fee_unit_price", + "electric_fee_unit_price", + "cost_fee_unit_price", + "activity_service_fee", + "activity_electric_fee", + "charge_startr_time", + "charge_end_time", + "charge_meal_electric_fee", + "charge_meal_service_fee", + "charge_meal_total_fee", + "electric_card_use_dgree", + "check_electric_unit_price", + "check_service_unit_price", + "time_flag", + "create_time", + "update_time", + "detail_more" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_equipment_charge_order_detail" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}", + "where": "" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "order_no", + "charge_degree", + "service_fee", + "electric_fee", + "cost_fee", + "service_fee_unit_price", + "electric_fee_unit_price", + "cost_fee_unit_price", + "activity_service_fee", + "activity_electric_fee", + "charge_startr_time", + "charge_end_time", + "charge_meal_electric_fee", + "charge_meal_service_fee", + "charge_meal_total_fee", + "electric_card_use_dgree", + "check_electric_unit_price", + "check_service_unit_price", + "time_flag", + "create_time", + "update_time", + "detail_more" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_equipment_charge_order_detail" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment_charge_order_detail" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_ext_hurry_quit.json b/DataX/json_full/t_ext_hurry_quit.json new file mode 100644 index 0000000..a2f5d8d --- /dev/null +++ b/DataX/json_full/t_ext_hurry_quit.json @@ -0,0 +1,120 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "order_no", + "telephone", + "user_id", + "message1", + "message2", + "create_time", + "station_id", + "equipment_id", + "connector_id", + "flag", + "close_time", + "finish_time", + "minutesDifference", + "reduce_money", + "beforeBalance", + "afterBalance", + "fee_station", + "manual_count", + "last_manual_time", + "last_manual_message", + "person_id", + "refund_status", + "refund_time", + "refund_person_id", + "refund_person_name", + "refund_amount", + "refund_type", + "unplug_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_ext_hurry_quit" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "order_no", + "telephone", + "user_id", + "message1", + "message2", + "create_time", + "station_id", + "equipment_id", + "connector_id", + "flag", + "close_time", + "finish_time", + "minutesDifference", + "reduce_money", + "beforeBalance", + "afterBalance", + "fee_station", + "manual_count", + "last_manual_time", + "last_manual_message", + "person_id", + "refund_status", + "refund_time", + "refund_person_id", + "refund_person_name", + "refund_amount", + "refund_type", + "unplug_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_ext_hurry_quit" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_ext_hurry_quit" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_station.json b/DataX/json_full/t_station.json new file mode 100644 index 0000000..b4de818 --- /dev/null +++ b/DataX/json_full/t_station.json @@ -0,0 +1,226 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "station_sn", + "operator_id", + "equioment_owner_id", + "station_name", + "country_code", + "region_id", + "region_code", + "regional_name", + "regional_code", + "address", + "station_tel", + "service_tel", + "station_type", + "station_type_msg", + "station_business_type", + "station_business_type_msg", + "station_status", + "station_status_msg", + "park_nums", + "station_lng", + "station_lat", + "site_guide", + "construction", + "construction_msg", + "pictures", + "match_cars", + "park_info", + "business_start_time", + "business_end_time", + "electricity_fee", + "service_fee", + "is_park_fee", + "park_fee", + "payment", + "payment_msg", + "support_order", + "support_order_msg", + "service_item", + "service_item_image_url", + "service_item_msg", + "station_location_type", + "station_location_type_msg", + "station_charge_type", + "station_charge_type_msg", + "station_power", + "station_voltage", + "station_current", + "is_open", + "is_open_msg", + "is_display", + "is_display_msg", + "sharing_code", + "sharing_applet_picture", + "theme_picture", + "is_barrier_flag", + "is_barrier_flag_msg", + "charge_settle_type", + "charge_settle_type_msg", + "fast_connector_nums", + "slow_connector_nums", + "transformer_capacity", + "installed_capacity", + "remark", + "record_unique_no", + "equipment_owner_name", + "supply_type", + "resident_no", + "watt_hour_meter_no", + "forward_power", + "operation_date", + "charge_replace_type", + "is_hlht", + "is_subsidy", + "create_time", + "update_time", + "area_code_countryside", + "business_expand_type", + "capacity", + "video_monitor", + "electricity_type", + "receive_shop" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_station" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "station_sn", + "operator_id", + "equioment_owner_id", + "station_name", + "country_code", + "region_id", + "region_code", + "regional_name", + "regional_code", + "address", + "station_tel", + "service_tel", + "station_type", + "station_type_msg", + "station_business_type", + "station_business_type_msg", + "station_status", + "station_status_msg", + "park_nums", + "station_lng", + "station_lat", + "site_guide", + "construction", + "construction_msg", + "pictures", + "match_cars", + "park_info", + "business_start_time", + "business_end_time", + "electricity_fee", + "service_fee", + "is_park_fee", + "park_fee", + "payment", + "payment_msg", + "support_order", + "support_order_msg", + "service_item", + "service_item_image_url", + "service_item_msg", + "station_location_type", + "station_location_type_msg", + "station_charge_type", + "station_charge_type_msg", + "station_power", + "station_voltage", + "station_current", + "is_open", + "is_open_msg", + "is_display", + "is_display_msg", + "sharing_code", + "sharing_applet_picture", + "theme_picture", + "is_barrier_flag", + "is_barrier_flag_msg", + "charge_settle_type", + "charge_settle_type_msg", + "fast_connector_nums", + "slow_connector_nums", + "transformer_capacity", + "installed_capacity", + "remark", + "record_unique_no", + "equipment_owner_name", + "supply_type", + "resident_no", + "watt_hour_meter_no", + "forward_power", + "operation_date", + "charge_replace_type", + "is_hlht", + "is_subsidy", + "create_time", + "update_time", + "area_code_countryside", + "business_expand_type", + "capacity", + "video_monitor", + "electricity_type", + "receive_shop" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_station" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_station" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_time_day.json b/DataX/json_full/t_time_day.json new file mode 100644 index 0000000..bebd21e --- /dev/null +++ b/DataX/json_full/t_time_day.json @@ -0,0 +1,72 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "create_time", + "update_time", + "time_fq", + "teme_ve" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_time_day" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "create_time", + "update_time", + "time_fq", + "teme_ve" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_time_day" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_time_day" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_user.json b/DataX/json_full/t_user.json new file mode 100644 index 0000000..f05352c --- /dev/null +++ b/DataX/json_full/t_user.json @@ -0,0 +1,152 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_name", + "nicker_name", + "sex", + "phone", + "qk_openid", + "open_id", + "ali_user_id", + "birthday", + "avatar", + "app_type", + "password", + "upper_user", + "master_flag", + "user_type", + "multi_charge", + "relation_phone", + "user_owner_id", + "union_id", + "state", + "reg_channel", + "reg_time", + "city_label_code", + "city_label", + "reg_source", + "create_time", + "update_time", + "operator", + "invite_code", + "identity_id", + "operator_id", + "address", + "vip_level", + "province_code", + "province", + "work_company", + "is_bind_wxinfo", + "company_min_charge_balance", + "enable_online_recharge", + "integral", + "shared_user", + "enable_online_refund", + "recommenderId", + "lx_wx_user_id", + "phone_area" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_user" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_name", + "nicker_name", + "sex", + "phone", + "qk_openid", + "open_id", + "ali_user_id", + "birthday", + "avatar", + "app_type", + "password", + "upper_user", + "master_flag", + "user_type", + "multi_charge", + "relation_phone", + "user_owner_id", + "union_id", + "state", + "reg_channel", + "reg_time", + "city_label_code", + "city_label", + "reg_source", + "create_time", + "update_time", + "operator", + "invite_code", + "identity_id", + "operator_id", + "address", + "vip_level", + "province_code", + "province", + "work_company", + "is_bind_wxinfo", + "company_min_charge_balance", + "enable_online_recharge", + "integral", + "shared_user", + "enable_online_refund", + "recommenderId", + "lx_wx_user_id", + "phone_area" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_user" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_user" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_user_account.json b/DataX/json_full/t_user_account.json new file mode 100644 index 0000000..d41b38e --- /dev/null +++ b/DataX/json_full/t_user_account.json @@ -0,0 +1,96 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_id", + "ua_balance", + "ua_deposit", + "ua_balance_time", + "ua_password", + "ua_state", + "ua_real_balance", + "ua_real_degree", + "create_time", + "update_time", + "donate_money", + "donate_able_money", + "ua_able_balance", + "refund_money", + "ua_consume_money", + "ua_clear_zero_money" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_user_account" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_id", + "ua_balance", + "ua_deposit", + "ua_balance_time", + "ua_password", + "ua_state", + "ua_real_balance", + "ua_real_degree", + "create_time", + "update_time", + "donate_money", + "donate_able_money", + "ua_able_balance", + "refund_money", + "ua_consume_money", + "ua_clear_zero_money" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_user_account" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_user_account" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_full/t_user_upload_fault.json b/DataX/json_full/t_user_upload_fault.json new file mode 100644 index 0000000..cb66f2d --- /dev/null +++ b/DataX/json_full/t_user_upload_fault.json @@ -0,0 +1,76 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "equipment_id", + "user_id", + "fault_config_id", + "remark", + "create_time", + "update_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "table": [ + "t_user_upload_fault" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "equipment_id", + "user_id", + "fault_config_id", + "remark", + "create_time", + "update_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [ + "TRUNCATE TABLE t_user_upload_fault" + ], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_user_upload_fault" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_account_recharge.json b/DataX/json_inc/t_account_recharge.json new file mode 100644 index 0000000..35d2510 --- /dev/null +++ b/DataX/json_inc/t_account_recharge.json @@ -0,0 +1,124 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "prid", + "order_id", + "outtradeno", + "pay_time", + "pay_money", + "refund_time", + "refund_money", + "refund_state", + "order_time", + "pay_xml", + "refund_xml", + "check_xml", + "pay_state", + "pay_type", + "phone", + "check_time", + "user_id", + "create_time", + "update_time", + "donate_money", + "memo", + "refund_memo", + "refund_outtradno", + "refund_image", + "refund_operer", + "rechage_operer", + "refund_reason", + "vip", + "meal_id", + "refund_apply_time", + "pay_method" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `prid`, `order_id`, `outtradeno`, `pay_time`, `pay_money`, `refund_time`, `refund_money`, `refund_state`, `order_time`, `pay_xml`, `refund_xml`, `check_xml`, `pay_state`, `pay_type`, `phone`, `check_time`, `user_id`, `create_time`, `update_time`, `donate_money`, `memo`, `refund_memo`, `refund_outtradno`, `refund_image`, `refund_operer`, `rechage_operer`, `refund_reason`, `vip`, `meal_id`, `refund_apply_time`, `pay_method` FROM `t_account_recharge` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "prid", + "order_id", + "outtradeno", + "pay_time", + "pay_money", + "refund_time", + "refund_money", + "refund_state", + "order_time", + "pay_xml", + "refund_xml", + "check_xml", + "pay_state", + "pay_type", + "phone", + "check_time", + "user_id", + "create_time", + "update_time", + "donate_money", + "memo", + "refund_memo", + "refund_outtradno", + "refund_image", + "refund_operer", + "rechage_operer", + "refund_reason", + "vip", + "meal_id", + "refund_apply_time", + "pay_method" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_account_recharge" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_account_water.json b/DataX/json_inc/t_account_water.json new file mode 100644 index 0000000..85f6fcb --- /dev/null +++ b/DataX/json_inc/t_account_water.json @@ -0,0 +1,102 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "cash", + "phone", + "txt_desc", + "sign_msg", + "reason", + "sign", + "type", + "order_id", + "treatment", + "user_id", + "pay_status", + "pay_method", + "update_time", + "create_time", + "pre_balance", + "bef_balance", + "memo", + "donate_money", + "ua_balance", + "to_user_id" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `cash`, `phone`, `txt_desc`, `sign_msg`, `reason`, `sign`, `type`, `order_id`, `treatment`, `user_id`, `pay_status`, `pay_method`, `update_time`, `create_time`, `pre_balance`, `bef_balance`, `memo`, `donate_money`, `ua_balance`, `to_user_id` FROM `t_account_water` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "cash", + "phone", + "txt_desc", + "sign_msg", + "reason", + "sign", + "type", + "order_id", + "treatment", + "user_id", + "pay_status", + "pay_method", + "update_time", + "create_time", + "pre_balance", + "bef_balance", + "memo", + "donate_money", + "ua_balance", + "to_user_id" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_account_water" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_car.json b/DataX/json_inc/t_car.json new file mode 100644 index 0000000..4a1ad53 --- /dev/null +++ b/DataX/json_inc/t_car.json @@ -0,0 +1,96 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `car_plate_no`, `car_sn`, `car_brand_name`, `car_model_name`, `car_vin`, `car_engine`, `car_type`, `car_owner_company_id`, `car_business`, `car_bus_path`, `is_auto_charge`, `user_id`, `user_card_no`, `is_disable`, `create_time`, `update_time`, `operator_id` FROM `t_car` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_car" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_company.json b/DataX/json_inc/t_company.json new file mode 100644 index 0000000..8d66cff --- /dev/null +++ b/DataX/json_inc/t_company.json @@ -0,0 +1,130 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "company_no", + "company_type", + "company_name", + "company_state", + "company_logo", + "company_admin_phone", + "company_admin_name", + "balance_remind", + "balance_remind_phone", + "agent_id", + "station_set_type", + "specified_range", + "specified_operator_ids", + "is_company_discount", + "is_overlay_coupons", + "station_ids", + "no_discount_stationids", + "remark", + "invoice_eligible", + "refuse_invoice_tip", + "is_share_master", + "settlement_method", + "charge_discount_type", + "charge_discount", + "settle_status", + "charge_unit_service", + "charge_unit_electric", + "is_custom_price", + "company_order_permission", + "open_station_ids", + "create_time", + "update_time", + "minimum_recharge_amount", + "inner_person" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `company_no`, `company_type`, `company_name`, `company_state`, `company_logo`, `company_admin_phone`, `company_admin_name`, `balance_remind`, `balance_remind_phone`, `agent_id`, `station_set_type`, `specified_range`, `specified_operator_ids`, `is_company_discount`, `is_overlay_coupons`, `station_ids`, `no_discount_stationids`, `remark`, `invoice_eligible`, `refuse_invoice_tip`, `is_share_master`, `settlement_method`, `charge_discount_type`, `charge_discount`, `settle_status`, `charge_unit_service`, `charge_unit_electric`, `is_custom_price`, `company_order_permission`, `open_station_ids`, `create_time`, `update_time`, `minimum_recharge_amount`, `inner_person` FROM `t_company` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "company_no", + "company_type", + "company_name", + "company_state", + "company_logo", + "company_admin_phone", + "company_admin_name", + "balance_remind", + "balance_remind_phone", + "agent_id", + "station_set_type", + "specified_range", + "specified_operator_ids", + "is_company_discount", + "is_overlay_coupons", + "station_ids", + "no_discount_stationids", + "remark", + "invoice_eligible", + "refuse_invoice_tip", + "is_share_master", + "settlement_method", + "charge_discount_type", + "charge_discount", + "settle_status", + "charge_unit_service", + "charge_unit_electric", + "is_custom_price", + "company_order_permission", + "open_station_ids", + "create_time", + "update_time", + "minimum_recharge_amount", + "inner_person" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_company" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_company_info_value.json b/DataX/json_inc/t_company_info_value.json new file mode 100644 index 0000000..e8fbaeb --- /dev/null +++ b/DataX/json_inc/t_company_info_value.json @@ -0,0 +1,74 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "info_value", + "type_id", + "create_time", + "update_time", + "remark", + "renzheng_flag" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `info_value`, `type_id`, `create_time`, `update_time`, `remark`, `renzheng_flag` FROM `t_company_info_value` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "info_value", + "type_id", + "create_time", + "update_time", + "remark", + "renzheng_flag" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_company_info_value" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_connector.json b/DataX/json_inc/t_connector.json new file mode 100644 index 0000000..a18460f --- /dev/null +++ b/DataX/json_inc/t_connector.json @@ -0,0 +1,110 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "equipment_id", + "connector_sn", + "connector_short_sn", + "connector_no", + "connector_name", + "connector_type", + "voltage_upper_limits", + "voltage_lower_limits", + "current", + "power", + "parkno", + "appointment_id", + "appointment_user_id", + "open_lock_user_id", + "detection_type", + "is_ground_lock", + "is_ground_lock_msg", + "is_geomagnetic", + "is_geomagnetic_msg", + "is_appointment", + "is_appointment_msg", + "auto_open_lock_time", + "create_time", + "update_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `equipment_id`, `connector_sn`, `connector_short_sn`, `connector_no`, `connector_name`, `connector_type`, `voltage_upper_limits`, `voltage_lower_limits`, `current`, `power`, `parkno`, `appointment_id`, `appointment_user_id`, `open_lock_user_id`, `detection_type`, `is_ground_lock`, `is_ground_lock_msg`, `is_geomagnetic`, `is_geomagnetic_msg`, `is_appointment`, `is_appointment_msg`, `auto_open_lock_time`, `create_time`, `update_time` FROM `t_connector` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "equipment_id", + "connector_sn", + "connector_short_sn", + "connector_no", + "connector_name", + "connector_type", + "voltage_upper_limits", + "voltage_lower_limits", + "current", + "power", + "parkno", + "appointment_id", + "appointment_user_id", + "open_lock_user_id", + "detection_type", + "is_ground_lock", + "is_ground_lock_msg", + "is_geomagnetic", + "is_geomagnetic_msg", + "is_appointment", + "is_appointment_msg", + "auto_open_lock_time", + "create_time", + "update_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_connector" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment.json b/DataX/json_inc/t_equipment.json new file mode 100644 index 0000000..ce1b5ec --- /dev/null +++ b/DataX/json_inc/t_equipment.json @@ -0,0 +1,120 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "station_id", + "agreement_id", + "equipment_name", + "equipment_sn", + "equipment_short_sn", + "equipment_no", + "manufacturer_id", + "manufacturer_sn", + "manufacturer_name", + "equipment_model", + "production_date", + "equipment_type", + "equipment_type_msg", + "equipment_lng", + "equipment_lat", + "connector_count", + "power", + "national_standard", + "national_standard_msg", + "status", + "equipment_business_type", + "equipment_business_type_msg", + "is_use_card", + "is_use_card_msg", + "charge_host_esn", + "create_time", + "update_time", + "aux_power", + "app_show" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `station_id`, `agreement_id`, `equipment_name`, `equipment_sn`, `equipment_short_sn`, `equipment_no`, `manufacturer_id`, `manufacturer_sn`, `manufacturer_name`, `equipment_model`, `production_date`, `equipment_type`, `equipment_type_msg`, `equipment_lng`, `equipment_lat`, `connector_count`, `power`, `national_standard`, `national_standard_msg`, `status`, `equipment_business_type`, `equipment_business_type_msg`, `is_use_card`, `is_use_card_msg`, `charge_host_esn`, `create_time`, `update_time`, `aux_power`, `app_show` FROM `t_equipment` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "station_id", + "agreement_id", + "equipment_name", + "equipment_sn", + "equipment_short_sn", + "equipment_no", + "manufacturer_id", + "manufacturer_sn", + "manufacturer_name", + "equipment_model", + "production_date", + "equipment_type", + "equipment_type_msg", + "equipment_lng", + "equipment_lat", + "connector_count", + "power", + "national_standard", + "national_standard_msg", + "status", + "equipment_business_type", + "equipment_business_type_msg", + "is_use_card", + "is_use_card_msg", + "charge_host_esn", + "create_time", + "update_time", + "aux_power", + "app_show" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment_charge_order.json b/DataX/json_inc/t_equipment_charge_order.json new file mode 100644 index 0000000..817438c --- /dev/null +++ b/DataX/json_inc/t_equipment_charge_order.json @@ -0,0 +1,282 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `user_id`, `company_id`, `station_id`, `equipment_id`, `connector_id`, `user_card_no`, `state`, `operate_state`, `order_no`, `platform_order_no`, `platform_sale_type`, `platform_driver_id`, `platform_phone`, `charging_amt`, `order_type`, `order_time`, `charge_begin_time`, `charge_end_time`, `charge_begin_degree`, `charge_end_degree`, `charge_degree`, `charge_ah`, `charge_times_degree`, `charge_begin_soc`, `charge_end_soc`, `charge_cur_soc`, `charge_duration`, `charge_settle_type`, `charge_unit_price`, `charge_unit_cost`, `charge_unit_service_fee`, `receivable_electric_fee`, `receivable_service_fee`, `receivable_total_fee`, `charge_elecfee_amount`, `charge_elecfee_cost_amount`, `charge_service_amount`, `actual_pay_amount`, `pay_order_no`, `pay_amount`, `pay_method`, `pay_status`, `pay_time`, `user_coupon_id`, `coupon_name`, `coupon_type`, `coupon_amount`, `elecfee_coupon_amount`, `servicefee_coupon_amount`, `invoice_id`, `invoice_fee`, `ticket_status`, `activity_electric_fee`, `activity_service_fee`, `charge_plate_no`, `charge_vin`, `charge_strategy`, `charge_strategy_param`, `operator_id`, `operator_income`, `plat_service_fee`, `charge_discount_type`, `settle_electric_fee`, `settle_service_fee`, `settle_fee`, `settle_method`, `settle_coupon_amount`, `settle_coupon_electric_fee`, `settle_coupon_service_fee`, `settle_status`, `cancel_type`, `cancel_code`, `cancel_msg`, `cancel_time`, `finish_type`, `finish_code`, `finish_msg`, `finish_time`, `boot_mode`, `settlement_type`, `is_parking_coupon`, `control_source`, `exception_flag`, `exception_msg`, `report_time`, `com_service_fee`, `com_electric_fee`, `com_total_fee`, `com_charge_discount_type`, `com_charge_discount`, `has_shallow_report`, `has_deep_report`, `is_exception`, `shallow_report_is_received`, `create_time`, `update_time`, `charge_market_id`, `coupon_total_amount`, `coupon_service_total_amount`, `coupon_elecfee_total_amount`, `activity_name`, `activity_total_fee`, `company_activity_id`, `company_activity_name`, `company_activity_total_fee`, `company_activity_service_fee`, `company_activity_electric_fee`, `subsidy_degree`, `subsidy_fee`, `special_channel_order_source` FROM `t_equipment_charge_order` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment_charge_order" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment_charge_order_detail.json b/DataX/json_inc/t_equipment_charge_order_detail.json new file mode 100644 index 0000000..c76e67e --- /dev/null +++ b/DataX/json_inc/t_equipment_charge_order_detail.json @@ -0,0 +1,106 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "order_no", + "charge_degree", + "service_fee", + "electric_fee", + "cost_fee", + "service_fee_unit_price", + "electric_fee_unit_price", + "cost_fee_unit_price", + "activity_service_fee", + "activity_electric_fee", + "charge_startr_time", + "charge_end_time", + "charge_meal_electric_fee", + "charge_meal_service_fee", + "charge_meal_total_fee", + "electric_card_use_dgree", + "check_electric_unit_price", + "check_service_unit_price", + "time_flag", + "create_time", + "update_time", + "detail_more" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `order_no`, `charge_degree`, `service_fee`, `electric_fee`, `cost_fee`, `service_fee_unit_price`, `electric_fee_unit_price`, `cost_fee_unit_price`, `activity_service_fee`, `activity_electric_fee`, `charge_startr_time`, `charge_end_time`, `charge_meal_electric_fee`, `charge_meal_service_fee`, `charge_meal_total_fee`, `electric_card_use_dgree`, `check_electric_unit_price`, `check_service_unit_price`, `time_flag`, `create_time`, `update_time`, `detail_more` FROM `t_equipment_charge_order_detail` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "order_no", + "charge_degree", + "service_fee", + "electric_fee", + "cost_fee", + "service_fee_unit_price", + "electric_fee_unit_price", + "cost_fee_unit_price", + "activity_service_fee", + "activity_electric_fee", + "charge_startr_time", + "charge_end_time", + "charge_meal_electric_fee", + "charge_meal_service_fee", + "charge_meal_total_fee", + "electric_card_use_dgree", + "check_electric_unit_price", + "check_service_unit_price", + "time_flag", + "create_time", + "update_time", + "detail_more" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment_charge_order_detail" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_ext_hurry_quit.json b/DataX/json_inc/t_ext_hurry_quit.json new file mode 100644 index 0000000..fd3b1f5 --- /dev/null +++ b/DataX/json_inc/t_ext_hurry_quit.json @@ -0,0 +1,118 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "order_no", + "telephone", + "user_id", + "message1", + "message2", + "create_time", + "station_id", + "equipment_id", + "connector_id", + "flag", + "close_time", + "finish_time", + "minutesDifference", + "reduce_money", + "beforeBalance", + "afterBalance", + "fee_station", + "manual_count", + "last_manual_time", + "last_manual_message", + "person_id", + "refund_status", + "refund_time", + "refund_person_id", + "refund_person_name", + "refund_amount", + "refund_type", + "unplug_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `order_no`, `telephone`, `user_id`, `message1`, `message2`, `create_time`, `station_id`, `equipment_id`, `connector_id`, `flag`, `close_time`, `finish_time`, `minutesDifference`, `reduce_money`, `beforeBalance`, `afterBalance`, `fee_station`, `manual_count`, `last_manual_time`, `last_manual_message`, `person_id`, `refund_status`, `refund_time`, `refund_person_id`, `refund_person_name`, `refund_amount`, `refund_type`, `unplug_time` FROM `t_ext_hurry_quit` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "order_no", + "telephone", + "user_id", + "message1", + "message2", + "create_time", + "station_id", + "equipment_id", + "connector_id", + "flag", + "close_time", + "finish_time", + "minutesDifference", + "reduce_money", + "beforeBalance", + "afterBalance", + "fee_station", + "manual_count", + "last_manual_time", + "last_manual_message", + "person_id", + "refund_status", + "refund_time", + "refund_person_id", + "refund_person_name", + "refund_amount", + "refund_type", + "unplug_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_ext_hurry_quit" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_station.json b/DataX/json_inc/t_station.json new file mode 100644 index 0000000..11dc396 --- /dev/null +++ b/DataX/json_inc/t_station.json @@ -0,0 +1,224 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "station_sn", + "operator_id", + "equioment_owner_id", + "station_name", + "country_code", + "region_id", + "region_code", + "regional_name", + "regional_code", + "address", + "station_tel", + "service_tel", + "station_type", + "station_type_msg", + "station_business_type", + "station_business_type_msg", + "station_status", + "station_status_msg", + "park_nums", + "station_lng", + "station_lat", + "site_guide", + "construction", + "construction_msg", + "pictures", + "match_cars", + "park_info", + "business_start_time", + "business_end_time", + "electricity_fee", + "service_fee", + "is_park_fee", + "park_fee", + "payment", + "payment_msg", + "support_order", + "support_order_msg", + "service_item", + "service_item_image_url", + "service_item_msg", + "station_location_type", + "station_location_type_msg", + "station_charge_type", + "station_charge_type_msg", + "station_power", + "station_voltage", + "station_current", + "is_open", + "is_open_msg", + "is_display", + "is_display_msg", + "sharing_code", + "sharing_applet_picture", + "theme_picture", + "is_barrier_flag", + "is_barrier_flag_msg", + "charge_settle_type", + "charge_settle_type_msg", + "fast_connector_nums", + "slow_connector_nums", + "transformer_capacity", + "installed_capacity", + "remark", + "record_unique_no", + "equipment_owner_name", + "supply_type", + "resident_no", + "watt_hour_meter_no", + "forward_power", + "operation_date", + "charge_replace_type", + "is_hlht", + "is_subsidy", + "create_time", + "update_time", + "area_code_countryside", + "business_expand_type", + "capacity", + "video_monitor", + "electricity_type", + "receive_shop" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `station_sn`, `operator_id`, `equioment_owner_id`, `station_name`, `country_code`, `region_id`, `region_code`, `regional_name`, `regional_code`, `address`, `station_tel`, `service_tel`, `station_type`, `station_type_msg`, `station_business_type`, `station_business_type_msg`, `station_status`, `station_status_msg`, `park_nums`, `station_lng`, `station_lat`, `site_guide`, `construction`, `construction_msg`, `pictures`, `match_cars`, `park_info`, `business_start_time`, `business_end_time`, `electricity_fee`, `service_fee`, `is_park_fee`, `park_fee`, `payment`, `payment_msg`, `support_order`, `support_order_msg`, `service_item`, `service_item_image_url`, `service_item_msg`, `station_location_type`, `station_location_type_msg`, `station_charge_type`, `station_charge_type_msg`, `station_power`, `station_voltage`, `station_current`, `is_open`, `is_open_msg`, `is_display`, `is_display_msg`, `sharing_code`, `sharing_applet_picture`, `theme_picture`, `is_barrier_flag`, `is_barrier_flag_msg`, `charge_settle_type`, `charge_settle_type_msg`, `fast_connector_nums`, `slow_connector_nums`, `transformer_capacity`, `installed_capacity`, `remark`, `record_unique_no`, `equipment_owner_name`, `supply_type`, `resident_no`, `watt_hour_meter_no`, `forward_power`, `operation_date`, `charge_replace_type`, `is_hlht`, `is_subsidy`, `create_time`, `update_time`, `area_code_countryside`, `business_expand_type`, `capacity`, `video_monitor`, `electricity_type`, `receive_shop` FROM `t_station` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "station_sn", + "operator_id", + "equioment_owner_id", + "station_name", + "country_code", + "region_id", + "region_code", + "regional_name", + "regional_code", + "address", + "station_tel", + "service_tel", + "station_type", + "station_type_msg", + "station_business_type", + "station_business_type_msg", + "station_status", + "station_status_msg", + "park_nums", + "station_lng", + "station_lat", + "site_guide", + "construction", + "construction_msg", + "pictures", + "match_cars", + "park_info", + "business_start_time", + "business_end_time", + "electricity_fee", + "service_fee", + "is_park_fee", + "park_fee", + "payment", + "payment_msg", + "support_order", + "support_order_msg", + "service_item", + "service_item_image_url", + "service_item_msg", + "station_location_type", + "station_location_type_msg", + "station_charge_type", + "station_charge_type_msg", + "station_power", + "station_voltage", + "station_current", + "is_open", + "is_open_msg", + "is_display", + "is_display_msg", + "sharing_code", + "sharing_applet_picture", + "theme_picture", + "is_barrier_flag", + "is_barrier_flag_msg", + "charge_settle_type", + "charge_settle_type_msg", + "fast_connector_nums", + "slow_connector_nums", + "transformer_capacity", + "installed_capacity", + "remark", + "record_unique_no", + "equipment_owner_name", + "supply_type", + "resident_no", + "watt_hour_meter_no", + "forward_power", + "operation_date", + "charge_replace_type", + "is_hlht", + "is_subsidy", + "create_time", + "update_time", + "area_code_countryside", + "business_expand_type", + "capacity", + "video_monitor", + "electricity_type", + "receive_shop" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_station" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_time_day.json b/DataX/json_inc/t_time_day.json new file mode 100644 index 0000000..6c17810 --- /dev/null +++ b/DataX/json_inc/t_time_day.json @@ -0,0 +1,70 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "create_time", + "update_time", + "time_fq", + "teme_ve" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `create_time`, `update_time`, `time_fq`, `teme_ve` FROM `t_time_day` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "create_time", + "update_time", + "time_fq", + "teme_ve" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_time_day" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_user.json b/DataX/json_inc/t_user.json new file mode 100644 index 0000000..aed8105 --- /dev/null +++ b/DataX/json_inc/t_user.json @@ -0,0 +1,150 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_name", + "nicker_name", + "sex", + "phone", + "qk_openid", + "open_id", + "ali_user_id", + "birthday", + "avatar", + "app_type", + "password", + "upper_user", + "master_flag", + "user_type", + "multi_charge", + "relation_phone", + "user_owner_id", + "union_id", + "state", + "reg_channel", + "reg_time", + "city_label_code", + "city_label", + "reg_source", + "create_time", + "update_time", + "operator", + "invite_code", + "identity_id", + "operator_id", + "address", + "vip_level", + "province_code", + "province", + "work_company", + "is_bind_wxinfo", + "company_min_charge_balance", + "enable_online_recharge", + "integral", + "shared_user", + "enable_online_refund", + "recommenderId", + "lx_wx_user_id", + "phone_area" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `user_name`, `nicker_name`, `sex`, `phone`, `qk_openid`, `open_id`, `ali_user_id`, `birthday`, `avatar`, `app_type`, `password`, `upper_user`, `master_flag`, `user_type`, `multi_charge`, `relation_phone`, `user_owner_id`, `union_id`, `state`, `reg_channel`, `reg_time`, `city_label_code`, `city_label`, `reg_source`, `create_time`, `update_time`, `operator`, `invite_code`, `identity_id`, `operator_id`, `address`, `vip_level`, `province_code`, `province`, `work_company`, `is_bind_wxinfo`, `company_min_charge_balance`, `enable_online_recharge`, `integral`, `shared_user`, `enable_online_refund`, `recommenderId`, `lx_wx_user_id`, `phone_area` FROM `t_user` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_name", + "nicker_name", + "sex", + "phone", + "qk_openid", + "open_id", + "ali_user_id", + "birthday", + "avatar", + "app_type", + "password", + "upper_user", + "master_flag", + "user_type", + "multi_charge", + "relation_phone", + "user_owner_id", + "union_id", + "state", + "reg_channel", + "reg_time", + "city_label_code", + "city_label", + "reg_source", + "create_time", + "update_time", + "operator", + "invite_code", + "identity_id", + "operator_id", + "address", + "vip_level", + "province_code", + "province", + "work_company", + "is_bind_wxinfo", + "company_min_charge_balance", + "enable_online_recharge", + "integral", + "shared_user", + "enable_online_refund", + "recommenderId", + "lx_wx_user_id", + "phone_area" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_user" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_user_account.json b/DataX/json_inc/t_user_account.json new file mode 100644 index 0000000..d44036e --- /dev/null +++ b/DataX/json_inc/t_user_account.json @@ -0,0 +1,94 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_id", + "ua_balance", + "ua_deposit", + "ua_balance_time", + "ua_password", + "ua_state", + "ua_real_balance", + "ua_real_degree", + "create_time", + "update_time", + "donate_money", + "donate_able_money", + "ua_able_balance", + "refund_money", + "ua_consume_money", + "ua_clear_zero_money" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `user_id`, `ua_balance`, `ua_deposit`, `ua_balance_time`, `ua_password`, `ua_state`, `ua_real_balance`, `ua_real_degree`, `create_time`, `update_time`, `donate_money`, `donate_able_money`, `ua_able_balance`, `refund_money`, `ua_consume_money`, `ua_clear_zero_money` FROM `t_user_account` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_id", + "ua_balance", + "ua_deposit", + "ua_balance_time", + "ua_password", + "ua_state", + "ua_real_balance", + "ua_real_degree", + "create_time", + "update_time", + "donate_money", + "donate_able_money", + "ua_able_balance", + "refund_money", + "ua_consume_money", + "ua_clear_zero_money" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_user_account" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_user_upload_fault.json b/DataX/json_inc/t_user_upload_fault.json new file mode 100644 index 0000000..e382b7f --- /dev/null +++ b/DataX/json_inc/t_user_upload_fault.json @@ -0,0 +1,74 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "equipment_id", + "user_id", + "fault_config_id", + "remark", + "create_time", + "update_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `equipment_id`, `user_id`, `fault_config_id`, `remark`, `create_time`, `update_time` FROM `t_user_upload_fault` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "equipment_id", + "user_id", + "fault_config_id", + "remark", + "create_time", + "update_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_user_upload_fault" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/run_jobs_full.sh b/DataX/run_jobs_full.sh index 64df98e..9ad45a1 100644 --- a/DataX/run_jobs_full.sh +++ b/DataX/run_jobs_full.sh @@ -1,36 +1,21 @@ #!/bin/bash -# 环境变量补丁 +# 环境变量补丁 (针对 Crontab 环境下可能找不到 java 的问题) export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin [ -f /etc/profile ] && . /etc/profile [ -f ~/.bash_profile ] && . ~/.bash_profile [ -f ~/.profile ] && . ~/.profile -# 检查参数 -TARGET=$1 -if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then - echo "使用方法: $0 [doris|mysql]" - exit 1 -fi - -# 配置文件 -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -CONF_DIR="$SCRIPT_DIR/json" -CONFIG_FILE="$SCRIPT_DIR/db_config.sh" - -# 加载数据库配置 -if [ -f "$CONFIG_FILE" ]; then - source "$CONFIG_FILE" -else - echo "错误: 找不到配置文件 $CONFIG_FILE" - exit 1 -fi - +# 配置路径 DATAX_HOME="/usr/local/datax" +# 获取当前脚本所在目录 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# JSON配置文件目录指向 json_full +CONF_DIR="$SCRIPT_DIR/json_full" BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" -# 明确指定要执行的15个文件 +# 明确指定要执行的15个文件（全量同步） JOBS=( "t_equipment_charge_order.json" "t_equipment_charge_order_detail.json" @@ -49,29 +34,20 @@ JOBS=( "t_user_upload_fault.json" ) -# 日志配置 -LOG_DIR="$SCRIPT_DIR/logs" -if [ ! -d "$LOG_DIR" ]; then - mkdir -p "$LOG_DIR" -fi -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -LOG_FILE="$LOG_DIR/run_full_${TARGET}_${TIMESTAMP}.log" -echo "日志将同时记录到: $LOG_FILE" +# 定义全局参数 +PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=192.168.10.250:8030 -Ddest_jdbc=jdbc:mysql://192.168.10.250:9030/yltcharge" -# 根据目标设置参数 -if [ "$TARGET" == "doris" ]; then - DEST_PARAMS="$DORIS_DEST_PARAMS" - TITLE="MySQL -> Doris Full Sync" -else - DEST_PARAMS="$MYSQL_DEST_PARAMS" - TITLE="MySQL -> MySQL Full Sync" -fi # 在脚本最开始记录开始时间 SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) -echo "DataX全量同步脚本 ($TITLE)" +echo "DataX全量同步脚本 (Full Sync)" +echo "==========================" +echo "DataX路径: $DATAX_PY" +echo "配置文件目录: $CONF_DIR" +echo "任务数量: ${#JOBS[@]}" +echo "脚本开始时间: $SCRIPT_START_TIME" echo "==========================" # 切换到配置文件目录 @@ -80,62 +56,138 @@ cd "$CONF_DIR" || { exit 1 } +echo "当前目录: $(pwd)" +echo + +# 执行计数器 SUCCESS_COUNT=0 FAIL_COUNT=0 TOTAL=${#JOBS[@]} +CURRENT=0 +# 数组存储每个任务的执行详情 declare -a TASK_RESULTS=() declare -a TASK_DURATIONS=() declare -a TASK_STATUSES=() for JOB in "${JOBS[@]}"; do - TABLE_NAME="${JOB%.json}" - echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" + CURRENT=$((CURRENT + 1)) - # 动态构造全量 SQL：SELECT * FROM table - QUERY_SQL="SELECT * FROM $TABLE_NAME" - # 动态构造清空 SQL：TRUNCATE TABLE table - PRE_SQL="TRUNCATE TABLE $TABLE_NAME" + echo "任务 [$CURRENT/$TOTAL] $JOB" + echo "任务开始时间: $(date '+%H:%M:%S')" + echo "----------------------------------------" - # 注意：这里需要仔细处理参数引用，将变量作为整体传递 - # 优化：全量同步时表已清空，强制使用 insert 模式以提升性能（覆盖 db_config.sh 中的 replace 模式） - OPTIMIZED_DEST_PARAMS="${DEST_PARAMS/dest_write_mode=replace/dest_write_mode=insert}" - - PARAMS="$SRC_BASE_PARAMS $OPTIMIZED_DEST_PARAMS -Dsrc_query_sql=\"$QUERY_SQL\" -Ddest_pre_sql=\"$PRE_SQL\"" - - # 打印最终的执行命令（去除敏感信息后），用于调试 - # echo "执行命令: python $DATAX_PY -p \"$PARAMS\" $JOB" + # 检查文件是否存在 + if [ ! -f "$JOB" ]; then + echo "✗ 错误: 文件不存在 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + TASK_RESULTS+=("$JOB") + TASK_STATUSES+=("文件不存在") + TASK_DURATIONS+=("0") + echo "----------------------------------------" + continue + fi + # 记录开始时间（用于计算耗时） START_TIME=$(date +%s) - # 使用 eval 来正确处理带引号的参数，或者直接修改调用方式 - # 由于PARAMS中包含空格和引号，直接引用可能会导致参数解析错误 - # 最稳妥的方式是将 -p 后面的参数用单引号包起来，但内部又有双引号，比较麻烦 - # 我们可以尝试直接将所有参数拼接好，让python去解析 - /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" - EXIT_CODE=${PIPESTATUS[0]} + # 执行任务 (添加 -p 参数传递变量) + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" + EXIT_CODE=$? + + # 计算耗时 END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) + # 保存任务详情 TASK_RESULTS+=("$JOB") TASK_DURATIONS+=("$DURATION") + # 处理执行结果 if [ $EXIT_CODE -eq 0 ]; then echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) TASK_STATUSES+=("成功") else echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" + echo "错误代码: $EXIT_CODE" FAIL_COUNT=$((FAIL_COUNT + 1)) TASK_STATUSES+=("失败") + + # 询问是否继续执行后续任务 + # 全量同步失败可能需要人工干预，但这里保持自动询问或直接继续 + # 为方便自动化，这里改为仅提示，不阻塞 (或者保持原样) + # 保持原样: + # echo "----------------------------------------" + # read -p "任务失败，是否继续执行下一个任务？(y/n): " -n 1 -r + # echo + # if [[ ! $REPLY =~ ^[Yy]$ ]]; then + # echo "用户选择停止执行" + # break + # fi fi + + echo "任务结束时间: $(date '+%H:%M:%S')" + echo "----------------------------------------" + echo + + # 可选：任务间等待时间（秒） + # sleep 1 done -# ... (汇总逻辑与之前相同) +# 记录脚本结束时间 +SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_END_TIMESTAMP=$(date +%s) +TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) + echo "==========================" -echo "执行结果汇总 (Full Sync to $TARGET)" +echo "执行结果汇总 (Full Sync)" echo "==========================" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "脚本结束时间: $SCRIPT_END_TIME" +echo "脚本总耗时: ${TOTAL_DURATION}秒" +echo "==========================" +echo "任务执行详情:" +echo "----------------------------------------" + +# 输出每个任务的执行结果 +for i in "${!TASK_RESULTS[@]}"; do + STATUS_ICON="✓" + if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then + STATUS_ICON="✗" + fi + printf "%-40s %-2s %-10s %-8s秒\n" \ + "${TASK_RESULTS[$i]}" \ + "$STATUS_ICON" \ + "${TASK_STATUSES[$i]}" \ + "${TASK_DURATIONS[$i]}" +done + +echo "----------------------------------------" +echo "汇总统计:" echo "总任务数: $TOTAL" echo "成功: $SUCCESS_COUNT" echo "失败: $FAIL_COUNT" +if [ $TOTAL -gt 0 ]; then + SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) + echo "成功率: ${SUCCESS_RATE}%" +fi + +# 计算平均耗时 +if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then + TOTAL_TASK_TIME=0 + VALID_TASKS=0 + for duration in "${TASK_DURATIONS[@]}"; do + if [ "$duration" -gt 0 ]; then + TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) + VALID_TASKS=$((VALID_TASKS + 1)) + fi + done + + if [ $VALID_TASKS -gt 0 ]; then + AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) + echo "平均任务耗时: ${AVG_DURATION}秒" + fi +fi + echo "==========================" diff --git a/DataX/run_jobs_inc.sh b/DataX/run_jobs_inc.sh index c20e361..a7eb8db 100644 --- a/DataX/run_jobs_inc.sh +++ b/DataX/run_jobs_inc.sh @@ -1,36 +1,21 @@ #!/bin/bash -# 环境变量补丁 +# 环境变量补丁 (针对 Crontab 环境下可能找不到 java 的问题) export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin [ -f /etc/profile ] && . /etc/profile [ -f ~/.bash_profile ] && . ~/.bash_profile [ -f ~/.profile ] && . ~/.profile -# 检查参数 -TARGET=$1 -if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then - echo "使用方法: $0 [doris|mysql]" - exit 1 -fi - -# 配置文件 -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -CONF_DIR="$SCRIPT_DIR/json" -CONFIG_FILE="$SCRIPT_DIR/db_config.sh" - -# 加载数据库配置 -if [ -f "$CONFIG_FILE" ]; then - source "$CONFIG_FILE" -else - echo "错误: 找不到配置文件 $CONFIG_FILE" - exit 1 -fi - +# 配置路径 DATAX_HOME="/usr/local/datax" +# 获取当前脚本所在目录 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# JSON配置文件目录指向 json_inc +CONF_DIR="$SCRIPT_DIR/json_inc" BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" -# 明确指定要执行的15个文件 +# 明确指定要执行的15个文件（增量同步） JOBS=( "t_equipment_charge_order.json" "t_equipment_charge_order_detail.json" @@ -49,29 +34,22 @@ JOBS=( "t_user_upload_fault.json" ) -# 日志配置 -LOG_DIR="$SCRIPT_DIR/logs" -if [ ! -d "$LOG_DIR" ]; then - mkdir -p "$LOG_DIR" -fi -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -LOG_FILE="$LOG_DIR/run_inc_${TARGET}_${TIMESTAMP}.log" -echo "日志将同时记录到: $LOG_FILE" +# 定义全局参数 +PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=192.168.10.250:8030 -Ddest_jdbc=jdbc:mysql://192.168.10.250:9030/yltcharge" -# 根据目标设置参数 -if [ "$TARGET" == "doris" ]; then - DEST_PARAMS="$DORIS_DEST_PARAMS" - TITLE="MySQL -> Doris Incremental Sync" -else - DEST_PARAMS="$MYSQL_DEST_PARAMS" - TITLE="MySQL -> MySQL Incremental Sync" -fi # 在脚本最开始记录开始时间 SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) -echo "DataX增量同步脚本 ($TITLE)" +echo "DataX增量同步脚本 (Incremental Sync)" +echo "==========================" +echo "DataX路径: $DATAX_PY" +echo "配置文件目录: $CONF_DIR" +echo "任务数量: ${#JOBS[@]}" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "注意: 增量模式下不执行 Truncate，请确保目标表为 Unique Key 模型以实现 Upsert 效果" +echo "策略: 每次同步最近10000条数据(按ID倒序)，利用Doris主键机制自动覆盖更新" echo "==========================" # 切换到配置文件目录 @@ -80,49 +58,127 @@ cd "$CONF_DIR" || { exit 1 } +echo "当前目录: $(pwd)" +echo + +# 执行计数器 SUCCESS_COUNT=0 FAIL_COUNT=0 TOTAL=${#JOBS[@]} +CURRENT=0 +# 数组存储每个任务的执行详情 declare -a TASK_RESULTS=() declare -a TASK_DURATIONS=() declare -a TASK_STATUSES=() for JOB in "${JOBS[@]}"; do - TABLE_NAME="${JOB%.json}" - echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" + CURRENT=$((CURRENT + 1)) - # 动态构造增量 SQL：按 id 倒序取 1000 条 - QUERY_SQL="SELECT * FROM $TABLE_NAME ORDER BY id DESC LIMIT 1000" - # 增量模式下 preSql 为空（不清空目标表） - PRE_SQL="" + echo "任务 [$CURRENT/$TOTAL] $JOB" + echo "任务开始时间: $(date '+%H:%M:%S')" + echo "----------------------------------------" - PARAMS="$SRC_BASE_PARAMS $DEST_PARAMS -Dsrc_query_sql=\"$QUERY_SQL\" -Ddest_pre_sql=\"$PRE_SQL\"" + # 检查文件是否存在 + if [ ! -f "$JOB" ]; then + echo "✗ 错误: 文件不存在 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + TASK_RESULTS+=("$JOB") + TASK_STATUSES+=("文件不存在") + TASK_DURATIONS+=("0") + echo "----------------------------------------" + continue + fi + # 记录开始时间（用于计算耗时） START_TIME=$(date +%s) - /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" - EXIT_CODE=${PIPESTATUS[0]} + + # 执行任务 (添加 -p 参数传递变量) + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" + EXIT_CODE=$? + + # 计算耗时 END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) + # 保存任务详情 TASK_RESULTS+=("$JOB") TASK_DURATIONS+=("$DURATION") + # 处理执行结果 if [ $EXIT_CODE -eq 0 ]; then echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) TASK_STATUSES+=("成功") else echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" + echo "错误代码: $EXIT_CODE" FAIL_COUNT=$((FAIL_COUNT + 1)) TASK_STATUSES+=("失败") + # 增量任务通常无人值守，失败不暂停 fi + + echo "任务结束时间: $(date '+%H:%M:%S')" + echo "----------------------------------------" + echo + + # 可选：任务间等待时间（秒） + # sleep 1 done +# 记录脚本结束时间 +SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_END_TIMESTAMP=$(date +%s) +TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) + echo "==========================" -echo "执行结果汇总 (Incremental Sync to $TARGET)" +echo "执行结果汇总 (Incremental Sync)" echo "==========================" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "脚本结束时间: $SCRIPT_END_TIME" +echo "脚本总耗时: ${TOTAL_DURATION}秒" +echo "==========================" +echo "任务执行详情:" +echo "----------------------------------------" + +# 输出每个任务的执行结果 +for i in "${!TASK_RESULTS[@]}"; do + STATUS_ICON="✓" + if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then + STATUS_ICON="✗" + fi + printf "%-40s %-2s %-10s %-8s秒\n" \ + "${TASK_RESULTS[$i]}" \ + "$STATUS_ICON" \ + "${TASK_STATUSES[$i]}" \ + "${TASK_DURATIONS[$i]}" +done + +echo "----------------------------------------" +echo "汇总统计:" echo "总任务数: $TOTAL" echo "成功: $SUCCESS_COUNT" echo "失败: $FAIL_COUNT" +if [ $TOTAL -gt 0 ]; then + SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) + echo "成功率: ${SUCCESS_RATE}%" +fi + +# 计算平均耗时 +if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then + TOTAL_TASK_TIME=0 + VALID_TASKS=0 + for duration in "${TASK_DURATIONS[@]}"; do + if [ "$duration" -gt 0 ]; then + TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) + VALID_TASKS=$((VALID_TASKS + 1)) + fi + done + + if [ $VALID_TASKS -gt 0 ]; then + AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) + echo "平均任务耗时: ${AVG_DURATION}秒" + fi +fi + echo "==========================" diff --git a/DataX/start_cron.sh b/DataX/start_cron.sh index d2497cb..d51ce3b 100644 --- a/DataX/start_cron.sh +++ b/DataX/start_cron.sh @@ -3,21 +3,16 @@ # ========================================== # DataX 增量同步定时任务配置脚本 # 适用系统: Rocky Linux 9 / CentOS / RHEL -# 功能: 添加每5分钟执行一次的 crontab 任务 +# 功能: 添加每10分钟执行一次的 crontab 任务 # ========================================== # 1. 基础配置 -TARGET=$1 -if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then - echo "使用方法: $0 [doris|mysql]" - exit 1 -fi - +# 假设您将整个文件夹上传到了 /usr/local/datax WORK_DIR="/usr/local/datax" SCRIPT_NAME="run_jobs_inc.sh" SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" -# 日志存放位置 -LOG_FILE="$WORK_DIR/logs/datax_inc_${TARGET}_cron.log" +# 日志存放位置，建议单独放在 logs 目录 +LOG_FILE="$WORK_DIR/logs/datax_inc_cron.log" # 2. 检查环境 echo "正在检查环境..." @@ -47,11 +42,13 @@ echo "正在赋予脚本执行权限..." chmod +x "$SCRIPT_PATH" # 4. 配置 Crontab -# 任务定义: 每5分钟执行一次 -CRON_CMD="*/5 * * * * /bin/bash $SCRIPT_PATH $TARGET >> $LOG_FILE 2>&1" +# 任务定义: 每10分钟执行一次 (*/10) +# >> $LOG_FILE 2>&1 : 将标准输出和错误输出都追加写入日志文件 +CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" -# 检查任务是否已存在 (使用脚本路径+目标参数来匹配) -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") +# 检查任务是否已存在 (避免重复添加) +# 使用 grep 查找包含脚本路径的行 +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") if [ -n "$EXISTING_JOB" ]; then echo "警告: 检测到该脚本的定时任务已存在:" @@ -64,8 +61,9 @@ if [ -n "$EXISTING_JOB" ]; then exit 0 fi - # 移除旧任务 - crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - + # 移除旧任务 (通过 grep -v 反向过滤掉包含脚本路径的行) + # 注意: 这会保留其他不相关的定时任务 + crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - echo "旧任务已移除" fi @@ -80,7 +78,7 @@ echo "----------------------------------------" echo "当前用户的 Crontab 列表:" crontab -l echo "----------------------------------------" -echo "执行策略: 每5分钟执行一次" +echo "执行策略: 每10分钟执行一次" echo "日志文件: $LOG_FILE" echo "----------------------------------------" echo "提示: 请确保 crond 服务正在运行" diff --git a/DataX/stop_cron.sh b/DataX/stop_cron.sh index 298cea7..ac633eb 100644 --- a/DataX/stop_cron.sh +++ b/DataX/stop_cron.sh @@ -7,27 +7,21 @@ # 场景: 全量同步前防止增量任务干扰 # ========================================== -# 1. 基础配置 -TARGET=$1 -if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then - echo "使用方法: $0 [doris|mysql]" - exit 1 -fi - +# 1. 基础配置 (需与 start_cron.sh 保持一致以确保匹配正确) WORK_DIR="/usr/local/datax" SCRIPT_NAME="run_jobs_inc.sh" SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" # 2. 检查任务是否存在 -echo "正在检查 $TARGET 定时任务..." +echo "正在检查定时任务..." -# 使用脚本路径和目标参数匹配 -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") +# 使用 grep 查找包含脚本路径的行 +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") if [ -z "$EXISTING_JOB" ]; then echo "----------------------------------------" - echo "提示: 未发现 $TARGET 的定时任务，无需停止。" - echo "目标脚本: $SCRIPT_PATH $TARGET" + echo "提示: 未发现该脚本的定时任务，无需停止。" + echo "目标脚本: $SCRIPT_PATH" echo "----------------------------------------" exit 0 fi @@ -37,18 +31,20 @@ echo " $EXISTING_JOB" echo "----------------------------------------" # 3. 移除任务 -echo "正在移除 $TARGET 任务..." +echo "正在移除任务..." # 备份当前的 crontab BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" crontab -l > "$BACKUP_FILE" 2>/dev/null echo "已备份当前 Crontab 到: $BACKUP_FILE" -# 核心逻辑: 移除包含脚本路径和目标的行 -crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - +# 核心逻辑: 移除包含脚本路径的行 +# grep -v "$SCRIPT_PATH" : 排除包含脚本路径的行 +# crontab - : 将过滤后的内容重新写入 crontab +crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - # 4. 验证结果 -REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") +REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") if [ -z "$REMAINING_JOB" ]; then echo "----------------------------------------" diff --git a/DataX/说明.md b/DataX/说明.md new file mode 100644 index 0000000..1e45ae1 --- /dev/null +++ b/DataX/说明.md @@ -0,0 +1,102 @@ +# DataX 同步部署与使用说明 + +#### 一、下载与安装 DataX + +1. **下载 DataX 包** + 官方下载地址：https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202309/datax.tar.gz + +2. **部署到服务器** + 将压缩包上传并解压到 `/usr/local/` 目录： + ```shell + cd /usr/local + tar zxvf datax.tar.gz + # 解压后应当存在 /usr/local/datax 目录 + ``` + +#### 二、配置部署 + +本方案包含**全量同步**和**增量同步**两套策略，请按以下步骤部署配置文件和脚本。 + +1. **上传文件** + 请将本地 `d:\dsWork\aiData\Doc\6、DataX配置\` 目录下的以下内容上传至服务器 `/usr/local/datax/` 根目录下： + + * 文件夹 `json_full/` (存放全量JSON配置) + * 文件夹 `json_inc/` (存放增量JSON配置) + * 脚本 `run_jobs_full.sh` + * 脚本 `run_jobs_inc.sh` + * 脚本 `setup_cron.sh` + + **上传后的服务器目录结构应如下所示：** + ```text + /usr/local/datax/ + ├── bin/ + ├── conf/ + ├── json_full/ <-- 新增：全量配置目录 + ├── json_inc/ <-- 新增：增量配置目录 + ├── run_jobs_full.sh <-- 新增：全量执行脚本 + ├── run_jobs_inc.sh <-- 新增：增量执行脚本 + └── setup_cron.sh <-- 新增：定时任务安装脚本 + ``` + +2. **赋予执行权限** + ```shell + cd /usr/local/datax + chmod +x *.sh + ``` + +3. **修改数据库密码 (如有需要)** + 脚本中已配置好默认的数据库连接信息。如需修改，请编辑 `run_jobs_full.sh` 和 `run_jobs_inc.sh` 中的 `PARAMS` 变量： + ```bash + PARAMS="-Dsrc_user=... -Dsrc_pwd=... -Ddest_user=... -Ddest_pwd=..." + ``` + +#### 三、运行策略说明 + +**1. 全量同步 (Full Sync)** +* **适用场景**：系统初始化，或数据出现严重偏差需要彻底重跑时。 +* **同步逻辑**： + 1. 执行 `TRUNCATE` 清空 Doris 目标表。 + 2. 从 MySQL 读取全量数据写入 Doris。 +* **手动执行命令**： + ```shell + cd /usr/local/datax + ./run_jobs_full.sh + ``` + +**2. 增量同步 (Incremental Sync)** +* **适用场景**：日常持续数据同步（高频）。 +* **同步逻辑**： + 1. **不删除**任何历史数据（无空窗期）。 + 2. 读取 MySQL 中 ID 最大的 **10000条** 记录（覆盖新增和最近更新的数据）。 + 3. 推送到 Doris，利用 Doris 的 **Unique Key** 主键模型自动完成“存在即更新，不存在即插入” (Upsert)。 +* **手动执行命令**： + ```shell + cd /usr/local/datax + ./run_jobs_inc.sh + ``` + +#### 四、自动化定时任务配置 + +我们提供了一键配置脚本，用于在 Rocky Linux 9 / CentOS 等系统中设置自动调度。 + +1. **执行安装脚本** + ```shell + cd /usr/local/datax + ./setup_cron.sh + ``` + *该脚本会自动添加 Crontab 任务，配置为每 10 分钟执行一次增量同步。* + +2. **验证与日志** + * **查看任务**：`crontab -l` + * **查看日志**：自动同步的日志会保存在 `/usr/local/datax/logs/datax_inc_cron.log` + ```shell + tail -f /usr/local/datax/logs/datax_inc_cron.log + ``` + +#### 五、重要注意事项 + +1. **Doris 表模型要求** + 目标 Doris 表必须使用 **Unique Key** 模型（建表时指定 `UNIQUE KEY(id)`），否则增量同步会导致大量重复数据。 + +2. **Python 环境** + DataX 依赖 Python 环境（推荐 Python 2.7 或 3.x）。脚本默认调用 `/usr/bin/python`，如果您的 Python 路径不同，请修改脚本头部的 `DATAX_PY` 变量。 diff --git a/DataX/Doc/说明.md b/DataX_Bad/Doc/说明.md similarity index 100% rename from DataX/Doc/说明.md rename to DataX_Bad/Doc/说明.md diff --git a/DataX/db_config.sh b/DataX_Bad/db_config.sh similarity index 84% rename from DataX/db_config.sh rename to DataX_Bad/db_config.sh index 79d5ef6..02e1fbe 100644 --- a/DataX/db_config.sh +++ b/DataX_Bad/db_config.sh @@ -8,7 +8,7 @@ # 1. 源端 MySQL 配置 (Source) SRC_USER="ylt" SRC_PWD="Ycharge666" -SRC_JDBC="jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge?useUnicode=true\&characterEncoding=utf-8\&useSSL=false\&serverTimezone=Asia/Shanghai\&yearIsDateType=false\&zeroDateTimeBehavior=convertToNull" +SRC_JDBC="jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai&yearIsDateType=false&zeroDateTimeBehavior=convertToNull" # 2. 目标端 Doris 配置 (Target Doris) DORIS_USER="root" @@ -20,7 +20,7 @@ DORIS_LOAD_URL="10.10.14.204:8030" # 如果目标 MySQL 的账号密码与源端一致，可以直接引用上面的变量 MYSQL_USER="ylt" MYSQL_PWD="Ycharge666" -MYSQL_JDBC="jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true\&characterEncoding=utf-8\&useSSL=false\&serverTimezone=Asia/Shanghai\&rewriteBatchedStatements=true" +MYSQL_JDBC="jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true&useConfigs=maxPerformance" # --- 内部参数构造 (无需修改) --- SRC_BASE_PARAMS="-Dsrc_user=$SRC_USER -Dsrc_pwd=$SRC_PWD -Dsrc_jdbc=\"$SRC_JDBC\"" diff --git a/DataX/json/t_account_recharge.json b/DataX_Bad/json/t_account_recharge.json similarity index 81% rename from DataX/json/t_account_recharge.json rename to DataX_Bad/json/t_account_recharge.json index 634b7fa..9864a81 100644 --- a/DataX/json/t_account_recharge.json +++ b/DataX_Bad/json/t_account_recharge.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -76,14 +80,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_account_water.json b/DataX_Bad/json/t_account_water.json similarity index 78% rename from DataX/json/t_account_water.json rename to DataX_Bad/json/t_account_water.json index e24c9b4..cb780e8 100644 --- a/DataX/json/t_account_water.json +++ b/DataX_Bad/json/t_account_water.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -65,14 +69,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_car.json b/DataX_Bad/json/t_car.json similarity index 78% rename from DataX/json/t_car.json rename to DataX_Bad/json/t_car.json index 2995bf2..ad0bea6 100644 --- a/DataX/json/t_car.json +++ b/DataX_Bad/json/t_car.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -62,14 +66,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_company.json b/DataX_Bad/json/t_company.json similarity index 82% rename from DataX/json/t_company.json rename to DataX_Bad/json/t_company.json index cfe1dab..ed7edd6 100644 --- a/DataX/json/t_company.json +++ b/DataX_Bad/json/t_company.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -79,14 +83,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_company_info_value.json b/DataX_Bad/json/t_company_info_value.json similarity index 74% rename from DataX/json/t_company_info_value.json rename to DataX_Bad/json/t_company_info_value.json index 9d24909..418c63c 100644 --- a/DataX/json/t_company_info_value.json +++ b/DataX_Bad/json/t_company_info_value.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -51,14 +55,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_connector.json b/DataX_Bad/json/t_connector.json similarity index 80% rename from DataX/json/t_connector.json rename to DataX_Bad/json/t_connector.json index 805ae18..bee6654 100644 --- a/DataX/json/t_connector.json +++ b/DataX_Bad/json/t_connector.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -69,14 +73,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_equipment.json b/DataX_Bad/json/t_equipment.json similarity index 81% rename from DataX/json/t_equipment.json rename to DataX_Bad/json/t_equipment.json index 7ef4860..29b63cf 100644 --- a/DataX/json/t_equipment.json +++ b/DataX_Bad/json/t_equipment.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -74,14 +78,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_equipment_charge_order.json b/DataX_Bad/json/t_equipment_charge_order.json similarity index 89% rename from DataX/json/t_equipment_charge_order.json rename to DataX_Bad/json/t_equipment_charge_order.json index 01ea753..938c1c4 100644 --- a/DataX/json/t_equipment_charge_order.json +++ b/DataX_Bad/json/t_equipment_charge_order.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -155,14 +159,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_equipment_charge_order_detail.json b/DataX_Bad/json/t_equipment_charge_order_detail.json similarity index 80% rename from DataX/json/t_equipment_charge_order_detail.json rename to DataX_Bad/json/t_equipment_charge_order_detail.json index e72415b..1d74351 100644 --- a/DataX/json/t_equipment_charge_order_detail.json +++ b/DataX_Bad/json/t_equipment_charge_order_detail.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -67,14 +71,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_ext_hurry_quit.json b/DataX_Bad/json/t_ext_hurry_quit.json similarity index 80% rename from DataX/json/t_ext_hurry_quit.json rename to DataX_Bad/json/t_ext_hurry_quit.json index e048ae8..b358650 100644 --- a/DataX/json/t_ext_hurry_quit.json +++ b/DataX_Bad/json/t_ext_hurry_quit.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -73,14 +77,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_station.json b/DataX_Bad/json/t_station.json similarity index 87% rename from DataX/json/t_station.json rename to DataX_Bad/json/t_station.json index f243c1e..3bcf189 100644 --- a/DataX/json/t_station.json +++ b/DataX_Bad/json/t_station.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -126,14 +130,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_time_day.json b/DataX_Bad/json/t_time_day.json similarity index 73% rename from DataX/json/t_time_day.json rename to DataX_Bad/json/t_time_day.json index f4c69e4..d6aeb64 100644 --- a/DataX/json/t_time_day.json +++ b/DataX_Bad/json/t_time_day.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -49,14 +53,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_user.json b/DataX_Bad/json/t_user.json similarity index 83% rename from DataX/json/t_user.json rename to DataX_Bad/json/t_user.json index 6876a5a..eb40743 100644 --- a/DataX/json/t_user.json +++ b/DataX_Bad/json/t_user.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -89,14 +93,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_user_account.json b/DataX_Bad/json/t_user_account.json similarity index 77% rename from DataX/json/t_user_account.json rename to DataX_Bad/json/t_user_account.json index 37144b6..02fa79f 100644 --- a/DataX/json/t_user_account.json +++ b/DataX_Bad/json/t_user_account.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -61,14 +65,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX/json/t_user_upload_fault.json b/DataX_Bad/json/t_user_upload_fault.json similarity index 74% rename from DataX/json/t_user_upload_fault.json rename to DataX_Bad/json/t_user_upload_fault.json index 5e35ff9..76102c5 100644 --- a/DataX/json/t_user_upload_fault.json +++ b/DataX_Bad/json/t_user_upload_fault.json @@ -10,13 +10,17 @@ "jdbcUrl": [ "${src_jdbc}" ], - "querySql": [ - "${src_query_sql}" + "table": [ + "${src_table}" ] } ], "username": "${src_user}", - "password": "${src_pwd}" + "password": "${src_pwd}", + "splitPk": "id", + "column": [ + "*" + ] } }, "writer": { @@ -51,14 +55,19 @@ "format": "json", "strip_outer_array": true }, - "writeMode": "${dest_write_mode}" + "writeMode": "${dest_write_mode}", + "batchSize": 1024, + "session": [ + "set session unique_checks=0", + "set session foreign_key_checks=0" + ] } } } ], "setting": { "speed": { - "channel": "1" + "channel": "4" } } } diff --git a/DataX_Bad/run_jobs_full.sh b/DataX_Bad/run_jobs_full.sh new file mode 100644 index 0000000..02fdbcb --- /dev/null +++ b/DataX_Bad/run_jobs_full.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +# 环境变量补丁 +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +# 检查参数 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + +# 配置文件 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +CONF_DIR="$SCRIPT_DIR/json" +CONFIG_FILE="$SCRIPT_DIR/db_config.sh" + +# 加载数据库配置 +if [ -f "$CONFIG_FILE" ]; then + source "$CONFIG_FILE" +else + echo "错误: 找不到配置文件 $CONFIG_FILE" + exit 1 +fi + +# 默认参数 +export DATAX_JVM="-Xms8g -Xmx8g" +DATAX_HOME="/usr/local/datax" +BIN_DIR="$DATAX_HOME/bin" +DATAX_PY="$BIN_DIR/datax.py" + +# 明确指定要执行的15个文件 +JOBS=( + "t_equipment_charge_order.json" + "t_equipment_charge_order_detail.json" + "t_account_recharge.json" + "t_account_water.json" + "t_car.json" + "t_company.json" + "t_company_info_value.json" + "t_connector.json" + "t_equipment.json" + "t_station.json" + "t_ext_hurry_quit.json" + "t_time_day.json" + "t_user.json" + "t_user_account.json" + "t_user_upload_fault.json" +) + +# 日志配置 +LOG_DIR="$SCRIPT_DIR/logs" +if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" +fi +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="$LOG_DIR/run_full_${TARGET}_${TIMESTAMP}.log" +echo "日志将同时记录到: $LOG_FILE" + +# 根据目标设置参数 +if [ "$TARGET" == "doris" ]; then + DEST_PARAMS="$DORIS_DEST_PARAMS" + TITLE="MySQL -> Doris Full Sync" +else + DEST_PARAMS="$MYSQL_DEST_PARAMS" + TITLE="MySQL -> MySQL Full Sync" +fi + +# 在脚本最开始记录开始时间 +SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_START_TIMESTAMP=$(date +%s) + +echo "DataX全量同步脚本 ($TITLE)" +echo "==========================" + +# 切换到配置文件目录 +cd "$CONF_DIR" || { + echo "错误: 无法进入配置文件目录 $CONF_DIR" + exit 1 +} + +SUCCESS_COUNT=0 +FAIL_COUNT=0 +TOTAL=${#JOBS[@]} + +declare -a TASK_RESULTS=() +declare -a TASK_DURATIONS=() +declare -a TASK_STATUSES=() + +for JOB in "${JOBS[@]}"; do + TABLE_NAME="${JOB%.json}" + echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" + + # 动态构造全量 SQL：SELECT * FROM table + QUERY_SQL="SELECT * FROM $TABLE_NAME" + # 动态构造清空 SQL：TRUNCATE TABLE table + PRE_SQL="TRUNCATE TABLE $TABLE_NAME" + + # 注意：这里需要仔细处理参数引用，将变量作为整体传递 + # 优化：全量同步时表已清空，强制使用 insert 模式以提升性能（覆盖 db_config.sh 中的 replace 模式） + OPTIMIZED_DEST_PARAMS="${DEST_PARAMS/dest_write_mode=replace/dest_write_mode=insert}" + + PARAMS="$SRC_BASE_PARAMS $OPTIMIZED_DEST_PARAMS -Dsrc_table=\"$TABLE_NAME\" -Ddest_pre_sql=\"$PRE_SQL\"" + + # 打印最终的执行命令（去除敏感信息后），用于调试 + # echo "执行命令: python $DATAX_PY -p \"$PARAMS\" $JOB" + + START_TIME=$(date +%s) + # 使用 eval 来正确处理带引号的参数，或者直接修改调用方式 + # 由于PARAMS中包含空格和引号，直接引用可能会导致参数解析错误 + # 最稳妥的方式是将 -p 后面的参数用单引号包起来，但内部又有双引号，比较麻烦 + # 我们可以尝试直接将所有参数拼接好，让python去解析 + + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" + EXIT_CODE=${PIPESTATUS[0]} + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + TASK_RESULTS+=("$JOB") + TASK_DURATIONS+=("$DURATION") + + if [ $EXIT_CODE -eq 0 ]; then + echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + TASK_STATUSES+=("成功") + else + echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" + FAIL_COUNT=$((FAIL_COUNT + 1)) + TASK_STATUSES+=("失败") + fi +done + +# ... (汇总逻辑与之前相同) +echo "==========================" +echo "执行结果汇总 (Full Sync to $TARGET)" +echo "==========================" +echo "总任务数: $TOTAL" +echo "成功: $SUCCESS_COUNT" +echo "失败: $FAIL_COUNT" +echo "==========================" diff --git a/DataX_Bad/run_jobs_inc.sh b/DataX_Bad/run_jobs_inc.sh new file mode 100644 index 0000000..c20e361 --- /dev/null +++ b/DataX_Bad/run_jobs_inc.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# 环境变量补丁 +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +[ -f /etc/profile ] && . /etc/profile +[ -f ~/.bash_profile ] && . ~/.bash_profile +[ -f ~/.profile ] && . ~/.profile + +# 检查参数 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + +# 配置文件 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +CONF_DIR="$SCRIPT_DIR/json" +CONFIG_FILE="$SCRIPT_DIR/db_config.sh" + +# 加载数据库配置 +if [ -f "$CONFIG_FILE" ]; then + source "$CONFIG_FILE" +else + echo "错误: 找不到配置文件 $CONFIG_FILE" + exit 1 +fi + +DATAX_HOME="/usr/local/datax" +BIN_DIR="$DATAX_HOME/bin" +DATAX_PY="$BIN_DIR/datax.py" + +# 明确指定要执行的15个文件 +JOBS=( + "t_equipment_charge_order.json" + "t_equipment_charge_order_detail.json" + "t_account_recharge.json" + "t_account_water.json" + "t_car.json" + "t_company.json" + "t_company_info_value.json" + "t_connector.json" + "t_equipment.json" + "t_station.json" + "t_ext_hurry_quit.json" + "t_time_day.json" + "t_user.json" + "t_user_account.json" + "t_user_upload_fault.json" +) + +# 日志配置 +LOG_DIR="$SCRIPT_DIR/logs" +if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" +fi +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="$LOG_DIR/run_inc_${TARGET}_${TIMESTAMP}.log" +echo "日志将同时记录到: $LOG_FILE" + +# 根据目标设置参数 +if [ "$TARGET" == "doris" ]; then + DEST_PARAMS="$DORIS_DEST_PARAMS" + TITLE="MySQL -> Doris Incremental Sync" +else + DEST_PARAMS="$MYSQL_DEST_PARAMS" + TITLE="MySQL -> MySQL Incremental Sync" +fi + +# 在脚本最开始记录开始时间 +SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_START_TIMESTAMP=$(date +%s) + +echo "DataX增量同步脚本 ($TITLE)" +echo "==========================" + +# 切换到配置文件目录 +cd "$CONF_DIR" || { + echo "错误: 无法进入配置文件目录 $CONF_DIR" + exit 1 +} + +SUCCESS_COUNT=0 +FAIL_COUNT=0 +TOTAL=${#JOBS[@]} + +declare -a TASK_RESULTS=() +declare -a TASK_DURATIONS=() +declare -a TASK_STATUSES=() + +for JOB in "${JOBS[@]}"; do + TABLE_NAME="${JOB%.json}" + echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" + + # 动态构造增量 SQL：按 id 倒序取 1000 条 + QUERY_SQL="SELECT * FROM $TABLE_NAME ORDER BY id DESC LIMIT 1000" + # 增量模式下 preSql 为空（不清空目标表） + PRE_SQL="" + + PARAMS="$SRC_BASE_PARAMS $DEST_PARAMS -Dsrc_query_sql=\"$QUERY_SQL\" -Ddest_pre_sql=\"$PRE_SQL\"" + + START_TIME=$(date +%s) + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" + EXIT_CODE=${PIPESTATUS[0]} + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + TASK_RESULTS+=("$JOB") + TASK_DURATIONS+=("$DURATION") + + if [ $EXIT_CODE -eq 0 ]; then + echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + TASK_STATUSES+=("成功") + else + echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" + FAIL_COUNT=$((FAIL_COUNT + 1)) + TASK_STATUSES+=("失败") + fi +done + +echo "==========================" +echo "执行结果汇总 (Incremental Sync to $TARGET)" +echo "==========================" +echo "总任务数: $TOTAL" +echo "成功: $SUCCESS_COUNT" +echo "失败: $FAIL_COUNT" +echo "==========================" diff --git a/DataX/scheme/README.md b/DataX_Bad/scheme/README.md similarity index 100% rename from DataX/scheme/README.md rename to DataX_Bad/scheme/README.md diff --git a/DataX/scheme/generate_schema.py b/DataX_Bad/scheme/generate_schema.py similarity index 100% rename from DataX/scheme/generate_schema.py rename to DataX_Bad/scheme/generate_schema.py diff --git a/DataX/scheme/output/t_account_recharge_doris.sql b/DataX_Bad/scheme/output/t_account_recharge_doris.sql similarity index 100% rename from DataX/scheme/output/t_account_recharge_doris.sql rename to DataX_Bad/scheme/output/t_account_recharge_doris.sql diff --git a/DataX/scheme/output/t_account_recharge_mysql.sql b/DataX_Bad/scheme/output/t_account_recharge_mysql.sql similarity index 100% rename from DataX/scheme/output/t_account_recharge_mysql.sql rename to DataX_Bad/scheme/output/t_account_recharge_mysql.sql diff --git a/DataX/scheme/output/t_account_water_doris.sql b/DataX_Bad/scheme/output/t_account_water_doris.sql similarity index 100% rename from DataX/scheme/output/t_account_water_doris.sql rename to DataX_Bad/scheme/output/t_account_water_doris.sql diff --git a/DataX/scheme/output/t_account_water_mysql.sql b/DataX_Bad/scheme/output/t_account_water_mysql.sql similarity index 100% rename from DataX/scheme/output/t_account_water_mysql.sql rename to DataX_Bad/scheme/output/t_account_water_mysql.sql diff --git a/DataX/scheme/output/t_car_doris.sql b/DataX_Bad/scheme/output/t_car_doris.sql similarity index 100% rename from DataX/scheme/output/t_car_doris.sql rename to DataX_Bad/scheme/output/t_car_doris.sql diff --git a/DataX/scheme/output/t_car_mysql.sql b/DataX_Bad/scheme/output/t_car_mysql.sql similarity index 100% rename from DataX/scheme/output/t_car_mysql.sql rename to DataX_Bad/scheme/output/t_car_mysql.sql diff --git a/DataX/scheme/output/t_company_doris.sql b/DataX_Bad/scheme/output/t_company_doris.sql similarity index 100% rename from DataX/scheme/output/t_company_doris.sql rename to DataX_Bad/scheme/output/t_company_doris.sql diff --git a/DataX/scheme/output/t_company_info_value_doris.sql b/DataX_Bad/scheme/output/t_company_info_value_doris.sql similarity index 100% rename from DataX/scheme/output/t_company_info_value_doris.sql rename to DataX_Bad/scheme/output/t_company_info_value_doris.sql diff --git a/DataX/scheme/output/t_company_info_value_mysql.sql b/DataX_Bad/scheme/output/t_company_info_value_mysql.sql similarity index 100% rename from DataX/scheme/output/t_company_info_value_mysql.sql rename to DataX_Bad/scheme/output/t_company_info_value_mysql.sql diff --git a/DataX/scheme/output/t_company_mysql.sql b/DataX_Bad/scheme/output/t_company_mysql.sql similarity index 100% rename from DataX/scheme/output/t_company_mysql.sql rename to DataX_Bad/scheme/output/t_company_mysql.sql diff --git a/DataX/scheme/output/t_connector_doris.sql b/DataX_Bad/scheme/output/t_connector_doris.sql similarity index 100% rename from DataX/scheme/output/t_connector_doris.sql rename to DataX_Bad/scheme/output/t_connector_doris.sql diff --git a/DataX/scheme/output/t_connector_mysql.sql b/DataX_Bad/scheme/output/t_connector_mysql.sql similarity index 100% rename from DataX/scheme/output/t_connector_mysql.sql rename to DataX_Bad/scheme/output/t_connector_mysql.sql diff --git a/DataX/scheme/output/t_equipment_charge_order_detail_doris.sql b/DataX_Bad/scheme/output/t_equipment_charge_order_detail_doris.sql similarity index 100% rename from DataX/scheme/output/t_equipment_charge_order_detail_doris.sql rename to DataX_Bad/scheme/output/t_equipment_charge_order_detail_doris.sql diff --git a/DataX/scheme/output/t_equipment_charge_order_detail_mysql.sql b/DataX_Bad/scheme/output/t_equipment_charge_order_detail_mysql.sql similarity index 100% rename from DataX/scheme/output/t_equipment_charge_order_detail_mysql.sql rename to DataX_Bad/scheme/output/t_equipment_charge_order_detail_mysql.sql diff --git a/DataX/scheme/output/t_equipment_charge_order_doris.sql b/DataX_Bad/scheme/output/t_equipment_charge_order_doris.sql similarity index 100% rename from DataX/scheme/output/t_equipment_charge_order_doris.sql rename to DataX_Bad/scheme/output/t_equipment_charge_order_doris.sql diff --git a/DataX/scheme/output/t_equipment_charge_order_mysql.sql b/DataX_Bad/scheme/output/t_equipment_charge_order_mysql.sql similarity index 100% rename from DataX/scheme/output/t_equipment_charge_order_mysql.sql rename to DataX_Bad/scheme/output/t_equipment_charge_order_mysql.sql diff --git a/DataX/scheme/output/t_equipment_doris.sql b/DataX_Bad/scheme/output/t_equipment_doris.sql similarity index 100% rename from DataX/scheme/output/t_equipment_doris.sql rename to DataX_Bad/scheme/output/t_equipment_doris.sql diff --git a/DataX/scheme/output/t_equipment_mysql.sql b/DataX_Bad/scheme/output/t_equipment_mysql.sql similarity index 100% rename from DataX/scheme/output/t_equipment_mysql.sql rename to DataX_Bad/scheme/output/t_equipment_mysql.sql diff --git a/DataX/scheme/output/t_ext_hurry_quit_doris.sql b/DataX_Bad/scheme/output/t_ext_hurry_quit_doris.sql similarity index 100% rename from DataX/scheme/output/t_ext_hurry_quit_doris.sql rename to DataX_Bad/scheme/output/t_ext_hurry_quit_doris.sql diff --git a/DataX/scheme/output/t_ext_hurry_quit_mysql.sql b/DataX_Bad/scheme/output/t_ext_hurry_quit_mysql.sql similarity index 100% rename from DataX/scheme/output/t_ext_hurry_quit_mysql.sql rename to DataX_Bad/scheme/output/t_ext_hurry_quit_mysql.sql diff --git a/DataX/scheme/output/t_station_doris.sql b/DataX_Bad/scheme/output/t_station_doris.sql similarity index 100% rename from DataX/scheme/output/t_station_doris.sql rename to DataX_Bad/scheme/output/t_station_doris.sql diff --git a/DataX/scheme/output/t_station_mysql.sql b/DataX_Bad/scheme/output/t_station_mysql.sql similarity index 100% rename from DataX/scheme/output/t_station_mysql.sql rename to DataX_Bad/scheme/output/t_station_mysql.sql diff --git a/DataX/scheme/output/t_time_day_doris.sql b/DataX_Bad/scheme/output/t_time_day_doris.sql similarity index 100% rename from DataX/scheme/output/t_time_day_doris.sql rename to DataX_Bad/scheme/output/t_time_day_doris.sql diff --git a/DataX/scheme/output/t_time_day_mysql.sql b/DataX_Bad/scheme/output/t_time_day_mysql.sql similarity index 100% rename from DataX/scheme/output/t_time_day_mysql.sql rename to DataX_Bad/scheme/output/t_time_day_mysql.sql diff --git a/DataX/scheme/output/t_user_account_doris.sql b/DataX_Bad/scheme/output/t_user_account_doris.sql similarity index 100% rename from DataX/scheme/output/t_user_account_doris.sql rename to DataX_Bad/scheme/output/t_user_account_doris.sql diff --git a/DataX/scheme/output/t_user_account_mysql.sql b/DataX_Bad/scheme/output/t_user_account_mysql.sql similarity index 100% rename from DataX/scheme/output/t_user_account_mysql.sql rename to DataX_Bad/scheme/output/t_user_account_mysql.sql diff --git a/DataX/scheme/output/t_user_doris.sql b/DataX_Bad/scheme/output/t_user_doris.sql similarity index 100% rename from DataX/scheme/output/t_user_doris.sql rename to DataX_Bad/scheme/output/t_user_doris.sql diff --git a/DataX/scheme/output/t_user_mysql.sql b/DataX_Bad/scheme/output/t_user_mysql.sql similarity index 100% rename from DataX/scheme/output/t_user_mysql.sql rename to DataX_Bad/scheme/output/t_user_mysql.sql diff --git a/DataX/scheme/output/t_user_upload_fault_doris.sql b/DataX_Bad/scheme/output/t_user_upload_fault_doris.sql similarity index 100% rename from DataX/scheme/output/t_user_upload_fault_doris.sql rename to DataX_Bad/scheme/output/t_user_upload_fault_doris.sql diff --git a/DataX/scheme/output/t_user_upload_fault_mysql.sql b/DataX_Bad/scheme/output/t_user_upload_fault_mysql.sql similarity index 100% rename from DataX/scheme/output/t_user_upload_fault_mysql.sql rename to DataX_Bad/scheme/output/t_user_upload_fault_mysql.sql diff --git a/DataX_Bad/start_cron.sh b/DataX_Bad/start_cron.sh new file mode 100644 index 0000000..d2497cb --- /dev/null +++ b/DataX_Bad/start_cron.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# ========================================== +# DataX 增量同步定时任务配置脚本 +# 适用系统: Rocky Linux 9 / CentOS / RHEL +# 功能: 添加每5分钟执行一次的 crontab 任务 +# ========================================== + +# 1. 基础配置 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" +# 日志存放位置 +LOG_FILE="$WORK_DIR/logs/datax_inc_${TARGET}_cron.log" + +# 2. 检查环境 +echo "正在检查环境..." + +# 检查工作目录 +if [ ! -d "$WORK_DIR" ]; then + echo "错误: 工作目录 $WORK_DIR 不存在" + echo "请先将 DataX 配置相关文件上传至服务器的 $WORK_DIR" + exit 1 +fi + +# 检查脚本是否存在 +if [ ! -f "$SCRIPT_PATH" ]; then + echo "错误: 执行脚本 $SCRIPT_PATH 不存在" + echo "请确保 $SCRIPT_NAME 位于 $WORK_DIR 目录下" + exit 1 +fi + +# 创建日志目录 +if [ ! -d "$(dirname "$LOG_FILE")" ]; then + mkdir -p "$(dirname "$LOG_FILE")" + echo "创建日志目录: $(dirname "$LOG_FILE")" +fi + +# 3. 赋予执行权限 +echo "正在赋予脚本执行权限..." +chmod +x "$SCRIPT_PATH" + +# 4. 配置 Crontab +# 任务定义: 每5分钟执行一次 +CRON_CMD="*/5 * * * * /bin/bash $SCRIPT_PATH $TARGET >> $LOG_FILE 2>&1" + +# 检查任务是否已存在 (使用脚本路径+目标参数来匹配) +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") + +if [ -n "$EXISTING_JOB" ]; then + echo "警告: 检测到该脚本的定时任务已存在:" + echo " $EXISTING_JOB" + echo "----------------------------------------" + read -p "是否先删除旧任务再重新添加? (y/n): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "已取消操作，未修改 Crontab。" + exit 0 + fi + + # 移除旧任务 + crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - + echo "旧任务已移除" +fi + +# 添加新任务 +# (crontab -l; echo "new job") | crontab - 这种写法可以保留原有任务并追加新任务 +(crontab -l 2>/dev/null; echo "$CRON_CMD") | crontab - + +# 5. 验证与提示 +echo "----------------------------------------" +echo "✅ 定时任务设置成功!" +echo "----------------------------------------" +echo "当前用户的 Crontab 列表:" +crontab -l +echo "----------------------------------------" +echo "执行策略: 每5分钟执行一次" +echo "日志文件: $LOG_FILE" +echo "----------------------------------------" +echo "提示: 请确保 crond 服务正在运行" +echo "检查命令: systemctl status crond" +echo "启动命令: systemctl start crond" diff --git a/DataX_Bad/stop_cron.sh b/DataX_Bad/stop_cron.sh new file mode 100644 index 0000000..298cea7 --- /dev/null +++ b/DataX_Bad/stop_cron.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# ========================================== +# DataX 增量同步定时任务停止脚本 +# 适用系统: Rocky Linux 9 / CentOS / RHEL +# 功能: 移除 run_jobs_inc.sh 的 crontab 任务 +# 场景: 全量同步前防止增量任务干扰 +# ========================================== + +# 1. 基础配置 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + +WORK_DIR="/usr/local/datax" +SCRIPT_NAME="run_jobs_inc.sh" +SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" + +# 2. 检查任务是否存在 +echo "正在检查 $TARGET 定时任务..." + +# 使用脚本路径和目标参数匹配 +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") + +if [ -z "$EXISTING_JOB" ]; then + echo "----------------------------------------" + echo "提示: 未发现 $TARGET 的定时任务，无需停止。" + echo "目标脚本: $SCRIPT_PATH $TARGET" + echo "----------------------------------------" + exit 0 +fi + +echo "发现正在运行的定时任务:" +echo " $EXISTING_JOB" +echo "----------------------------------------" + +# 3. 移除任务 +echo "正在移除 $TARGET 任务..." + +# 备份当前的 crontab +BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" +crontab -l > "$BACKUP_FILE" 2>/dev/null +echo "已备份当前 Crontab 到: $BACKUP_FILE" + +# 核心逻辑: 移除包含脚本路径和目标的行 +crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - + +# 4. 验证结果 +REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") + +if [ -z "$REMAINING_JOB" ]; then + echo "----------------------------------------" + echo "✅ 增量同步定时任务已成功停止 (移除)!" + echo "----------------------------------------" + echo "当前用户的 Crontab 列表:" + crontab -l + echo "----------------------------------------" + echo "现在您可以安全地执行全量同步操作了。" +else + echo "❌ 停止失败，任务仍然存在。请尝试手动编辑: crontab -e" + exit 1 +fi