From 2e03f4344529ba51d9fb893a5f18183e4df068c8 Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Tue, 20 Jan 2026 11:57:40 +0800 Subject: [PATCH] 'commit' --- .../{json => json_full}/t_account_recharge.json | 0 .../{json => json_full}/t_account_water.json | 0 .../{json => json_full}/t_car.json | 0 .../{json => json_full}/t_company.json | 0 .../t_company_info_value.json | 0 .../{json => json_full}/t_connector.json | 0 .../{json => json_full}/t_equipment.json | 0 .../t_equipment_charge_order.json | 0 .../t_equipment_charge_order_detail.json | 0 .../json_inc/t_account_recharge.json | 124 ++++++++ .../json_inc/t_account_water.json | 102 +++++++ Doc/6、DataX配置/json_inc/t_car.json | 96 ++++++ Doc/6、DataX配置/json_inc/t_company.json | 130 ++++++++ .../json_inc/t_company_info_value.json | 74 +++++ Doc/6、DataX配置/json_inc/t_connector.json | 110 +++++++ Doc/6、DataX配置/json_inc/t_equipment.json | 120 ++++++++ .../json_inc/t_equipment_charge_order.json | 282 ++++++++++++++++++ .../t_equipment_charge_order_detail.json | 106 +++++++ Doc/6、DataX配置/run_jobs_full.sh | 181 +++++++++++ .../{run_jobs.sh => run_jobs_inc.sh} | 25 +- 20 files changed, 1336 insertions(+), 14 deletions(-) rename Doc/6、DataX配置/{json => json_full}/t_account_recharge.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_account_water.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_car.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_company.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_company_info_value.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_connector.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_equipment.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_equipment_charge_order.json (100%) rename Doc/6、DataX配置/{json => json_full}/t_equipment_charge_order_detail.json (100%) create mode 100644 Doc/6、DataX配置/json_inc/t_account_recharge.json create mode 100644 Doc/6、DataX配置/json_inc/t_account_water.json create mode 100644 Doc/6、DataX配置/json_inc/t_car.json create mode 100644 Doc/6、DataX配置/json_inc/t_company.json create mode 100644 Doc/6、DataX配置/json_inc/t_company_info_value.json create mode 100644 Doc/6、DataX配置/json_inc/t_connector.json create mode 100644 Doc/6、DataX配置/json_inc/t_equipment.json create mode 100644 Doc/6、DataX配置/json_inc/t_equipment_charge_order.json create mode 100644 Doc/6、DataX配置/json_inc/t_equipment_charge_order_detail.json create mode 100644 Doc/6、DataX配置/run_jobs_full.sh rename Doc/6、DataX配置/{run_jobs.sh => run_jobs_inc.sh} (87%) diff --git a/Doc/6、DataX配置/json/t_account_recharge.json b/Doc/6、DataX配置/json_full/t_account_recharge.json similarity index 100% rename from Doc/6、DataX配置/json/t_account_recharge.json rename to Doc/6、DataX配置/json_full/t_account_recharge.json diff --git a/Doc/6、DataX配置/json/t_account_water.json b/Doc/6、DataX配置/json_full/t_account_water.json similarity index 100% rename from Doc/6、DataX配置/json/t_account_water.json rename to Doc/6、DataX配置/json_full/t_account_water.json diff --git a/Doc/6、DataX配置/json/t_car.json b/Doc/6、DataX配置/json_full/t_car.json similarity index 100% rename from Doc/6、DataX配置/json/t_car.json rename to Doc/6、DataX配置/json_full/t_car.json diff --git a/Doc/6、DataX配置/json/t_company.json b/Doc/6、DataX配置/json_full/t_company.json similarity index 100% rename from Doc/6、DataX配置/json/t_company.json rename to Doc/6、DataX配置/json_full/t_company.json diff --git a/Doc/6、DataX配置/json/t_company_info_value.json b/Doc/6、DataX配置/json_full/t_company_info_value.json similarity index 100% rename from Doc/6、DataX配置/json/t_company_info_value.json rename to Doc/6、DataX配置/json_full/t_company_info_value.json diff --git a/Doc/6、DataX配置/json/t_connector.json b/Doc/6、DataX配置/json_full/t_connector.json similarity index 100% rename from Doc/6、DataX配置/json/t_connector.json rename to Doc/6、DataX配置/json_full/t_connector.json diff --git a/Doc/6、DataX配置/json/t_equipment.json b/Doc/6、DataX配置/json_full/t_equipment.json similarity index 100% rename from Doc/6、DataX配置/json/t_equipment.json rename to Doc/6、DataX配置/json_full/t_equipment.json diff --git a/Doc/6、DataX配置/json/t_equipment_charge_order.json b/Doc/6、DataX配置/json_full/t_equipment_charge_order.json similarity index 100% rename from Doc/6、DataX配置/json/t_equipment_charge_order.json rename to Doc/6、DataX配置/json_full/t_equipment_charge_order.json diff --git a/Doc/6、DataX配置/json/t_equipment_charge_order_detail.json b/Doc/6、DataX配置/json_full/t_equipment_charge_order_detail.json similarity index 100% rename from Doc/6、DataX配置/json/t_equipment_charge_order_detail.json rename to Doc/6、DataX配置/json_full/t_equipment_charge_order_detail.json diff --git a/Doc/6、DataX配置/json_inc/t_account_recharge.json b/Doc/6、DataX配置/json_inc/t_account_recharge.json new file mode 100644 index 0000000..35d2510 --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_account_recharge.json @@ -0,0 +1,124 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "prid", + "order_id", + "outtradeno", + "pay_time", + "pay_money", + "refund_time", + "refund_money", + "refund_state", + "order_time", + "pay_xml", + "refund_xml", + "check_xml", + "pay_state", + "pay_type", + "phone", + "check_time", + "user_id", + "create_time", + "update_time", + "donate_money", + "memo", + "refund_memo", + "refund_outtradno", + "refund_image", + "refund_operer", + "rechage_operer", + "refund_reason", + "vip", + "meal_id", + "refund_apply_time", + "pay_method" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `prid`, `order_id`, `outtradeno`, `pay_time`, `pay_money`, `refund_time`, `refund_money`, `refund_state`, `order_time`, `pay_xml`, `refund_xml`, `check_xml`, `pay_state`, `pay_type`, `phone`, `check_time`, `user_id`, `create_time`, `update_time`, `donate_money`, `memo`, `refund_memo`, `refund_outtradno`, `refund_image`, `refund_operer`, `rechage_operer`, `refund_reason`, `vip`, `meal_id`, `refund_apply_time`, `pay_method` FROM `t_account_recharge` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "prid", + "order_id", + "outtradeno", + "pay_time", + "pay_money", + "refund_time", + "refund_money", + "refund_state", + "order_time", + "pay_xml", + "refund_xml", + "check_xml", + "pay_state", + "pay_type", + "phone", + "check_time", + "user_id", + "create_time", + "update_time", + "donate_money", + "memo", + "refund_memo", + "refund_outtradno", + "refund_image", + "refund_operer", + "rechage_operer", + "refund_reason", + "vip", + "meal_id", + "refund_apply_time", + "pay_method" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_account_recharge" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_account_water.json b/Doc/6、DataX配置/json_inc/t_account_water.json new file mode 100644 index 0000000..85f6fcb --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_account_water.json @@ -0,0 +1,102 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "cash", + "phone", + "txt_desc", + "sign_msg", + "reason", + "sign", + "type", + "order_id", + "treatment", + "user_id", + "pay_status", + "pay_method", + "update_time", + "create_time", + "pre_balance", + "bef_balance", + "memo", + "donate_money", + "ua_balance", + "to_user_id" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `cash`, `phone`, `txt_desc`, `sign_msg`, `reason`, `sign`, `type`, `order_id`, `treatment`, `user_id`, `pay_status`, `pay_method`, `update_time`, `create_time`, `pre_balance`, `bef_balance`, `memo`, `donate_money`, `ua_balance`, `to_user_id` FROM `t_account_water` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "cash", + "phone", + "txt_desc", + "sign_msg", + "reason", + "sign", + "type", + "order_id", + "treatment", + "user_id", + "pay_status", + "pay_method", + "update_time", + "create_time", + "pre_balance", + "bef_balance", + "memo", + "donate_money", + "ua_balance", + "to_user_id" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_account_water" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_car.json b/Doc/6、DataX配置/json_inc/t_car.json new file mode 100644 index 0000000..4a1ad53 --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_car.json @@ -0,0 +1,96 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `car_plate_no`, `car_sn`, `car_brand_name`, `car_model_name`, `car_vin`, `car_engine`, `car_type`, `car_owner_company_id`, `car_business`, `car_bus_path`, `is_auto_charge`, `user_id`, `user_card_no`, `is_disable`, `create_time`, `update_time`, `operator_id` FROM `t_car` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_car" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_company.json b/Doc/6、DataX配置/json_inc/t_company.json new file mode 100644 index 0000000..8d66cff --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_company.json @@ -0,0 +1,130 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "company_no", + "company_type", + "company_name", + "company_state", + "company_logo", + "company_admin_phone", + "company_admin_name", + "balance_remind", + "balance_remind_phone", + "agent_id", + "station_set_type", + "specified_range", + "specified_operator_ids", + "is_company_discount", + "is_overlay_coupons", + "station_ids", + "no_discount_stationids", + "remark", + "invoice_eligible", + "refuse_invoice_tip", + "is_share_master", + "settlement_method", + "charge_discount_type", + "charge_discount", + "settle_status", + "charge_unit_service", + "charge_unit_electric", + "is_custom_price", + "company_order_permission", + "open_station_ids", + "create_time", + "update_time", + "minimum_recharge_amount", + "inner_person" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `company_no`, `company_type`, `company_name`, `company_state`, `company_logo`, `company_admin_phone`, `company_admin_name`, `balance_remind`, `balance_remind_phone`, `agent_id`, `station_set_type`, `specified_range`, `specified_operator_ids`, `is_company_discount`, `is_overlay_coupons`, `station_ids`, `no_discount_stationids`, `remark`, `invoice_eligible`, `refuse_invoice_tip`, `is_share_master`, `settlement_method`, `charge_discount_type`, `charge_discount`, `settle_status`, `charge_unit_service`, `charge_unit_electric`, `is_custom_price`, `company_order_permission`, `open_station_ids`, `create_time`, `update_time`, `minimum_recharge_amount`, `inner_person` FROM `t_company` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "company_no", + "company_type", + "company_name", + "company_state", + "company_logo", + "company_admin_phone", + "company_admin_name", + "balance_remind", + "balance_remind_phone", + "agent_id", + "station_set_type", + "specified_range", + "specified_operator_ids", + "is_company_discount", + "is_overlay_coupons", + "station_ids", + "no_discount_stationids", + "remark", + "invoice_eligible", + "refuse_invoice_tip", + "is_share_master", + "settlement_method", + "charge_discount_type", + "charge_discount", + "settle_status", + "charge_unit_service", + "charge_unit_electric", + "is_custom_price", + "company_order_permission", + "open_station_ids", + "create_time", + "update_time", + "minimum_recharge_amount", + "inner_person" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_company" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_company_info_value.json b/Doc/6、DataX配置/json_inc/t_company_info_value.json new file mode 100644 index 0000000..e8fbaeb --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_company_info_value.json @@ -0,0 +1,74 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "info_value", + "type_id", + "create_time", + "update_time", + "remark", + "renzheng_flag" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `info_value`, `type_id`, `create_time`, `update_time`, `remark`, `renzheng_flag` FROM `t_company_info_value` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "info_value", + "type_id", + "create_time", + "update_time", + "remark", + "renzheng_flag" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_company_info_value" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_connector.json b/Doc/6、DataX配置/json_inc/t_connector.json new file mode 100644 index 0000000..a18460f --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_connector.json @@ -0,0 +1,110 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "equipment_id", + "connector_sn", + "connector_short_sn", + "connector_no", + "connector_name", + "connector_type", + "voltage_upper_limits", + "voltage_lower_limits", + "current", + "power", + "parkno", + "appointment_id", + "appointment_user_id", + "open_lock_user_id", + "detection_type", + "is_ground_lock", + "is_ground_lock_msg", + "is_geomagnetic", + "is_geomagnetic_msg", + "is_appointment", + "is_appointment_msg", + "auto_open_lock_time", + "create_time", + "update_time" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `equipment_id`, `connector_sn`, `connector_short_sn`, `connector_no`, `connector_name`, `connector_type`, `voltage_upper_limits`, `voltage_lower_limits`, `current`, `power`, `parkno`, `appointment_id`, `appointment_user_id`, `open_lock_user_id`, `detection_type`, `is_ground_lock`, `is_ground_lock_msg`, `is_geomagnetic`, `is_geomagnetic_msg`, `is_appointment`, `is_appointment_msg`, `auto_open_lock_time`, `create_time`, `update_time` FROM `t_connector` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "equipment_id", + "connector_sn", + "connector_short_sn", + "connector_no", + "connector_name", + "connector_type", + "voltage_upper_limits", + "voltage_lower_limits", + "current", + "power", + "parkno", + "appointment_id", + "appointment_user_id", + "open_lock_user_id", + "detection_type", + "is_ground_lock", + "is_ground_lock_msg", + "is_geomagnetic", + "is_geomagnetic_msg", + "is_appointment", + "is_appointment_msg", + "auto_open_lock_time", + "create_time", + "update_time" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_connector" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_equipment.json b/Doc/6、DataX配置/json_inc/t_equipment.json new file mode 100644 index 0000000..ce1b5ec --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_equipment.json @@ -0,0 +1,120 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "station_id", + "agreement_id", + "equipment_name", + "equipment_sn", + "equipment_short_sn", + "equipment_no", + "manufacturer_id", + "manufacturer_sn", + "manufacturer_name", + "equipment_model", + "production_date", + "equipment_type", + "equipment_type_msg", + "equipment_lng", + "equipment_lat", + "connector_count", + "power", + "national_standard", + "national_standard_msg", + "status", + "equipment_business_type", + "equipment_business_type_msg", + "is_use_card", + "is_use_card_msg", + "charge_host_esn", + "create_time", + "update_time", + "aux_power", + "app_show" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `station_id`, `agreement_id`, `equipment_name`, `equipment_sn`, `equipment_short_sn`, `equipment_no`, `manufacturer_id`, `manufacturer_sn`, `manufacturer_name`, `equipment_model`, `production_date`, `equipment_type`, `equipment_type_msg`, `equipment_lng`, `equipment_lat`, `connector_count`, `power`, `national_standard`, `national_standard_msg`, `status`, `equipment_business_type`, `equipment_business_type_msg`, `is_use_card`, `is_use_card_msg`, `charge_host_esn`, `create_time`, `update_time`, `aux_power`, `app_show` FROM `t_equipment` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "station_id", + "agreement_id", + "equipment_name", + "equipment_sn", + "equipment_short_sn", + "equipment_no", + "manufacturer_id", + "manufacturer_sn", + "manufacturer_name", + "equipment_model", + "production_date", + "equipment_type", + "equipment_type_msg", + "equipment_lng", + "equipment_lat", + "connector_count", + "power", + "national_standard", + "national_standard_msg", + "status", + "equipment_business_type", + "equipment_business_type_msg", + "is_use_card", + "is_use_card_msg", + "charge_host_esn", + "create_time", + "update_time", + "aux_power", + "app_show" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_equipment_charge_order.json b/Doc/6、DataX配置/json_inc/t_equipment_charge_order.json new file mode 100644 index 0000000..817438c --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_equipment_charge_order.json @@ -0,0 +1,282 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `user_id`, `company_id`, `station_id`, `equipment_id`, `connector_id`, `user_card_no`, `state`, `operate_state`, `order_no`, `platform_order_no`, `platform_sale_type`, `platform_driver_id`, `platform_phone`, `charging_amt`, `order_type`, `order_time`, `charge_begin_time`, `charge_end_time`, `charge_begin_degree`, `charge_end_degree`, `charge_degree`, `charge_ah`, `charge_times_degree`, `charge_begin_soc`, `charge_end_soc`, `charge_cur_soc`, `charge_duration`, `charge_settle_type`, `charge_unit_price`, `charge_unit_cost`, `charge_unit_service_fee`, `receivable_electric_fee`, `receivable_service_fee`, `receivable_total_fee`, `charge_elecfee_amount`, `charge_elecfee_cost_amount`, `charge_service_amount`, `actual_pay_amount`, `pay_order_no`, `pay_amount`, `pay_method`, `pay_status`, `pay_time`, `user_coupon_id`, `coupon_name`, `coupon_type`, `coupon_amount`, `elecfee_coupon_amount`, `servicefee_coupon_amount`, `invoice_id`, `invoice_fee`, `ticket_status`, `activity_electric_fee`, `activity_service_fee`, `charge_plate_no`, `charge_vin`, `charge_strategy`, `charge_strategy_param`, `operator_id`, `operator_income`, `plat_service_fee`, `charge_discount_type`, `settle_electric_fee`, `settle_service_fee`, `settle_fee`, `settle_method`, `settle_coupon_amount`, `settle_coupon_electric_fee`, `settle_coupon_service_fee`, `settle_status`, `cancel_type`, `cancel_code`, `cancel_msg`, `cancel_time`, `finish_type`, `finish_code`, `finish_msg`, `finish_time`, `boot_mode`, `settlement_type`, `is_parking_coupon`, `control_source`, `exception_flag`, `exception_msg`, `report_time`, `com_service_fee`, `com_electric_fee`, `com_total_fee`, `com_charge_discount_type`, `com_charge_discount`, `has_shallow_report`, `has_deep_report`, `is_exception`, `shallow_report_is_received`, `create_time`, `update_time`, `charge_market_id`, `coupon_total_amount`, `coupon_service_total_amount`, `coupon_elecfee_total_amount`, `activity_name`, `activity_total_fee`, `company_activity_id`, `company_activity_name`, `company_activity_total_fee`, `company_activity_service_fee`, `company_activity_electric_fee`, `subsidy_degree`, `subsidy_fee`, `special_channel_order_source` FROM `t_equipment_charge_order` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment_charge_order" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/json_inc/t_equipment_charge_order_detail.json b/Doc/6、DataX配置/json_inc/t_equipment_charge_order_detail.json new file mode 100644 index 0000000..c76e67e --- /dev/null +++ b/Doc/6、DataX配置/json_inc/t_equipment_charge_order_detail.json @@ -0,0 +1,106 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "column": [ + "id", + "order_no", + "charge_degree", + "service_fee", + "electric_fee", + "cost_fee", + "service_fee_unit_price", + "electric_fee_unit_price", + "cost_fee_unit_price", + "activity_service_fee", + "activity_electric_fee", + "charge_startr_time", + "charge_end_time", + "charge_meal_electric_fee", + "charge_meal_service_fee", + "charge_meal_total_fee", + "electric_card_use_dgree", + "check_electric_unit_price", + "check_service_unit_price", + "time_flag", + "create_time", + "update_time", + "detail_more" + ], + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "SELECT `id`, `order_no`, `charge_degree`, `service_fee`, `electric_fee`, `cost_fee`, `service_fee_unit_price`, `electric_fee_unit_price`, `cost_fee_unit_price`, `activity_service_fee`, `activity_electric_fee`, `charge_startr_time`, `charge_end_time`, `charge_meal_electric_fee`, `charge_meal_service_fee`, `charge_meal_total_fee`, `electric_card_use_dgree`, `check_electric_unit_price`, `check_service_unit_price`, `time_flag`, `create_time`, `update_time`, `detail_more` FROM `t_equipment_charge_order_detail` ORDER BY id DESC LIMIT 10000" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "doriswriter", + "parameter": { + "loadUrl": [ + "${dest_load_url}" + ], + "column": [ + "id", + "order_no", + "charge_degree", + "service_fee", + "electric_fee", + "cost_fee", + "service_fee_unit_price", + "electric_fee_unit_price", + "cost_fee_unit_price", + "activity_service_fee", + "activity_electric_fee", + "charge_startr_time", + "charge_end_time", + "charge_meal_electric_fee", + "charge_meal_service_fee", + "charge_meal_total_fee", + "electric_card_use_dgree", + "check_electric_unit_price", + "check_service_unit_price", + "time_flag", + "create_time", + "update_time", + "detail_more" + ], + "username": "${dest_user}", + "password": "${dest_pwd}", + "postSql": [], + "preSql": [], + "flushInterval": 30000, + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "selectedDatabase": "yltcharge", + "table": [ + "t_equipment_charge_order_detail" + ] + } + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + } + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/Doc/6、DataX配置/run_jobs_full.sh b/Doc/6、DataX配置/run_jobs_full.sh new file mode 100644 index 0000000..c8e0ae6 --- /dev/null +++ b/Doc/6、DataX配置/run_jobs_full.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# 配置路径 +DATAX_HOME="/usr/local/datax" +# 获取当前脚本所在目录 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# JSON配置文件目录指向 json_full +CONF_DIR="$SCRIPT_DIR/json_full" +BIN_DIR="$DATAX_HOME/bin" +DATAX_PY="$BIN_DIR/datax.py" + +# 明确指定要执行的9个文件（全量同步） +JOBS=( + "t_equipment_charge_order.json" + "t_equipment_charge_order_detail.json" + "t_account_recharge.json" + "t_account_water.json" + "t_car.json" + "t_company.json" + "t_company_info_value.json" + "t_connector.json" + "t_equipment.json" +) + +# 定义全局参数 +PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" + + +# 在脚本最开始记录开始时间 +SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_START_TIMESTAMP=$(date +%s) + +echo "DataX全量同步脚本 (Full Sync)" +echo "==========================" +echo "DataX路径: $DATAX_PY" +echo "配置文件目录: $CONF_DIR" +echo "任务数量: ${#JOBS[@]}" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "==========================" + +# 切换到配置文件目录 +cd "$CONF_DIR" || { + echo "错误: 无法进入配置文件目录 $CONF_DIR" + exit 1 +} + +echo "当前目录: $(pwd)" +echo + +# 执行计数器 +SUCCESS_COUNT=0 +FAIL_COUNT=0 +TOTAL=${#JOBS[@]} +CURRENT=0 + +# 数组存储每个任务的执行详情 +declare -a TASK_RESULTS=() +declare -a TASK_DURATIONS=() +declare -a TASK_STATUSES=() + +for JOB in "${JOBS[@]}"; do + CURRENT=$((CURRENT + 1)) + + echo "任务 [$CURRENT/$TOTAL] $JOB" + echo "任务开始时间: $(date '+%H:%M:%S')" + echo "----------------------------------------" + + # 检查文件是否存在 + if [ ! -f "$JOB" ]; then + echo "✗ 错误: 文件不存在 - $JOB" + FAIL_COUNT=$((FAIL_COUNT + 1)) + TASK_RESULTS+=("$JOB") + TASK_STATUSES+=("文件不存在") + TASK_DURATIONS+=("0") + echo "----------------------------------------" + continue + fi + + # 记录开始时间（用于计算耗时） + START_TIME=$(date +%s) + + # 执行任务 (添加 -p 参数传递变量) + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" + EXIT_CODE=$? + + # 计算耗时 + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + # 保存任务详情 + TASK_RESULTS+=("$JOB") + TASK_DURATIONS+=("$DURATION") + + # 处理执行结果 + if [ $EXIT_CODE -eq 0 ]; then + echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + TASK_STATUSES+=("成功") + else + echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" + echo "错误代码: $EXIT_CODE" + FAIL_COUNT=$((FAIL_COUNT + 1)) + TASK_STATUSES+=("失败") + + # 询问是否继续执行后续任务 + # 全量同步失败可能需要人工干预，但这里保持自动询问或直接继续 + # 为方便自动化，这里改为仅提示，不阻塞 (或者保持原样) + # 保持原样: + # echo "----------------------------------------" + # read -p "任务失败，是否继续执行下一个任务？(y/n): " -n 1 -r + # echo + # if [[ ! $REPLY =~ ^[Yy]$ ]]; then + # echo "用户选择停止执行" + # break + # fi + fi + + echo "任务结束时间: $(date '+%H:%M:%S')" + echo "----------------------------------------" + echo + + # 可选：任务间等待时间（秒） + # sleep 1 +done + +# 记录脚本结束时间 +SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') +SCRIPT_END_TIMESTAMP=$(date +%s) +TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) + +echo "==========================" +echo "执行结果汇总 (Full Sync)" +echo "==========================" +echo "脚本开始时间: $SCRIPT_START_TIME" +echo "脚本结束时间: $SCRIPT_END_TIME" +echo "脚本总耗时: ${TOTAL_DURATION}秒" +echo "==========================" +echo "任务执行详情:" +echo "----------------------------------------" + +# 输出每个任务的执行结果 +for i in "${!TASK_RESULTS[@]}"; do + STATUS_ICON="✓" + if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then + STATUS_ICON="✗" + fi + printf "%-40s %-2s %-10s %-8s秒\n" \ + "${TASK_RESULTS[$i]}" \ + "$STATUS_ICON" \ + "${TASK_STATUSES[$i]}" \ + "${TASK_DURATIONS[$i]}" +done + +echo "----------------------------------------" +echo "汇总统计:" +echo "总任务数: $TOTAL" +echo "成功: $SUCCESS_COUNT" +echo "失败: $FAIL_COUNT" +if [ $TOTAL -gt 0 ]; then + SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) + echo "成功率: ${SUCCESS_RATE}%" +fi + +# 计算平均耗时 +if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then + TOTAL_TASK_TIME=0 + VALID_TASKS=0 + for duration in "${TASK_DURATIONS[@]}"; do + if [ "$duration" -gt 0 ]; then + TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) + VALID_TASKS=$((VALID_TASKS + 1)) + fi + done + + if [ $VALID_TASKS -gt 0 ]; then + AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) + echo "平均任务耗时: ${AVG_DURATION}秒" + fi +fi + +echo "==========================" diff --git a/Doc/6、DataX配置/run_jobs.sh b/Doc/6、DataX配置/run_jobs_inc.sh similarity index 87% rename from Doc/6、DataX配置/run_jobs.sh rename to Doc/6、DataX配置/run_jobs_inc.sh index 4061d66..c4d1100 100644 --- a/Doc/6、DataX配置/run_jobs.sh +++ b/Doc/6、DataX配置/run_jobs_inc.sh @@ -2,11 +2,14 @@ # 配置路径 DATAX_HOME="/usr/local/datax" -CONF_DIR="$DATAX_HOME/conf" +# 获取当前脚本所在目录 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +# JSON配置文件目录指向 json_inc +CONF_DIR="$SCRIPT_DIR/json_inc" BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" -# 明确指定要执行的10个文件（按你需要的顺序） +# 明确指定要执行的9个文件（增量同步） JOBS=( "t_equipment_charge_order.json" "t_equipment_charge_order_detail.json" @@ -27,12 +30,14 @@ PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) -echo "DataX顺序执行脚本" +echo "DataX增量同步脚本 (Incremental Sync)" echo "==========================" echo "DataX路径: $DATAX_PY" echo "配置文件目录: $CONF_DIR" echo "任务数量: ${#JOBS[@]}" echo "脚本开始时间: $SCRIPT_START_TIME" +echo "注意: 增量模式下不执行 Truncate，请确保目标表为 Unique Key 模型以实现 Upsert 效果" +echo "策略: 每次同步最近10000条数据(按ID倒序)，利用Doris主键机制自动覆盖更新" echo "==========================" # 切换到配置文件目录 @@ -98,15 +103,7 @@ for JOB in "${JOBS[@]}"; do echo "错误代码: $EXIT_CODE" FAIL_COUNT=$((FAIL_COUNT + 1)) TASK_STATUSES+=("失败") - - # 询问是否继续执行后续任务 - echo "----------------------------------------" - read -p "任务失败，是否继续执行下一个任务？(y/n): " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "用户选择停止执行" - break - fi + # 增量任务通常无人值守，失败不暂停 fi echo "任务结束时间: $(date '+%H:%M:%S')" @@ -123,7 +120,7 @@ SCRIPT_END_TIMESTAMP=$(date +%s) TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) echo "==========================" -echo "执行结果汇总" +echo "执行结果汇总 (Incremental Sync)" echo "==========================" echo "脚本开始时间: $SCRIPT_START_TIME" echo "脚本结束时间: $SCRIPT_END_TIME" @@ -172,4 +169,4 @@ if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then fi fi -echo "==========================" \ No newline at end of file +echo "=========================="