diff --git a/DataX/db_config.sh b/DataX/db_config.sh new file mode 100644 index 0000000..6c21532 --- /dev/null +++ b/DataX/db_config.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ========================================== +# DataX 数据库连接集中配置文件 +# 修改此文件即可同步更新所有全量/增量脚本 +# ========================================== + +# 1. 源端 MySQL 配置 (Source) +SRC_USER="ylt" +SRC_PWD="Ycharge666" +SRC_JDBC="jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge" + +# 2. 目标端 Doris 配置 (Target Doris) +DORIS_USER="root" +DORIS_PWD="DsideaL147258369" +DORIS_JDBC="jdbc:mysql://10.10.14.204:9030/yltcharge" +DORIS_LOAD_URL="10.10.14.204:8030" + +# 3. 目标端 MySQL 配置 (Target MySQL) +# 如果目标 MySQL 的账号密码与源端一致，可以直接引用上面的变量 +MYSQL_USER="ylt" +MYSQL_PWD="Ycharge666" +MYSQL_JDBC="jdbc:mysql://10.10.14.210:22066/yltcharge?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai" + +# --- 内部参数构造 (无需修改) --- +SRC_BASE_PARAMS="-Dsrc_user=$SRC_USER -Dsrc_pwd=$SRC_PWD -Dsrc_jdbc=$SRC_JDBC" + +DORIS_DEST_PARAMS="-Ddest_writer_name=doriswriter -Ddest_user=$DORIS_USER -Ddest_pwd=$DORIS_PWD -Ddest_load_url=$DORIS_LOAD_URL -Ddest_jdbc=$DORIS_JDBC -Ddest_write_mode=replace" + +# 注意：MYSQL_JDBC 可能包含特殊字符（如&），如果不加引号引用，在shell中拼接时可能会出问题 +# 但在这里定义字符串是安全的。问题出在 run_jobs_full.sh 中引用 PARAMS 的时候。 +# 为了保险起见，我们在 JDBC URL 两侧加上转义的双引号 +MYSQL_DEST_PARAMS="-Ddest_writer_name=mysqlwriter -Ddest_user=$MYSQL_USER -Ddest_pwd=$MYSQL_PWD -Ddest_jdbc=\"$MYSQL_JDBC\" -Ddest_write_mode=replace -Ddest_load_url=none" diff --git a/DataX/json_inc/t_account_recharge.json b/DataX/json/t_account_recharge.json similarity index 56% rename from DataX/json_inc/t_account_recharge.json rename to DataX/json/t_account_recharge.json index 35d2510..634b7fa 100644 --- a/DataX/json_inc/t_account_recharge.json +++ b/DataX/json/t_account_recharge.json @@ -5,47 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "prid", - "order_id", - "outtradeno", - "pay_time", - "pay_money", - "refund_time", - "refund_money", - "refund_state", - "order_time", - "pay_xml", - "refund_xml", - "check_xml", - "pay_state", - "pay_type", - "phone", - "check_time", - "user_id", - "create_time", - "update_time", - "donate_money", - "memo", - "refund_memo", - "refund_outtradno", - "refund_image", - "refund_operer", - "rechage_operer", - "refund_reason", - "vip", - "meal_id", - "refund_apply_time", - "pay_method" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `prid`, `order_id`, `outtradeno`, `pay_time`, `pay_money`, `refund_time`, `refund_money`, `refund_state`, `order_time`, `pay_xml`, `refund_xml`, `check_xml`, `pay_state`, `pay_type`, `phone`, `check_time`, `user_id`, `create_time`, `update_time`, `donate_money`, `memo`, `refund_memo`, `refund_outtradno`, `refund_image`, `refund_operer`, `rechage_operer`, `refund_reason`, `vip`, `meal_id`, `refund_apply_time`, `pay_method` FROM `t_account_recharge` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -54,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "prid", @@ -93,24 +58,25 @@ "refund_apply_time", "pay_method" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_account_recharge" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_inc/t_account_water.json b/DataX/json/t_account_water.json similarity index 60% rename from DataX/json_inc/t_account_water.json rename to DataX/json/t_account_water.json index 85f6fcb..e24c9b4 100644 --- a/DataX/json_inc/t_account_water.json +++ b/DataX/json/t_account_water.json @@ -5,36 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "cash", - "phone", - "txt_desc", - "sign_msg", - "reason", - "sign", - "type", - "order_id", - "treatment", - "user_id", - "pay_status", - "pay_method", - "update_time", - "create_time", - "pre_balance", - "bef_balance", - "memo", - "donate_money", - "ua_balance", - "to_user_id" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `cash`, `phone`, `txt_desc`, `sign_msg`, `reason`, `sign`, `type`, `order_id`, `treatment`, `user_id`, `pay_status`, `pay_method`, `update_time`, `create_time`, `pre_balance`, `bef_balance`, `memo`, `donate_money`, `ua_balance`, `to_user_id` FROM `t_account_water` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -43,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "cash", @@ -71,24 +47,25 @@ "ua_balance", "to_user_id" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_account_water" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_inc/t_time_day.json b/DataX/json/t_car.json similarity index 61% rename from DataX/json_inc/t_time_day.json rename to DataX/json/t_car.json index 6c17810..2995bf2 100644 --- a/DataX/json_inc/t_time_day.json +++ b/DataX/json/t_car.json @@ -5,20 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "create_time", - "update_time", - "time_fq", - "teme_ve" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `create_time`, `update_time`, `time_fq`, `teme_ve` FROM `t_time_day` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -27,36 +20,49 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "create_time", - "update_time", - "time_fq", - "teme_ve" - ], "username": "${dest_user}", "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "column": [ + "id", + "car_plate_no", + "car_sn", + "car_brand_name", + "car_model_name", + "car_vin", + "car_engine", + "car_type", + "car_owner_company_id", + "car_business", + "car_bus_path", + "is_auto_charge", + "user_id", + "user_card_no", + "is_disable", + "create_time", + "update_time", + "operator_id" + ], + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ - "t_time_day" + "t_car" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_inc/t_company.json b/DataX/json/t_company.json similarity index 53% rename from DataX/json_inc/t_company.json rename to DataX/json/t_company.json index 8d66cff..cfe1dab 100644 --- a/DataX/json_inc/t_company.json +++ b/DataX/json/t_company.json @@ -5,50 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "company_no", - "company_type", - "company_name", - "company_state", - "company_logo", - "company_admin_phone", - "company_admin_name", - "balance_remind", - "balance_remind_phone", - "agent_id", - "station_set_type", - "specified_range", - "specified_operator_ids", - "is_company_discount", - "is_overlay_coupons", - "station_ids", - "no_discount_stationids", - "remark", - "invoice_eligible", - "refuse_invoice_tip", - "is_share_master", - "settlement_method", - "charge_discount_type", - "charge_discount", - "settle_status", - "charge_unit_service", - "charge_unit_electric", - "is_custom_price", - "company_order_permission", - "open_station_ids", - "create_time", - "update_time", - "minimum_recharge_amount", - "inner_person" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `company_no`, `company_type`, `company_name`, `company_state`, `company_logo`, `company_admin_phone`, `company_admin_name`, `balance_remind`, `balance_remind_phone`, `agent_id`, `station_set_type`, `specified_range`, `specified_operator_ids`, `is_company_discount`, `is_overlay_coupons`, `station_ids`, `no_discount_stationids`, `remark`, `invoice_eligible`, `refuse_invoice_tip`, `is_share_master`, `settlement_method`, `charge_discount_type`, `charge_discount`, `settle_status`, `charge_unit_service`, `charge_unit_electric`, `is_custom_price`, `company_order_permission`, `open_station_ids`, `create_time`, `update_time`, `minimum_recharge_amount`, `inner_person` FROM `t_company` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -57,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "company_no", @@ -99,24 +61,25 @@ "minimum_recharge_amount", "inner_person" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_company" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_inc/t_company_info_value.json b/DataX/json/t_company_info_value.json similarity index 69% rename from DataX/json_inc/t_company_info_value.json rename to DataX/json/t_company_info_value.json index e8fbaeb..9d24909 100644 --- a/DataX/json_inc/t_company_info_value.json +++ b/DataX/json/t_company_info_value.json @@ -5,22 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "info_value", - "type_id", - "create_time", - "update_time", - "remark", - "renzheng_flag" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `info_value`, `type_id`, `create_time`, `update_time`, `remark`, `renzheng_flag` FROM `t_company_info_value` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -29,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "info_value", @@ -43,24 +33,25 @@ "remark", "renzheng_flag" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_company_info_value" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_inc/t_connector.json b/DataX/json/t_connector.json similarity index 57% rename from DataX/json_inc/t_connector.json rename to DataX/json/t_connector.json index a18460f..805ae18 100644 --- a/DataX/json_inc/t_connector.json +++ b/DataX/json/t_connector.json @@ -5,40 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "equipment_id", - "connector_sn", - "connector_short_sn", - "connector_no", - "connector_name", - "connector_type", - "voltage_upper_limits", - "voltage_lower_limits", - "current", - "power", - "parkno", - "appointment_id", - "appointment_user_id", - "open_lock_user_id", - "detection_type", - "is_ground_lock", - "is_ground_lock_msg", - "is_geomagnetic", - "is_geomagnetic_msg", - "is_appointment", - "is_appointment_msg", - "auto_open_lock_time", - "create_time", - "update_time" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `equipment_id`, `connector_sn`, `connector_short_sn`, `connector_no`, `connector_name`, `connector_type`, `voltage_upper_limits`, `voltage_lower_limits`, `current`, `power`, `parkno`, `appointment_id`, `appointment_user_id`, `open_lock_user_id`, `detection_type`, `is_ground_lock`, `is_ground_lock_msg`, `is_geomagnetic`, `is_geomagnetic_msg`, `is_appointment`, `is_appointment_msg`, `auto_open_lock_time`, `create_time`, `update_time` FROM `t_connector` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -47,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "equipment_id", @@ -79,24 +51,25 @@ "create_time", "update_time" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_connector" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_inc/t_equipment.json b/DataX/json/t_equipment.json similarity index 55% rename from DataX/json_inc/t_equipment.json rename to DataX/json/t_equipment.json index ce1b5ec..7ef4860 100644 --- a/DataX/json_inc/t_equipment.json +++ b/DataX/json/t_equipment.json @@ -5,45 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "station_id", - "agreement_id", - "equipment_name", - "equipment_sn", - "equipment_short_sn", - "equipment_no", - "manufacturer_id", - "manufacturer_sn", - "manufacturer_name", - "equipment_model", - "production_date", - "equipment_type", - "equipment_type_msg", - "equipment_lng", - "equipment_lat", - "connector_count", - "power", - "national_standard", - "national_standard_msg", - "status", - "equipment_business_type", - "equipment_business_type_msg", - "is_use_card", - "is_use_card_msg", - "charge_host_esn", - "create_time", - "update_time", - "aux_power", - "app_show" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `station_id`, `agreement_id`, `equipment_name`, `equipment_sn`, `equipment_short_sn`, `equipment_no`, `manufacturer_id`, `manufacturer_sn`, `manufacturer_name`, `equipment_model`, `production_date`, `equipment_type`, `equipment_type_msg`, `equipment_lng`, `equipment_lat`, `connector_count`, `power`, `national_standard`, `national_standard_msg`, `status`, `equipment_business_type`, `equipment_business_type_msg`, `is_use_card`, `is_use_card_msg`, `charge_host_esn`, `create_time`, `update_time`, `aux_power`, `app_show` FROM `t_equipment` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -52,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "station_id", @@ -89,24 +56,25 @@ "aux_power", "app_show" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_equipment" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json/t_equipment_charge_order.json b/DataX/json/t_equipment_charge_order.json new file mode 100644 index 0000000..01ea753 --- /dev/null +++ b/DataX/json/t_equipment_charge_order.json @@ -0,0 +1,169 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "connection": [ + { + "jdbcUrl": [ + "${src_jdbc}" + ], + "querySql": [ + "${src_query_sql}" + ] + } + ], + "username": "${src_user}", + "password": "${src_pwd}" + } + }, + "writer": { + "name": "${dest_writer_name}", + "parameter": { + "username": "${dest_user}", + "password": "${dest_pwd}", + "column": [ + "id", + "user_id", + "company_id", + "station_id", + "equipment_id", + "connector_id", + "user_card_no", + "state", + "operate_state", + "order_no", + "platform_order_no", + "platform_sale_type", + "platform_driver_id", + "platform_phone", + "charging_amt", + "order_type", + "order_time", + "charge_begin_time", + "charge_end_time", + "charge_begin_degree", + "charge_end_degree", + "charge_degree", + "charge_ah", + "charge_times_degree", + "charge_begin_soc", + "charge_end_soc", + "charge_cur_soc", + "charge_duration", + "charge_settle_type", + "charge_unit_price", + "charge_unit_cost", + "charge_unit_service_fee", + "receivable_electric_fee", + "receivable_service_fee", + "receivable_total_fee", + "charge_elecfee_amount", + "charge_elecfee_cost_amount", + "charge_service_amount", + "actual_pay_amount", + "pay_order_no", + "pay_amount", + "pay_method", + "pay_status", + "pay_time", + "user_coupon_id", + "coupon_name", + "coupon_type", + "coupon_amount", + "elecfee_coupon_amount", + "servicefee_coupon_amount", + "invoice_id", + "invoice_fee", + "ticket_status", + "activity_electric_fee", + "activity_service_fee", + "charge_plate_no", + "charge_vin", + "charge_strategy", + "charge_strategy_param", + "operator_id", + "operator_income", + "plat_service_fee", + "charge_discount_type", + "settle_electric_fee", + "settle_service_fee", + "settle_fee", + "settle_method", + "settle_coupon_amount", + "settle_coupon_electric_fee", + "settle_coupon_service_fee", + "settle_status", + "cancel_type", + "cancel_code", + "cancel_msg", + "cancel_time", + "finish_type", + "finish_code", + "finish_msg", + "finish_time", + "boot_mode", + "settlement_type", + "is_parking_coupon", + "control_source", + "exception_flag", + "exception_msg", + "report_time", + "com_service_fee", + "com_electric_fee", + "com_total_fee", + "com_charge_discount_type", + "com_charge_discount", + "has_shallow_report", + "has_deep_report", + "is_exception", + "shallow_report_is_received", + "create_time", + "update_time", + "charge_market_id", + "coupon_total_amount", + "coupon_service_total_amount", + "coupon_elecfee_total_amount", + "activity_name", + "activity_total_fee", + "company_activity_id", + "company_activity_name", + "company_activity_total_fee", + "company_activity_service_fee", + "company_activity_electric_fee", + "subsidy_degree", + "subsidy_fee", + "special_channel_order_source" + ], + "preSql": [ + "${dest_pre_sql}" + ], + "connection": [ + { + "jdbcUrl": "${dest_jdbc}", + "table": [ + "t_equipment_charge_order" + ] + } + ], + "loadUrl": [ + "${dest_load_url}" + ], + "loadProps": { + "format": "json", + "strip_outer_array": true + }, + "writeMode": "${dest_write_mode}" + } + } + } + ], + "setting": { + "speed": { + "channel": "1" + } + } + } +} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment_charge_order_detail.json b/DataX/json/t_equipment_charge_order_detail.json similarity index 56% rename from DataX/json_inc/t_equipment_charge_order_detail.json rename to DataX/json/t_equipment_charge_order_detail.json index c76e67e..e72415b 100644 --- a/DataX/json_inc/t_equipment_charge_order_detail.json +++ b/DataX/json/t_equipment_charge_order_detail.json @@ -5,38 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "order_no", - "charge_degree", - "service_fee", - "electric_fee", - "cost_fee", - "service_fee_unit_price", - "electric_fee_unit_price", - "cost_fee_unit_price", - "activity_service_fee", - "activity_electric_fee", - "charge_startr_time", - "charge_end_time", - "charge_meal_electric_fee", - "charge_meal_service_fee", - "charge_meal_total_fee", - "electric_card_use_dgree", - "check_electric_unit_price", - "check_service_unit_price", - "time_flag", - "create_time", - "update_time", - "detail_more" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], "querySql": [ - "SELECT `id`, `order_no`, `charge_degree`, `service_fee`, `electric_fee`, `cost_fee`, `service_fee_unit_price`, `electric_fee_unit_price`, `cost_fee_unit_price`, `activity_service_fee`, `activity_electric_fee`, `charge_startr_time`, `charge_end_time`, `charge_meal_electric_fee`, `charge_meal_service_fee`, `charge_meal_total_fee`, `electric_card_use_dgree`, `check_electric_unit_price`, `check_service_unit_price`, `time_flag`, `create_time`, `update_time`, `detail_more` FROM `t_equipment_charge_order_detail` ORDER BY id DESC LIMIT 10000" + "${src_query_sql}" ] } ], @@ -45,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "order_no", @@ -75,24 +49,25 @@ "update_time", "detail_more" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, + "preSql": [ + "${dest_pre_sql}" + ], "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_equipment_charge_order_detail" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_ext_hurry_quit.json b/DataX/json/t_ext_hurry_quit.json similarity index 61% rename from DataX/json_full/t_ext_hurry_quit.json rename to DataX/json/t_ext_hurry_quit.json index a2f5d8d..e048ae8 100644 --- a/DataX/json_full/t_ext_hurry_quit.json +++ b/DataX/json/t_ext_hurry_quit.json @@ -5,44 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "order_no", - "telephone", - "user_id", - "message1", - "message2", - "create_time", - "station_id", - "equipment_id", - "connector_id", - "flag", - "close_time", - "finish_time", - "minutesDifference", - "reduce_money", - "beforeBalance", - "afterBalance", - "fee_station", - "manual_count", - "last_manual_time", - "last_manual_message", - "person_id", - "refund_status", - "refund_time", - "refund_person_id", - "refund_person_name", - "refund_amount", - "refund_type", - "unplug_time" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], - "table": [ - "t_ext_hurry_quit" + "querySql": [ + "${src_query_sql}" ] } ], @@ -51,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "order_no", @@ -87,26 +55,25 @@ "refund_type", "unplug_time" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], "preSql": [ - "TRUNCATE TABLE t_ext_hurry_quit" + "${dest_pre_sql}" ], - "flushInterval": 30000, "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_ext_hurry_quit" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_station.json b/DataX/json/t_station.json similarity index 54% rename from DataX/json_full/t_station.json rename to DataX/json/t_station.json index b4de818..f243c1e 100644 --- a/DataX/json_full/t_station.json +++ b/DataX/json/t_station.json @@ -5,97 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "station_sn", - "operator_id", - "equioment_owner_id", - "station_name", - "country_code", - "region_id", - "region_code", - "regional_name", - "regional_code", - "address", - "station_tel", - "service_tel", - "station_type", - "station_type_msg", - "station_business_type", - "station_business_type_msg", - "station_status", - "station_status_msg", - "park_nums", - "station_lng", - "station_lat", - "site_guide", - "construction", - "construction_msg", - "pictures", - "match_cars", - "park_info", - "business_start_time", - "business_end_time", - "electricity_fee", - "service_fee", - "is_park_fee", - "park_fee", - "payment", - "payment_msg", - "support_order", - "support_order_msg", - "service_item", - "service_item_image_url", - "service_item_msg", - "station_location_type", - "station_location_type_msg", - "station_charge_type", - "station_charge_type_msg", - "station_power", - "station_voltage", - "station_current", - "is_open", - "is_open_msg", - "is_display", - "is_display_msg", - "sharing_code", - "sharing_applet_picture", - "theme_picture", - "is_barrier_flag", - "is_barrier_flag_msg", - "charge_settle_type", - "charge_settle_type_msg", - "fast_connector_nums", - "slow_connector_nums", - "transformer_capacity", - "installed_capacity", - "remark", - "record_unique_no", - "equipment_owner_name", - "supply_type", - "resident_no", - "watt_hour_meter_no", - "forward_power", - "operation_date", - "charge_replace_type", - "is_hlht", - "is_subsidy", - "create_time", - "update_time", - "area_code_countryside", - "business_expand_type", - "capacity", - "video_monitor", - "electricity_type", - "receive_shop" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], - "table": [ - "t_station" + "querySql": [ + "${src_query_sql}" ] } ], @@ -104,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "station_sn", @@ -193,26 +108,25 @@ "electricity_type", "receive_shop" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], "preSql": [ - "TRUNCATE TABLE t_station" + "${dest_pre_sql}" ], - "flushInterval": 30000, "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_station" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_time_day.json b/DataX/json/t_time_day.json similarity index 72% rename from DataX/json_full/t_time_day.json rename to DataX/json/t_time_day.json index bebd21e..f4c69e4 100644 --- a/DataX/json_full/t_time_day.json +++ b/DataX/json/t_time_day.json @@ -5,20 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "create_time", - "update_time", - "time_fq", - "teme_ve" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], - "table": [ - "t_time_day" + "querySql": [ + "${src_query_sql}" ] } ], @@ -27,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "create_time", @@ -39,26 +31,25 @@ "time_fq", "teme_ve" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], "preSql": [ - "TRUNCATE TABLE t_time_day" + "${dest_pre_sql}" ], - "flushInterval": 30000, "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_time_day" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_user.json b/DataX/json/t_user.json similarity index 58% rename from DataX/json_full/t_user.json rename to DataX/json/t_user.json index f05352c..6876a5a 100644 --- a/DataX/json_full/t_user.json +++ b/DataX/json/t_user.json @@ -5,60 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "user_name", - "nicker_name", - "sex", - "phone", - "qk_openid", - "open_id", - "ali_user_id", - "birthday", - "avatar", - "app_type", - "password", - "upper_user", - "master_flag", - "user_type", - "multi_charge", - "relation_phone", - "user_owner_id", - "union_id", - "state", - "reg_channel", - "reg_time", - "city_label_code", - "city_label", - "reg_source", - "create_time", - "update_time", - "operator", - "invite_code", - "identity_id", - "operator_id", - "address", - "vip_level", - "province_code", - "province", - "work_company", - "is_bind_wxinfo", - "company_min_charge_balance", - "enable_online_recharge", - "integral", - "shared_user", - "enable_online_refund", - "recommenderId", - "lx_wx_user_id", - "phone_area" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], - "table": [ - "t_user" + "querySql": [ + "${src_query_sql}" ] } ], @@ -67,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "user_name", @@ -119,26 +71,25 @@ "lx_wx_user_id", "phone_area" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], "preSql": [ - "TRUNCATE TABLE t_user" + "${dest_pre_sql}" ], - "flushInterval": 30000, "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_user" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_user_account.json b/DataX/json/t_user_account.json similarity index 64% rename from DataX/json_full/t_user_account.json rename to DataX/json/t_user_account.json index d41b38e..37144b6 100644 --- a/DataX/json_full/t_user_account.json +++ b/DataX/json/t_user_account.json @@ -5,32 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "user_id", - "ua_balance", - "ua_deposit", - "ua_balance_time", - "ua_password", - "ua_state", - "ua_real_balance", - "ua_real_degree", - "create_time", - "update_time", - "donate_money", - "donate_able_money", - "ua_able_balance", - "refund_money", - "ua_consume_money", - "ua_clear_zero_money" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], - "table": [ - "t_user_account" + "querySql": [ + "${src_query_sql}" ] } ], @@ -39,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "user_id", @@ -63,26 +43,25 @@ "ua_consume_money", "ua_clear_zero_money" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], "preSql": [ - "TRUNCATE TABLE t_user_account" + "${dest_pre_sql}" ], - "flushInterval": 30000, "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_user_account" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_user_upload_fault.json b/DataX/json/t_user_upload_fault.json similarity index 70% rename from DataX/json_full/t_user_upload_fault.json rename to DataX/json/t_user_upload_fault.json index cb66f2d..5e35ff9 100644 --- a/DataX/json_full/t_user_upload_fault.json +++ b/DataX/json/t_user_upload_fault.json @@ -5,22 +5,13 @@ "reader": { "name": "mysqlreader", "parameter": { - "column": [ - "id", - "equipment_id", - "user_id", - "fault_config_id", - "remark", - "create_time", - "update_time" - ], "connection": [ { "jdbcUrl": [ "${src_jdbc}" ], - "table": [ - "t_user_upload_fault" + "querySql": [ + "${src_query_sql}" ] } ], @@ -29,11 +20,10 @@ } }, "writer": { - "name": "doriswriter", + "name": "${dest_writer_name}", "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], + "username": "${dest_user}", + "password": "${dest_pwd}", "column": [ "id", "equipment_id", @@ -43,26 +33,25 @@ "create_time", "update_time" ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], "preSql": [ - "TRUNCATE TABLE t_user_upload_fault" + "${dest_pre_sql}" ], - "flushInterval": 30000, "connection": [ { "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", "table": [ "t_user_upload_fault" ] } ], + "loadUrl": [ + "${dest_load_url}" + ], "loadProps": { "format": "json", "strip_outer_array": true - } + }, + "writeMode": "${dest_write_mode}" } } } diff --git a/DataX/json_full/t_account_recharge.json b/DataX/json_full/t_account_recharge.json deleted file mode 100644 index 727ce5f..0000000 --- a/DataX/json_full/t_account_recharge.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "prid", - "order_id", - "outtradeno", - "pay_time", - "pay_money", - "refund_time", - "refund_money", - "refund_state", - "order_time", - "pay_xml", - "refund_xml", - "check_xml", - "pay_state", - "pay_type", - "phone", - "check_time", - "user_id", - "create_time", - "update_time", - "donate_money", - "memo", - "refund_memo", - "refund_outtradno", - "refund_image", - "refund_operer", - "rechage_operer", - "refund_reason", - "vip", - "meal_id", - "refund_apply_time", - "pay_method" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_account_recharge" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "prid", - "order_id", - "outtradeno", - "pay_time", - "pay_money", - "refund_time", - "refund_money", - "refund_state", - "order_time", - "pay_xml", - "refund_xml", - "check_xml", - "pay_state", - "pay_type", - "phone", - "check_time", - "user_id", - "create_time", - "update_time", - "donate_money", - "memo", - "refund_memo", - "refund_outtradno", - "refund_image", - "refund_operer", - "rechage_operer", - "refund_reason", - "vip", - "meal_id", - "refund_apply_time", - "pay_method" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_account_recharge" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_account_recharge" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_account_water.json b/DataX/json_full/t_account_water.json deleted file mode 100644 index 54b63f8..0000000 --- a/DataX/json_full/t_account_water.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "cash", - "phone", - "txt_desc", - "sign_msg", - "reason", - "sign", - "type", - "order_id", - "treatment", - "user_id", - "pay_status", - "pay_method", - "update_time", - "create_time", - "pre_balance", - "bef_balance", - "memo", - "donate_money", - "ua_balance", - "to_user_id" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_account_water" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "cash", - "phone", - "txt_desc", - "sign_msg", - "reason", - "sign", - "type", - "order_id", - "treatment", - "user_id", - "pay_status", - "pay_method", - "update_time", - "create_time", - "pre_balance", - "bef_balance", - "memo", - "donate_money", - "ua_balance", - "to_user_id" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_account_water" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_account_water" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_car.json b/DataX/json_full/t_car.json deleted file mode 100644 index be62364..0000000 --- a/DataX/json_full/t_car.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "car_plate_no", - "car_sn", - "car_brand_name", - "car_model_name", - "car_vin", - "car_engine", - "car_type", - "car_owner_company_id", - "car_business", - "car_bus_path", - "is_auto_charge", - "user_id", - "user_card_no", - "is_disable", - "create_time", - "update_time", - "operator_id" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_car" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "car_plate_no", - "car_sn", - "car_brand_name", - "car_model_name", - "car_vin", - "car_engine", - "car_type", - "car_owner_company_id", - "car_business", - "car_bus_path", - "is_auto_charge", - "user_id", - "user_card_no", - "is_disable", - "create_time", - "update_time", - "operator_id" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_car" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_car" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_company.json b/DataX/json_full/t_company.json deleted file mode 100644 index 9c82856..0000000 --- a/DataX/json_full/t_company.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "company_no", - "company_type", - "company_name", - "company_state", - "company_logo", - "company_admin_phone", - "company_admin_name", - "balance_remind", - "balance_remind_phone", - "agent_id", - "station_set_type", - "specified_range", - "specified_operator_ids", - "is_company_discount", - "is_overlay_coupons", - "station_ids", - "no_discount_stationids", - "remark", - "invoice_eligible", - "refuse_invoice_tip", - "is_share_master", - "settlement_method", - "charge_discount_type", - "charge_discount", - "settle_status", - "charge_unit_service", - "charge_unit_electric", - "is_custom_price", - "company_order_permission", - "open_station_ids", - "create_time", - "update_time", - "minimum_recharge_amount", - "inner_person" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_company" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "company_no", - "company_type", - "company_name", - "company_state", - "company_logo", - "company_admin_phone", - "company_admin_name", - "balance_remind", - "balance_remind_phone", - "agent_id", - "station_set_type", - "specified_range", - "specified_operator_ids", - "is_company_discount", - "is_overlay_coupons", - "station_ids", - "no_discount_stationids", - "remark", - "invoice_eligible", - "refuse_invoice_tip", - "is_share_master", - "settlement_method", - "charge_discount_type", - "charge_discount", - "settle_status", - "charge_unit_service", - "charge_unit_electric", - "is_custom_price", - "company_order_permission", - "open_station_ids", - "create_time", - "update_time", - "minimum_recharge_amount", - "inner_person" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_company" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_company" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_company_info_value.json b/DataX/json_full/t_company_info_value.json deleted file mode 100644 index 4147538..0000000 --- a/DataX/json_full/t_company_info_value.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "info_value", - "type_id", - "create_time", - "update_time", - "remark", - "renzheng_flag" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_company_info_value" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "info_value", - "type_id", - "create_time", - "update_time", - "remark", - "renzheng_flag" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_company_info_value" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_company_info_value" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_connector.json b/DataX/json_full/t_connector.json deleted file mode 100644 index 9e915c1..0000000 --- a/DataX/json_full/t_connector.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "equipment_id", - "connector_sn", - "connector_short_sn", - "connector_no", - "connector_name", - "connector_type", - "voltage_upper_limits", - "voltage_lower_limits", - "current", - "power", - "parkno", - "appointment_id", - "appointment_user_id", - "open_lock_user_id", - "detection_type", - "is_ground_lock", - "is_ground_lock_msg", - "is_geomagnetic", - "is_geomagnetic_msg", - "is_appointment", - "is_appointment_msg", - "auto_open_lock_time", - "create_time", - "update_time" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_connector" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "equipment_id", - "connector_sn", - "connector_short_sn", - "connector_no", - "connector_name", - "connector_type", - "voltage_upper_limits", - "voltage_lower_limits", - "current", - "power", - "parkno", - "appointment_id", - "appointment_user_id", - "open_lock_user_id", - "detection_type", - "is_ground_lock", - "is_ground_lock_msg", - "is_geomagnetic", - "is_geomagnetic_msg", - "is_appointment", - "is_appointment_msg", - "auto_open_lock_time", - "create_time", - "update_time" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_connector" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_connector" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_equipment.json b/DataX/json_full/t_equipment.json deleted file mode 100644 index bf81d8b..0000000 --- a/DataX/json_full/t_equipment.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "station_id", - "agreement_id", - "equipment_name", - "equipment_sn", - "equipment_short_sn", - "equipment_no", - "manufacturer_id", - "manufacturer_sn", - "manufacturer_name", - "equipment_model", - "production_date", - "equipment_type", - "equipment_type_msg", - "equipment_lng", - "equipment_lat", - "connector_count", - "power", - "national_standard", - "national_standard_msg", - "status", - "equipment_business_type", - "equipment_business_type_msg", - "is_use_card", - "is_use_card_msg", - "charge_host_esn", - "create_time", - "update_time", - "aux_power", - "app_show" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_equipment" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "station_id", - "agreement_id", - "equipment_name", - "equipment_sn", - "equipment_short_sn", - "equipment_no", - "manufacturer_id", - "manufacturer_sn", - "manufacturer_name", - "equipment_model", - "production_date", - "equipment_type", - "equipment_type_msg", - "equipment_lng", - "equipment_lat", - "connector_count", - "power", - "national_standard", - "national_standard_msg", - "status", - "equipment_business_type", - "equipment_business_type_msg", - "is_use_card", - "is_use_card_msg", - "charge_host_esn", - "create_time", - "update_time", - "aux_power", - "app_show" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_equipment" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_equipment_charge_order.json b/DataX/json_full/t_equipment_charge_order.json deleted file mode 100644 index aef7367..0000000 --- a/DataX/json_full/t_equipment_charge_order.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_id", - "company_id", - "station_id", - "equipment_id", - "connector_id", - "user_card_no", - "state", - "operate_state", - "order_no", - "platform_order_no", - "platform_sale_type", - "platform_driver_id", - "platform_phone", - "charging_amt", - "order_type", - "order_time", - "charge_begin_time", - "charge_end_time", - "charge_begin_degree", - "charge_end_degree", - "charge_degree", - "charge_ah", - "charge_times_degree", - "charge_begin_soc", - "charge_end_soc", - "charge_cur_soc", - "charge_duration", - "charge_settle_type", - "charge_unit_price", - "charge_unit_cost", - "charge_unit_service_fee", - "receivable_electric_fee", - "receivable_service_fee", - "receivable_total_fee", - "charge_elecfee_amount", - "charge_elecfee_cost_amount", - "charge_service_amount", - "actual_pay_amount", - "pay_order_no", - "pay_amount", - "pay_method", - "pay_status", - "pay_time", - "user_coupon_id", - "coupon_name", - "coupon_type", - "coupon_amount", - "elecfee_coupon_amount", - "servicefee_coupon_amount", - "invoice_id", - "invoice_fee", - "ticket_status", - "activity_electric_fee", - "activity_service_fee", - "charge_plate_no", - "charge_vin", - "charge_strategy", - "charge_strategy_param", - "operator_id", - "operator_income", - "plat_service_fee", - "charge_discount_type", - "settle_electric_fee", - "settle_service_fee", - "settle_fee", - "settle_method", - "settle_coupon_amount", - "settle_coupon_electric_fee", - "settle_coupon_service_fee", - "settle_status", - "cancel_type", - "cancel_code", - "cancel_msg", - "cancel_time", - "finish_type", - "finish_code", - "finish_msg", - "finish_time", - "boot_mode", - "settlement_type", - "is_parking_coupon", - "control_source", - "exception_flag", - "exception_msg", - "report_time", - "com_service_fee", - "com_electric_fee", - "com_total_fee", - "com_charge_discount_type", - "com_charge_discount", - "has_shallow_report", - "has_deep_report", - "is_exception", - "shallow_report_is_received", - "create_time", - "update_time", - "charge_market_id", - "coupon_total_amount", - "coupon_service_total_amount", - "coupon_elecfee_total_amount", - "activity_name", - "activity_total_fee", - "company_activity_id", - "company_activity_name", - "company_activity_total_fee", - "company_activity_service_fee", - "company_activity_electric_fee", - "subsidy_degree", - "subsidy_fee", - "special_channel_order_source" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_equipment_charge_order" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_id", - "company_id", - "station_id", - "equipment_id", - "connector_id", - "user_card_no", - "state", - "operate_state", - "order_no", - "platform_order_no", - "platform_sale_type", - "platform_driver_id", - "platform_phone", - "charging_amt", - "order_type", - "order_time", - "charge_begin_time", - "charge_end_time", - "charge_begin_degree", - "charge_end_degree", - "charge_degree", - "charge_ah", - "charge_times_degree", - "charge_begin_soc", - "charge_end_soc", - "charge_cur_soc", - "charge_duration", - "charge_settle_type", - "charge_unit_price", - "charge_unit_cost", - "charge_unit_service_fee", - "receivable_electric_fee", - "receivable_service_fee", - "receivable_total_fee", - "charge_elecfee_amount", - "charge_elecfee_cost_amount", - "charge_service_amount", - "actual_pay_amount", - "pay_order_no", - "pay_amount", - "pay_method", - "pay_status", - "pay_time", - "user_coupon_id", - "coupon_name", - "coupon_type", - "coupon_amount", - "elecfee_coupon_amount", - "servicefee_coupon_amount", - "invoice_id", - "invoice_fee", - "ticket_status", - "activity_electric_fee", - "activity_service_fee", - "charge_plate_no", - "charge_vin", - "charge_strategy", - "charge_strategy_param", - "operator_id", - "operator_income", - "plat_service_fee", - "charge_discount_type", - "settle_electric_fee", - "settle_service_fee", - "settle_fee", - "settle_method", - "settle_coupon_amount", - "settle_coupon_electric_fee", - "settle_coupon_service_fee", - "settle_status", - "cancel_type", - "cancel_code", - "cancel_msg", - "cancel_time", - "finish_type", - "finish_code", - "finish_msg", - "finish_time", - "boot_mode", - "settlement_type", - "is_parking_coupon", - "control_source", - "exception_flag", - "exception_msg", - "report_time", - "com_service_fee", - "com_electric_fee", - "com_total_fee", - "com_charge_discount_type", - "com_charge_discount", - "has_shallow_report", - "has_deep_report", - "is_exception", - "shallow_report_is_received", - "create_time", - "update_time", - "charge_market_id", - "coupon_total_amount", - "coupon_service_total_amount", - "coupon_elecfee_total_amount", - "activity_name", - "activity_total_fee", - "company_activity_id", - "company_activity_name", - "company_activity_total_fee", - "company_activity_service_fee", - "company_activity_electric_fee", - "subsidy_degree", - "subsidy_fee", - "special_channel_order_source" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_equipment_charge_order" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment_charge_order" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_full/t_equipment_charge_order_detail.json b/DataX/json_full/t_equipment_charge_order_detail.json deleted file mode 100644 index d5d1808..0000000 --- a/DataX/json_full/t_equipment_charge_order_detail.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "order_no", - "charge_degree", - "service_fee", - "electric_fee", - "cost_fee", - "service_fee_unit_price", - "electric_fee_unit_price", - "cost_fee_unit_price", - "activity_service_fee", - "activity_electric_fee", - "charge_startr_time", - "charge_end_time", - "charge_meal_electric_fee", - "charge_meal_service_fee", - "charge_meal_total_fee", - "electric_card_use_dgree", - "check_electric_unit_price", - "check_service_unit_price", - "time_flag", - "create_time", - "update_time", - "detail_more" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "table": [ - "t_equipment_charge_order_detail" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}", - "where": "" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "order_no", - "charge_degree", - "service_fee", - "electric_fee", - "cost_fee", - "service_fee_unit_price", - "electric_fee_unit_price", - "cost_fee_unit_price", - "activity_service_fee", - "activity_electric_fee", - "charge_startr_time", - "charge_end_time", - "charge_meal_electric_fee", - "charge_meal_service_fee", - "charge_meal_total_fee", - "electric_card_use_dgree", - "check_electric_unit_price", - "check_service_unit_price", - "time_flag", - "create_time", - "update_time", - "detail_more" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [ - "TRUNCATE TABLE t_equipment_charge_order_detail" - ], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment_charge_order_detail" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_car.json b/DataX/json_inc/t_car.json deleted file mode 100644 index 4a1ad53..0000000 --- a/DataX/json_inc/t_car.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "car_plate_no", - "car_sn", - "car_brand_name", - "car_model_name", - "car_vin", - "car_engine", - "car_type", - "car_owner_company_id", - "car_business", - "car_bus_path", - "is_auto_charge", - "user_id", - "user_card_no", - "is_disable", - "create_time", - "update_time", - "operator_id" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `car_plate_no`, `car_sn`, `car_brand_name`, `car_model_name`, `car_vin`, `car_engine`, `car_type`, `car_owner_company_id`, `car_business`, `car_bus_path`, `is_auto_charge`, `user_id`, `user_card_no`, `is_disable`, `create_time`, `update_time`, `operator_id` FROM `t_car` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "car_plate_no", - "car_sn", - "car_brand_name", - "car_model_name", - "car_vin", - "car_engine", - "car_type", - "car_owner_company_id", - "car_business", - "car_bus_path", - "is_auto_charge", - "user_id", - "user_card_no", - "is_disable", - "create_time", - "update_time", - "operator_id" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_car" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_equipment_charge_order.json b/DataX/json_inc/t_equipment_charge_order.json deleted file mode 100644 index 817438c..0000000 --- a/DataX/json_inc/t_equipment_charge_order.json +++ /dev/null @@ -1,282 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_id", - "company_id", - "station_id", - "equipment_id", - "connector_id", - "user_card_no", - "state", - "operate_state", - "order_no", - "platform_order_no", - "platform_sale_type", - "platform_driver_id", - "platform_phone", - "charging_amt", - "order_type", - "order_time", - "charge_begin_time", - "charge_end_time", - "charge_begin_degree", - "charge_end_degree", - "charge_degree", - "charge_ah", - "charge_times_degree", - "charge_begin_soc", - "charge_end_soc", - "charge_cur_soc", - "charge_duration", - "charge_settle_type", - "charge_unit_price", - "charge_unit_cost", - "charge_unit_service_fee", - "receivable_electric_fee", - "receivable_service_fee", - "receivable_total_fee", - "charge_elecfee_amount", - "charge_elecfee_cost_amount", - "charge_service_amount", - "actual_pay_amount", - "pay_order_no", - "pay_amount", - "pay_method", - "pay_status", - "pay_time", - "user_coupon_id", - "coupon_name", - "coupon_type", - "coupon_amount", - "elecfee_coupon_amount", - "servicefee_coupon_amount", - "invoice_id", - "invoice_fee", - "ticket_status", - "activity_electric_fee", - "activity_service_fee", - "charge_plate_no", - "charge_vin", - "charge_strategy", - "charge_strategy_param", - "operator_id", - "operator_income", - "plat_service_fee", - "charge_discount_type", - "settle_electric_fee", - "settle_service_fee", - "settle_fee", - "settle_method", - "settle_coupon_amount", - "settle_coupon_electric_fee", - "settle_coupon_service_fee", - "settle_status", - "cancel_type", - "cancel_code", - "cancel_msg", - "cancel_time", - "finish_type", - "finish_code", - "finish_msg", - "finish_time", - "boot_mode", - "settlement_type", - "is_parking_coupon", - "control_source", - "exception_flag", - "exception_msg", - "report_time", - "com_service_fee", - "com_electric_fee", - "com_total_fee", - "com_charge_discount_type", - "com_charge_discount", - "has_shallow_report", - "has_deep_report", - "is_exception", - "shallow_report_is_received", - "create_time", - "update_time", - "charge_market_id", - "coupon_total_amount", - "coupon_service_total_amount", - "coupon_elecfee_total_amount", - "activity_name", - "activity_total_fee", - "company_activity_id", - "company_activity_name", - "company_activity_total_fee", - "company_activity_service_fee", - "company_activity_electric_fee", - "subsidy_degree", - "subsidy_fee", - "special_channel_order_source" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `user_id`, `company_id`, `station_id`, `equipment_id`, `connector_id`, `user_card_no`, `state`, `operate_state`, `order_no`, `platform_order_no`, `platform_sale_type`, `platform_driver_id`, `platform_phone`, `charging_amt`, `order_type`, `order_time`, `charge_begin_time`, `charge_end_time`, `charge_begin_degree`, `charge_end_degree`, `charge_degree`, `charge_ah`, `charge_times_degree`, `charge_begin_soc`, `charge_end_soc`, `charge_cur_soc`, `charge_duration`, `charge_settle_type`, `charge_unit_price`, `charge_unit_cost`, `charge_unit_service_fee`, `receivable_electric_fee`, `receivable_service_fee`, `receivable_total_fee`, `charge_elecfee_amount`, `charge_elecfee_cost_amount`, `charge_service_amount`, `actual_pay_amount`, `pay_order_no`, `pay_amount`, `pay_method`, `pay_status`, `pay_time`, `user_coupon_id`, `coupon_name`, `coupon_type`, `coupon_amount`, `elecfee_coupon_amount`, `servicefee_coupon_amount`, `invoice_id`, `invoice_fee`, `ticket_status`, `activity_electric_fee`, `activity_service_fee`, `charge_plate_no`, `charge_vin`, `charge_strategy`, `charge_strategy_param`, `operator_id`, `operator_income`, `plat_service_fee`, `charge_discount_type`, `settle_electric_fee`, `settle_service_fee`, `settle_fee`, `settle_method`, `settle_coupon_amount`, `settle_coupon_electric_fee`, `settle_coupon_service_fee`, `settle_status`, `cancel_type`, `cancel_code`, `cancel_msg`, `cancel_time`, `finish_type`, `finish_code`, `finish_msg`, `finish_time`, `boot_mode`, `settlement_type`, `is_parking_coupon`, `control_source`, `exception_flag`, `exception_msg`, `report_time`, `com_service_fee`, `com_electric_fee`, `com_total_fee`, `com_charge_discount_type`, `com_charge_discount`, `has_shallow_report`, `has_deep_report`, `is_exception`, `shallow_report_is_received`, `create_time`, `update_time`, `charge_market_id`, `coupon_total_amount`, `coupon_service_total_amount`, `coupon_elecfee_total_amount`, `activity_name`, `activity_total_fee`, `company_activity_id`, `company_activity_name`, `company_activity_total_fee`, `company_activity_service_fee`, `company_activity_electric_fee`, `subsidy_degree`, `subsidy_fee`, `special_channel_order_source` FROM `t_equipment_charge_order` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_id", - "company_id", - "station_id", - "equipment_id", - "connector_id", - "user_card_no", - "state", - "operate_state", - "order_no", - "platform_order_no", - "platform_sale_type", - "platform_driver_id", - "platform_phone", - "charging_amt", - "order_type", - "order_time", - "charge_begin_time", - "charge_end_time", - "charge_begin_degree", - "charge_end_degree", - "charge_degree", - "charge_ah", - "charge_times_degree", - "charge_begin_soc", - "charge_end_soc", - "charge_cur_soc", - "charge_duration", - "charge_settle_type", - "charge_unit_price", - "charge_unit_cost", - "charge_unit_service_fee", - "receivable_electric_fee", - "receivable_service_fee", - "receivable_total_fee", - "charge_elecfee_amount", - "charge_elecfee_cost_amount", - "charge_service_amount", - "actual_pay_amount", - "pay_order_no", - "pay_amount", - "pay_method", - "pay_status", - "pay_time", - "user_coupon_id", - "coupon_name", - "coupon_type", - "coupon_amount", - "elecfee_coupon_amount", - "servicefee_coupon_amount", - "invoice_id", - "invoice_fee", - "ticket_status", - "activity_electric_fee", - "activity_service_fee", - "charge_plate_no", - "charge_vin", - "charge_strategy", - "charge_strategy_param", - "operator_id", - "operator_income", - "plat_service_fee", - "charge_discount_type", - "settle_electric_fee", - "settle_service_fee", - "settle_fee", - "settle_method", - "settle_coupon_amount", - "settle_coupon_electric_fee", - "settle_coupon_service_fee", - "settle_status", - "cancel_type", - "cancel_code", - "cancel_msg", - "cancel_time", - "finish_type", - "finish_code", - "finish_msg", - "finish_time", - "boot_mode", - "settlement_type", - "is_parking_coupon", - "control_source", - "exception_flag", - "exception_msg", - "report_time", - "com_service_fee", - "com_electric_fee", - "com_total_fee", - "com_charge_discount_type", - "com_charge_discount", - "has_shallow_report", - "has_deep_report", - "is_exception", - "shallow_report_is_received", - "create_time", - "update_time", - "charge_market_id", - "coupon_total_amount", - "coupon_service_total_amount", - "coupon_elecfee_total_amount", - "activity_name", - "activity_total_fee", - "company_activity_id", - "company_activity_name", - "company_activity_total_fee", - "company_activity_service_fee", - "company_activity_electric_fee", - "subsidy_degree", - "subsidy_fee", - "special_channel_order_source" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_equipment_charge_order" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_ext_hurry_quit.json b/DataX/json_inc/t_ext_hurry_quit.json deleted file mode 100644 index fd3b1f5..0000000 --- a/DataX/json_inc/t_ext_hurry_quit.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "order_no", - "telephone", - "user_id", - "message1", - "message2", - "create_time", - "station_id", - "equipment_id", - "connector_id", - "flag", - "close_time", - "finish_time", - "minutesDifference", - "reduce_money", - "beforeBalance", - "afterBalance", - "fee_station", - "manual_count", - "last_manual_time", - "last_manual_message", - "person_id", - "refund_status", - "refund_time", - "refund_person_id", - "refund_person_name", - "refund_amount", - "refund_type", - "unplug_time" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `order_no`, `telephone`, `user_id`, `message1`, `message2`, `create_time`, `station_id`, `equipment_id`, `connector_id`, `flag`, `close_time`, `finish_time`, `minutesDifference`, `reduce_money`, `beforeBalance`, `afterBalance`, `fee_station`, `manual_count`, `last_manual_time`, `last_manual_message`, `person_id`, `refund_status`, `refund_time`, `refund_person_id`, `refund_person_name`, `refund_amount`, `refund_type`, `unplug_time` FROM `t_ext_hurry_quit` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "order_no", - "telephone", - "user_id", - "message1", - "message2", - "create_time", - "station_id", - "equipment_id", - "connector_id", - "flag", - "close_time", - "finish_time", - "minutesDifference", - "reduce_money", - "beforeBalance", - "afterBalance", - "fee_station", - "manual_count", - "last_manual_time", - "last_manual_message", - "person_id", - "refund_status", - "refund_time", - "refund_person_id", - "refund_person_name", - "refund_amount", - "refund_type", - "unplug_time" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_ext_hurry_quit" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_station.json b/DataX/json_inc/t_station.json deleted file mode 100644 index 11dc396..0000000 --- a/DataX/json_inc/t_station.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "station_sn", - "operator_id", - "equioment_owner_id", - "station_name", - "country_code", - "region_id", - "region_code", - "regional_name", - "regional_code", - "address", - "station_tel", - "service_tel", - "station_type", - "station_type_msg", - "station_business_type", - "station_business_type_msg", - "station_status", - "station_status_msg", - "park_nums", - "station_lng", - "station_lat", - "site_guide", - "construction", - "construction_msg", - "pictures", - "match_cars", - "park_info", - "business_start_time", - "business_end_time", - "electricity_fee", - "service_fee", - "is_park_fee", - "park_fee", - "payment", - "payment_msg", - "support_order", - "support_order_msg", - "service_item", - "service_item_image_url", - "service_item_msg", - "station_location_type", - "station_location_type_msg", - "station_charge_type", - "station_charge_type_msg", - "station_power", - "station_voltage", - "station_current", - "is_open", - "is_open_msg", - "is_display", - "is_display_msg", - "sharing_code", - "sharing_applet_picture", - "theme_picture", - "is_barrier_flag", - "is_barrier_flag_msg", - "charge_settle_type", - "charge_settle_type_msg", - "fast_connector_nums", - "slow_connector_nums", - "transformer_capacity", - "installed_capacity", - "remark", - "record_unique_no", - "equipment_owner_name", - "supply_type", - "resident_no", - "watt_hour_meter_no", - "forward_power", - "operation_date", - "charge_replace_type", - "is_hlht", - "is_subsidy", - "create_time", - "update_time", - "area_code_countryside", - "business_expand_type", - "capacity", - "video_monitor", - "electricity_type", - "receive_shop" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `station_sn`, `operator_id`, `equioment_owner_id`, `station_name`, `country_code`, `region_id`, `region_code`, `regional_name`, `regional_code`, `address`, `station_tel`, `service_tel`, `station_type`, `station_type_msg`, `station_business_type`, `station_business_type_msg`, `station_status`, `station_status_msg`, `park_nums`, `station_lng`, `station_lat`, `site_guide`, `construction`, `construction_msg`, `pictures`, `match_cars`, `park_info`, `business_start_time`, `business_end_time`, `electricity_fee`, `service_fee`, `is_park_fee`, `park_fee`, `payment`, `payment_msg`, `support_order`, `support_order_msg`, `service_item`, `service_item_image_url`, `service_item_msg`, `station_location_type`, `station_location_type_msg`, `station_charge_type`, `station_charge_type_msg`, `station_power`, `station_voltage`, `station_current`, `is_open`, `is_open_msg`, `is_display`, `is_display_msg`, `sharing_code`, `sharing_applet_picture`, `theme_picture`, `is_barrier_flag`, `is_barrier_flag_msg`, `charge_settle_type`, `charge_settle_type_msg`, `fast_connector_nums`, `slow_connector_nums`, `transformer_capacity`, `installed_capacity`, `remark`, `record_unique_no`, `equipment_owner_name`, `supply_type`, `resident_no`, `watt_hour_meter_no`, `forward_power`, `operation_date`, `charge_replace_type`, `is_hlht`, `is_subsidy`, `create_time`, `update_time`, `area_code_countryside`, `business_expand_type`, `capacity`, `video_monitor`, `electricity_type`, `receive_shop` FROM `t_station` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "station_sn", - "operator_id", - "equioment_owner_id", - "station_name", - "country_code", - "region_id", - "region_code", - "regional_name", - "regional_code", - "address", - "station_tel", - "service_tel", - "station_type", - "station_type_msg", - "station_business_type", - "station_business_type_msg", - "station_status", - "station_status_msg", - "park_nums", - "station_lng", - "station_lat", - "site_guide", - "construction", - "construction_msg", - "pictures", - "match_cars", - "park_info", - "business_start_time", - "business_end_time", - "electricity_fee", - "service_fee", - "is_park_fee", - "park_fee", - "payment", - "payment_msg", - "support_order", - "support_order_msg", - "service_item", - "service_item_image_url", - "service_item_msg", - "station_location_type", - "station_location_type_msg", - "station_charge_type", - "station_charge_type_msg", - "station_power", - "station_voltage", - "station_current", - "is_open", - "is_open_msg", - "is_display", - "is_display_msg", - "sharing_code", - "sharing_applet_picture", - "theme_picture", - "is_barrier_flag", - "is_barrier_flag_msg", - "charge_settle_type", - "charge_settle_type_msg", - "fast_connector_nums", - "slow_connector_nums", - "transformer_capacity", - "installed_capacity", - "remark", - "record_unique_no", - "equipment_owner_name", - "supply_type", - "resident_no", - "watt_hour_meter_no", - "forward_power", - "operation_date", - "charge_replace_type", - "is_hlht", - "is_subsidy", - "create_time", - "update_time", - "area_code_countryside", - "business_expand_type", - "capacity", - "video_monitor", - "electricity_type", - "receive_shop" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_station" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_user.json b/DataX/json_inc/t_user.json deleted file mode 100644 index aed8105..0000000 --- a/DataX/json_inc/t_user.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_name", - "nicker_name", - "sex", - "phone", - "qk_openid", - "open_id", - "ali_user_id", - "birthday", - "avatar", - "app_type", - "password", - "upper_user", - "master_flag", - "user_type", - "multi_charge", - "relation_phone", - "user_owner_id", - "union_id", - "state", - "reg_channel", - "reg_time", - "city_label_code", - "city_label", - "reg_source", - "create_time", - "update_time", - "operator", - "invite_code", - "identity_id", - "operator_id", - "address", - "vip_level", - "province_code", - "province", - "work_company", - "is_bind_wxinfo", - "company_min_charge_balance", - "enable_online_recharge", - "integral", - "shared_user", - "enable_online_refund", - "recommenderId", - "lx_wx_user_id", - "phone_area" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `user_name`, `nicker_name`, `sex`, `phone`, `qk_openid`, `open_id`, `ali_user_id`, `birthday`, `avatar`, `app_type`, `password`, `upper_user`, `master_flag`, `user_type`, `multi_charge`, `relation_phone`, `user_owner_id`, `union_id`, `state`, `reg_channel`, `reg_time`, `city_label_code`, `city_label`, `reg_source`, `create_time`, `update_time`, `operator`, `invite_code`, `identity_id`, `operator_id`, `address`, `vip_level`, `province_code`, `province`, `work_company`, `is_bind_wxinfo`, `company_min_charge_balance`, `enable_online_recharge`, `integral`, `shared_user`, `enable_online_refund`, `recommenderId`, `lx_wx_user_id`, `phone_area` FROM `t_user` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_name", - "nicker_name", - "sex", - "phone", - "qk_openid", - "open_id", - "ali_user_id", - "birthday", - "avatar", - "app_type", - "password", - "upper_user", - "master_flag", - "user_type", - "multi_charge", - "relation_phone", - "user_owner_id", - "union_id", - "state", - "reg_channel", - "reg_time", - "city_label_code", - "city_label", - "reg_source", - "create_time", - "update_time", - "operator", - "invite_code", - "identity_id", - "operator_id", - "address", - "vip_level", - "province_code", - "province", - "work_company", - "is_bind_wxinfo", - "company_min_charge_balance", - "enable_online_recharge", - "integral", - "shared_user", - "enable_online_refund", - "recommenderId", - "lx_wx_user_id", - "phone_area" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_user" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_user_account.json b/DataX/json_inc/t_user_account.json deleted file mode 100644 index d44036e..0000000 --- a/DataX/json_inc/t_user_account.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "user_id", - "ua_balance", - "ua_deposit", - "ua_balance_time", - "ua_password", - "ua_state", - "ua_real_balance", - "ua_real_degree", - "create_time", - "update_time", - "donate_money", - "donate_able_money", - "ua_able_balance", - "refund_money", - "ua_consume_money", - "ua_clear_zero_money" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `user_id`, `ua_balance`, `ua_deposit`, `ua_balance_time`, `ua_password`, `ua_state`, `ua_real_balance`, `ua_real_degree`, `create_time`, `update_time`, `donate_money`, `donate_able_money`, `ua_able_balance`, `refund_money`, `ua_consume_money`, `ua_clear_zero_money` FROM `t_user_account` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "user_id", - "ua_balance", - "ua_deposit", - "ua_balance_time", - "ua_password", - "ua_state", - "ua_real_balance", - "ua_real_degree", - "create_time", - "update_time", - "donate_money", - "donate_able_money", - "ua_able_balance", - "refund_money", - "ua_consume_money", - "ua_clear_zero_money" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_user_account" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/json_inc/t_user_upload_fault.json b/DataX/json_inc/t_user_upload_fault.json deleted file mode 100644 index e382b7f..0000000 --- a/DataX/json_inc/t_user_upload_fault.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "column": [ - "id", - "equipment_id", - "user_id", - "fault_config_id", - "remark", - "create_time", - "update_time" - ], - "connection": [ - { - "jdbcUrl": [ - "${src_jdbc}" - ], - "querySql": [ - "SELECT `id`, `equipment_id`, `user_id`, `fault_config_id`, `remark`, `create_time`, `update_time` FROM `t_user_upload_fault` ORDER BY id DESC LIMIT 10000" - ] - } - ], - "username": "${src_user}", - "password": "${src_pwd}" - } - }, - "writer": { - "name": "doriswriter", - "parameter": { - "loadUrl": [ - "${dest_load_url}" - ], - "column": [ - "id", - "equipment_id", - "user_id", - "fault_config_id", - "remark", - "create_time", - "update_time" - ], - "username": "${dest_user}", - "password": "${dest_pwd}", - "postSql": [], - "preSql": [], - "flushInterval": 30000, - "connection": [ - { - "jdbcUrl": "${dest_jdbc}", - "selectedDatabase": "yltcharge", - "table": [ - "t_user_upload_fault" - ] - } - ], - "loadProps": { - "format": "json", - "strip_outer_array": true - } - } - } - } - ], - "setting": { - "speed": { - "channel": "1" - } - } - } -} \ No newline at end of file diff --git a/DataX/run_jobs_full.sh b/DataX/run_jobs_full.sh index ae8d830..43f6ff2 100644 --- a/DataX/run_jobs_full.sh +++ b/DataX/run_jobs_full.sh @@ -1,21 +1,36 @@ #!/bin/bash -# 环境变量补丁 (针对 Crontab 环境下可能找不到 java 的问题) +# 环境变量补丁 export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin [ -f /etc/profile ] && . /etc/profile [ -f ~/.bash_profile ] && . ~/.bash_profile [ -f ~/.profile ] && . ~/.profile -# 配置路径 -DATAX_HOME="/usr/local/datax" -# 获取当前脚本所在目录 +# 检查参数 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + +# 配置文件 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# JSON配置文件目录指向 json_full -CONF_DIR="$SCRIPT_DIR/json_full" +CONF_DIR="$SCRIPT_DIR/json" +CONFIG_FILE="$SCRIPT_DIR/db_config.sh" + +# 加载数据库配置 +if [ -f "$CONFIG_FILE" ]; then + source "$CONFIG_FILE" +else + echo "错误: 找不到配置文件 $CONFIG_FILE" + exit 1 +fi + +DATAX_HOME="/usr/local/datax" BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" -# 明确指定要执行的15个文件（全量同步） +# 明确指定要执行的15个文件 JOBS=( "t_equipment_charge_order.json" "t_equipment_charge_order_detail.json" @@ -34,20 +49,29 @@ JOBS=( "t_user_upload_fault.json" ) -# 定义全局参数 -PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" +# 日志配置 +LOG_DIR="$SCRIPT_DIR/logs" +if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" +fi +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="$LOG_DIR/run_full_${TARGET}_${TIMESTAMP}.log" +echo "日志将同时记录到: $LOG_FILE" +# 根据目标设置参数 +if [ "$TARGET" == "doris" ]; then + DEST_PARAMS="$DORIS_DEST_PARAMS" + TITLE="MySQL -> Doris Full Sync" +else + DEST_PARAMS="$MYSQL_DEST_PARAMS" + TITLE="MySQL -> MySQL Full Sync" +fi # 在脚本最开始记录开始时间 SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) -echo "DataX全量同步脚本 (Full Sync)" -echo "==========================" -echo "DataX路径: $DATAX_PY" -echo "配置文件目录: $CONF_DIR" -echo "任务数量: ${#JOBS[@]}" -echo "脚本开始时间: $SCRIPT_START_TIME" +echo "DataX全量同步脚本 ($TITLE)" echo "==========================" # 切换到配置文件目录 @@ -56,138 +80,59 @@ cd "$CONF_DIR" || { exit 1 } -echo "当前目录: $(pwd)" -echo - -# 执行计数器 SUCCESS_COUNT=0 FAIL_COUNT=0 TOTAL=${#JOBS[@]} -CURRENT=0 -# 数组存储每个任务的执行详情 declare -a TASK_RESULTS=() declare -a TASK_DURATIONS=() declare -a TASK_STATUSES=() for JOB in "${JOBS[@]}"; do - CURRENT=$((CURRENT + 1)) + TABLE_NAME="${JOB%.json}" + echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" - echo "任务 [$CURRENT/$TOTAL] $JOB" - echo "任务开始时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" + # 动态构造全量 SQL：SELECT * FROM table + QUERY_SQL="SELECT * FROM $TABLE_NAME" + # 动态构造清空 SQL：TRUNCATE TABLE table + PRE_SQL="TRUNCATE TABLE $TABLE_NAME" - # 检查文件是否存在 - if [ ! -f "$JOB" ]; then - echo "✗ 错误: 文件不存在 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - TASK_RESULTS+=("$JOB") - TASK_STATUSES+=("文件不存在") - TASK_DURATIONS+=("0") - echo "----------------------------------------" - continue - fi + # 注意：这里需要仔细处理参数引用，将变量作为整体传递 + PARAMS="$SRC_BASE_PARAMS $DEST_PARAMS -Dsrc_query_sql=\"$QUERY_SQL\" -Ddest_pre_sql=\"$PRE_SQL\"" + + # 打印最终的执行命令（去除敏感信息后），用于调试 + # echo "执行命令: python $DATAX_PY -p \"$PARAMS\" $JOB" - # 记录开始时间（用于计算耗时） START_TIME=$(date +%s) + # 使用 eval 来正确处理带引号的参数，或者直接修改调用方式 + # 由于PARAMS中包含空格和引号，直接引用可能会导致参数解析错误 + # 最稳妥的方式是将 -p 后面的参数用单引号包起来，但内部又有双引号，比较麻烦 + # 我们可以尝试直接将所有参数拼接好，让python去解析 - # 执行任务 (添加 -p 参数传递变量) - /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" - EXIT_CODE=$? - - # 计算耗时 + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" + EXIT_CODE=${PIPESTATUS[0]} END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) - # 保存任务详情 TASK_RESULTS+=("$JOB") TASK_DURATIONS+=("$DURATION") - # 处理执行结果 if [ $EXIT_CODE -eq 0 ]; then echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) TASK_STATUSES+=("成功") else echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" - echo "错误代码: $EXIT_CODE" FAIL_COUNT=$((FAIL_COUNT + 1)) TASK_STATUSES+=("失败") - - # 询问是否继续执行后续任务 - # 全量同步失败可能需要人工干预，但这里保持自动询问或直接继续 - # 为方便自动化，这里改为仅提示，不阻塞 (或者保持原样) - # 保持原样: - # echo "----------------------------------------" - # read -p "任务失败，是否继续执行下一个任务？(y/n): " -n 1 -r - # echo - # if [[ ! $REPLY =~ ^[Yy]$ ]]; then - # echo "用户选择停止执行" - # break - # fi fi - - echo "任务结束时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" - echo - - # 可选：任务间等待时间（秒） - # sleep 1 done -# 记录脚本结束时间 -SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_END_TIMESTAMP=$(date +%s) -TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) - +# ... (汇总逻辑与之前相同) echo "==========================" -echo "执行结果汇总 (Full Sync)" +echo "执行结果汇总 (Full Sync to $TARGET)" echo "==========================" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "脚本结束时间: $SCRIPT_END_TIME" -echo "脚本总耗时: ${TOTAL_DURATION}秒" -echo "==========================" -echo "任务执行详情:" -echo "----------------------------------------" - -# 输出每个任务的执行结果 -for i in "${!TASK_RESULTS[@]}"; do - STATUS_ICON="✓" - if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then - STATUS_ICON="✗" - fi - printf "%-40s %-2s %-10s %-8s秒\n" \ - "${TASK_RESULTS[$i]}" \ - "$STATUS_ICON" \ - "${TASK_STATUSES[$i]}" \ - "${TASK_DURATIONS[$i]}" -done - -echo "----------------------------------------" -echo "汇总统计:" echo "总任务数: $TOTAL" echo "成功: $SUCCESS_COUNT" echo "失败: $FAIL_COUNT" -if [ $TOTAL -gt 0 ]; then - SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) - echo "成功率: ${SUCCESS_RATE}%" -fi - -# 计算平均耗时 -if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then - TOTAL_TASK_TIME=0 - VALID_TASKS=0 - for duration in "${TASK_DURATIONS[@]}"; do - if [ "$duration" -gt 0 ]; then - TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) - VALID_TASKS=$((VALID_TASKS + 1)) - fi - done - - if [ $VALID_TASKS -gt 0 ]; then - AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) - echo "平均任务耗时: ${AVG_DURATION}秒" - fi -fi - echo "==========================" diff --git a/DataX/run_jobs_inc.sh b/DataX/run_jobs_inc.sh index 0080f5f..b30d3f5 100644 --- a/DataX/run_jobs_inc.sh +++ b/DataX/run_jobs_inc.sh @@ -1,21 +1,36 @@ #!/bin/bash -# 环境变量补丁 (针对 Crontab 环境下可能找不到 java 的问题) +# 环境变量补丁 export PATH=$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin [ -f /etc/profile ] && . /etc/profile [ -f ~/.bash_profile ] && . ~/.bash_profile [ -f ~/.profile ] && . ~/.profile -# 配置路径 -DATAX_HOME="/usr/local/datax" -# 获取当前脚本所在目录 +# 检查参数 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + +# 配置文件 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# JSON配置文件目录指向 json_inc -CONF_DIR="$SCRIPT_DIR/json_inc" +CONF_DIR="$SCRIPT_DIR/json" +CONFIG_FILE="$SCRIPT_DIR/db_config.sh" + +# 加载数据库配置 +if [ -f "$CONFIG_FILE" ]; then + source "$CONFIG_FILE" +else + echo "错误: 找不到配置文件 $CONFIG_FILE" + exit 1 +fi + +DATAX_HOME="/usr/local/datax" BIN_DIR="$DATAX_HOME/bin" DATAX_PY="$BIN_DIR/datax.py" -# 明确指定要执行的15个文件（增量同步） +# 明确指定要执行的15个文件 JOBS=( "t_equipment_charge_order.json" "t_equipment_charge_order_detail.json" @@ -34,22 +49,29 @@ JOBS=( "t_user_upload_fault.json" ) -# 定义全局参数 -PARAMS="-Dsrc_user=ylt -Dsrc_pwd=Ycharge666 -Dsrc_jdbc=jdbc:mysql://rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com:3306/yltcharge -Ddest_user=root -Ddest_pwd=DsideaL147258369 -Ddest_load_url=10.10.14.204:8030 -Ddest_jdbc=jdbc:mysql://10.10.14.204:9030/yltcharge" +# 日志配置 +LOG_DIR="$SCRIPT_DIR/logs" +if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" +fi +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="$LOG_DIR/run_inc_${TARGET}_${TIMESTAMP}.log" +echo "日志将同时记录到: $LOG_FILE" +# 根据目标设置参数 +if [ "$TARGET" == "doris" ]; then + DEST_PARAMS="$DORIS_DEST_PARAMS" + TITLE="MySQL -> Doris Incremental Sync" +else + DEST_PARAMS="$MYSQL_DEST_PARAMS" + TITLE="MySQL -> MySQL Incremental Sync" +fi # 在脚本最开始记录开始时间 SCRIPT_START_TIME=$(date '+%Y-%m-%d %H:%M:%S') SCRIPT_START_TIMESTAMP=$(date +%s) -echo "DataX增量同步脚本 (Incremental Sync)" -echo "==========================" -echo "DataX路径: $DATAX_PY" -echo "配置文件目录: $CONF_DIR" -echo "任务数量: ${#JOBS[@]}" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "注意: 增量模式下不执行 Truncate，请确保目标表为 Unique Key 模型以实现 Upsert 效果" -echo "策略: 每次同步最近10000条数据(按ID倒序)，利用Doris主键机制自动覆盖更新" +echo "DataX增量同步脚本 ($TITLE)" echo "==========================" # 切换到配置文件目录 @@ -58,127 +80,49 @@ cd "$CONF_DIR" || { exit 1 } -echo "当前目录: $(pwd)" -echo - -# 执行计数器 SUCCESS_COUNT=0 FAIL_COUNT=0 TOTAL=${#JOBS[@]} -CURRENT=0 -# 数组存储每个任务的执行详情 declare -a TASK_RESULTS=() declare -a TASK_DURATIONS=() declare -a TASK_STATUSES=() for JOB in "${JOBS[@]}"; do - CURRENT=$((CURRENT + 1)) + TABLE_NAME="${JOB%.json}" + echo "任务 [$((SUCCESS_COUNT + FAIL_COUNT + 1))/$TOTAL] $JOB" - echo "任务 [$CURRENT/$TOTAL] $JOB" - echo "任务开始时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" + # 动态构造增量 SQL：按 id 倒序取 10000 条 + QUERY_SQL="SELECT * FROM $TABLE_NAME ORDER BY id DESC LIMIT 10000" + # 增量模式下 preSql 为空（不清空目标表） + PRE_SQL="" - # 检查文件是否存在 - if [ ! -f "$JOB" ]; then - echo "✗ 错误: 文件不存在 - $JOB" - FAIL_COUNT=$((FAIL_COUNT + 1)) - TASK_RESULTS+=("$JOB") - TASK_STATUSES+=("文件不存在") - TASK_DURATIONS+=("0") - echo "----------------------------------------" - continue - fi + PARAMS="$SRC_BASE_PARAMS $DEST_PARAMS -Dsrc_query_sql=\"$QUERY_SQL\" -Ddest_pre_sql=\"$PRE_SQL\"" - # 记录开始时间（用于计算耗时） START_TIME=$(date +%s) - - # 执行任务 (添加 -p 参数传递变量) - /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" - EXIT_CODE=$? - - # 计算耗时 + /usr/bin/python "$DATAX_PY" -p "$PARAMS" "$JOB" 2>&1 | tee -a "$LOG_FILE" + EXIT_CODE=${PIPESTATUS[0]} END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) - # 保存任务详情 TASK_RESULTS+=("$JOB") TASK_DURATIONS+=("$DURATION") - # 处理执行结果 if [ $EXIT_CODE -eq 0 ]; then echo "✓ 成功: $JOB (耗时: ${DURATION}秒)" SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) TASK_STATUSES+=("成功") else echo "✗ 失败: $JOB (耗时: ${DURATION}秒)" - echo "错误代码: $EXIT_CODE" FAIL_COUNT=$((FAIL_COUNT + 1)) TASK_STATUSES+=("失败") - # 增量任务通常无人值守，失败不暂停 fi - - echo "任务结束时间: $(date '+%H:%M:%S')" - echo "----------------------------------------" - echo - - # 可选：任务间等待时间（秒） - # sleep 1 done -# 记录脚本结束时间 -SCRIPT_END_TIME=$(date '+%Y-%m-%d %H:%M:%S') -SCRIPT_END_TIMESTAMP=$(date +%s) -TOTAL_DURATION=$((SCRIPT_END_TIMESTAMP - SCRIPT_START_TIMESTAMP)) - echo "==========================" -echo "执行结果汇总 (Incremental Sync)" +echo "执行结果汇总 (Incremental Sync to $TARGET)" echo "==========================" -echo "脚本开始时间: $SCRIPT_START_TIME" -echo "脚本结束时间: $SCRIPT_END_TIME" -echo "脚本总耗时: ${TOTAL_DURATION}秒" -echo "==========================" -echo "任务执行详情:" -echo "----------------------------------------" - -# 输出每个任务的执行结果 -for i in "${!TASK_RESULTS[@]}"; do - STATUS_ICON="✓" - if [ "${TASK_STATUSES[$i]}" = "失败" ] || [ "${TASK_STATUSES[$i]}" = "文件不存在" ]; then - STATUS_ICON="✗" - fi - printf "%-40s %-2s %-10s %-8s秒\n" \ - "${TASK_RESULTS[$i]}" \ - "$STATUS_ICON" \ - "${TASK_STATUSES[$i]}" \ - "${TASK_DURATIONS[$i]}" -done - -echo "----------------------------------------" -echo "汇总统计:" echo "总任务数: $TOTAL" echo "成功: $SUCCESS_COUNT" echo "失败: $FAIL_COUNT" -if [ $TOTAL -gt 0 ]; then - SUCCESS_RATE=$((SUCCESS_COUNT * 100 / TOTAL)) - echo "成功率: ${SUCCESS_RATE}%" -fi - -# 计算平均耗时 -if [ ${#TASK_DURATIONS[@]} -gt 0 ]; then - TOTAL_TASK_TIME=0 - VALID_TASKS=0 - for duration in "${TASK_DURATIONS[@]}"; do - if [ "$duration" -gt 0 ]; then - TOTAL_TASK_TIME=$((TOTAL_TASK_TIME + duration)) - VALID_TASKS=$((VALID_TASKS + 1)) - fi - done - - if [ $VALID_TASKS -gt 0 ]; then - AVG_DURATION=$((TOTAL_TASK_TIME / VALID_TASKS)) - echo "平均任务耗时: ${AVG_DURATION}秒" - fi -fi - echo "==========================" diff --git a/DataX/scheme/README.md b/DataX/scheme/README.md new file mode 100644 index 0000000..2c6cdb7 --- /dev/null +++ b/DataX/scheme/README.md @@ -0,0 +1,42 @@ +# 表结构生成工具 + +该工具用于连接 MySQL 数据库，自动获取表结构元数据，并生成适用于 MySQL 和 Doris 的 `CREATE TABLE` 语句。 + +## 功能特点 +1. **元数据提取**：从 `information_schema` 获取字段名、类型、长度、是否为空、默认值及注释。 +2. **MySQL DDL 生成**：生成标准的 MySQL 建表语句。 +3. **Doris DDL 生成**： + - 自动映射 MySQL 类型到 Doris 类型（如 `text` -> `string`）。 + - 使用 **Unique Key** 模型（支持幂等写入/覆盖更新）。 + - 默认配置为单副本（`replication_num = 1`），适用于单机部署。 + - 默认设置 1 个分桶（`BUCKETS 1`）。 + +## 使用方法 + +### 1. 配置数据库 +编辑 `generate_schema.py` 文件开头的 `DB_CONFIG` 变量： +```python +DB_CONFIG = { + "host": "your_host", + "port": 3306, + "user": "your_user", + "password": "your_password", + "database": "your_db", + "charset": "utf8mb4" +} +``` + +### 2. 运行脚本 +- **自动处理**：如果不传参数，脚本将自动读取 `../json/` 目录下的所有 `.json` 文件名作为表名进行批量处理。 + ```bash + python generate_schema.py + ``` +- **指定表名**：也可以手动传入需要生成结构的表名（支持多个）： + ```bash + python generate_schema.py 表名1 [表名2] ... + ``` + +### 3. 查看输出 +生成的 SQL 文件将存放在 `output/` 目录下： +- `表名_mysql.sql`: MySQL 建表语句 +- `表名_doris.sql`: Doris 建表语句 diff --git a/DataX/scheme/generate_schema.py b/DataX/scheme/generate_schema.py new file mode 100644 index 0000000..b21729d --- /dev/null +++ b/DataX/scheme/generate_schema.py @@ -0,0 +1,182 @@ +import pymysql +import sys +import os +import re + +# 默认数据库配置 +DB_CONFIG = { + "host": "rm-bp1ux6tuk49er80t9xo.mysql.rds.aliyuncs.com", + "port": 3306, + "user": "ylt", + "password": "Ycharge666", + "database": "yltcharge", + "charset": "utf8mb4" +} + +# MySQL 到 Doris 类型映射 +TYPE_MAPPING = { + "int": "INT", + "tinyint": "TINYINT", + "smallint": "SMALLINT", + "mediumint": "INT", + "bigint": "BIGINT", + "float": "FLOAT", + "double": "DOUBLE", + "decimal": "DECIMAL", + "char": "CHAR", + "varchar": "VARCHAR", + "text": "STRING", + "mediumtext": "STRING", + "longtext": "STRING", + "date": "DATE", + "datetime": "DATETIME", + "timestamp": "DATETIME", + "bit": "BOOLEAN" +} + +def get_table_schema(table_name): + connection = pymysql.connect(**DB_CONFIG) + try: + with connection.cursor(pymysql.cursors.DictCursor) as cursor: + # 获取列信息 + sql = """ + SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, COLUMN_COMMENT, EXTRA + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s + ORDER BY ORDINAL_POSITION + """ + cursor.execute(sql, (DB_CONFIG['database'], table_name)) + columns = cursor.fetchall() + + # 获取主键信息 + cursor.execute(f"SHOW KEYS FROM {table_name} WHERE Key_name = 'PRIMARY'") + pk_info = cursor.fetchall() + pk_columns = [info['Column_name'] for info in pk_info] + + return columns, pk_columns + finally: + connection.close() + +def generate_mysql_ddl(table_name, columns, pk_columns): + ddl = [f"CREATE TABLE `{table_name}` ("] + col_defs = [] + for col in columns: + line = f" `{col['COLUMN_NAME']}` {col['COLUMN_TYPE']}" + if col['IS_NULLABLE'] == 'NO': + line += " NOT NULL" + if col['COLUMN_DEFAULT'] is not None: + if isinstance(col['COLUMN_DEFAULT'], str): + line += f" DEFAULT '{col['COLUMN_DEFAULT']}'" + else: + line += f" DEFAULT {col['COLUMN_DEFAULT']}" + if col['EXTRA']: + line += f" {col['EXTRA']}" + if col['COLUMN_COMMENT']: + line += f" COMMENT '{col['COLUMN_COMMENT']}'" + col_defs.append(line) + + if pk_columns: + col_defs.append(f" PRIMARY KEY ({', '.join([f'`{c}`' for c in pk_columns])})") + + ddl.append(",\n".join(col_defs)) + ddl.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;") + return "\n".join(ddl) + +def generate_doris_ddl(table_name, columns, pk_columns): + # Doris 建议主键放在前面 + ddl = [f"CREATE TABLE `{table_name}` ("] + col_defs = [] + + # 确定 Unique Key 列表（如果没有主键，则取第一个字段，但用户要求唯一主键） + keys = pk_columns if pk_columns else [columns[0]['COLUMN_NAME']] + + for col in columns: + data_type = col['DATA_TYPE'].lower() + doris_type = TYPE_MAPPING.get(data_type, "STRING") + + # 处理长度信息 + if doris_type in ["VARCHAR", "CHAR"]: + # 从 COLUMN_TYPE 中提取长度，如 varchar(255) + match = re.search(r'\((\d+)\)', col['COLUMN_TYPE']) + if match: + length = match.group(1) + # Doris VARCHAR 最大支持 65533 + if int(length) > 65533: + doris_type = "STRING" + else: + doris_type = f"{doris_type}({length})" + else: + doris_type = f"{doris_type}(255)" + elif doris_type == "DECIMAL": + match = re.search(r'\((\d+,\d+)\)', col['COLUMN_TYPE']) + if match: + doris_type = f"DECIMAL({match.group(1)})" + + line = f" `{col['COLUMN_NAME']}` {doris_type}" + if col['IS_NULLABLE'] == 'NO': + line += " NOT NULL" + # Doris 默认值处理较简化，暂时只处理简单字符串和数字 + if col['COLUMN_DEFAULT'] is not None: + if isinstance(col['COLUMN_DEFAULT'], str) and '(' not in col['COLUMN_DEFAULT']: + line += f" DEFAULT '{col['COLUMN_DEFAULT']}'" + + if col['COLUMN_COMMENT']: + line += f" COMMENT '{col['COLUMN_COMMENT']}'" + col_defs.append(line) + + ddl.append(",\n".join(col_defs)) + ddl.append(")") + ddl.append("UNIQUE KEY(" + ", ".join([f"`{k}`" for k in keys]) + ")") + ddl.append(f"DISTRIBUTED BY HASH(`{keys[0]}`) BUCKETS 1") + ddl.append('PROPERTIES (') + ddl.append(' "replication_num" = "1"') + ddl.append(');') + + return "\n".join(ddl) + +def main(): + output_dir = os.path.join(os.path.dirname(__file__), "output") + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + # 自动获取表名列表 + if len(sys.argv) < 2: + print("未指定表名，将自动从 json 目录获取...") + json_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "json")) + if not os.path.exists(json_dir): + print(f"错误: 找不到目录 {json_dir}") + sys.exit(1) + + tables = [f.replace('.json', '') for f in os.listdir(json_dir) if f.endswith('.json')] + if not tables: + print(f"错误: 在 {json_dir} 中没有找到 .json 文件") + sys.exit(1) + print(f"发现 {len(tables)} 张表: {', '.join(tables)}") + else: + tables = sys.argv[1:] + + for table in tables: + print(f"正在获取表 {table} 的结构...") + try: + columns, pk_columns = get_table_schema(table) + + mysql_ddl = generate_mysql_ddl(table, columns, pk_columns) + doris_ddl = generate_doris_ddl(table, columns, pk_columns) + + mysql_file = os.path.join(output_dir, f"{table}_mysql.sql") + doris_file = os.path.join(output_dir, f"{table}_doris.sql") + + with open(mysql_file, "w", encoding="utf-8") as f: + f.write(mysql_ddl) + + with open(doris_file, "w", encoding="utf-8") as f: + f.write(doris_ddl) + + print(f" - MySQL DDL 已生成: {mysql_file}") + print(f" - Doris DDL 已生成: {doris_file}") + + except Exception as e: + print(f" - 处理表 {table} 时出错: {e}") + +if __name__ == "__main__": + main() diff --git a/DataX/scheme/output/t_account_recharge_doris.sql b/DataX/scheme/output/t_account_recharge_doris.sql new file mode 100644 index 0000000..398bcc3 --- /dev/null +++ b/DataX/scheme/output/t_account_recharge_doris.sql @@ -0,0 +1,40 @@ +CREATE TABLE `t_account_recharge` ( + `id` BIGINT NOT NULL, + `prid` VARCHAR(36) NOT NULL COMMENT '唯一编号', + `order_id` VARCHAR(64) COMMENT '订单编号', + `outtradeno` VARCHAR(50) COMMENT '支付业务交易号', + `pay_time` DATETIME COMMENT '支付时间', + `pay_money` DOUBLE DEFAULT '0.00' COMMENT '支付金额', + `pay_money_real` DOUBLE COMMENT '实收金额', + `refund_time` VARCHAR(20) COMMENT '退款时间', + `refund_money` DOUBLE DEFAULT '0' COMMENT '退款金额', + `refund_state` VARCHAR(50) COMMENT '退款状态（0未退款 1已退款 2 退款失败 3退款申请审核 4未退款驳回 5建帮活动不可退款）', + `order_time` VARCHAR(20) COMMENT '单下时间', + `pay_xml` STRING COMMENT '支付报文', + `refund_xml` STRING COMMENT '退款报文', + `check_xml` STRING COMMENT '对账报文', + `pay_state` VARCHAR(2) COMMENT '支付状态（0下单成功 1已支付 2 支付失败）', + `pay_type` VARCHAR(2) COMMENT '0 微信小程序 1微信 app支付 2阿里支付 3现金充值 4转账充值', + `phone` VARCHAR(160) COMMENT '电话', + `check_time` VARCHAR(18) COMMENT '对账时间', + `user_id` BIGINT COMMENT '用户id', + `create_time` DATETIME, + `update_time` DATETIME, + `donate_money` DOUBLE DEFAULT '0.00' COMMENT '赠送金额', + `memo` VARCHAR(500), + `refund_memo` VARCHAR(500) COMMENT '退款备注', + `refund_outtradno` VARCHAR(64) COMMENT '退款单号', + `refund_image` VARCHAR(255) COMMENT '退款证明文件url', + `refund_operer` VARCHAR(64) COMMENT '退款负责人', + `rechage_operer` VARCHAR(64) COMMENT '充值操作人', + `refund_reason` VARCHAR(255), + `vip` INT DEFAULT '0' COMMENT '0不是 1是', + `meal_id` VARCHAR(32), + `refund_apply_time` VARCHAR(32), + `pay_method` INT DEFAULT '0' COMMENT '0充值 1充值付费 ' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_account_recharge_mysql.sql b/DataX/scheme/output/t_account_recharge_mysql.sql new file mode 100644 index 0000000..7c13725 --- /dev/null +++ b/DataX/scheme/output/t_account_recharge_mysql.sql @@ -0,0 +1,36 @@ +CREATE TABLE `t_account_recharge` ( + `id` bigint(20) NOT NULL auto_increment, + `prid` varchar(36) NOT NULL COMMENT '唯一编号', + `order_id` varchar(64) COMMENT '订单编号', + `outtradeno` varchar(50) COMMENT '支付业务交易号', + `pay_time` datetime COMMENT '支付时间', + `pay_money` double(20,2) DEFAULT '0.00' COMMENT '支付金额', + `pay_money_real` double(20,2) COMMENT '实收金额', + `refund_time` varchar(20) COMMENT '退款时间', + `refund_money` double DEFAULT '0' COMMENT '退款金额', + `refund_state` varchar(50) COMMENT '退款状态（0未退款 1已退款 2 退款失败 3退款申请审核 4未退款驳回 5建帮活动不可退款）', + `order_time` varchar(20) COMMENT '单下时间', + `pay_xml` text COMMENT '支付报文', + `refund_xml` text COMMENT '退款报文', + `check_xml` text COMMENT '对账报文', + `pay_state` varchar(2) COMMENT '支付状态（0下单成功 1已支付 2 支付失败）', + `pay_type` varchar(2) COMMENT '0 微信小程序 1微信 app支付 2阿里支付 3现金充值 4转账充值', + `phone` varchar(160) COMMENT '电话', + `check_time` varchar(18) COMMENT '对账时间', + `user_id` bigint(50) COMMENT '用户id', + `create_time` datetime, + `update_time` datetime, + `donate_money` double(20,2) DEFAULT '0.00' COMMENT '赠送金额', + `memo` varchar(500), + `refund_memo` varchar(500) COMMENT '退款备注', + `refund_outtradno` varchar(64) COMMENT '退款单号', + `refund_image` varchar(255) COMMENT '退款证明文件url', + `refund_operer` varchar(64) COMMENT '退款负责人', + `rechage_operer` varchar(64) COMMENT '充值操作人', + `refund_reason` varchar(255), + `vip` int(5) DEFAULT '0' COMMENT '0不是 1是', + `meal_id` varchar(32), + `refund_apply_time` varchar(32), + `pay_method` int(11) DEFAULT '0' COMMENT '0充值 1充值付费 ', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_account_water_doris.sql b/DataX/scheme/output/t_account_water_doris.sql new file mode 100644 index 0000000..9d0a58c --- /dev/null +++ b/DataX/scheme/output/t_account_water_doris.sql @@ -0,0 +1,30 @@ +CREATE TABLE `t_account_water` ( + `id` BIGINT NOT NULL, + `cash` DECIMAL(20,2) COMMENT '资金', + `phone` VARCHAR(256) COMMENT '电话', + `txt_desc` VARCHAR(250) COMMENT '文本说明', + `sign_msg` VARCHAR(4) DEFAULT '+' COMMENT '符号说明', + `reason` VARCHAR(255) DEFAULT '' COMMENT '理由', + `sign` TINYINT DEFAULT '0' COMMENT '号符 0加 1减去', + `type` CHAR(2) DEFAULT '0' COMMENT '0充值 1充电扣款 2充值赠送 3后台充值 4退款 5后台转出 6后台转入 7账户清零 8用户申请退款 9退款驳回 10退款-消费平账 11充值-消费平账 12占位费余额支付 13退占位费', + `order_id` VARCHAR(125) COMMENT '订单id（关联充值表 ，充电消费记录）', + `treatment` CHAR(1) COMMENT '处理方式 0 系统自动处理 1人工处理', + `user_id` BIGINT COMMENT '用户id', + `pay_status` CHAR(1) DEFAULT '1' COMMENT '状态 0失败 1成功 2待审核', + `pay_method` CHAR(2) NOT NULL DEFAULT '4' COMMENT '支付方式 0现金支付 1支付宝 2微信小程序 3微信app 4后台充值 5用户申请退款6第三方扣款 7套餐扣费 8电卡结算 9联合结算', + `update_time` DATETIME, + `create_time` DATETIME COMMENT '创建时间', + `pre_balance` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '操作前金额', + `bef_balance` DECIMAL(20,2) DEFAULT '0.00' COMMENT '操作后金额', + `memo` VARCHAR(520) COMMENT '备注', + `donate_money` DECIMAL(20,2) DEFAULT '0.00' COMMENT '赠送金额', + `ua_balance` DECIMAL(20,2) DEFAULT '0.00' COMMENT '可用金额', + `to_user_id` BIGINT COMMENT '如果是华航转账或者扣回，是给了哪个用户', + `wx_person_id` CHAR(36) COMMENT '微信上登录的管理员，在公众号上转账的人员ID', + `wx_person_name` VARCHAR(255) COMMENT '微信上登录的管理员，在公众号上转账的人员姓名' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_account_water_mysql.sql b/DataX/scheme/output/t_account_water_mysql.sql new file mode 100644 index 0000000..f273dec --- /dev/null +++ b/DataX/scheme/output/t_account_water_mysql.sql @@ -0,0 +1,26 @@ +CREATE TABLE `t_account_water` ( + `id` bigint(20) NOT NULL auto_increment, + `cash` decimal(20,2) COMMENT '资金', + `phone` varchar(256) COMMENT '电话', + `txt_desc` varchar(250) COMMENT '文本说明', + `sign_msg` varchar(4) DEFAULT '+' COMMENT '符号说明', + `reason` varchar(255) DEFAULT '' COMMENT '理由', + `sign` tinyint(2) DEFAULT '0' COMMENT '号符 0加 1减去', + `type` char(2) DEFAULT '0' COMMENT '0充值 1充电扣款 2充值赠送 3后台充值 4退款 5后台转出 6后台转入 7账户清零 8用户申请退款 9退款驳回 10退款-消费平账 11充值-消费平账 12占位费余额支付 13退占位费', + `order_id` varchar(125) COMMENT '订单id（关联充值表 ，充电消费记录）', + `treatment` char(1) COMMENT '处理方式 0 系统自动处理 1人工处理', + `user_id` bigint(50) COMMENT '用户id', + `pay_status` char(1) DEFAULT '1' COMMENT '状态 0失败 1成功 2待审核', + `pay_method` char(2) NOT NULL DEFAULT '4' COMMENT '支付方式 0现金支付 1支付宝 2微信小程序 3微信app 4后台充值 5用户申请退款6第三方扣款 7套餐扣费 8电卡结算 9联合结算', + `update_time` datetime, + `create_time` datetime COMMENT '创建时间', + `pre_balance` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '操作前金额', + `bef_balance` decimal(20,2) DEFAULT '0.00' COMMENT '操作后金额', + `memo` varchar(520) COMMENT '备注', + `donate_money` decimal(20,2) DEFAULT '0.00' COMMENT '赠送金额', + `ua_balance` decimal(20,2) DEFAULT '0.00' COMMENT '可用金额', + `to_user_id` bigint(20) COMMENT '如果是华航转账或者扣回，是给了哪个用户', + `wx_person_id` char(36) COMMENT '微信上登录的管理员，在公众号上转账的人员ID', + `wx_person_name` varchar(255) COMMENT '微信上登录的管理员，在公众号上转账的人员姓名', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_car_doris.sql b/DataX/scheme/output/t_car_doris.sql new file mode 100644 index 0000000..9cf1004 --- /dev/null +++ b/DataX/scheme/output/t_car_doris.sql @@ -0,0 +1,25 @@ +CREATE TABLE `t_car` ( + `id` BIGINT NOT NULL, + `car_plate_no` VARCHAR(10) NOT NULL COMMENT '车牌号', + `car_sn` VARCHAR(25) COMMENT '车辆编码 (当行业为公交时必填)', + `car_brand_name` VARCHAR(50) COMMENT '品牌名称', + `car_model_name` VARCHAR(32) COMMENT '车型名称', + `car_vin` VARCHAR(32) COMMENT '车辆vin', + `car_engine` VARCHAR(32) COMMENT '公里数', + `car_type` TINYINT COMMENT '0私家车 1公交车', + `car_owner_company_id` BIGINT COMMENT '企业id', + `car_business` TINYINT NOT NULL DEFAULT '7' COMMENT '行业： 1 公交 2 物流 3 环卫 4 出租 5 网约 6 公务 7 其他', + `car_bus_path` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '公交路线（只有行业为公交时，这个字段才生效）', + `is_auto_charge` TINYINT DEFAULT '0' COMMENT '是否能vin充电', + `user_id` BIGINT COMMENT '用户id', + `user_card_no` VARCHAR(50) COMMENT '用户卡号', + `is_disable` TINYINT DEFAULT '1' COMMENT '状态 0 禁用 1启用', + `create_time` DATETIME, + `update_time` DATETIME, + `operator_id` CHAR(36) COMMENT '操作者ID' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_car_mysql.sql b/DataX/scheme/output/t_car_mysql.sql new file mode 100644 index 0000000..6d259a0 --- /dev/null +++ b/DataX/scheme/output/t_car_mysql.sql @@ -0,0 +1,21 @@ +CREATE TABLE `t_car` ( + `id` bigint(20) NOT NULL auto_increment, + `car_plate_no` varchar(10) NOT NULL COMMENT '车牌号', + `car_sn` varchar(25) COMMENT '车辆编码 (当行业为公交时必填)', + `car_brand_name` varchar(50) COMMENT '品牌名称', + `car_model_name` varchar(32) COMMENT '车型名称', + `car_vin` varchar(32) COMMENT '车辆vin', + `car_engine` varchar(32) COMMENT '公里数', + `car_type` tinyint(4) COMMENT '0私家车 1公交车', + `car_owner_company_id` bigint(20) COMMENT '企业id', + `car_business` tinyint(3) NOT NULL DEFAULT '7' COMMENT '行业： 1 公交 2 物流 3 环卫 4 出租 5 网约 6 公务 7 其他', + `car_bus_path` varchar(100) NOT NULL DEFAULT '' COMMENT '公交路线（只有行业为公交时，这个字段才生效）', + `is_auto_charge` tinyint(4) DEFAULT '0' COMMENT '是否能vin充电', + `user_id` bigint(20) COMMENT '用户id', + `user_card_no` varchar(50) COMMENT '用户卡号', + `is_disable` tinyint(4) DEFAULT '1' COMMENT '状态 0 禁用 1启用', + `create_time` datetime, + `update_time` datetime, + `operator_id` char(36) COMMENT '操作者ID', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_company_doris.sql b/DataX/scheme/output/t_company_doris.sql new file mode 100644 index 0000000..3de3fda --- /dev/null +++ b/DataX/scheme/output/t_company_doris.sql @@ -0,0 +1,54 @@ +CREATE TABLE `t_company` ( + `id` BIGINT NOT NULL, + `company_no` VARCHAR(32) NOT NULL, + `company_type` TINYINT COMMENT '0、 普通企业 +1、 私营公交 +2、 国营公交 +3 、流量平台 +4、网约车认证企业 +5、大巴车企业 +6、VIP企业 +7、物流车 +8、特种车辆', + `company_name` VARCHAR(128) NOT NULL COMMENT '公司名称', + `company_state` CHAR(1) NOT NULL DEFAULT '0' COMMENT '公司状态：0=正常；1=删除', + `company_logo` VARCHAR(100) COMMENT '公司logo', + `company_admin_phone` VARCHAR(50) COMMENT '管理员电话', + `company_admin_name` VARCHAR(100) COMMENT '管理姓名', + `balance_remind` DOUBLE DEFAULT '0.00' COMMENT '提醒额度', + `balance_remind_phone` VARCHAR(20) COMMENT '余额提醒人电话', + `agent_id` BIGINT, + `station_set_type` TINYINT DEFAULT '0' COMMENT '站点设置类型 0设置优惠站点 1设置不折扣站点', + `specified_range` TINYINT DEFAULT '0' COMMENT '指定范围 0指定站点 1指定运营商', + `specified_operator_ids` VARCHAR(500) DEFAULT '' COMMENT '指定优惠的运营商id 以逗号隔开', + `is_company_discount` TINYINT DEFAULT '1' COMMENT '是否开启企业优惠折扣 0否 1是', + `is_overlay_coupons` TINYINT DEFAULT '0' COMMENT '是否叠加优惠券 0不是 1是', + `station_ids` VARCHAR(300) COMMENT '开放站点id', + `no_discount_stationids` VARCHAR(50) COMMENT '不参与结算的站点', + `remark` VARCHAR(200) COMMENT '备注', + `invoice_eligible` VARCHAR(255) DEFAULT '1' COMMENT '是否可以开票，1可以0不可以', + `unify_invoice_title_flag` TINYINT DEFAULT '0' COMMENT '是否统一开票名头 0否 1是', + `unify_invoice_title` VARCHAR(255) COMMENT '统一开票名头（公司抬头）', + `unify_tax_no` VARCHAR(64) COMMENT '统一开票税号', + `main_account_unify_invoice` TINYINT DEFAULT '0' COMMENT '主帐号统一开票 0否 1是', + `refuse_invoice_tip` VARCHAR(255) COMMENT '拒绝开票提示', + `is_share_master` TINYINT DEFAULT '1' COMMENT '共享主账号账号金额 0 不共享 1共享', + `settlement_method` TINYINT DEFAULT '0' COMMENT '0预付 1 后结 2实时', + `charge_discount_type` TINYINT DEFAULT '0' COMMENT '充电优惠类型 0 服务费折扣（0%）； 1 固定服务费（元/度）；2整体折扣 （0%）3固定价格基础上服务费在进行折扣 4固定价格（电费单价+服务费单价）；', + `charge_discount` DOUBLE DEFAULT '0.0000000' COMMENT '优惠折扣', + `settle_status` TINYINT DEFAULT '0' COMMENT '是否开启结算 0关闭 1开通', + `charge_unit_service` DECIMAL(10,7) DEFAULT '0.0000000' COMMENT '充电服务费单价', + `charge_unit_electric` DECIMAL(10,7) DEFAULT '0.0000000' COMMENT '充电电费单价', + `is_custom_price` INT DEFAULT '0' COMMENT '是否开启自定义价格 0不开启 1开启', + `company_order_permission` TINYINT DEFAULT '0' COMMENT '企业订单权限 0-全部不允许 1-仅主账户 2-主账户+子账户', + `open_station_ids` VARCHAR(2000) COMMENT '企业开放站点(允许企业在哪些站点启动充电)', + `create_time` DATETIME, + `update_time` DATETIME, + `minimum_recharge_amount` DECIMAL(10,2) COMMENT '充值最小金额（针对车管、司管企业），空不限制', + `inner_person` VARCHAR(255) +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_company_info_value_doris.sql b/DataX/scheme/output/t_company_info_value_doris.sql new file mode 100644 index 0000000..197f2da --- /dev/null +++ b/DataX/scheme/output/t_company_info_value_doris.sql @@ -0,0 +1,14 @@ +CREATE TABLE `t_company_info_value` ( + `id` VARCHAR(255) NOT NULL, + `info_value` VARCHAR(255), + `type_id` VARCHAR(255), + `create_time` DATETIME, + `update_time` DATETIME, + `remark` VARCHAR(255), + `renzheng_flag` INT DEFAULT '0' COMMENT '认证标识' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_company_info_value_mysql.sql b/DataX/scheme/output/t_company_info_value_mysql.sql new file mode 100644 index 0000000..43e7f1d --- /dev/null +++ b/DataX/scheme/output/t_company_info_value_mysql.sql @@ -0,0 +1,10 @@ +CREATE TABLE `t_company_info_value` ( + `id` varchar(255) NOT NULL, + `info_value` varchar(255), + `type_id` varchar(255), + `create_time` datetime, + `update_time` datetime, + `remark` varchar(255), + `renzheng_flag` int(11) DEFAULT '0' COMMENT '认证标识', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_company_mysql.sql b/DataX/scheme/output/t_company_mysql.sql new file mode 100644 index 0000000..d23c5b4 --- /dev/null +++ b/DataX/scheme/output/t_company_mysql.sql @@ -0,0 +1,50 @@ +CREATE TABLE `t_company` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `company_no` varchar(32) NOT NULL, + `company_type` tinyint(4) COMMENT '0、 普通企业 +1、 私营公交 +2、 国营公交 +3 、流量平台 +4、网约车认证企业 +5、大巴车企业 +6、VIP企业 +7、物流车 +8、特种车辆', + `company_name` varchar(128) NOT NULL COMMENT '公司名称', + `company_state` char(1) NOT NULL DEFAULT '0' COMMENT '公司状态：0=正常；1=删除', + `company_logo` varchar(100) COMMENT '公司logo', + `company_admin_phone` varchar(50) COMMENT '管理员电话', + `company_admin_name` varchar(100) COMMENT '管理姓名', + `balance_remind` double(10,2) DEFAULT '0.00' COMMENT '提醒额度', + `balance_remind_phone` varchar(20) COMMENT '余额提醒人电话', + `agent_id` bigint(20) unsigned, + `station_set_type` tinyint(4) DEFAULT '0' COMMENT '站点设置类型 0设置优惠站点 1设置不折扣站点', + `specified_range` tinyint(4) DEFAULT '0' COMMENT '指定范围 0指定站点 1指定运营商', + `specified_operator_ids` varchar(500) DEFAULT '' COMMENT '指定优惠的运营商id 以逗号隔开', + `is_company_discount` tinyint(4) DEFAULT '1' COMMENT '是否开启企业优惠折扣 0否 1是', + `is_overlay_coupons` tinyint(4) DEFAULT '0' COMMENT '是否叠加优惠券 0不是 1是', + `station_ids` varchar(300) COMMENT '开放站点id', + `no_discount_stationids` varchar(50) COMMENT '不参与结算的站点', + `remark` varchar(200) COMMENT '备注', + `invoice_eligible` varchar(255) DEFAULT '1' COMMENT '是否可以开票，1可以0不可以', + `unify_invoice_title_flag` tinyint(1) DEFAULT '0' COMMENT '是否统一开票名头 0否 1是', + `unify_invoice_title` varchar(255) COMMENT '统一开票名头（公司抬头）', + `unify_tax_no` varchar(64) COMMENT '统一开票税号', + `main_account_unify_invoice` tinyint(1) DEFAULT '0' COMMENT '主帐号统一开票 0否 1是', + `refuse_invoice_tip` varchar(255) COMMENT '拒绝开票提示', + `is_share_master` tinyint(4) DEFAULT '1' COMMENT '共享主账号账号金额 0 不共享 1共享', + `settlement_method` tinyint(4) DEFAULT '0' COMMENT '0预付 1 后结 2实时', + `charge_discount_type` tinyint(4) DEFAULT '0' COMMENT '充电优惠类型 0 服务费折扣（0%）； 1 固定服务费（元/度）；2整体折扣 （0%）3固定价格基础上服务费在进行折扣 4固定价格（电费单价+服务费单价）；', + `charge_discount` double(10,7) DEFAULT '0.0000000' COMMENT '优惠折扣', + `settle_status` tinyint(4) DEFAULT '0' COMMENT '是否开启结算 0关闭 1开通', + `charge_unit_service` decimal(10,7) DEFAULT '0.0000000' COMMENT '充电服务费单价', + `charge_unit_electric` decimal(10,7) DEFAULT '0.0000000' COMMENT '充电电费单价', + `is_custom_price` int(2) DEFAULT '0' COMMENT '是否开启自定义价格 0不开启 1开启', + `company_order_permission` tinyint(4) DEFAULT '0' COMMENT '企业订单权限 0-全部不允许 1-仅主账户 2-主账户+子账户', + `open_station_ids` varchar(2000) COMMENT '企业开放站点(允许企业在哪些站点启动充电)', + `create_time` datetime, + `update_time` datetime, + `minimum_recharge_amount` decimal(10,2) COMMENT '充值最小金额（针对车管、司管企业），空不限制', + `inner_person` varchar(255), + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_connector_doris.sql b/DataX/scheme/output/t_connector_doris.sql new file mode 100644 index 0000000..af136d1 --- /dev/null +++ b/DataX/scheme/output/t_connector_doris.sql @@ -0,0 +1,38 @@ +CREATE TABLE `t_connector` ( + `id` BIGINT NOT NULL, + `equipment_id` BIGINT NOT NULL COMMENT '所属设备编码id', + `connector_sn` VARCHAR(50) NOT NULL COMMENT '接口编码', + `connector_short_sn` VARCHAR(50) NOT NULL COMMENT '接口短编号', + `connector_no` TINYINT, + `connector_name` VARCHAR(30) COMMENT '接口名称', + `connector_type` TINYINT NOT NULL COMMENT '接口类型 1：家用插座；2：交流接口插座；3：交流接口插头；4：直流接口枪头 5：无线充电座；6：其它 +1：家用插座； +2：交流接口插座； +3：交流接口插头； +4：直流接口枪头 +5：无线充电座； +6：其它', + `voltage_upper_limits` SMALLINT NOT NULL COMMENT '额定电压上限', + `voltage_lower_limits` SMALLINT NOT NULL COMMENT '额定电压下限', + `current` SMALLINT NOT NULL DEFAULT '0' COMMENT '额定电流', + `power` DOUBLE NOT NULL DEFAULT '0.0' COMMENT '额定功率', + `parkno` VARCHAR(20) COMMENT '车位号', + `appointment_id` BIGINT COMMENT '预约记录id', + `appointment_user_id` BIGINT COMMENT '预约记录的用户id', + `open_lock_user_id` BIGINT COMMENT '开锁的用户id', + `detection_type` TINYINT DEFAULT '0' COMMENT '检测类型 0 无 1 检测平台 2 CNTE检测', + `is_ground_lock` TINYINT DEFAULT '0' COMMENT '0没有地锁 1有地锁', + `is_ground_lock_msg` VARCHAR(20) COMMENT '是否配有地锁解释', + `is_geomagnetic` TINYINT DEFAULT '0' COMMENT '0没有地磁 1有地磁', + `is_geomagnetic_msg` VARCHAR(20) COMMENT '是否配有地磁', + `is_appointment` TINYINT DEFAULT '0' COMMENT '0可预约 1已预约 2不可预约 ', + `is_appointment_msg` VARCHAR(20) COMMENT '预约状态解释', + `auto_open_lock_time` DATETIME COMMENT '预约的时间截止点', + `create_time` DATETIME COMMENT '创建时间', + `update_time` DATETIME COMMENT '更新时间' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_connector_mysql.sql b/DataX/scheme/output/t_connector_mysql.sql new file mode 100644 index 0000000..4ac32af --- /dev/null +++ b/DataX/scheme/output/t_connector_mysql.sql @@ -0,0 +1,34 @@ +CREATE TABLE `t_connector` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `equipment_id` bigint(20) NOT NULL COMMENT '所属设备编码id', + `connector_sn` varchar(50) NOT NULL COMMENT '接口编码', + `connector_short_sn` varchar(50) NOT NULL COMMENT '接口短编号', + `connector_no` tinyint(4), + `connector_name` varchar(30) COMMENT '接口名称', + `connector_type` tinyint(4) NOT NULL COMMENT '接口类型 1：家用插座；2：交流接口插座；3：交流接口插头；4：直流接口枪头 5：无线充电座；6：其它 +1：家用插座； +2：交流接口插座； +3：交流接口插头； +4：直流接口枪头 +5：无线充电座； +6：其它', + `voltage_upper_limits` smallint(8) NOT NULL COMMENT '额定电压上限', + `voltage_lower_limits` smallint(8) NOT NULL COMMENT '额定电压下限', + `current` smallint(4) NOT NULL DEFAULT '0' COMMENT '额定电流', + `power` double(8,1) NOT NULL DEFAULT '0.0' COMMENT '额定功率', + `parkno` varchar(20) COMMENT '车位号', + `appointment_id` bigint(20) COMMENT '预约记录id', + `appointment_user_id` bigint(20) COMMENT '预约记录的用户id', + `open_lock_user_id` bigint(20) COMMENT '开锁的用户id', + `detection_type` tinyint(4) DEFAULT '0' COMMENT '检测类型 0 无 1 检测平台 2 CNTE检测', + `is_ground_lock` tinyint(4) DEFAULT '0' COMMENT '0没有地锁 1有地锁', + `is_ground_lock_msg` varchar(20) COMMENT '是否配有地锁解释', + `is_geomagnetic` tinyint(4) DEFAULT '0' COMMENT '0没有地磁 1有地磁', + `is_geomagnetic_msg` varchar(20) COMMENT '是否配有地磁', + `is_appointment` tinyint(4) DEFAULT '0' COMMENT '0可预约 1已预约 2不可预约 ', + `is_appointment_msg` varchar(20) COMMENT '预约状态解释', + `auto_open_lock_time` datetime COMMENT '预约的时间截止点', + `create_time` datetime COMMENT '创建时间', + `update_time` datetime COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_equipment_charge_order_detail_doris.sql b/DataX/scheme/output/t_equipment_charge_order_detail_doris.sql new file mode 100644 index 0000000..b173106 --- /dev/null +++ b/DataX/scheme/output/t_equipment_charge_order_detail_doris.sql @@ -0,0 +1,30 @@ +CREATE TABLE `t_equipment_charge_order_detail` ( + `id` BIGINT NOT NULL, + `order_no` VARCHAR(50) COMMENT '订单号', + `charge_degree` DOUBLE DEFAULT '0.000' COMMENT '充电电量', + `service_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '服务费', + `electric_fee` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '电费', + `cost_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '成本费', + `service_fee_unit_price` DECIMAL(20,2) DEFAULT '0.00' COMMENT '服务费单价', + `electric_fee_unit_price` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '电费单价', + `cost_fee_unit_price` DECIMAL(20,2) DEFAULT '0.00' COMMENT '成本费单价', + `activity_service_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '活动时优惠了的服务费', + `activity_electric_fee` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '活动时优惠了的电费', + `charge_startr_time` DATETIME COMMENT '分时充电开始时间', + `charge_end_time` DATETIME COMMENT '分时结束时间', + `charge_meal_electric_fee` DOUBLE DEFAULT '0.00' COMMENT '电卡套餐电费', + `charge_meal_service_fee` DOUBLE DEFAULT '0.00' COMMENT '电卡套餐服务费', + `charge_meal_total_fee` DOUBLE DEFAULT '0.00' COMMENT '电卡套餐总金额', + `electric_card_use_dgree` DOUBLE DEFAULT '0.00' COMMENT '电卡套餐消耗的度数', + `check_electric_unit_price` DOUBLE DEFAULT '0.00' COMMENT '核算电费单价', + `check_service_unit_price` DOUBLE DEFAULT '0.00' COMMENT '核算服务费单价', + `time_flag` TINYINT DEFAULT '1' COMMENT '尖峰平谷', + `create_time` DATETIME, + `update_time` DATETIME, + `detail_more` STRING COMMENT '活动详情信息补充' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_equipment_charge_order_detail_mysql.sql b/DataX/scheme/output/t_equipment_charge_order_detail_mysql.sql new file mode 100644 index 0000000..88e195e --- /dev/null +++ b/DataX/scheme/output/t_equipment_charge_order_detail_mysql.sql @@ -0,0 +1,26 @@ +CREATE TABLE `t_equipment_charge_order_detail` ( + `id` bigint(20) NOT NULL auto_increment, + `order_no` varchar(50) COMMENT '订单号', + `charge_degree` double(20,3) DEFAULT '0.000' COMMENT '充电电量', + `service_fee` decimal(20,2) DEFAULT '0.00' COMMENT '服务费', + `electric_fee` decimal(20,7) DEFAULT '0.0000000' COMMENT '电费', + `cost_fee` decimal(20,2) DEFAULT '0.00' COMMENT '成本费', + `service_fee_unit_price` decimal(20,2) DEFAULT '0.00' COMMENT '服务费单价', + `electric_fee_unit_price` decimal(20,7) DEFAULT '0.0000000' COMMENT '电费单价', + `cost_fee_unit_price` decimal(20,2) DEFAULT '0.00' COMMENT '成本费单价', + `activity_service_fee` decimal(20,2) DEFAULT '0.00' COMMENT '活动时优惠了的服务费', + `activity_electric_fee` decimal(20,7) DEFAULT '0.0000000' COMMENT '活动时优惠了的电费', + `charge_startr_time` datetime COMMENT '分时充电开始时间', + `charge_end_time` datetime COMMENT '分时结束时间', + `charge_meal_electric_fee` double(20,2) DEFAULT '0.00' COMMENT '电卡套餐电费', + `charge_meal_service_fee` double(20,2) DEFAULT '0.00' COMMENT '电卡套餐服务费', + `charge_meal_total_fee` double(20,2) DEFAULT '0.00' COMMENT '电卡套餐总金额', + `electric_card_use_dgree` double(20,2) DEFAULT '0.00' COMMENT '电卡套餐消耗的度数', + `check_electric_unit_price` double(20,2) DEFAULT '0.00' COMMENT '核算电费单价', + `check_service_unit_price` double(20,2) DEFAULT '0.00' COMMENT '核算服务费单价', + `time_flag` tinyint(4) DEFAULT '1' COMMENT '尖峰平谷', + `create_time` datetime, + `update_time` datetime, + `detail_more` text COMMENT '活动详情信息补充', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_equipment_charge_order_doris.sql b/DataX/scheme/output/t_equipment_charge_order_doris.sql new file mode 100644 index 0000000..5236cca --- /dev/null +++ b/DataX/scheme/output/t_equipment_charge_order_doris.sql @@ -0,0 +1,118 @@ +CREATE TABLE `t_equipment_charge_order` ( + `id` BIGINT NOT NULL, + `user_id` BIGINT NOT NULL COMMENT '用户id', + `company_id` BIGINT COMMENT '企业id', + `station_id` BIGINT COMMENT '充电站id', + `equipment_id` BIGINT NOT NULL COMMENT '充电桩id', + `connector_id` BIGINT NOT NULL COMMENT '充电枪id', + `user_card_no` VARCHAR(32) COMMENT '用户充电卡号', + `state` TINYINT COMMENT '订单状态 0 创建中 1准备充电中 2 充电中 3完成 4 取消 5挂起 6需要手动补单 7结算中', + `operate_state` TINYINT DEFAULT '0' COMMENT '订单操作状态 0无 1启动充电 2结束充电 ', + `order_no` VARCHAR(20) NOT NULL COMMENT '订单号', + `platform_order_no` VARCHAR(50) COMMENT '平台订单号【互通互联】', + `platform_sale_type` TINYINT DEFAULT '1' COMMENT '平台转售模式【互通互联】1：转售模式（该模式下由用户运营商向用户开票，由设备运营商给用户运营商开票）2：代收代付模式（该模式下，由设备运营商直接给用户开票）', + `platform_driver_id` VARCHAR(50) COMMENT '平台用户ID【互通互联】', + `platform_phone` VARCHAR(50) COMMENT '平台用户手机号【互通互联】', + `charging_amt` DECIMAL(20,2) DEFAULT '0.00' COMMENT '预充金额，电量充到该金额自动停止充电', + `order_type` TINYINT COMMENT '订单类型 1普通用户订单 3 企业用户订单 4第三方平台订单', + `order_time` DATETIME COMMENT '订单时间', + `charge_begin_time` DATETIME COMMENT '充电开始时间', + `charge_end_time` DATETIME COMMENT '充电结束时间', + `charge_begin_degree` DOUBLE DEFAULT '0.00' COMMENT '充电开始度数', + `charge_end_degree` DOUBLE DEFAULT '0.00' COMMENT '充电结束度数', + `charge_degree` DOUBLE DEFAULT '0.000' COMMENT '充电度数', + `charge_ah` DOUBLE DEFAULT '0.00' COMMENT '充电电量', + `charge_times_degree` VARCHAR(1000) COMMENT '充电分时度数', + `charge_begin_soc` INT DEFAULT '0' COMMENT '充电开始soc', + `charge_end_soc` INT DEFAULT '0' COMMENT '充电结束soc', + `charge_cur_soc` INT DEFAULT '0' COMMENT '充电当前soc', + `charge_duration` INT DEFAULT '0' COMMENT '充电时长', + `charge_settle_type` TINYINT, + `charge_unit_price` DECIMAL(10,7) COMMENT '电费单价', + `charge_unit_cost` DECIMAL(10,7) COMMENT '电费成本单价', + `charge_unit_service_fee` DECIMAL(20,7) COMMENT '服务费单价', + `receivable_electric_fee` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '应收电费', + `receivable_service_fee` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '应收服务费', + `receivable_total_fee` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '应收总金额', + `charge_elecfee_amount` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '电费(实收电费)', + `charge_elecfee_cost_amount` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '电费成本', + `charge_service_amount` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '服务费(实收服务费)', + `actual_pay_amount` DECIMAL(20,2) DEFAULT '0.00' COMMENT '实际支付金额', + `pay_order_no` VARCHAR(30) COMMENT '支付订单号', + `pay_amount` DECIMAL(20,2) DEFAULT '0.00' COMMENT '本次订单付款金额', + `pay_method` TINYINT COMMENT '支付方式 0 余额支付 1 支付宝 2微信小程序 3微信app支付', + `pay_status` TINYINT DEFAULT '0' COMMENT '支付状态 0未支付 1待支付 2已支付 3支付失败', + `pay_time` DATETIME COMMENT '支付时间', + `user_coupon_id` BIGINT COMMENT '用户 优惠劵id', + `coupon_name` VARCHAR(30) COMMENT '优惠券名称', + `coupon_type` TINYINT COMMENT '优惠券类型', + `coupon_amount` DECIMAL(20,2) DEFAULT '0.00' COMMENT '优惠券总金额', + `elecfee_coupon_amount` DECIMAL(20,2) DEFAULT '0.00' COMMENT '电费优惠券金额', + `servicefee_coupon_amount` DECIMAL(20,2) DEFAULT '0.00' COMMENT '服务费优惠券金额', + `invoice_id` BIGINT COMMENT '发票id', + `invoice_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '可以开票金额', + `ticket_status` TINYINT DEFAULT '0' COMMENT '0未开票 1代开票 2已开票', + `activity_electric_fee` DECIMAL(20,7) DEFAULT '0.0000000' COMMENT '活动时优惠了的电费 [降价活动]', + `activity_service_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '活动时优惠了的服务费[降价活动]', + `charge_plate_no` VARCHAR(20) COMMENT '车牌号', + `charge_vin` VARCHAR(100) COMMENT '车辆vin码', + `charge_strategy` VARCHAR(20) COMMENT '充电策略', + `charge_strategy_param` VARCHAR(30) COMMENT '充电策略参数', + `operator_id` BIGINT COMMENT '运营商id', + `operator_income` DECIMAL(20,2) DEFAULT '0.00' COMMENT '运营商收入金额', + `plat_service_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '平台服务费', + `charge_discount_type` TINYINT DEFAULT '0' COMMENT '充电优惠类型 0 服务费折扣（0%） 1 固定服务费（元/度）2整体折扣 （0%）3固定价格在进行折扣 4固定电费服务费价格', + `settle_electric_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '算结电费', + `settle_service_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '算结服务费', + `settle_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '结算总费用', + `settle_method` TINYINT DEFAULT '2' COMMENT '0预付 1 后结 2实时', + `settle_coupon_amount` DECIMAL(20,2) DEFAULT '0.00' COMMENT '算结优惠金额', + `settle_coupon_electric_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '结算电费金额', + `settle_coupon_service_fee` DECIMAL(20,2) DEFAULT '0.00' COMMENT '算结服务费', + `settle_status` TINYINT DEFAULT '0' COMMENT '是否开启结算 0关闭 1开通', + `cancel_type` INT COMMENT '取消类型 1（后台异常取消） 2（设备异常取消）,3(订单挂起),4(已结束订单又被唤起)', + `cancel_code` INT COMMENT '0 创建中订单(状态0)超过60秒自动取消, 1 创建中订单(状态1)超过60秒自动取消,2 充电中订单(状态2)超过24小时自动取消,3 充电中订单(开始时间异常)超过3分钟自动取消,4卡单，5订单挂起 6已结束订单又被唤起为充电中', + `cancel_msg` VARCHAR(100) COMMENT '取消原因', + `cancel_time` DATETIME COMMENT '取消时间', + `finish_type` INT COMMENT '结束类型', + `finish_code` INT COMMENT '结束编码', + `finish_msg` VARCHAR(100) COMMENT '结束原因', + `finish_time` DATETIME COMMENT '订单结束时间', + `boot_mode` TINYINT DEFAULT '1' COMMENT '启动方式 0本地刷卡启动 1后台启动 2本地管理员启动', + `settlement_type` TINYINT DEFAULT '0' COMMENT '结算类型0余额结算 1电卡结算 2电卡结算+余额结算', + `is_parking_coupon` TINYINT DEFAULT '0' COMMENT '领取停车券状态 0未领取停车券 1领券停车券', + `control_source` VARCHAR(50) COMMENT '渠道 wx 微信, bk 后台 ,app ', + `exception_flag` TINYINT DEFAULT '0' COMMENT '订单异常标志 0正常 1电量负数 2 vin相同,在一个有交集的时间段内出现的订单异常 3电量异常修正后订单', + `exception_msg` VARCHAR(255) COMMENT '订单异常描述', + `report_time` DATETIME COMMENT '订单上报时间', + `com_service_fee` DECIMAL(10,2) DEFAULT '0.00' COMMENT '企业服务费', + `com_electric_fee` DECIMAL(10,2) DEFAULT '0.00' COMMENT '企业电费', + `com_total_fee` DECIMAL(10,2) DEFAULT '0.00' COMMENT '企业费用', + `com_charge_discount_type` TINYINT COMMENT '企业结算折扣方式 充电优惠类型 0 服务费折扣（0%）； 1 固定服务费（元/度）；2整体折扣 （0%）3固定价格基础上服务费在进行折扣 4固定价格（电费单价+服务费单价）；', + `com_charge_discount` DOUBLE DEFAULT '0.00' COMMENT '优惠折扣', + `has_shallow_report` TINYINT DEFAULT '0' COMMENT '有无浅检报告 0无 1有', + `has_deep_report` TINYINT DEFAULT '0' COMMENT '有无深检报告 0无 1有', + `is_exception` TINYINT DEFAULT '0' COMMENT '是否为异常订单 0否 1是', + `shallow_report_is_received` TINYINT DEFAULT '0' COMMENT '是否已领取浅检报告 0未领取 1已领取', + `create_time` DATETIME COMMENT '创建时间', + `update_time` DATETIME COMMENT '更新时间', + `charge_market_id` BIGINT, + `coupon_total_amount` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '总优惠金额', + `coupon_service_total_amount` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '总优惠服务费', + `coupon_elecfee_total_amount` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '总优惠服务费', + `activity_name` VARCHAR(255) NOT NULL DEFAULT '' COMMENT '活动名称', + `activity_total_fee` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '活动时优惠了的总金额[充电折扣活动]', + `company_activity_id` BIGINT COMMENT '企业活动id', + `company_activity_name` VARCHAR(1000) COMMENT '企业活动名称', + `company_activity_total_fee` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '企业优惠总金额', + `company_activity_service_fee` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '企业优惠服务费', + `company_activity_electric_fee` DECIMAL(20,7) NOT NULL DEFAULT '0.0000000' COMMENT '企业优惠电费', + `subsidy_degree` DECIMAL(20,3) NOT NULL DEFAULT '0.000' COMMENT '补贴度数', + `subsidy_fee` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '补贴金额', + `special_channel_order_source` TINYINT COMMENT '特殊渠道订单来源 1启明' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_equipment_charge_order_mysql.sql b/DataX/scheme/output/t_equipment_charge_order_mysql.sql new file mode 100644 index 0000000..05781aa --- /dev/null +++ b/DataX/scheme/output/t_equipment_charge_order_mysql.sql @@ -0,0 +1,114 @@ +CREATE TABLE `t_equipment_charge_order` ( + `id` bigint(20) NOT NULL auto_increment, + `user_id` bigint(20) NOT NULL COMMENT '用户id', + `company_id` bigint(20) COMMENT '企业id', + `station_id` bigint(20) COMMENT '充电站id', + `equipment_id` bigint(20) NOT NULL COMMENT '充电桩id', + `connector_id` bigint(20) NOT NULL COMMENT '充电枪id', + `user_card_no` varchar(32) COMMENT '用户充电卡号', + `state` tinyint(4) COMMENT '订单状态 0 创建中 1准备充电中 2 充电中 3完成 4 取消 5挂起 6需要手动补单 7结算中', + `operate_state` tinyint(4) DEFAULT '0' COMMENT '订单操作状态 0无 1启动充电 2结束充电 ', + `order_no` varchar(20) NOT NULL COMMENT '订单号', + `platform_order_no` varchar(50) COMMENT '平台订单号【互通互联】', + `platform_sale_type` tinyint(4) DEFAULT '1' COMMENT '平台转售模式【互通互联】1：转售模式（该模式下由用户运营商向用户开票，由设备运营商给用户运营商开票）2：代收代付模式（该模式下，由设备运营商直接给用户开票）', + `platform_driver_id` varchar(50) COMMENT '平台用户ID【互通互联】', + `platform_phone` varchar(50) COMMENT '平台用户手机号【互通互联】', + `charging_amt` decimal(20,2) DEFAULT '0.00' COMMENT '预充金额，电量充到该金额自动停止充电', + `order_type` tinyint(4) COMMENT '订单类型 1普通用户订单 3 企业用户订单 4第三方平台订单', + `order_time` datetime COMMENT '订单时间', + `charge_begin_time` datetime COMMENT '充电开始时间', + `charge_end_time` datetime COMMENT '充电结束时间', + `charge_begin_degree` double(10,2) DEFAULT '0.00' COMMENT '充电开始度数', + `charge_end_degree` double(10,2) DEFAULT '0.00' COMMENT '充电结束度数', + `charge_degree` double(10,3) DEFAULT '0.000' COMMENT '充电度数', + `charge_ah` double(50,2) DEFAULT '0.00' COMMENT '充电电量', + `charge_times_degree` varchar(1000) COMMENT '充电分时度数', + `charge_begin_soc` int(20) DEFAULT '0' COMMENT '充电开始soc', + `charge_end_soc` int(20) DEFAULT '0' COMMENT '充电结束soc', + `charge_cur_soc` int(20) DEFAULT '0' COMMENT '充电当前soc', + `charge_duration` int(20) DEFAULT '0' COMMENT '充电时长', + `charge_settle_type` tinyint(4), + `charge_unit_price` decimal(10,7) COMMENT '电费单价', + `charge_unit_cost` decimal(10,7) COMMENT '电费成本单价', + `charge_unit_service_fee` decimal(20,7) COMMENT '服务费单价', + `receivable_electric_fee` decimal(20,7) DEFAULT '0.0000000' COMMENT '应收电费', + `receivable_service_fee` decimal(20,7) DEFAULT '0.0000000' COMMENT '应收服务费', + `receivable_total_fee` decimal(20,7) DEFAULT '0.0000000' COMMENT '应收总金额', + `charge_elecfee_amount` decimal(20,7) DEFAULT '0.0000000' COMMENT '电费(实收电费)', + `charge_elecfee_cost_amount` decimal(20,7) DEFAULT '0.0000000' COMMENT '电费成本', + `charge_service_amount` decimal(20,7) DEFAULT '0.0000000' COMMENT '服务费(实收服务费)', + `actual_pay_amount` decimal(20,2) DEFAULT '0.00' COMMENT '实际支付金额', + `pay_order_no` varchar(30) COMMENT '支付订单号', + `pay_amount` decimal(20,2) DEFAULT '0.00' COMMENT '本次订单付款金额', + `pay_method` tinyint(4) COMMENT '支付方式 0 余额支付 1 支付宝 2微信小程序 3微信app支付', + `pay_status` tinyint(4) DEFAULT '0' COMMENT '支付状态 0未支付 1待支付 2已支付 3支付失败', + `pay_time` datetime COMMENT '支付时间', + `user_coupon_id` bigint(20) COMMENT '用户 优惠劵id', + `coupon_name` varchar(30) COMMENT '优惠券名称', + `coupon_type` tinyint(4) COMMENT '优惠券类型', + `coupon_amount` decimal(20,2) DEFAULT '0.00' COMMENT '优惠券总金额', + `elecfee_coupon_amount` decimal(20,2) DEFAULT '0.00' COMMENT '电费优惠券金额', + `servicefee_coupon_amount` decimal(20,2) DEFAULT '0.00' COMMENT '服务费优惠券金额', + `invoice_id` bigint(20) COMMENT '发票id', + `invoice_fee` decimal(20,2) DEFAULT '0.00' COMMENT '可以开票金额', + `ticket_status` tinyint(4) DEFAULT '0' COMMENT '0未开票 1代开票 2已开票', + `activity_electric_fee` decimal(20,7) DEFAULT '0.0000000' COMMENT '活动时优惠了的电费 [降价活动]', + `activity_service_fee` decimal(20,2) DEFAULT '0.00' COMMENT '活动时优惠了的服务费[降价活动]', + `charge_plate_no` varchar(20) COMMENT '车牌号', + `charge_vin` varchar(100) COMMENT '车辆vin码', + `charge_strategy` varchar(20) COMMENT '充电策略', + `charge_strategy_param` varchar(30) COMMENT '充电策略参数', + `operator_id` bigint(20) COMMENT '运营商id', + `operator_income` decimal(20,2) DEFAULT '0.00' COMMENT '运营商收入金额', + `plat_service_fee` decimal(20,2) DEFAULT '0.00' COMMENT '平台服务费', + `charge_discount_type` tinyint(4) DEFAULT '0' COMMENT '充电优惠类型 0 服务费折扣（0%） 1 固定服务费（元/度）2整体折扣 （0%）3固定价格在进行折扣 4固定电费服务费价格', + `settle_electric_fee` decimal(20,2) DEFAULT '0.00' COMMENT '算结电费', + `settle_service_fee` decimal(20,2) DEFAULT '0.00' COMMENT '算结服务费', + `settle_fee` decimal(20,2) DEFAULT '0.00' COMMENT '结算总费用', + `settle_method` tinyint(4) DEFAULT '2' COMMENT '0预付 1 后结 2实时', + `settle_coupon_amount` decimal(20,2) DEFAULT '0.00' COMMENT '算结优惠金额', + `settle_coupon_electric_fee` decimal(20,2) DEFAULT '0.00' COMMENT '结算电费金额', + `settle_coupon_service_fee` decimal(20,2) DEFAULT '0.00' COMMENT '算结服务费', + `settle_status` tinyint(4) DEFAULT '0' COMMENT '是否开启结算 0关闭 1开通', + `cancel_type` int(4) COMMENT '取消类型 1（后台异常取消） 2（设备异常取消）,3(订单挂起),4(已结束订单又被唤起)', + `cancel_code` int(4) COMMENT '0 创建中订单(状态0)超过60秒自动取消, 1 创建中订单(状态1)超过60秒自动取消,2 充电中订单(状态2)超过24小时自动取消,3 充电中订单(开始时间异常)超过3分钟自动取消,4卡单，5订单挂起 6已结束订单又被唤起为充电中', + `cancel_msg` varchar(100) COMMENT '取消原因', + `cancel_time` datetime COMMENT '取消时间', + `finish_type` int(10) COMMENT '结束类型', + `finish_code` int(10) COMMENT '结束编码', + `finish_msg` varchar(100) COMMENT '结束原因', + `finish_time` datetime COMMENT '订单结束时间', + `boot_mode` tinyint(4) DEFAULT '1' COMMENT '启动方式 0本地刷卡启动 1后台启动 2本地管理员启动', + `settlement_type` tinyint(4) DEFAULT '0' COMMENT '结算类型0余额结算 1电卡结算 2电卡结算+余额结算', + `is_parking_coupon` tinyint(4) DEFAULT '0' COMMENT '领取停车券状态 0未领取停车券 1领券停车券', + `control_source` varchar(50) COMMENT '渠道 wx 微信, bk 后台 ,app ', + `exception_flag` tinyint(4) DEFAULT '0' COMMENT '订单异常标志 0正常 1电量负数 2 vin相同,在一个有交集的时间段内出现的订单异常 3电量异常修正后订单', + `exception_msg` varchar(255) COMMENT '订单异常描述', + `report_time` datetime COMMENT '订单上报时间', + `com_service_fee` decimal(10,2) DEFAULT '0.00' COMMENT '企业服务费', + `com_electric_fee` decimal(10,2) DEFAULT '0.00' COMMENT '企业电费', + `com_total_fee` decimal(10,2) DEFAULT '0.00' COMMENT '企业费用', + `com_charge_discount_type` tinyint(4) COMMENT '企业结算折扣方式 充电优惠类型 0 服务费折扣（0%）； 1 固定服务费（元/度）；2整体折扣 （0%）3固定价格基础上服务费在进行折扣 4固定价格（电费单价+服务费单价）；', + `com_charge_discount` double(10,2) DEFAULT '0.00' COMMENT '优惠折扣', + `has_shallow_report` tinyint(4) DEFAULT '0' COMMENT '有无浅检报告 0无 1有', + `has_deep_report` tinyint(4) DEFAULT '0' COMMENT '有无深检报告 0无 1有', + `is_exception` tinyint(4) DEFAULT '0' COMMENT '是否为异常订单 0否 1是', + `shallow_report_is_received` tinyint(4) DEFAULT '0' COMMENT '是否已领取浅检报告 0未领取 1已领取', + `create_time` datetime COMMENT '创建时间', + `update_time` datetime COMMENT '更新时间', + `charge_market_id` bigint(20), + `coupon_total_amount` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '总优惠金额', + `coupon_service_total_amount` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '总优惠服务费', + `coupon_elecfee_total_amount` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '总优惠服务费', + `activity_name` varchar(255) NOT NULL DEFAULT '' COMMENT '活动名称', + `activity_total_fee` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '活动时优惠了的总金额[充电折扣活动]', + `company_activity_id` bigint(20) COMMENT '企业活动id', + `company_activity_name` varchar(1000) COMMENT '企业活动名称', + `company_activity_total_fee` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '企业优惠总金额', + `company_activity_service_fee` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '企业优惠服务费', + `company_activity_electric_fee` decimal(20,7) NOT NULL DEFAULT '0.0000000' COMMENT '企业优惠电费', + `subsidy_degree` decimal(20,3) NOT NULL DEFAULT '0.000' COMMENT '补贴度数', + `subsidy_fee` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '补贴金额', + `special_channel_order_source` tinyint(4) COMMENT '特殊渠道订单来源 1启明', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_equipment_doris.sql b/DataX/scheme/output/t_equipment_doris.sql new file mode 100644 index 0000000..b39ed3e --- /dev/null +++ b/DataX/scheme/output/t_equipment_doris.sql @@ -0,0 +1,37 @@ +CREATE TABLE `t_equipment` ( + `id` BIGINT NOT NULL COMMENT '流水号', + `station_id` BIGINT COMMENT '所属充电站编码', + `agreement_id` TINYINT COMMENT '对接协议id', + `equipment_name` VARCHAR(50) COMMENT '设备名称', + `equipment_sn` VARCHAR(30) NOT NULL COMMENT '设备编码', + `equipment_short_sn` VARCHAR(30) NOT NULL COMMENT '设备短编号(二维码)', + `equipment_no` BIGINT COMMENT '充电桩编号', + `manufacturer_id` VARCHAR(100) COMMENT '设备生产商组织机构代码', + `manufacturer_sn` VARCHAR(32) COMMENT '设备出厂编号', + `manufacturer_name` VARCHAR(30) COMMENT '设备生产商名称', + `equipment_model` VARCHAR(30) COMMENT '设备型号', + `production_date` DATETIME COMMENT '生产日期', + `equipment_type` TINYINT NOT NULL COMMENT '设备类型1：直流设备；2：交流设备；3：交直流一体设备；4：无线设备；5：其它；6：V2G；7：有序', + `equipment_type_msg` VARCHAR(20) COMMENT '设备类型解释', + `equipment_lng` DOUBLE COMMENT '经度', + `equipment_lat` DOUBLE COMMENT '纬度', + `connector_count` TINYINT DEFAULT '0' COMMENT '充电枪数量', + `power` DOUBLE NOT NULL DEFAULT '0.0' COMMENT '设备总功率', + `national_standard` TINYINT DEFAULT '2' COMMENT '国家标准 1:2011 2:2015 3:兼容2011和2015', + `national_standard_msg` VARCHAR(20) COMMENT '国家标准解释', + `status` TINYINT COMMENT '充电桩状态：0：未知；1：建设中；5：关闭下线；6：维护中；50：正常使用', + `equipment_business_type` TINYINT DEFAULT '1' COMMENT '充电桩运营类型 1：公共；50：个人；100：公交专用；101：环卫专用；102：物流专用103：出租车专用；255：其它', + `equipment_business_type_msg` VARCHAR(20) COMMENT '充电桩运营类型 ', + `is_use_card` TINYINT NOT NULL DEFAULT '0' COMMENT '0不允许刷卡 1允许刷卡', + `is_use_card_msg` VARCHAR(20) COMMENT '刷卡允许解释', + `charge_host_esn` VARCHAR(255) DEFAULT '' COMMENT '充电设备ESN', + `create_time` DATETIME COMMENT '创建时间', + `update_time` DATETIME COMMENT '更新时间', + `aux_power` TINYINT DEFAULT '1' COMMENT '辅助电源 1:12v 2:24v 3:兼容12v和24v', + `app_show` TINYINT DEFAULT '1' COMMENT '1:展示 其他不展示' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_equipment_mysql.sql b/DataX/scheme/output/t_equipment_mysql.sql new file mode 100644 index 0000000..6e1bb9b --- /dev/null +++ b/DataX/scheme/output/t_equipment_mysql.sql @@ -0,0 +1,33 @@ +CREATE TABLE `t_equipment` ( + `id` bigint(20) unsigned NOT NULL auto_increment COMMENT '流水号', + `station_id` bigint(20) COMMENT '所属充电站编码', + `agreement_id` tinyint(4) COMMENT '对接协议id', + `equipment_name` varchar(50) COMMENT '设备名称', + `equipment_sn` varchar(30) NOT NULL COMMENT '设备编码', + `equipment_short_sn` varchar(30) NOT NULL COMMENT '设备短编号(二维码)', + `equipment_no` bigint(20) COMMENT '充电桩编号', + `manufacturer_id` varchar(100) COMMENT '设备生产商组织机构代码', + `manufacturer_sn` varchar(32) COMMENT '设备出厂编号', + `manufacturer_name` varchar(30) COMMENT '设备生产商名称', + `equipment_model` varchar(30) COMMENT '设备型号', + `production_date` datetime COMMENT '生产日期', + `equipment_type` tinyint(4) NOT NULL COMMENT '设备类型1：直流设备；2：交流设备；3：交直流一体设备；4：无线设备；5：其它；6：V2G；7：有序', + `equipment_type_msg` varchar(20) COMMENT '设备类型解释', + `equipment_lng` double(10,6) COMMENT '经度', + `equipment_lat` double(10,6) COMMENT '纬度', + `connector_count` tinyint(4) DEFAULT '0' COMMENT '充电枪数量', + `power` double(8,1) NOT NULL DEFAULT '0.0' COMMENT '设备总功率', + `national_standard` tinyint(4) DEFAULT '2' COMMENT '国家标准 1:2011 2:2015 3:兼容2011和2015', + `national_standard_msg` varchar(20) COMMENT '国家标准解释', + `status` tinyint(2) COMMENT '充电桩状态：0：未知；1：建设中；5：关闭下线；6：维护中；50：正常使用', + `equipment_business_type` tinyint(4) DEFAULT '1' COMMENT '充电桩运营类型 1：公共；50：个人；100：公交专用；101：环卫专用；102：物流专用103：出租车专用；255：其它', + `equipment_business_type_msg` varchar(20) COMMENT '充电桩运营类型 ', + `is_use_card` tinyint(4) NOT NULL DEFAULT '0' COMMENT '0不允许刷卡 1允许刷卡', + `is_use_card_msg` varchar(20) COMMENT '刷卡允许解释', + `charge_host_esn` varchar(255) DEFAULT '' COMMENT '充电设备ESN', + `create_time` datetime COMMENT '创建时间', + `update_time` datetime COMMENT '更新时间', + `aux_power` tinyint(4) DEFAULT '1' COMMENT '辅助电源 1:12v 2:24v 3:兼容12v和24v', + `app_show` tinyint(4) DEFAULT '1' COMMENT '1:展示 其他不展示', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_ext_hurry_quit_doris.sql b/DataX/scheme/output/t_ext_hurry_quit_doris.sql new file mode 100644 index 0000000..ede6533 --- /dev/null +++ b/DataX/scheme/output/t_ext_hurry_quit_doris.sql @@ -0,0 +1,39 @@ +CREATE TABLE `t_ext_hurry_quit` ( + `id` INT NOT NULL COMMENT '主键', + `order_no` VARCHAR(255) NOT NULL COMMENT '订单号', + `telephone` VARCHAR(11) NOT NULL COMMENT '用户手机号', + `user_id` INT NOT NULL COMMENT '用户id', + `message1` VARCHAR(1024) NOT NULL COMMENT '第一次发送的短信内容', + `message2` VARCHAR(1024) COMMENT '第二次发送短信的内容', + `create_time` DATETIME NOT NULL COMMENT '第一次自动发送时间', + `station_id` INT NOT NULL COMMENT '场站号', + `equipment_id` INT NOT NULL COMMENT '充电桩号', + `connector_id` INT NOT NULL COMMENT '充电枪号', + `flag` INT NOT NULL DEFAULT '0' COMMENT '1:发送1轮短信 +2:发送2轮短信 +3:待结算 +4:已结算，-1: 重新开始充电，占桩费重新计算 ,-2:用户在规定时间内离开，停止计费', + `close_time` DATETIME COMMENT '关闭计费时间', + `finish_time` DATETIME COMMENT '用户拔枪时间', + `minutesDifference` INT COMMENT '占位时长，单位分钟', + `reduce_money` DECIMAL(10,2) COMMENT '扣款金额', + `beforeBalance` DECIMAL(10,2) COMMENT '扣款前金额', + `afterBalance` DECIMAL(10,2) COMMENT '扣款后金额', + `fee_station` INT NOT NULL DEFAULT '0' COMMENT '当时是不是收费停车场，1：是，0：否', + `manual_count` INT NOT NULL DEFAULT '0' COMMENT '手动发送短信数量', + `last_manual_time` DATETIME COMMENT '最后一次手动发送短信的时间', + `last_manual_message` VARCHAR(1024) COMMENT '最后一次手动发送短信的内容', + `person_id` CHAR(36) COMMENT '最后一次手动发送短的人员ID', + `refund_status` INT DEFAULT '0' COMMENT '是否已退款', + `refund_time` DATETIME COMMENT '退款时间', + `refund_person_id` CHAR(36) COMMENT '退款人ID', + `refund_person_name` VARCHAR(255) COMMENT '退款人姓名', + `refund_amount` DECIMAL(10,2) COMMENT '退款金额', + `refund_type` VARCHAR(255) COMMENT '退款类型：全额，部分', + `unplug_time` DATETIME COMMENT '实际拔枪时间' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_ext_hurry_quit_mysql.sql b/DataX/scheme/output/t_ext_hurry_quit_mysql.sql new file mode 100644 index 0000000..0f8cc0d --- /dev/null +++ b/DataX/scheme/output/t_ext_hurry_quit_mysql.sql @@ -0,0 +1,35 @@ +CREATE TABLE `t_ext_hurry_quit` ( + `id` int(11) NOT NULL auto_increment COMMENT '主键', + `order_no` varchar(255) NOT NULL COMMENT '订单号', + `telephone` varchar(11) NOT NULL COMMENT '用户手机号', + `user_id` int(11) NOT NULL COMMENT '用户id', + `message1` varchar(1024) NOT NULL COMMENT '第一次发送的短信内容', + `message2` varchar(1024) COMMENT '第二次发送短信的内容', + `create_time` datetime NOT NULL COMMENT '第一次自动发送时间', + `station_id` int(11) NOT NULL COMMENT '场站号', + `equipment_id` int(11) NOT NULL COMMENT '充电桩号', + `connector_id` int(11) NOT NULL COMMENT '充电枪号', + `flag` int(11) NOT NULL DEFAULT '0' COMMENT '1:发送1轮短信 +2:发送2轮短信 +3:待结算 +4:已结算，-1: 重新开始充电，占桩费重新计算 ,-2:用户在规定时间内离开，停止计费', + `close_time` datetime COMMENT '关闭计费时间', + `finish_time` datetime COMMENT '用户拔枪时间', + `minutesDifference` int(11) COMMENT '占位时长，单位分钟', + `reduce_money` decimal(10,2) COMMENT '扣款金额', + `beforeBalance` decimal(10,2) COMMENT '扣款前金额', + `afterBalance` decimal(10,2) COMMENT '扣款后金额', + `fee_station` int(11) NOT NULL DEFAULT '0' COMMENT '当时是不是收费停车场，1：是，0：否', + `manual_count` int(11) NOT NULL DEFAULT '0' COMMENT '手动发送短信数量', + `last_manual_time` datetime COMMENT '最后一次手动发送短信的时间', + `last_manual_message` varchar(1024) COMMENT '最后一次手动发送短信的内容', + `person_id` char(36) COMMENT '最后一次手动发送短的人员ID', + `refund_status` int(11) DEFAULT '0' COMMENT '是否已退款', + `refund_time` datetime COMMENT '退款时间', + `refund_person_id` char(36) COMMENT '退款人ID', + `refund_person_name` varchar(255) COMMENT '退款人姓名', + `refund_amount` decimal(10,2) COMMENT '退款金额', + `refund_type` varchar(255) COMMENT '退款类型：全额，部分', + `unplug_time` datetime COMMENT '实际拔枪时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_station_doris.sql b/DataX/scheme/output/t_station_doris.sql new file mode 100644 index 0000000..04bbd6c --- /dev/null +++ b/DataX/scheme/output/t_station_doris.sql @@ -0,0 +1,113 @@ +CREATE TABLE `t_station` ( + `id` BIGINT NOT NULL COMMENT '流水号', + `station_sn` VARCHAR(50) NOT NULL COMMENT '充电站ID', + `operator_id` BIGINT NOT NULL COMMENT '运营商Id', + `equioment_owner_id` VARCHAR(50) NOT NULL COMMENT '设备所属方编码', + `station_name` VARCHAR(50) NOT NULL COMMENT '充电站名称', + `country_code` VARCHAR(2) NOT NULL COMMENT '国家代码', + `region_id` BIGINT COMMENT '省市辖区Id', + `region_code` VARCHAR(20) NOT NULL COMMENT '省市辖区编码', + `regional_name` VARCHAR(150) COMMENT '站点所在的区域名称', + `regional_code` VARCHAR(150) COMMENT '站点区域编码', + `address` VARCHAR(50) NOT NULL COMMENT '详细地址', + `station_tel` VARCHAR(30) COMMENT '站点电话', + `service_tel` VARCHAR(30) NOT NULL COMMENT '客服电话', + `station_type` SMALLINT NOT NULL COMMENT '站点类型 +1：公共； +50：个人； +100：公交专用； +101：环卫专用； +102：物流专用 +103：出租车专用； +255：其它 104:分时租赁（专用） 105:小区共享（专用） 106:单位（专用） 107:私人共享桩（专用）', + `station_type_msg` VARCHAR(20) COMMENT '站点类型解释', + `station_business_type` TINYINT DEFAULT '0' COMMENT '充电站运营类型 0自运营 1 加盟 2合资 3代运营 4=互联接入', + `station_business_type_msg` VARCHAR(20) COMMENT '充电站运营类型解释', + `station_status` SMALLINT NOT NULL COMMENT '站点状态 +0：未知； +1：建设中； +5：关闭下线； +6：维护中； +50：正常使用', + `station_status_msg` VARCHAR(20) COMMENT '站点状态解释', + `park_nums` SMALLINT NOT NULL COMMENT '车位数量', + `station_lng` DOUBLE NOT NULL COMMENT '经度', + `station_lat` DOUBLE NOT NULL COMMENT '纬度', + `site_guide` VARCHAR(100) COMMENT '站点引导', + `construction` SMALLINT NOT NULL COMMENT '建设场所 +1：居民区 ； +2：公共机构； +3：企事业单位； +4：写字楼； +5：工业园区； +6：交通枢纽； +7：大型文体设施； +8：城市绿地； +9：大型建筑配建停车场； +10：路边停车位； +11：城际高速服务区； +255：其它 12:风景区 13:公交场站 14:加油加气站 15:出租车', + `construction_msg` VARCHAR(20) COMMENT '充电站建设场所解释', + `pictures` STRING COMMENT '站点照片', + `match_cars` VARCHAR(100) COMMENT '使用车型描述', + `park_info` VARCHAR(100) COMMENT '车位楼层及数量描述', + `business_start_time` SMALLINT COMMENT '营业开始时间', + `business_end_time` SMALLINT COMMENT '营业结束时间', + `electricity_fee` VARCHAR(256) COMMENT '充电费率描述', + `service_fee` VARCHAR(100) COMMENT '服务费率描述', + `is_park_fee` TINYINT DEFAULT '0' COMMENT '是否停车免费 0停车收费 1限时免费停车 2免费 3充电限时减免 10参考场地实际收费标准', + `park_fee` VARCHAR(100) COMMENT '停车费描述', + `payment` TINYINT DEFAULT '1' COMMENT '支付方式：1余额支付 2 线上支付', + `payment_msg` VARCHAR(20) COMMENT '支付方式解释', + `support_order` TINYINT DEFAULT '0' COMMENT '是否支持预约 0：不支持；1：支持', + `support_order_msg` VARCHAR(20) COMMENT '是否支持预约解释', + `service_item` VARCHAR(50) COMMENT '提供的服务 卫生间,休息室,便利店,有人值守,雨棚', + `service_item_image_url` VARCHAR(300) COMMENT '服务项目图片url', + `service_item_msg` VARCHAR(100) COMMENT '提供的服务解释', + `station_location_type` TINYINT DEFAULT '1' COMMENT '站点位置类型 1 地上 2地下', + `station_location_type_msg` VARCHAR(20) COMMENT '站点位置类型', + `station_charge_type` TINYINT DEFAULT '1' COMMENT '站点充电类型 1 快充 2慢充', + `station_charge_type_msg` VARCHAR(20) COMMENT '站点充电类型 ', + `station_power` VARCHAR(30) COMMENT '充电站功率', + `station_voltage` VARCHAR(30) COMMENT '充电站电压', + `station_current` VARCHAR(30) COMMENT '充电站电流', + `is_open` TINYINT DEFAULT '0' COMMENT '对外开放 0 不开放 1开放', + `is_open_msg` VARCHAR(20) COMMENT '对外开放解释', + `is_display` TINYINT DEFAULT '0' COMMENT '地图上是否显示 0不显示 1显示', + `is_display_msg` VARCHAR(20) COMMENT '地图上是否显示解释', + `sharing_code` VARCHAR(250) COMMENT '二维码图片七牛地址', + `sharing_applet_picture` VARCHAR(250) COMMENT '分享小程序节点大图', + `theme_picture` VARCHAR(150) COMMENT '充电站主题图片', + `is_barrier_flag` TINYINT DEFAULT '0' COMMENT '是否有道闸 0无 1有', + `is_barrier_flag_msg` VARCHAR(20) COMMENT '是否有道闸解释', + `charge_settle_type` TINYINT DEFAULT '1' COMMENT '结算类型 1 全时段电费 2 分时段电费', + `charge_settle_type_msg` VARCHAR(20) COMMENT '结算类型 1 全时段电费 2 分时段电费', + `fast_connector_nums` INT DEFAULT '0' COMMENT '快充枪数量', + `slow_connector_nums` INT DEFAULT '0' COMMENT '慢充枪数量', + `transformer_capacity` DOUBLE DEFAULT '0.000' COMMENT '变压器容量', + `installed_capacity` DOUBLE DEFAULT '0.000' COMMENT '装机容量', + `remark` VARCHAR(100) COMMENT '备注', + `record_unique_no` VARCHAR(24) NOT NULL DEFAULT '' COMMENT '监管备案号(电站全省唯一备案号)', + `equipment_owner_name` VARCHAR(255) NOT NULL DEFAULT '' COMMENT '设备运营商名称', + `supply_type` TINYINT NOT NULL DEFAULT '1' COMMENT '供电类型 1.直供电 2.转供电', + `resident_no` VARCHAR(12) NOT NULL DEFAULT '' COMMENT '供电局户号', + `watt_hour_meter_no` VARCHAR(24) NOT NULL DEFAULT '' COMMENT '供电表号', + `forward_power` VARCHAR(10) NOT NULL DEFAULT '' COMMENT '外电功率，保留小数点后两位，单位千瓦', + `operation_date` DATETIME NOT NULL DEFAULT '2020-01-01 00:00:00' COMMENT '投运日期', + `charge_replace_type` TINYINT NOT NULL DEFAULT '0001' COMMENT '充换电类型 1充电 2换电 3充换电一体', + `is_hlht` TINYINT NOT NULL DEFAULT '0' COMMENT '是否是互联互通站点 1是 0不是', + `is_subsidy` TINYINT NOT NULL DEFAULT '0' COMMENT '是否参与补贴 0否 1是', + `create_time` DATETIME COMMENT '创建时间', + `update_time` DATETIME COMMENT '更新时间', + `area_code_countryside` VARCHAR(20) DEFAULT '111111111111' COMMENT '充换电站所在县以下行政区划代码', + `business_expand_type` TINYINT DEFAULT '0' COMMENT '报装类型,是否独立报装，0 否 1是', + `capacity` DOUBLE NOT NULL DEFAULT '0.00' COMMENT '报装电源容量', + `video_monitor` TINYINT DEFAULT '1' COMMENT '视频监控配套情况,0无 1有', + `electricity_type` TINYINT DEFAULT '1' COMMENT '电费类型 1：商业用电 2：普通工业用电 3：大工业用电 4：其他用电', + `receive_shop` TINYINT NOT NULL DEFAULT '0' COMMENT '是不是商城领取商品的场站，1：是，0：否' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_station_mysql.sql b/DataX/scheme/output/t_station_mysql.sql new file mode 100644 index 0000000..fb43487 --- /dev/null +++ b/DataX/scheme/output/t_station_mysql.sql @@ -0,0 +1,109 @@ +CREATE TABLE `t_station` ( + `id` bigint(20) unsigned NOT NULL auto_increment COMMENT '流水号', + `station_sn` varchar(50) NOT NULL COMMENT '充电站ID', + `operator_id` bigint(20) NOT NULL COMMENT '运营商Id', + `equioment_owner_id` varchar(50) NOT NULL COMMENT '设备所属方编码', + `station_name` varchar(50) NOT NULL COMMENT '充电站名称', + `country_code` varchar(2) NOT NULL COMMENT '国家代码', + `region_id` bigint(20) COMMENT '省市辖区Id', + `region_code` varchar(20) NOT NULL COMMENT '省市辖区编码', + `regional_name` varchar(150) COMMENT '站点所在的区域名称', + `regional_code` varchar(150) COMMENT '站点区域编码', + `address` varchar(50) NOT NULL COMMENT '详细地址', + `station_tel` varchar(30) COMMENT '站点电话', + `service_tel` varchar(30) NOT NULL COMMENT '客服电话', + `station_type` smallint(4) NOT NULL COMMENT '站点类型 +1：公共； +50：个人； +100：公交专用； +101：环卫专用； +102：物流专用 +103：出租车专用； +255：其它 104:分时租赁（专用） 105:小区共享（专用） 106:单位（专用） 107:私人共享桩（专用）', + `station_type_msg` varchar(20) COMMENT '站点类型解释', + `station_business_type` tinyint(4) DEFAULT '0' COMMENT '充电站运营类型 0自运营 1 加盟 2合资 3代运营 4=互联接入', + `station_business_type_msg` varchar(20) COMMENT '充电站运营类型解释', + `station_status` smallint(4) NOT NULL COMMENT '站点状态 +0：未知； +1：建设中； +5：关闭下线； +6：维护中； +50：正常使用', + `station_status_msg` varchar(20) COMMENT '站点状态解释', + `park_nums` smallint(4) NOT NULL COMMENT '车位数量', + `station_lng` double(10,6) NOT NULL COMMENT '经度', + `station_lat` double(10,6) NOT NULL COMMENT '纬度', + `site_guide` varchar(100) COMMENT '站点引导', + `construction` smallint(4) NOT NULL COMMENT '建设场所 +1：居民区 ； +2：公共机构； +3：企事业单位； +4：写字楼； +5：工业园区； +6：交通枢纽； +7：大型文体设施； +8：城市绿地； +9：大型建筑配建停车场； +10：路边停车位； +11：城际高速服务区； +255：其它 12:风景区 13:公交场站 14:加油加气站 15:出租车', + `construction_msg` varchar(20) COMMENT '充电站建设场所解释', + `pictures` text COMMENT '站点照片', + `match_cars` varchar(100) COMMENT '使用车型描述', + `park_info` varchar(100) COMMENT '车位楼层及数量描述', + `business_start_time` smallint(4) COMMENT '营业开始时间', + `business_end_time` smallint(4) COMMENT '营业结束时间', + `electricity_fee` varchar(256) COMMENT '充电费率描述', + `service_fee` varchar(100) COMMENT '服务费率描述', + `is_park_fee` tinyint(4) DEFAULT '0' COMMENT '是否停车免费 0停车收费 1限时免费停车 2免费 3充电限时减免 10参考场地实际收费标准', + `park_fee` varchar(100) COMMENT '停车费描述', + `payment` tinyint(4) DEFAULT '1' COMMENT '支付方式：1余额支付 2 线上支付', + `payment_msg` varchar(20) COMMENT '支付方式解释', + `support_order` tinyint(4) DEFAULT '0' COMMENT '是否支持预约 0：不支持；1：支持', + `support_order_msg` varchar(20) COMMENT '是否支持预约解释', + `service_item` varchar(50) COMMENT '提供的服务 卫生间,休息室,便利店,有人值守,雨棚', + `service_item_image_url` varchar(300) COMMENT '服务项目图片url', + `service_item_msg` varchar(100) COMMENT '提供的服务解释', + `station_location_type` tinyint(4) DEFAULT '1' COMMENT '站点位置类型 1 地上 2地下', + `station_location_type_msg` varchar(20) COMMENT '站点位置类型', + `station_charge_type` tinyint(4) DEFAULT '1' COMMENT '站点充电类型 1 快充 2慢充', + `station_charge_type_msg` varchar(20) COMMENT '站点充电类型 ', + `station_power` varchar(30) COMMENT '充电站功率', + `station_voltage` varchar(30) COMMENT '充电站电压', + `station_current` varchar(30) COMMENT '充电站电流', + `is_open` tinyint(4) DEFAULT '0' COMMENT '对外开放 0 不开放 1开放', + `is_open_msg` varchar(20) COMMENT '对外开放解释', + `is_display` tinyint(4) DEFAULT '0' COMMENT '地图上是否显示 0不显示 1显示', + `is_display_msg` varchar(20) COMMENT '地图上是否显示解释', + `sharing_code` varchar(250) COMMENT '二维码图片七牛地址', + `sharing_applet_picture` varchar(250) COMMENT '分享小程序节点大图', + `theme_picture` varchar(150) COMMENT '充电站主题图片', + `is_barrier_flag` tinyint(4) unsigned DEFAULT '0' COMMENT '是否有道闸 0无 1有', + `is_barrier_flag_msg` varchar(20) COMMENT '是否有道闸解释', + `charge_settle_type` tinyint(4) DEFAULT '1' COMMENT '结算类型 1 全时段电费 2 分时段电费', + `charge_settle_type_msg` varchar(20) COMMENT '结算类型 1 全时段电费 2 分时段电费', + `fast_connector_nums` int(50) DEFAULT '0' COMMENT '快充枪数量', + `slow_connector_nums` int(50) DEFAULT '0' COMMENT '慢充枪数量', + `transformer_capacity` double(50,3) DEFAULT '0.000' COMMENT '变压器容量', + `installed_capacity` double(50,3) DEFAULT '0.000' COMMENT '装机容量', + `remark` varchar(100) COMMENT '备注', + `record_unique_no` varchar(24) NOT NULL DEFAULT '' COMMENT '监管备案号(电站全省唯一备案号)', + `equipment_owner_name` varchar(255) NOT NULL DEFAULT '' COMMENT '设备运营商名称', + `supply_type` tinyint(4) NOT NULL DEFAULT '1' COMMENT '供电类型 1.直供电 2.转供电', + `resident_no` varchar(12) NOT NULL DEFAULT '' COMMENT '供电局户号', + `watt_hour_meter_no` varchar(24) NOT NULL DEFAULT '' COMMENT '供电表号', + `forward_power` varchar(10) NOT NULL DEFAULT '' COMMENT '外电功率，保留小数点后两位，单位千瓦', + `operation_date` datetime NOT NULL DEFAULT '2020-01-01 00:00:00' COMMENT '投运日期', + `charge_replace_type` tinyint(4) unsigned zerofill NOT NULL DEFAULT '0001' COMMENT '充换电类型 1充电 2换电 3充换电一体', + `is_hlht` tinyint(4) NOT NULL DEFAULT '0' COMMENT '是否是互联互通站点 1是 0不是', + `is_subsidy` tinyint(4) NOT NULL DEFAULT '0' COMMENT '是否参与补贴 0否 1是', + `create_time` datetime COMMENT '创建时间', + `update_time` datetime COMMENT '更新时间', + `area_code_countryside` varchar(20) DEFAULT '111111111111' COMMENT '充换电站所在县以下行政区划代码', + `business_expand_type` tinyint(4) DEFAULT '0' COMMENT '报装类型,是否独立报装，0 否 1是', + `capacity` double(8,2) NOT NULL DEFAULT '0.00' COMMENT '报装电源容量', + `video_monitor` tinyint(4) DEFAULT '1' COMMENT '视频监控配套情况,0无 1有', + `electricity_type` tinyint(4) DEFAULT '1' COMMENT '电费类型 1：商业用电 2：普通工业用电 3：大工业用电 4：其他用电', + `receive_shop` tinyint(4) NOT NULL DEFAULT '0' COMMENT '是不是商城领取商品的场站，1：是，0：否', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_time_day_doris.sql b/DataX/scheme/output/t_time_day_doris.sql new file mode 100644 index 0000000..e5f1820 --- /dev/null +++ b/DataX/scheme/output/t_time_day_doris.sql @@ -0,0 +1,12 @@ +CREATE TABLE `t_time_day` ( + `id` BIGINT NOT NULL, + `create_time` DATETIME, + `update_time` DATETIME, + `time_fq` VARCHAR(24), + `teme_ve` VARCHAR(25) +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_time_day_mysql.sql b/DataX/scheme/output/t_time_day_mysql.sql new file mode 100644 index 0000000..7a218e2 --- /dev/null +++ b/DataX/scheme/output/t_time_day_mysql.sql @@ -0,0 +1,8 @@ +CREATE TABLE `t_time_day` ( + `id` bigint(20) NOT NULL auto_increment, + `create_time` datetime, + `update_time` datetime, + `time_fq` varchar(24), + `teme_ve` varchar(25), + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_user_account_doris.sql b/DataX/scheme/output/t_user_account_doris.sql new file mode 100644 index 0000000..efe794f --- /dev/null +++ b/DataX/scheme/output/t_user_account_doris.sql @@ -0,0 +1,24 @@ +CREATE TABLE `t_user_account` ( + `id` BIGINT NOT NULL, + `user_id` BIGINT NOT NULL, + `ua_balance` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '户用充值金额', + `ua_deposit` DECIMAL(20,2) DEFAULT '0.00' COMMENT '押金', + `ua_balance_time` DATETIME COMMENT '户用创建日期时间', + `ua_password` VARCHAR(32), + `ua_state` CHAR(1) NOT NULL DEFAULT '0' COMMENT '用户状态 0 可用', + `ua_real_balance` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '用户账户可用金额', + `ua_real_degree` DOUBLE DEFAULT '0.00' COMMENT '户用可用充电度数', + `create_time` DATETIME, + `update_time` DATETIME, + `donate_money` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '赠送金额', + `donate_able_money` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '赠送可用金额', + `ua_able_balance` DECIMAL(20,2) NOT NULL DEFAULT '0.00' COMMENT '账户可用金额', + `refund_money` DECIMAL(20,2) DEFAULT '0.00' COMMENT '退款金额', + `ua_consume_money` DECIMAL(20,2) DEFAULT '0.00' COMMENT '费消金额', + `ua_clear_zero_money` DECIMAL(20,2) DEFAULT '0.00' COMMENT '用户清零金额' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_user_account_mysql.sql b/DataX/scheme/output/t_user_account_mysql.sql new file mode 100644 index 0000000..27c78ec --- /dev/null +++ b/DataX/scheme/output/t_user_account_mysql.sql @@ -0,0 +1,20 @@ +CREATE TABLE `t_user_account` ( + `id` bigint(20) NOT NULL auto_increment, + `user_id` bigint(20) NOT NULL, + `ua_balance` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '户用充值金额', + `ua_deposit` decimal(20,2) unsigned DEFAULT '0.00' COMMENT '押金', + `ua_balance_time` datetime COMMENT '户用创建日期时间', + `ua_password` varchar(32), + `ua_state` char(1) NOT NULL DEFAULT '0' COMMENT '用户状态 0 可用', + `ua_real_balance` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '用户账户可用金额', + `ua_real_degree` double(20,2) DEFAULT '0.00' COMMENT '户用可用充电度数', + `create_time` datetime, + `update_time` datetime, + `donate_money` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '赠送金额', + `donate_able_money` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '赠送可用金额', + `ua_able_balance` decimal(20,2) NOT NULL DEFAULT '0.00' COMMENT '账户可用金额', + `refund_money` decimal(20,2) DEFAULT '0.00' COMMENT '退款金额', + `ua_consume_money` decimal(20,2) DEFAULT '0.00' COMMENT '费消金额', + `ua_clear_zero_money` decimal(20,2) DEFAULT '0.00' COMMENT '用户清零金额', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_user_doris.sql b/DataX/scheme/output/t_user_doris.sql new file mode 100644 index 0000000..72b0b1f --- /dev/null +++ b/DataX/scheme/output/t_user_doris.sql @@ -0,0 +1,52 @@ +CREATE TABLE `t_user` ( + `id` BIGINT NOT NULL, + `user_name` VARCHAR(100) COMMENT '用户名', + `nicker_name` VARCHAR(100) COMMENT '昵称（用于保存微信名）', + `sex` VARCHAR(10) COMMENT '性别 1 男 0 女 ', + `phone` VARCHAR(35) COMMENT '电话', + `qk_openid` VARCHAR(30) COMMENT '微信快捷登录的 openid （app）', + `open_id` VARCHAR(30) COMMENT '微信openid', + `ali_user_id` VARCHAR(30) COMMENT '支付宝小程序的 user_id', + `birthday` DATE, + `avatar` VARCHAR(200) COMMENT '头像url', + `app_type` VARCHAR(50) COMMENT 'app 类型 ios android (用于推送区分)', + `password` VARCHAR(50), + `upper_user` BIGINT COMMENT '上级企业用户 id', + `master_flag` INT COMMENT '是否是主账户标识 1 主账户 0 子账户', + `user_type` VARCHAR(10) NOT NULL COMMENT '区分用户 1普通用户 0代理商 3企业账户 2运营商', + `multi_charge` INT NOT NULL DEFAULT '1' COMMENT '一号多冲标识 默认1 ', + `relation_phone` VARCHAR(50) COMMENT '关联的账户电话号码 （针对企业用户）', + `user_owner_id` BIGINT COMMENT '第三方id(所属企业id)', + `union_id` VARCHAR(50) COMMENT '微信同一用户标识', + `state` VARCHAR(10) NOT NULL COMMENT '1 禁用 0启用 2刪除', + `reg_channel` VARCHAR(50) COMMENT '注册渠道 wx 微信 bk 后台 app', + `reg_time` DATETIME COMMENT '注册时间', + `city_label_code` VARCHAR(100) COMMENT '所属城市代码', + `city_label` VARCHAR(100) COMMENT '所属城市名称', + `reg_source` VARCHAR(100) COMMENT '注册来源 red_packet_share 红包分享 invite_new 新人邀请', + `create_time` DATETIME, + `update_time` DATETIME, + `operator` VARCHAR(50) COMMENT '操作人', + `invite_code` VARCHAR(10) COMMENT '邀请码', + `identity_id` VARCHAR(36) COMMENT '身份id (一般用户用手机号)', + `operator_id` BIGINT, + `address` VARCHAR(500) COMMENT '地址', + `vip_level` VARCHAR(20) DEFAULT '0' COMMENT '0 普通用户 vip1 会员', + `province_code` VARCHAR(50), + `province` VARCHAR(200), + `work_company` VARCHAR(500) COMMENT '工作单位', + `is_bind_wxinfo` INT DEFAULT '0' COMMENT '1 绑定 0 未绑定', + `company_min_charge_balance` DECIMAL(10,2) DEFAULT '0.00' COMMENT '企业用户最低充电限额', + `enable_online_recharge` INT DEFAULT '0' COMMENT '是否可在线充值（针对企业用户） 0 不允许 1 允许 ', + `integral` DECIMAL(10,2) DEFAULT '0.00' COMMENT '积分', + `shared_user` TINYINT DEFAULT '0' COMMENT '共享用户：0，不是，1，启明共享', + `enable_online_refund` INT DEFAULT '0' COMMENT '是否可在线退款（针对企业用户） 0 不允许 1 允许 ', + `recommenderId` BIGINT DEFAULT '0' COMMENT '推荐人ID', + `lx_wx_user_id` BIGINT DEFAULT '0' COMMENT '拉新人的user_id', + `phone_area` VARCHAR(255) COMMENT '手机号隶属行政区划' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_user_mysql.sql b/DataX/scheme/output/t_user_mysql.sql new file mode 100644 index 0000000..dc18127 --- /dev/null +++ b/DataX/scheme/output/t_user_mysql.sql @@ -0,0 +1,48 @@ +CREATE TABLE `t_user` ( + `id` bigint(20) NOT NULL auto_increment, + `user_name` varchar(100) COMMENT '用户名', + `nicker_name` varchar(100) COMMENT '昵称（用于保存微信名）', + `sex` varchar(10) COMMENT '性别 1 男 0 女 ', + `phone` varchar(35) COMMENT '电话', + `qk_openid` varchar(30) COMMENT '微信快捷登录的 openid （app）', + `open_id` varchar(30) COMMENT '微信openid', + `ali_user_id` varchar(30) COMMENT '支付宝小程序的 user_id', + `birthday` date, + `avatar` varchar(200) COMMENT '头像url', + `app_type` varchar(50) COMMENT 'app 类型 ios android (用于推送区分)', + `password` varchar(50), + `upper_user` bigint(20) COMMENT '上级企业用户 id', + `master_flag` int(4) COMMENT '是否是主账户标识 1 主账户 0 子账户', + `user_type` varchar(10) NOT NULL COMMENT '区分用户 1普通用户 0代理商 3企业账户 2运营商', + `multi_charge` int(10) NOT NULL DEFAULT '1' COMMENT '一号多冲标识 默认1 ', + `relation_phone` varchar(50) COMMENT '关联的账户电话号码 （针对企业用户）', + `user_owner_id` bigint(20) COMMENT '第三方id(所属企业id)', + `union_id` varchar(50) COMMENT '微信同一用户标识', + `state` varchar(10) NOT NULL COMMENT '1 禁用 0启用 2刪除', + `reg_channel` varchar(50) COMMENT '注册渠道 wx 微信 bk 后台 app', + `reg_time` datetime COMMENT '注册时间', + `city_label_code` varchar(100) COMMENT '所属城市代码', + `city_label` varchar(100) COMMENT '所属城市名称', + `reg_source` varchar(100) COMMENT '注册来源 red_packet_share 红包分享 invite_new 新人邀请', + `create_time` datetime, + `update_time` datetime, + `operator` varchar(50) COMMENT '操作人', + `invite_code` varchar(10) COMMENT '邀请码', + `identity_id` varchar(36) COMMENT '身份id (一般用户用手机号)', + `operator_id` bigint(20), + `address` varchar(500) COMMENT '地址', + `vip_level` varchar(20) DEFAULT '0' COMMENT '0 普通用户 vip1 会员', + `province_code` varchar(50), + `province` varchar(200), + `work_company` varchar(500) COMMENT '工作单位', + `is_bind_wxinfo` int(1) DEFAULT '0' COMMENT '1 绑定 0 未绑定', + `company_min_charge_balance` decimal(10,2) DEFAULT '0.00' COMMENT '企业用户最低充电限额', + `enable_online_recharge` int(2) DEFAULT '0' COMMENT '是否可在线充值（针对企业用户） 0 不允许 1 允许 ', + `integral` decimal(10,2) DEFAULT '0.00' COMMENT '积分', + `shared_user` tinyint(4) DEFAULT '0' COMMENT '共享用户：0，不是，1，启明共享', + `enable_online_refund` int(2) DEFAULT '0' COMMENT '是否可在线退款（针对企业用户） 0 不允许 1 允许 ', + `recommenderId` bigint(20) DEFAULT '0' COMMENT '推荐人ID', + `lx_wx_user_id` bigint(20) DEFAULT '0' COMMENT '拉新人的user_id', + `phone_area` varchar(255) COMMENT '手机号隶属行政区划', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/scheme/output/t_user_upload_fault_doris.sql b/DataX/scheme/output/t_user_upload_fault_doris.sql new file mode 100644 index 0000000..8f0573d --- /dev/null +++ b/DataX/scheme/output/t_user_upload_fault_doris.sql @@ -0,0 +1,14 @@ +CREATE TABLE `t_user_upload_fault` ( + `id` BIGINT NOT NULL, + `equipment_id` INT NOT NULL COMMENT '桩id', + `user_id` INT NOT NULL COMMENT '用户id', + `fault_config_id` INT COMMENT '故障原因id', + `remark` VARCHAR(200) DEFAULT '' COMMENT '备注', + `create_time` DATETIME COMMENT '创建时间', + `update_time` DATETIME COMMENT '更新时间' +) +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +); \ No newline at end of file diff --git a/DataX/scheme/output/t_user_upload_fault_mysql.sql b/DataX/scheme/output/t_user_upload_fault_mysql.sql new file mode 100644 index 0000000..f7d9cbb --- /dev/null +++ b/DataX/scheme/output/t_user_upload_fault_mysql.sql @@ -0,0 +1,10 @@ +CREATE TABLE `t_user_upload_fault` ( + `id` bigint(20) NOT NULL auto_increment, + `equipment_id` int(20) NOT NULL COMMENT '桩id', + `user_id` int(20) NOT NULL COMMENT '用户id', + `fault_config_id` int(20) COMMENT '故障原因id', + `remark` varchar(200) DEFAULT '' COMMENT '备注', + `create_time` datetime COMMENT '创建时间', + `update_time` datetime COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/DataX/start_cron.sh b/DataX/start_cron.sh index d51ce3b..4b32e8d 100644 --- a/DataX/start_cron.sh +++ b/DataX/start_cron.sh @@ -7,12 +7,17 @@ # ========================================== # 1. 基础配置 -# 假设您将整个文件夹上传到了 /usr/local/datax +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + WORK_DIR="/usr/local/datax" SCRIPT_NAME="run_jobs_inc.sh" SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" -# 日志存放位置，建议单独放在 logs 目录 -LOG_FILE="$WORK_DIR/logs/datax_inc_cron.log" +# 日志存放位置 +LOG_FILE="$WORK_DIR/logs/datax_inc_${TARGET}_cron.log" # 2. 检查环境 echo "正在检查环境..." @@ -42,13 +47,11 @@ echo "正在赋予脚本执行权限..." chmod +x "$SCRIPT_PATH" # 4. 配置 Crontab -# 任务定义: 每10分钟执行一次 (*/10) -# >> $LOG_FILE 2>&1 : 将标准输出和错误输出都追加写入日志文件 -CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH >> $LOG_FILE 2>&1" +# 任务定义: 每10分钟执行一次 +CRON_CMD="*/10 * * * * /bin/bash $SCRIPT_PATH $TARGET >> $LOG_FILE 2>&1" -# 检查任务是否已存在 (避免重复添加) -# 使用 grep 查找包含脚本路径的行 -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +# 检查任务是否已存在 (使用脚本路径+目标参数来匹配) +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") if [ -n "$EXISTING_JOB" ]; then echo "警告: 检测到该脚本的定时任务已存在:" @@ -61,9 +64,8 @@ if [ -n "$EXISTING_JOB" ]; then exit 0 fi - # 移除旧任务 (通过 grep -v 反向过滤掉包含脚本路径的行) - # 注意: 这会保留其他不相关的定时任务 - crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - + # 移除旧任务 + crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - echo "旧任务已移除" fi diff --git a/DataX/stop_cron.sh b/DataX/stop_cron.sh index ac633eb..298cea7 100644 --- a/DataX/stop_cron.sh +++ b/DataX/stop_cron.sh @@ -7,21 +7,27 @@ # 场景: 全量同步前防止增量任务干扰 # ========================================== -# 1. 基础配置 (需与 start_cron.sh 保持一致以确保匹配正确) +# 1. 基础配置 +TARGET=$1 +if [ "$TARGET" != "doris" ] && [ "$TARGET" != "mysql" ]; then + echo "使用方法: $0 [doris|mysql]" + exit 1 +fi + WORK_DIR="/usr/local/datax" SCRIPT_NAME="run_jobs_inc.sh" SCRIPT_PATH="$WORK_DIR/$SCRIPT_NAME" # 2. 检查任务是否存在 -echo "正在检查定时任务..." +echo "正在检查 $TARGET 定时任务..." -# 使用 grep 查找包含脚本路径的行 -EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +# 使用脚本路径和目标参数匹配 +EXISTING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") if [ -z "$EXISTING_JOB" ]; then echo "----------------------------------------" - echo "提示: 未发现该脚本的定时任务，无需停止。" - echo "目标脚本: $SCRIPT_PATH" + echo "提示: 未发现 $TARGET 的定时任务，无需停止。" + echo "目标脚本: $SCRIPT_PATH $TARGET" echo "----------------------------------------" exit 0 fi @@ -31,20 +37,18 @@ echo " $EXISTING_JOB" echo "----------------------------------------" # 3. 移除任务 -echo "正在移除任务..." +echo "正在移除 $TARGET 任务..." # 备份当前的 crontab BACKUP_FILE="/tmp/crontab_backup_$(date +%Y%m%d_%H%M%S).txt" crontab -l > "$BACKUP_FILE" 2>/dev/null echo "已备份当前 Crontab 到: $BACKUP_FILE" -# 核心逻辑: 移除包含脚本路径的行 -# grep -v "$SCRIPT_PATH" : 排除包含脚本路径的行 -# crontab - : 将过滤后的内容重新写入 crontab -crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH" | crontab - +# 核心逻辑: 移除包含脚本路径和目标的行 +crontab -l 2>/dev/null | grep -v "$SCRIPT_PATH $TARGET" | crontab - # 4. 验证结果 -REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH") +REMAINING_JOB=$(crontab -l 2>/dev/null | grep "$SCRIPT_PATH $TARGET") if [ -z "$REMAINING_JOB" ]; then echo "----------------------------------------" diff --git a/DataX/说明.md b/DataX/说明.md index 1e45ae1..2052121 100644 --- a/DataX/说明.md +++ b/DataX/说明.md @@ -1,102 +1,120 @@ -# DataX 同步部署与使用说明 - -#### 一、下载与安装 DataX - -1. **下载 DataX 包** - 官方下载地址：https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202309/datax.tar.gz - -2. **部署到服务器** - 将压缩包上传并解压到 `/usr/local/` 目录： - ```shell - cd /usr/local - tar zxvf datax.tar.gz - # 解压后应当存在 /usr/local/datax 目录 - ``` - -#### 二、配置部署 - -本方案包含**全量同步**和**增量同步**两套策略，请按以下步骤部署配置文件和脚本。 - -1. **上传文件** - 请将本地 `d:\dsWork\aiData\Doc\6、DataX配置\` 目录下的以下内容上传至服务器 `/usr/local/datax/` 根目录下： - - * 文件夹 `json_full/` (存放全量JSON配置) - * 文件夹 `json_inc/` (存放增量JSON配置) - * 脚本 `run_jobs_full.sh` - * 脚本 `run_jobs_inc.sh` - * 脚本 `setup_cron.sh` - - **上传后的服务器目录结构应如下所示：** - ```text - /usr/local/datax/ - ├── bin/ - ├── conf/ - ├── json_full/ <-- 新增：全量配置目录 - ├── json_inc/ <-- 新增：增量配置目录 - ├── run_jobs_full.sh <-- 新增：全量执行脚本 - ├── run_jobs_inc.sh <-- 新增：增量执行脚本 - └── setup_cron.sh <-- 新增：定时任务安装脚本 - ``` - -2. **赋予执行权限** - ```shell - cd /usr/local/datax - chmod +x *.sh - ``` - -3. **修改数据库密码 (如有需要)** - 脚本中已配置好默认的数据库连接信息。如需修改，请编辑 `run_jobs_full.sh` 和 `run_jobs_inc.sh` 中的 `PARAMS` 变量： - ```bash - PARAMS="-Dsrc_user=... -Dsrc_pwd=... -Ddest_user=... -Ddest_pwd=..." - ``` - -#### 三、运行策略说明 - -**1. 全量同步 (Full Sync)** -* **适用场景**：系统初始化，或数据出现严重偏差需要彻底重跑时。 -* **同步逻辑**： - 1. 执行 `TRUNCATE` 清空 Doris 目标表。 - 2. 从 MySQL 读取全量数据写入 Doris。 -* **手动执行命令**： - ```shell - cd /usr/local/datax - ./run_jobs_full.sh - ``` - -**2. 增量同步 (Incremental Sync)** -* **适用场景**：日常持续数据同步（高频）。 -* **同步逻辑**： - 1. **不删除**任何历史数据（无空窗期）。 - 2. 读取 MySQL 中 ID 最大的 **10000条** 记录（覆盖新增和最近更新的数据）。 - 3. 推送到 Doris，利用 Doris 的 **Unique Key** 主键模型自动完成“存在即更新，不存在即插入” (Upsert)。 -* **手动执行命令**： - ```shell - cd /usr/local/datax - ./run_jobs_inc.sh - ``` - -#### 四、自动化定时任务配置 - -我们提供了一键配置脚本，用于在 Rocky Linux 9 / CentOS 等系统中设置自动调度。 - -1. **执行安装脚本** - ```shell - cd /usr/local/datax - ./setup_cron.sh - ``` - *该脚本会自动添加 Crontab 任务，配置为每 10 分钟执行一次增量同步。* - -2. **验证与日志** - * **查看任务**：`crontab -l` - * **查看日志**：自动同步的日志会保存在 `/usr/local/datax/logs/datax_inc_cron.log` - ```shell - tail -f /usr/local/datax/logs/datax_inc_cron.log - ``` - -#### 五、重要注意事项 - -1. **Doris 表模型要求** - 目标 Doris 表必须使用 **Unique Key** 模型（建表时指定 `UNIQUE KEY(id)`），否则增量同步会导致大量重复数据。 - -2. **Python 环境** - DataX 依赖 Python 环境（推荐 Python 2.7 或 3.x）。脚本默认调用 `/usr/bin/python`，如果您的 Python 路径不同，请修改脚本头部的 `DATAX_PY` 变量。 +# DataX 同步部署与使用说明 + +本方案基于 DataX 实现从 MySQL 源端数据库到 **Doris** 或 **MySQL** 目标端的同步。支持全量和增量两种同步模式，并提供了一套模板化的配置与脚本。 + +--- + +#### 一、 目录结构 + +```text +/usr/local/datax/ +├── bin/ # DataX 二进制程序 +├── json/ # 统一的配置模板目录 (包含 15 个表的 JSON 模板) +├── db_config.sh # 数据库连接配置文件 (所有配置在此修改) +├── logs/ # 自动创建的同步日志目录 +├── run_jobs_full.sh # 全量同步脚本 (支持参数: doris | mysql) +├── run_jobs_inc.sh # 增量同步脚本 (支持参数: doris | mysql) +├── start_cron.sh # 开启增量定时同步 (支持参数: doris | mysql) +├── stop_cron.sh # 停止增量定时同步 (支持参数: doris | mysql) +├── scheme/ # 表结构生成工具目录 +│ ├── generate_schema.py # 自动生成 DDL 脚本 +│ └── output/ # 生成的 SQL 文件存放处 +└── 说明.md # 本说明文档 +``` + +--- + +#### 二、 部署步骤 + +1. **部署 DataX** + - 下载并解压 DataX 到 `/usr/local/datax`。 + - 确保系统已安装 Python (2.7 或 3.x)。 + +2. **上传配置文件** + - 将本地 `DataX/` 目录下的 `json/`, `*.sh`, `scheme/` 等文件上传至服务器 `/usr/local/datax/`。 + +3. **赋予权限** + ```bash + cd /usr/local/datax + chmod +x *.sh + ``` + +--- + +#### 三、 运行同步任务 + +所有脚本均支持通过第一个参数指定目标系统：`doris` 或 `mysql`。 + +##### 1. 全量同步 (Full Sync) +全量同步会先执行 `TRUNCATE TABLE` 清空目标表，然后导入全部数据。 +```bash +# 同步到 Doris +./run_jobs_full.sh doris + +# 同步到 MySQL +./run_jobs_full.sh mysql +``` + +##### 2. 增量同步 (Incremental Sync) +增量同步默认采集最近 10000 条数据（按 ID 倒序），利用目标端的 **Unique Key (Doris)** 或 **Primary Key (MySQL)** 机制实现自动覆盖更新。 +```bash +# 同步到 Doris +./run_jobs_inc.sh doris + +# 同步到 MySQL +./run_jobs_inc.sh mysql +``` + +--- + +#### 四、 自动化调度 (Crontab) + +使用提供的管理脚本一键配置每 10 分钟一次的增量同步任务。 + +1. **启动定时任务** + ```bash + # 为 Doris 同步启动定时任务 + ./start_cron.sh doris + + # 为 MySQL 同步启动定时任务 + ./start_cron.sh mysql + ``` + +2. **停止定时任务** + ```bash + ./stop_cron.sh doris # 或 mysql + ``` + +3. **查看日志** + 日志保存在 `logs/` 目录下，文件名为 `datax_inc_{target}_cron.log`。 + ```bash + tail -f logs/datax_inc_doris_cron.log + ``` + +--- + +#### 五、 辅助工具：表结构生成 + +在同步数据前，如果目标端尚未建表，可以使用 `scheme/` 下的工具生成建表语句。 + +1. **生成 DDL** + ```bash + cd scheme + # 自动处理 json/ 目录下的所有表 + python generate_schema.py + ``` + +2. **获取 SQL** + 生成的 SQL 文件位于 `scheme/output/`，包含 MySQL 和 Doris 两个版本。 + - **Doris 建表特性**：采用 Unique Key 模型，单副本 (`replication_num=1`)，单分桶，适合单机部署。 + +--- + +#### 六、 常见问题与注意事项 + +1. **数据库密码修改**： + - 所有数据库连接信息（源端 MySQL、目标端 Doris、目标端 MySQL）均集中在 [db_config.sh](file:///d:/dsWork/aiData/DataX/db_config.sh) 中。**修改该文件即可同步应用到所有同步任务**，无需修改 `.sh` 脚本。 +2. **Doris 负载地址**： + - Doris 同步依赖 `loadUrl` (FE 的 8030 端口) 和 JDBC (FE 的 9030 端口)。 +3. **网络连通性**： + - 请确保 DataX 部署机能够访问源端 RDS 和目标端数据库。