通过datax将mysql数据导入到clickhouse
·
需求:将mysql中的历史数据迁移到clickhouse中。
1、datax环境要求:Python2.x/Java8+
2、job.json:
{
"job":{
"content":[
{
"reader":{
"name":"mysqlreader",
"parameter":{
"username":"root",
"password":"xxxx",
"column":[
"id",
"user_id",
"user_name",
"org_code",
"org_name"
],
"connection":[
{
"jdbcUrl":["jdbc:mysql://10.0.0.127:3306/test"],
"table":["analysis_app_log_tb"]
}
],
"splitPk":"id"
}
},
"writer":{
"name":"clickhousewriter",
"parameter":{
"username":"xxx",
"password":"xxx",
"column":[
"*"
],
"preSql":[
"truncate table 'default'.analysis_app_log_tb ON CLUSTER default_cluster sync"
],
"connection":[
{
"table":[
"analysis_app_log_tb"
],
"jdbcUrl":"jdbc:clickhouse://10.0.0.127:21422/default?ssl=true&sslmode=none"
}
]
}
}
}
],
"setting":{
"speed":{
"channel":"1"
}
}
}
}
3、执行任务
bin/datax.py job.json
4、如果迁移表的部分数据需要将MysqlReader部分替换为sql查询。
示例:
{
"job":{
"content":[
{
"reader":{
"name":"mysqlreader",
"parameter":{
"username":"root",
"password":"xxxx",
"encoding":"UTF-8",
"mandatoryEncoding":"UTF-8",
"connection":[
{
"jdbcUrl":["jdbc:mysql://10.0.0.127:3306/test"],
"querySql":[
"select id,user_id,_user_name,org_code from analysis_app_log_tb where opt_start_time<'2025-07-24 00:00:00'"
]
}
]
}
},
"writer":{
"name":"clickhousewriter",
"parameter":{
"username":"xxx",
"password":"xxx",
"encoding":"UTF-8",
"column":[
"*"
],
"connection":[
{
"table":[
"analysis_app_log_tb"
],
"jdbcUrl":"jdbc:clickhouse://10.0.0.127:21422/default?ssl=true&sslmode=none"
}
],
"loadProps":{"format":"json","strip_outer_array":true}
}
}
}
],
"setting":{
"speed":{
"channel":"5"
}
}
}
}
更多推荐
所有评论(0)