需求:将mysql中的历史数据迁移到clickhouse中。
1、datax环境要求:Python2.x/Java8+
2、job.json:

{
"job":{
	"content":[
		{
			"reader":{
				"name":"mysqlreader",
				"parameter":{
					"username":"root",
					"password":"xxxx",
					"column":[
						"id",
						"user_id",
						"user_name",
						"org_code",
						"org_name"
					],
					"connection":[
						{
						"jdbcUrl":["jdbc:mysql://10.0.0.127:3306/test"],
						"table":["analysis_app_log_tb"]
						}
					],
					"splitPk":"id"
				}
		},
		"writer":{
			"name":"clickhousewriter",
			"parameter":{
				"username":"xxx",
				"password":"xxx",
				"column":[
					"*"
				],
				"preSql":[
					"truncate table 'default'.analysis_app_log_tb ON CLUSTER default_cluster sync"
				],
				"connection":[
				{
					"table":[
						"analysis_app_log_tb"
					],
					"jdbcUrl":"jdbc:clickhouse://10.0.0.127:21422/default?ssl=true&sslmode=none"
				}
				]
				}
			}
		}
	],
	"setting":{
		"speed":{
			"channel":"1"
		}
	}
}
}

3、执行任务

bin/datax.py job.json

4、如果迁移表的部分数据需要将MysqlReader部分替换为sql查询。
示例:

{
"job":{
	"content":[
		{
			"reader":{
				"name":"mysqlreader",
				"parameter":{
					"username":"root",
					"password":"xxxx",
					"encoding":"UTF-8",
					"mandatoryEncoding":"UTF-8",
					"connection":[
						{
						"jdbcUrl":["jdbc:mysql://10.0.0.127:3306/test"],
						"querySql":[
							"select id,user_id,_user_name,org_code from analysis_app_log_tb where opt_start_time<'2025-07-24 00:00:00'"
						]
						}
					]
				}
		},
		"writer":{
			"name":"clickhousewriter",
			"parameter":{
				"username":"xxx",
				"password":"xxx",
				"encoding":"UTF-8",
				"column":[
					"*"
				],
				"connection":[
				{
					"table":[
						"analysis_app_log_tb"
					],
					"jdbcUrl":"jdbc:clickhouse://10.0.0.127:21422/default?ssl=true&sslmode=none"
				}
				],
				"loadProps":{"format":"json","strip_outer_array":true}
				}
			}
		}
	],
	"setting":{
		"speed":{
			"channel":"5"
		}
	}
}
}

更多推荐