MongoShake跨mongodb分片集群数据同步

下载安装包

wget https://github.com/alibaba/MongoShake/releases/download/release-v2.8.6-20250825/mongo-shake-v2.8.6.tgz

解压安装包

tar xf mongo-shake-v2.8.6.tgz -C /opt/

编辑配置文件/opt/mongo-shake-v2.8.6/collector.conf

conf.version = 11
id = mongoshake
master_quorum = false
full_sync.http_port = 9101
incr_sync.http_port = 9102
system_profile_port = 9202
log.level = info
log.dir = ./logs
log.file = collector.log
log.flush = false
# 同步模式
sync_mode = all
# 源mongo分片地址
mongo_urls = mongodb://root:xxxxxxxxxxxx@192.168.1.1:27108,192.168.1.2:27108,192.168.1.3:27108;mongodb://root:xxxxxxxxxxxx@192.168.1.1:27208,192.168.1.2:27208,192.168.1.3:27208;mongodb://root:xxxxxxxxxxxx@192.168.1.1:27308,192.168.1.2:27308,192.168.1.3:27308
# 源mongos地址
mongo_cs_url = mongodb://root:xxxxxxxxxxxx@192.168.1.1:27000,192.168.1.2:27000,192.168.1.3:27000
tunnel = direct
# 目标mongodb地址
tunnel.address = mongodb://root:xxxxxxx@192.168.1.6:27000,192.168.1.7:27000,192.168.1.8:27000
tunnel.message = raw
tunnel.kafka.partition_number = 1
tunnel.json.format =
tunnel.mongo_ssl_root_ca_file =
mongo_connect_mode = secondaryPreferred
filter.namespace.black =
filter.namespace.white =
filter.pass.special.db =
filter.ddl_enable = false
filter.oplog.gids = false
# 断点续传配置,目标mongodb地址
checkpoint.storage.url = mongodb://root:xxxxxxxx@192.168.1.6:27000
checkpoint.storage.db = mongoshake
checkpoint.storage.collection = ckpt_default
checkpoint.storage.url.mongo_ssl_root_ca_file =
checkpoint.start_position = 1970-01-01T00:00:00Z
transform.namespace =
full_sync.reader.collection_parallel = 6
full_sync.reader.write_document_parallel = 8
full_sync.reader.document_batch_size = 128
full_sync.reader.fetch_batch_size = 8192
full_sync.reader.parallel_thread = 1
full_sync.reader.parallel_index = _id
full_sync.collection_exist_drop = true
full_sync.create_index = background
full_sync.executor.insert_on_dup_update = false
full_sync.executor.filter.orphan_document = false
full_sync.executor.majority_enable = false
full_sync.do_not_shard_destination = false
incr_sync.mongo_fetch_method = oplog
incr_sync.change_stream.watch_full_document = false
incr_sync.oplog.gids =
incr_sync.shard_key = collection
incr_sync.shard_by_object_id_whitelist =
incr_sync.worker = 8
incr_sync.tunnel.write_thread = 8
incr_sync.target_delay = 0
incr_sync.worker.batch_queue_size = 64
incr_sync.adaptive.batching_max_size = 1024
incr_sync.fetcher.buffer_capacity = 64
incr_sync.fetcher.buffer_size_threshold_in_kb = 512
incr_sync.reader.fetch_batch_size = 8192
incr_sync.executor.upsert = false
incr_sync.executor.insert_on_dup_update = false
incr_sync.conflict_write_to = none
incr_sync.executor.majority_enable = false
special.source.db.flag =
skip.nsshardkey.verify =

迁移之前需要关闭源集群Balancer,两个命令都要返回false

sh.getBalancerState()
sh.isBalancerRunning()
# 如果getBalancerState()返回true,则关闭Balancer
sh.stopBalancer()

启动MongoShake

/opt/mongo-shake-v2.8.6/start.sh /opt/mongo-shake-v2.8.6/collector.conf

启动之后查看进程

$ ps -ef |grep coll
root 2740124 1 0 14:49 ? 00:00:00 ./hypervisor --daemon --exec=./collector.linux -conf=collector.conf 1>> collector.linux.output 2>&1
root 2740125 2740124 99 14:49 ? 00:22:45 ./collector.linux -conf=collector.conf 1>> collector.linux.output 2>&1

查看日志

tail -f /opt/mongo-shake-v2.8.6/logs/collector.log

停止MongoShake

/opt/mongo-shake-v2.8.6/stop.sh /opt/mongo-shake-v2.8.6/logs/mongoshake.pid
文章作者: 慕容峻才
文章链接: https://www.acaiblog.top/MongoShake跨mongodb分片集群数据同步/
版权声明: 本博客所有文章除特别声明外,均采用 CC BY-NC-SA 4.0 许可协议。转载请注明来自 阿才的博客
微信打赏
支付宝打赏