we start task use yaml. when we restart this worker, panic occurs.
---
name: p17_incr4 # global unique
task-mode: incremental # full/incremental/all
is-sharding: false # whether multi dm-worker do one sharding job
meta-schema: "dm_meta" # meta schema in downstreaming database to store meta informaton of dm
remove-meta: false # remove meta from downstreaming database, now we delete checkpoint and online ddl information
enable-heartbeat: false # whether to enable heartbeat for calculating lag between master and syncer
# timezone: "Asia/Shanghai" # target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone
target-database:
host: "10.19.XX.XX"
port: 4000
user: "root"
password: "XXXXX"
mysql-instances: # one or more source database, config more source database for sharding merge
-
source-id: "10.19.65.17" # unique in all instances, used as id when save checkpoints, configs, etc.
# binlog pos used to as start pos for syncer, for different task-mode, this maybe used or not
# `full` / `all`:
# never be used
# `incremental`:
# if `remove-meta` is true, this will be used
# else if checkpoints already exists in `meta-schema`, this will not be used
# otherwise, this will be used
meta:
binlog-name: mysql-bin.004429
binlog-pos: 577611518
route-rules: ["user-route-rules-schema1"]
filter-rules: ["user-filter-1","user-filter-2"]
#column-mapping-rules: ["instance-1"]
black-white-list: "instance"
# `mydumper-config-name` and `mydumper` should only set one
mydumper-config-name: "global" # ref `mydumpers` config
# `loader-config-name` and `loader` should only set one
loader-config-name: "global" # ref `loaders` config
syncer-config-name: "global" # ref `syncers` config
# other common configs shared by all instances
routes: # schema/table route mapping
user-route-rules-schema1:
schema-pattern: "db_lfds"
target-schema: "db_lfds"
filters: # filter rules, mysql instance can ref rules in it
user-filter-1:
schema-pattern: "db_lfds"
events: ["truncate table", "drop table"] # ignore truncate/drop table ddl
action: Ignore
user-filter-2:
schema-pattern: "db_lfds"
table-pattern: "~.*"
events: ["all dml"] # only do all DML events
action: Do
black-white-list:
instance:
do-dbs: ["db_lfds"]
ignore-dbs: ["mysql", "information_schema","performance_schema"]
do-tables:
- db-name: "db_lfds"
tbl-name: "~.*"
column-mappings: # column mapping rules, mysql instance can ref rules in it
instance-1:
schema-pattern: "test_*"
table-pattern: "t_*"
expression: "partition id" # handle sharding partition id
source-column: "id"
target-column: "id"
arguments: ["1", "test_", "t_"]
instance-2:
schema-pattern: "test_*"
table-pattern: "t_*"
expression: "partition id" # handle sharding partition id
source-column: "id"
target-column: "id"
arguments: ["2", "test_", "t_"]
mydumpers: # mydumper process unit specific configs, mysql instance can ref one config in it
global:
mydumper-path: "./bin/mydumper"
threads: 4
chunk-filesize: 64
skip-tz-utc: true
#extra-args: "-B test -T t1,t2 --no-locks"
extra-args: "-B db_lfds --no-locks"
#extra-args: "-x db_lfds.*|db_lums.*|lkl_job.* --no-locks"
loaders: # loader process unit specific configs, mysql instance can ref one config in it
global:
pool-size: 16
dir: "./dumped_data"
syncers: # syncer process unit specific configs, mysql instance can ref one config in it
global:
worker-count: 16
batch: 100
max-retry: 100
safe-mode: false
Versions of the cluster
[tidb@devops-deploy-8014 dmctl]$ ./dmctl -V
Release Version: v1.0.0-alpha-46-g6855ea4
Git Commit Hash: 6855ea4e40bb5e3775709054a59a55c628a0922f
Git Branch: master
UTC Build Time: 2019-04-02 12:41:38
Go Version: go version go1.12 linux/amd64
[tidb@tidb2 bin]$ ./dm-master -V
Release Version: v1.0.0-alpha-46-g6855ea4
Git Commit Hash: 6855ea4e40bb5e3775709054a59a55c628a0922f
Git Branch: master
UTC Build Time: 2019-04-02 12:41:48
Go Version: go version go1.12 linux/amd64
[tidb@tidb3 bin]$ ./dm-worker -V
Release Version: v1.0.0-alpha-46-g6855ea4
Git Commit Hash: 6855ea4e40bb5e3775709054a59a55c628a0922f
Git Branch: master
UTC Build Time: 2019-04-02 12:41:42
Go Version: go version go1.12 linux/amd64
dm-worker error log
goroutine 16 [running]:
github.com/pingcap/dm/dm/worker.(*Worker).Start(0xc0002d59d0)
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/worker.go:119 +0x4ea
github.com/pingcap/dm/dm/worker.(*Server).Start.func1(0xc000188c00)
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/server.go:83 +0x5b
created by github.com/pingcap/dm/dm/worker.(*Server).Start
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/server.go:80 +0x13c
panic: restore task p17_incr4 ({"is-sharding":false,"online-ddl-scheme":"","case-sensitive":false,"name":"p17_incr4","mode":"incremental","ignore-checking-items":null,"sou
rce-id":"10.19.65.17","server-id":19040801,"flavor":"mysql","meta-schema":"dm_meta","remove-meta":false,"disable-heartbeat":true,"heartbeat-update-interval":1,"heartbeat-r
eport-interval":10,"enable-heartbeat":false,"meta":{"BinLogName":"mysql-bin.004429","BinLogPos":577611518},"Timezone":"","binlog-type":"local","relay-dir":"/data1/dm-worke
r1/deploy/relay_log","from":{"host":"10.19.65.17","port":54321,"user":"dmworker","max-allowed-packet":67108864},"to":{"host":"10.19.65.35","port":4000,"user":"root","max-a
llowed-packet":67108864},"route-rules":[{"schema-pattern":"db_lfds","table-pattern":"","target-schema":"db_lfds","target-table":""}],"filter-rules":[{"schema-pattern":"db_
lfds","table-pattern":"","events":["truncate table","drop table"],"sql-pattern":null,"action":"Ignore"},{"schema-pattern":"db_lfds","table-pattern":"","events":["all dml"]
,"sql-pattern":null,"action":"Do"}],"mapping-rule":[],"black-white-list":{"do-tables":[{"db-name":"db_lfds","tbl-name":"~.*"}],"do-dbs":["db_lfds"],"ignore-tables":null,"i
gnore-dbs":["mysql","information_schema","performance_schema"]},"mydumper-path":"./bin/mydumper","threads":4,"chunk-filesize":64,"skip-tz-utc":true,"extra-args":"-B db_lfd
s --no-locks","pool-size":16,"dir":"./dumped_data","meta-file":"","worker-count":16,"batch":100,"max-retry":100,"auto-fix-gtid":false,"enable-gtid":true,"disable-detect":f
alse,"safe-mode":false,"enable-ansi-quotes":false,"log-level":"info","log-file":"/data1/dm-worker1/deploy/log/dm-worker.log","log-rotate":"","pprof-addr":"","status-addr":
"","config-file":""}) in worker starting: sub task p17_incr4 init dm-unit error pattern db_lfds already exists
github.com/pingcap/errors.AlreadyExistsf
/go/pkg/mod/github.com/pingcap/[email protected]/juju_adaptor.go:97
github.com/pingcap/tidb-tools/pkg/table-rule-selector.(*trieSelector).insert
/go/pkg/mod/github.com/pingcap/[email protected]+incompatible/pkg/table-rule-selector/trie_selector.go:186
github.com/pingcap/tidb-tools/pkg/table-rule-selector.(*trieSelector).insertSchema
/go/pkg/mod/github.com/pingcap/[email protected]+incompatible/pkg/table-rule-selector/trie_selector.go:119
github.com/pingcap/tidb-tools/pkg/table-rule-selector.(*trieSelector).Insert
/go/pkg/mod/github.com/pingcap/[email protected]+incompatible/pkg/table-rule-selector/trie_selector.go:109
github.com/pingcap/tidb-tools/pkg/binlog-filter.(*BinlogEvent).AddRule
/go/pkg/mod/github.com/pingcap/[email protected]+incompatible/pkg/binlog-filter/filter.go:155
github.com/pingcap/tidb-tools/pkg/binlog-filter.NewBinlogEvent
/go/pkg/mod/github.com/pingcap/[email protected]+incompatible/pkg/binlog-filter/filter.go:134
github.com/pingcap/dm/syncer.(*Syncer).Init
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/syncer/syncer.go:258
github.com/pingcap/dm/dm/worker.(*SubTask).Init
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/subtask.go:111
github.com/pingcap/dm/dm/worker.(*Worker).StartSubTask
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/worker.go:195
github.com/pingcap/dm/dm/worker.(*Worker).Start
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/worker.go:118
github.com/pingcap/dm/dm/worker.(*Server).Start.func1
/home/jenkins/workspace/build_dm_master/go/src/github.com/pingcap/dm/dm/worker/server.go:83
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1337
insert into schema selector
add rule &{SchemaPattern:db_lfds TablePattern: Events:[all dml] SQLPattern:[] sqlRegularExp:<nil> Action:Do} into binlog event filter
initial rule &{SchemaPattern:db_lfds TablePattern: Events:[all dml] SQLPattern:[] sqlRegularExp:<nil> Action:Do} in binlog event filter
task read from meta file and formated
{
"is-sharding":false,
"online-ddl-scheme":"",
"case-sensitive":false,
"name":"p17_incr4",
"mode":"incremental",
"ignore-checking-items":null,
"source-id":"10.19.65.17",
"server-id":19040801,
"flavor":"mysql",
"meta-schema":"dm_meta",
"remove-meta":false,
"disable-heartbeat":true,
"heartbeat-update-interval":1,
"heartbeat-report-interval":10,
"enable-heartbeat":false,
"meta":{
"BinLogName":"mysql-bin.004429",
"BinLogPos":577611518
},
"Timezone":"",
"binlog-type":"local",
"relay-dir":"/data1/dm-worker1/deploy/relay_log",
"from":{
"host":"10.19.65.17",
"port":54321,
"user":"dmworker",
"max-allowed-packet":67108864
},
"to":{
"host":"10.19.65.35",
"port":4000,
"user":"root",
"max-allowed-packet":67108864
},
"route-rules":[
{
"schema-pattern":"db_lfds",
"table-pattern":"",
"target-schema":"db_lfds",
"target-table":""
}
],
"filter-rules":[
{
"schema-pattern":"db_lfds",
"table-pattern":"",
"events":[
"truncate table",
"drop table"
],
"sql-pattern":null,
"action":"Ignore"
},
{
"schema-pattern":"db_lfds",
"table-pattern":"",
"events":[
"all dml"
],
"sql-pattern":null,
"action":"Do"
}
],
"mapping-rule":[
],
"black-white-list":{
"do-tables":[
{
"db-name":"db_lfds",
"tbl-name":"~.*"
}
],
"do-dbs":[
"db_lfds"
],
"ignore-tables":null,
"ignore-dbs":[
"mysql",
"information_schema",
"performance_schema"
]
},
"mydumper-path":"./bin/mydumper",
"threads":4,
"chunk-filesize":64,
"skip-tz-utc":true,
"extra-args":"-B db_lfds --no-locks",
"pool-size":16,
"dir":"./dumped_data",
"meta-file":"",
"worker-count":16,
"batch":100,
"max-retry":100,
"auto-fix-gtid":false,
"enable-gtid":true,
"disable-detect":false,
"safe-mode":false,
"enable-ansi-quotes":false,
"log-level":"info",
"log-file":"/data1/dm-worker1/deploy/log/dm-worker.log",
"log-rotate":"",
"pprof-addr":"",
"status-addr":"",
"config-file":""
}
{
"schema-pattern":"db_lfds",
"table-pattern":"", # ~. is lost, so two rules schema-pattern and table-pattern is duplicate*
"events":[
"all dml"
],
"sql-pattern":null,
"action":"Do"
}