changefeed status of primary is warning but do not include error info

[Lily2025]

What did you do?

1、br restore data two db for primary and secondary 2、create changefeed for primary and secondary separately 3、set bdr role for primary and secondary separately 4、run tpcc for primary and run sysbench for secondary 5、add index for primary table 6、inject network partition between ticdc of primary and all tidb of secondary last for 5mins

What did you expect to see?

after fault recover: 1、changefeed status is normal 2、ddl can sync success 3、changefeed lag can be less than 3s after fault recover for a time

What did you see instead?

changefeed status of primary is warning but do not include error info { "upstream_id": 7340490029962833542, "namespace": "default", "id": "ticdc-task1", "sink_uri": "mysql://root:xxxxx@cdc-downstream-tc-tidb.endless-ha-test-ticdc-tps-7080582-1-976:4000", "create_time": "2024-02-28 11:58:46.930", "start_ts": 448028403768492048, "resolved_ts": 448031855228223506, "target_ts": 0, "checkpoint_tso": 448031745311244373, "checkpoint_time": "2024-02-28 15:31:13.752", "sort_engine": "unified", "state": "warning", "error": null, "error_history": null, "creator_version": "v8.0.0-alpha" }

after a while，the warning disappeared

[2024/02/28 15:37:44.094 +08:00] [WARN] [mysql_ddl_sink.go:133] ["Execute DDL with error, retry later"] [startTs=448031745311244289] [ddl="ALTER TABLE sbtest1 DROP INDEX index_test_1709105306801"] [namespace=default] [changefeed=ticdc-task1] [error="dial tcp 10.106.67.45:4000: i/o timeout"] [2024/02/28 15:37:44.095 +08:00] [ERROR] [ddl_sink.go:253] ["Execute DDL failed"] [namespace=default] [changefeed=ticdc-task1] [DDL="{"StartTs":448031745311244289,"CommitTs":448031745311244373,"Query":"ALTER TABLE sbtest1 DROP INDEX index_test_1709105306801","TableInfo":{"id":245,"name":{"O":"sbtest1","L":"sbtest1"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":515,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"k","L":"k"},"offset":1,"origin_default":null,"origin_default_bit":null,"default":"0","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":9,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"c","L":"c"},"offset":2,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":1,"Flen":120,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"pad","L":"pad"},"offset":3,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":1,"Flen":60,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":[{"id":1,"idx_name":{"O":"k_1","L":"k_1"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"k","L":"k"},"offset":1,"length":-1}],"state":5,"backfill_state":0,"comment":"","index_type":1,"is_unique":false,"is_primary":false,"is_invisible":false,"is_global":false,"mv_index":false}],"constraint_info":null,"fk_info":[],"state":5,"pk_is_handle":true,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":71663638,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":4,"max_idx_id":44,"max_fk_id":0,"max_cst_id":0,"update_timestamp":448031745311244289,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":4,"tiflash_replica":null,"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"SchemaID":104,"TableName":{"Schema":"sysbench_64_7000w","Table":"sbtest1","TableID":245,"IsPartition":false},"Version":448031745311244373,"RowColumnsOffset":{"1":0,"2":1,"3":2,"4":3},"ColumnsFlag":{"1":11,"2":33,"3":0,"4":0},"HandleIndexID":-1,"IndexColumnsOffset":[[0]]},"PreTableInfo":{"id":245,"name":{"O":"sbtest1","L":"sbtest1"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":515,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"k","L":"k"},"offset":1,"origin_default":null,"origin_default_bit":null,"default":"0","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":9,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"c","L":"c"},"offset":2,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":9,"Flen":120,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"pad","L":"pad"},"offset":3,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":1,"Flen":60,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":[{"id":1,"idx_name":{"O":"k_1","L":"k_1"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"k","L":"k"},"offset":1,"length":-1}],"state":5,"backfill_state":0,"comment":"","index_type":1,"is_unique":false,"is_primary":false,"is_invisible":false,"is_global":false,"mv_index":false},{"id":44,"idx_name":{"O":"index_test_1709105306801","L":"index_test_1709105306801"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"c","L":"c"},"offset":2,"length":-1}],"state":5,"backfill_state":0,"comment":"","index_type":1,"is_unique":false,"is_primary":false,"is_invisible":false,"is_global":false,"mv_index":false}],"constraint_info":null,"fk_info":[],"state":5,"pk_is_handle":true,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":71663638,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":4,"max_idx_id":44,"max_fk_id":0,"max_cst_id":0,"update_timestamp":448031732623474728,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":4,"tiflash_replica":null,"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"SchemaID":104,"TableName":{"Schema":"sysbench_64_7000w","Table":"sbtest1","TableID":245,"IsPartition":false},"Version":448031732793868385,"RowColumnsOffset":{"1":0,"2":1,"3":2,"4":3},"ColumnsFlag":{"1":11,"2":33,"3":32,"4":0},"HandleIndexID":-1,"IndexColumnsOffset":[[0]]},"Type":8,"Done":{},"Charset":"","Collate":"","IsBootstrap":false,"BDRRole":"primary"}"] [error="dial tcp 10.106.67.45:4000: i/o timeout"] [2024/02/28 15:37:44.095 +08:00] [WARN] [ddl_sink.go:171] ["owner ddl sink fails on action"] [namespace=default] [changefeed=ticdc-task1] [action=writeDDLEvent] [retryable=true] [error="dial tcp 10.106.67.45:4000: i/o timeout"] [2024/02/28 15:37:44.140 +08:00] [WARN] [changefeed.go:325] ["an warning occurred in Owner"] [namespace=default] [changefeed=ticdc-task1] [error="dial tcp 10.106.67.45:4000: i/o timeout"]

[2024/02/28 15:38:10.140 +08:00] [INFO] [ddl_manager.go:562] ["execute a ddl event successfully"] [ddl="ALTER TABLE sbtest1 DROP INDEX index_test_1709105306801"] [namespace=default] [changefeed=ticdc-task1] [bdrRole=primary] [ddlEvent="{"StartTs":448031745311244289,"CommitTs":448031745311244373,"Query":"ALTER TABLE sbtest1 DROP INDEX index_test_1709105306801","TableInfo":{"id":245,"name":{"O":"sbtest1","L":"sbtest1"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":515,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"k","L":"k"},"offset":1,"origin_default":null,"origin_default_bit":null,"default":"0","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":9,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"c","L":"c"},"offset":2,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":1,"Flen":120,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"pad","L":"pad"},"offset":3,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":1,"Flen":60,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":[{"id":1,"idx_name":{"O":"k_1","L":"k_1"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"k","L":"k"},"offset":1,"length":-1}],"state":5,"backfill_state":0,"comment":"","index_type":1,"is_unique":false,"is_primary":false,"is_invisible":false,"is_global":false,"mv_index":false}],"constraint_info":null,"fk_info":[],"state":5,"pk_is_handle":true,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":71663638,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":4,"max_idx_id":44,"max_fk_id":0,"max_cst_id":0,"update_timestamp":448031745311244289,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":4,"tiflash_replica":null,"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"SchemaID":104,"TableName":{"Schema":"sysbench_64_7000w","Table":"sbtest1","TableID":245,"IsPartition":false},"Version":448031745311244373,"RowColumnsOffset":{"1":0,"2":1,"3":2,"4":3},"ColumnsFlag":{"1":11,"2":33,"3":0,"4":0},"HandleIndexID":-1,"IndexColumnsOffset":[[0]]},"PreTableInfo":{"id":245,"name":{"O":"sbtest1","L":"sbtest1"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":515,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"k","L":"k"},"offset":1,"origin_default":null,"origin_default_bit":null,"default":"0","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":9,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"c","L":"c"},"offset":2,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":9,"Flen":120,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"pad","L":"pad"},"offset":3,"origin_default":null,"origin_default_bit":null,"default":"","default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":1,"Flen":60,"Decimal":0,"Charset":"utf8mb4","Collate":"utf8mb4_bin","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":[{"id":1,"idx_name":{"O":"k_1","L":"k_1"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"k","L":"k"},"offset":1,"length":-1}],"state":5,"backfill_state":0,"comment":"","index_type":1,"is_unique":false,"is_primary":false,"is_invisible":false,"is_global":false,"mv_index":false},{"id":44,"idx_name":{"O":"index_test_1709105306801","L":"index_test_1709105306801"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"c","L":"c"},"offset":2,"length":-1}],"state":5,"backfill_state":0,"comment":"","index_type":1,"is_unique":false,"is_primary":false,"is_invisible":false,"is_global":false,"mv_index":false}],"constraint_info":null,"fk_info":[],"state":5,"pk_is_handle":true,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":71663638,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":4,"max_idx_id":44,"max_fk_id":0,"max_cst_id":0,"update_timestamp":448031732623474728,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":4,"tiflash_replica":null,"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"SchemaID":104,"TableName":{"Schema":"sysbench_64_7000w","Table":"sbtest1","TableID":245,"IsPartition":false},"Version":448031732793868385,"RowColumnsOffset":{"1":0,"2":1,"3":2,"4":3},"ColumnsFlag":{"1":11,"2":33,"3":32,"4":0},"HandleIndexID":-1,"IndexColumnsOffset":[[0]]},"Type":8,"Done":{},"Charset":"","Collate":"","IsBootstrap":false,"BDRRole":"primary"}"]

Versions of the cluster

./cdc version Release Version: v8.0.0-alpha Git Commit Hash: 967475532744e4bfe21abbed22a3dfb2086570f1 Git Branch: heads/refs/tags/v8.0.0-alpha UTC Build Time: 2024-02-21 11:37:45 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: false

[Lily2025]

/remove-area dm /area ticdc

[Lily2025]

/assign asddongmen

[Lily2025]

/severity moderate

[asddongmen]

I try but can not reproduce this issue. Need more investigations.

[asddongmen]

Not reproduce again, close it.