dtle icon indicating copy to clipboard operation
dtle copied to clipboard

Published 20 hours ago • verified publisher icon actiontech

kafka incr replication job: When the event contains multiple statements, the row value is always 0

Open asiroliu opened this issue 2 years ago • 0 comments

Description

kafka incr replication job: When the event contains multiple statements, the row value is always 0

Steps to reproduce the issue

  1. create mysql-kakfka job
{
  "job_id": "kafka_table_level",
  "is_password_encrypted": false,
  "task_step_name": "all",
  "failover": true,
  "retry": 2,
  "src_task": {
    "task_name": "src",
    "node_id": "ccfc2d7c-7c85-9ba4-03ea-17dd530576d3",
    "mysql_src_task_config": {
      "gtid": "",
      "binlog_relay": false
    },
    "drop_table_if_exists": true,
    "skip_create_db_table": false,
    "repl_chan_buffer_size": 120,
    "chunk_size": 2000,
    "group_max_size": 1,
    "group_timeout": 100,
    "connection_config": {
      "database_type": "MySQL",
      "host": "172.100.9.3",
      "port": 3306,
      "user": "test_src",
      "password": "test_src"
    },
    "replicate_do_db": [
      {
        "table_schema": "action_db_1",
        "tables": [
          {
            "table_name": "sbtest1"
          }
        ]
      }
    ]
  },
  "dest_task": {
    "task_name": "dest",
    "node_id": "be858969-db0d-5d57-15a4-946d35fb9a4e",
    "parallel_workers": 1,
    "kafka_topic": "dtle",
    "kafka_broker_addrs": [
      "172.100.9.21:9092"
    ]
  }
}
  1. insert data on src MySQL
shell> sysbench /usr/share/sysbench/oltp_common.lua --mysql-host=172.100.9.3 --mysql-port=3306 --mysql-user=test --mysql-password=test --create_secondary=off --report-interval=10 --time=0 --mysql-db=action_db_1 --tables=2 --table_size=10 prepare

  1. check kafka message, the row is always 0 image

  2. dbz define https://debezium.io/documentation/reference/0.9/connectors/mysql.html#change-events-value image

  3. src MySQL general log

2022-04-13T07:58:15.052337Z        34 Query     INSERT INTO sbtest1(k, c, pad) VALUES(5, '83868641912-28773972837-60736120486-75162659906-27563526494-20381887404-41576422241-93426793964-56405065102-33518432330', '67847967377-48000963322-62604785301-91415491898-96926520291'),(6, '38014276128-25250245652-62722561801-27818678124-24890218270-18312424692-92565570600-36243745486-21199862476-38576014630', '23183251411-36241541236-31706421314-92007079971-60663066966'),(6, '33973744704-80540844748-72700647445-87330233173-87249600839-07301471459-22846777364-58808996678-64607045326-48799346817', '38615512647-91458489257-90681424432-95014675832-60408598704'),(6, '37002370280-58842166667-00026392672-77506866252-09658311935-56926959306-83464667271-94685475868-28264244556-14550208498', '63947013338-98809887124-59806726763-79831528812-45582457048'),(5, '44257470806-17967007152-32809666989-26174672567-29883439075-95767161284-94957565003-35708767253-53935174705-16168070783', '34551750492-67990399350-81179284955-79299808058-21257255869'),(6, '37216201353-39109531021-11197415756-87798784755-02463049870-83329763120-57551308766-61100580113-80090253566-30971527105', '05161542529-00085727016-35134775864-52531204064-98744439797'),(6, '33071042495-29920376648-91343430102-79082003121-73317691963-02846712788-88069761578-14885283975-44409837760-90760298045', '91798303270-64988107984-08161247972-12116454627-22996445111'),(5, '73754818686-04889373966-18668178968-56957589012-31352882173-91882653509-59577900152-88962682169-52981807259-62646890059', '76460662325-41613089656-42706083314-81833284991-17063140920'),(5, '26482547570-00155460224-12388481921-23289186371-78242522654-77998886134-73270876420-50821093220-31442690639-11588920653', '30508501104-50823269125-88107014550-70202920684-95842308929'),(6, '05677017559-47107518969-97509137401-28934334557-14497052050-61906823704-44077628507-24840441785-05187301456-27797851637', '29489382504-13697582598-09964978366-26554639515-36136545002')
  1. src MySQL binlog
BEGIN
/*!*/;
# at 26003
#220413 15:58:15 server id 3306  end_log_pos 26067 CRC32 0x9e21889c 	Table_map: `action_db_1`.`sbtest1` mapped to number 119
# at 26067
#220413 15:58:15 server id 3306  end_log_pos 27992 CRC32 0xf0108904 	Write_rows: table id 119 flags: STMT_END_F

BINLOG '
l4JWYhPqDAAAQAAAANNlAAAAAHcAAAAAAAEAC2FjdGlvbl9kYl8xAAdzYnRlc3QxAAQDA/7+BP54
/jwAnIghng==
l4JWYh7qDAAAhQcAAFhtAAAAAHcAAAAAAAEAAgAE//ABAAAABQAAAHc4Mzg2ODY0MTkxMi0yODc3
Mzk3MjgzNy02MDczNjEyMDQ4Ni03NTE2MjY1OTkwNi0yNzU2MzUyNjQ5NC0yMDM4MTg4NzQwNC00
MTU3NjQyMjI0MS05MzQyNjc5Mzk2NC01NjQwNTA2NTEwMi0zMzUxODQzMjMzMDs2Nzg0Nzk2NzM3
Ny00ODAwMDk2MzMyMi02MjYwNDc4NTMwMS05MTQxNTQ5MTg5OC05NjkyNjUyMDI5MfACAAAABgAA
AHczODAxNDI3NjEyOC0yNTI1MDI0NTY1Mi02MjcyMjU2MTgwMS0yNzgxODY3ODEyNC0yNDg5MDIx
ODI3MC0xODMxMjQyNDY5Mi05MjU2NTU3MDYwMC0zNjI0Mzc0NTQ4Ni0yMTE5OTg2MjQ3Ni0zODU3
NjAxNDYzMDsyMzE4MzI1MTQxMS0zNjI0MTU0MTIzNi0zMTcwNjQyMTMxNC05MjAwNzA3OTk3MS02
MDY2MzA2Njk2NvADAAAABgAAAHczMzk3Mzc0NDcwNC04MDU0MDg0NDc0OC03MjcwMDY0NzQ0NS04
NzMzMDIzMzE3My04NzI0OTYwMDgzOS0wNzMwMTQ3MTQ1OS0yMjg0Njc3NzM2NC01ODgwODk5NjY3
OC02NDYwNzA0NTMyNi00ODc5OTM0NjgxNzszODYxNTUxMjY0Ny05MTQ1ODQ4OTI1Ny05MDY4MTQy
NDQzMi05NTAxNDY3NTgzMi02MDQwODU5ODcwNPAEAAAABgAAAHczNzAwMjM3MDI4MC01ODg0MjE2
NjY2Ny0wMDAyNjM5MjY3Mi03NzUwNjg2NjI1Mi0wOTY1ODMxMTkzNS01NjkyNjk1OTMwNi04MzQ2
NDY2NzI3MS05NDY4NTQ3NTg2OC0yODI2NDI0NDU1Ni0xNDU1MDIwODQ5ODs2Mzk0NzAxMzMzOC05
ODgwOTg4NzEyNC01OTgwNjcyNjc2My03OTgzMTUyODgxMi00NTU4MjQ1NzA0OPAFAAAABQAAAHc0
NDI1NzQ3MDgwNi0xNzk2NzAwNzE1Mi0zMjgwOTY2Njk4OS0yNjE3NDY3MjU2Ny0yOTg4MzQzOTA3
NS05NTc2NzE2MTI4NC05NDk1NzU2NTAwMy0zNTcwODc2NzI1My01MzkzNTE3NDcwNS0xNjE2ODA3
MDc4MzszNDU1MTc1MDQ5Mi02Nzk5MDM5OTM1MC04MTE3OTI4NDk1NS03OTI5OTgwODA1OC0yMTI1
NzI1NTg2OfAGAAAABgAAAHczNzIxNjIwMTM1My0zOTEwOTUzMTAyMS0xMTE5NzQxNTc1Ni04Nzc5
ODc4NDc1NS0wMjQ2MzA0OTg3MC04MzMyOTc2MzEyMC01NzU1MTMwODc2Ni02MTEwMDU4MDExMy04
MDA5MDI1MzU2Ni0zMDk3MTUyNzEwNTswNTE2MTU0MjUyOS0wMDA4NTcyNzAxNi0zNTEzNDc3NTg2
NC01MjUzMTIwNDA2NC05ODc0NDQzOTc5N/AHAAAABgAAAHczMzA3MTA0MjQ5NS0yOTkyMDM3NjY0
OC05MTM0MzQzMDEwMi03OTA4MjAwMzEyMS03MzMxNzY5MTk2My0wMjg0NjcxMjc4OC04ODA2OTc2
MTU3OC0xNDg4NTI4Mzk3NS00NDQwOTgzNzc2MC05MDc2MDI5ODA0NTs5MTc5ODMwMzI3MC02NDk4
ODEwNzk4NC0wODE2MTI0Nzk3Mi0xMjExNjQ1NDYyNy0yMjk5NjQ0NTExMfAIAAAABQAAAHc3Mzc1
NDgxODY4Ni0wNDg4OTM3Mzk2Ni0xODY2ODE3ODk2OC01Njk1NzU4OTAxMi0zMTM1Mjg4MjE3My05
MTg4MjY1MzUwOS01OTU3NzkwMDE1Mi04ODk2MjY4MjE2OS01Mjk4MTgwNzI1OS02MjY0Njg5MDA1
OTs3NjQ2MDY2MjMyNS00MTYxMzA4OTY1Ni00MjcwNjA4MzMxNC04MTgzMzI4NDk5MS0xNzA2MzE0
MDkyMPAJAAAABQAAAHcyNjQ4MjU0NzU3MC0wMDE1NTQ2MDIyNC0xMjM4ODQ4MTkyMS0yMzI4OTE4
NjM3MS03ODI0MjUyMjY1NC03Nzk5ODg4NjEzNC03MzI3MDg3NjQyMC01MDgyMTA5MzIyMC0zMTQ0
MjY5MDYzOS0xMTU4ODkyMDY1MzszMDUwODUwMTEwNC01MDgyMzI2OTEyNS04ODEwNzAxNDU1MC03
MDIwMjkyMDY4NC05NTg0MjMwODkyOfAKAAAABgAAAHcwNTY3NzAxNzU1OS00NzEwNzUxODk2OS05
NzUwOTEzNzQwMS0yODkzNDMzNDU1Ny0xNDQ5NzA1MjA1MC02MTkwNjgyMzcwNC00NDA3NzYyODUw
Ny0yNDg0MDQ0MTc4NS0wNTE4NzMwMTQ1Ni0yNzc5Nzg1MTYzNzsyOTQ4OTM4MjUwNC0xMzY5NzU4
MjU5OC0wOTk2NDk3ODM2Ni0yNjU1NDYzOTUxNS0zNjEzNjU0NTAwMgSJEPA=
'/*!*/;
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=1 /* INT meta=0 nullable=0 is_null=0 */
###   @2=5 /* INT meta=0 nullable=0 is_null=0 */
###   @3='83868641912-28773972837-60736120486-75162659906-27563526494-20381887404-41576422241-93426793964-56405065102-33518432330' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='67847967377-48000963322-62604785301-91415491898-96926520291' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=2 /* INT meta=0 nullable=0 is_null=0 */
###   @2=6 /* INT meta=0 nullable=0 is_null=0 */
###   @3='38014276128-25250245652-62722561801-27818678124-24890218270-18312424692-92565570600-36243745486-21199862476-38576014630' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='23183251411-36241541236-31706421314-92007079971-60663066966' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=3 /* INT meta=0 nullable=0 is_null=0 */
###   @2=6 /* INT meta=0 nullable=0 is_null=0 */
###   @3='33973744704-80540844748-72700647445-87330233173-87249600839-07301471459-22846777364-58808996678-64607045326-48799346817' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='38615512647-91458489257-90681424432-95014675832-60408598704' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=4 /* INT meta=0 nullable=0 is_null=0 */
###   @2=6 /* INT meta=0 nullable=0 is_null=0 */
###   @3='37002370280-58842166667-00026392672-77506866252-09658311935-56926959306-83464667271-94685475868-28264244556-14550208498' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='63947013338-98809887124-59806726763-79831528812-45582457048' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=5 /* INT meta=0 nullable=0 is_null=0 */
###   @2=5 /* INT meta=0 nullable=0 is_null=0 */
###   @3='44257470806-17967007152-32809666989-26174672567-29883439075-95767161284-94957565003-35708767253-53935174705-16168070783' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='34551750492-67990399350-81179284955-79299808058-21257255869' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=6 /* INT meta=0 nullable=0 is_null=0 */
###   @2=6 /* INT meta=0 nullable=0 is_null=0 */
###   @3='37216201353-39109531021-11197415756-87798784755-02463049870-83329763120-57551308766-61100580113-80090253566-30971527105' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='05161542529-00085727016-35134775864-52531204064-98744439797' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=7 /* INT meta=0 nullable=0 is_null=0 */
###   @2=6 /* INT meta=0 nullable=0 is_null=0 */
###   @3='33071042495-29920376648-91343430102-79082003121-73317691963-02846712788-88069761578-14885283975-44409837760-90760298045' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='91798303270-64988107984-08161247972-12116454627-22996445111' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=8 /* INT meta=0 nullable=0 is_null=0 */
###   @2=5 /* INT meta=0 nullable=0 is_null=0 */
###   @3='73754818686-04889373966-18668178968-56957589012-31352882173-91882653509-59577900152-88962682169-52981807259-62646890059' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='76460662325-41613089656-42706083314-81833284991-17063140920' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=9 /* INT meta=0 nullable=0 is_null=0 */
###   @2=5 /* INT meta=0 nullable=0 is_null=0 */
###   @3='26482547570-00155460224-12388481921-23289186371-78242522654-77998886134-73270876420-50821093220-31442690639-11588920653' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='30508501104-50823269125-88107014550-70202920684-95842308929' /* STRING(60) meta=65084 nullable=0 is_null=0 */
### INSERT INTO `action_db_1`.`sbtest1`
### SET
###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
###   @2=6 /* INT meta=0 nullable=0 is_null=0 */
###   @3='05677017559-47107518969-97509137401-28934334557-14497052050-61906823704-44077628507-24840441785-05187301456-27797851637' /* STRING(120) meta=65144 nullable=0 is_null=0 */
###   @4='29489382504-13697582598-09964978366-26554639515-36136545002' /* STRING(60) meta=65084 nullable=0 is_null=0 */
# at 27992
#220413 15:58:15 server id 3306  end_log_pos 28023 CRC32 0x81eba6de 	Xid = 247
COMMIT/*!*/;

Output of ./dtle version:**

9.9.9.9-master-a18f86d

Additional details (log, config, job config etc):

kafka_table_level_cud_2.html.zip

asiroliu avatar Apr 13 '22 08:04 asiroliu