seatunnel
seatunnel copied to clipboard
[Bug] [Module Name] When use connector mysqlcdc to mysql or doris , error
Search before asking
- [X] I had searched in the issues and found no similar issues.
What happened
When use dolphinscheduler to seatunnel task, connector mysqlcdc to mysql or doris , error
SeaTunnel Version
2.3.1
SeaTunnel Config
env {
execution.parallelism = 1
job.mode = "STREAMING"
checkpoint.interval = 5000
#execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
source {
MySQL-CDC {
result_table_name = "dsp_device"
parallelism = 1
server-id = 5657
username = "root"
password = "root"
table-names = ["parking_iot1.dsp_device","parking_iot1.dsp_device1"]
base-url = "jdbc:mysql://10.1.4.8:3306/parking_iot1?useSSL=false&useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
}
}
transform {
}
sink {
jdbc {
source_table_name = "dsp_device"
url = "jdbc:mysql://10.1.4.8:3306/parking_iot2?useSSL=false&useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "root"
password = "root"
query = "insert into dsp_device(id,`name`,`type`,manufacturer_id,device_id,serial_num,`status`,install_time,remark,tenant_id,is_del,create_time,update_time) values(?,?,?,?,?,?,?,?,?,?,?,?,?)"
}
}
Running Command
/opt/software/seatunnel-2.3.1/bin/seatunnel.sh --config /opt/software/seatunnel-2.3.1/job/mysqlcdc_to_mysql.conf --deploy-mode local
Error Exception
Exception in thread "main" org.apache.seatunnel.core.starter.exception.CommandExecuteException: SeaTunnel job executed failed
at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:181)
at org.apache.seatunnel.core.starter.SeaTunnel.run(SeaTunnel.java:40)
at org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient.main(SeaTunnelClient.java:34)
Caused by: org.apache.seatunnel.engine.common.exception.SeaTunnelEngineException: java.util.concurrent.CompletionException: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
at com.hazelcast.spi.impl.AbstractInvocationFuture.returnOrThrowWithJoinConventions(AbstractInvocationFuture.java:819)
at com.hazelcast.spi.impl.AbstractInvocationFuture.resolveAndThrowWithJoinConvention(AbstractInvocationFuture.java:835)
at com.hazelcast.spi.impl.AbstractInvocationFuture.join(AbstractInvocationFuture.java:545)
at org.apache.seatunnel.engine.server.task.context.SeaTunnelSplitEnumeratorContext.assignSplit(SeaTunnelSplitEnumeratorContext.java:69)
at org.apache.seatunnel.api.source.SourceSplitEnumerator$Context.assignSplit(SourceSplitEnumerator.java:101)
at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.assignSplits(IncrementalSourceEnumerator.java:159)
at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.run(IncrementalSourceEnumerator.java:69)
at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.stateProcess(SourceSplitEnumeratorTask.java:276)
at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.call(SourceSplitEnumeratorTask.java:128)
at org.apache.seatunnel.engine.server.TaskExecutionService$BlockingWorker.run(TaskExecutionService.java:526)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
at java.base/java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2190)
at java.base/java.io.ObjectStreamClass$FieldReflector.checkObjectFieldValueTypes(ObjectStreamClass.java:2153)
at java.base/java.io.ObjectStreamClass.checkObjFieldValueTypes(ObjectStreamClass.java:1407)
at java.base/java.io.ObjectInputStream.defaultCheckFieldValues(ObjectInputStream.java:2371)
at java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2278)
at java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2087)
at java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1594)
at java.base/java.io.ObjectInputStream.readArray(ObjectInputStream.java:1993)
at java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1588)
at java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:430)
at org.apache.seatunnel.common.utils.SerializationUtils.deserialize(SerializationUtils.java:96)
at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.lambda$run$1(AssignSplitOperation.java:63)
at org.apache.seatunnel.common.utils.RetryUtils.retryWithException(RetryUtils.java:48)
at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.run(AssignSplitOperation.java:55)
at com.hazelcast.spi.impl.operationservice.Operation.call(Operation.java:189)
at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:273)
at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248)
at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213)
at com.hazelcast.spi.impl.operationexecutor.impl.OperationExecutorImpl.run(OperationExecutorImpl.java:411)
at com.hazelcast.spi.impl.operationexecutor.impl.OperationExecutorImpl.runOrExecute(OperationExecutorImpl.java:438)
at com.hazelcast.spi.impl.operationservice.impl.Invocation.doInvokeLocal(Invocation.java:601)
at com.hazelcast.spi.impl.operationservice.impl.Invocation.doInvoke(Invocation.java:580)
at com.hazelcast.spi.impl.operationservice.impl.Invocation.invoke0(Invocation.java:541)
at com.hazelcast.spi.impl.operationservice.impl.Invocation.invoke(Invocation.java:241)
at com.hazelcast.spi.impl.operationservice.impl.InvocationBuilderImpl.invoke(InvocationBuilderImpl.java:61)
at org.apache.seatunnel.engine.server.utils.NodeEngineUtil.sendOperationToMemberNode(NodeEngineUtil.java:51)
at org.apache.seatunnel.engine.server.execution.TaskExecutionContext.sendToMember(TaskExecutionContext.java:47)
at org.apache.seatunnel.engine.server.task.context.SeaTunnelSplitEnumeratorContext.assignSplit(SeaTunnelSplitEnumeratorContext.java:64)
... 11 more
at org.apache.seatunnel.engine.client.job.ClientJobProxy.waitForJobComplete(ClientJobProxy.java:119)
at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:174)
... 2 more
### Flink or Spark Version
_No response_
### Java or Scala Version
jdk 1.8
### Screenshots
_No response_
### Are you willing to submit PR?
- [X] Yes I am willing to submit a PR!
### Code of Conduct
- [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
Hi, could you please provide the DDL and test data of these two tables at your convenience?
This issue has been automatically marked as stale because it has not had recent activity for 30 days. It will be closed in next 7 days if no further activity occurs.
This issue has been closed because it has not received response for too long time. You could reopen it if you encountered similar problems in the future.