incubator-xtable icon indicating copy to clipboard operation
incubator-xtable copied to clipboard

sync fails with Could not initialize class org.apache.hudi.avro.model.DecimalWrapper error

Open sagarlakshmipathy opened this issue 1 year ago • 0 comments

Steps to reproduce:

With a delta dataset having decimal field, run sync for HUDI and ICEBERG targetFormats

sourceFormat: DELTA
targetFormats:
  - HUDI
  - ICEBERG
datasets:
  -
    tableBasePath: s3://onetable-bucket/delta/unpartitioned # or even local path
    tableName: unpartitioned

Stacktrace:

Exception in thread "main" java.lang.NoClassDefFoundError
        at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
        at java.base/java.util.concurrent.ForkJoinTask.getThrowableException(ForkJoinTask.java:603)
        at java.base/java.util.concurrent.ForkJoinTask.reportException(ForkJoinTask.java:678)
        at java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:737)
        at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateParallel(ReduceOps.java:919)
        at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233)
        at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
        at org.apache.hudi.common.data.HoodieBaseListData.<init>(HoodieBaseListData.java:46)
        at org.apache.hudi.common.data.HoodieListData.<init>(HoodieListData.java:69)
        at org.apache.hudi.common.data.HoodieListData.flatMap(HoodieListData.java:136)
        at org.apache.hudi.metadata.HoodieTableMetadataUtil.convertMetadataToColumnStatsRecords(HoodieTableMetadataUtil.java:1048)
        at org.apache.hudi.metadata.HoodieTableMetadataUtil.convertMetadataToRecords(HoodieTableMetadataUtil.java:328)
        at org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.lambda$update$21(HoodieBackedTableMetadataWriter.java:912)
        at org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.processAndCommit(HoodieBackedTableMetadataWriter.java:853)
        at org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.update(HoodieBackedTableMetadataWriter.java:910)
        at org.apache.hudi.client.BaseHoodieWriteClient.writeTableMetadata(BaseHoodieWriteClient.java:362)
        at org.apache.hudi.client.BaseHoodieWriteClient.commit(BaseHoodieWriteClient.java:285)
        at org.apache.hudi.client.BaseHoodieWriteClient.commitStats(BaseHoodieWriteClient.java:236)
        at org.apache.hudi.client.HoodieJavaWriteClient.commit(HoodieJavaWriteClient.java:92)
        at org.apache.hudi.client.HoodieJavaWriteClient.commit(HoodieJavaWriteClient.java:53)
        at org.apache.hudi.client.BaseHoodieWriteClient.commit(BaseHoodieWriteClient.java:201)
        at io.onetable.hudi.HudiTargetClient$CommitState.commit(HudiTargetClient.java:355)
        at io.onetable.hudi.HudiTargetClient.completeSync(HudiTargetClient.java:255)
        at io.onetable.spi.sync.TableFormatSync.getSyncResult(TableFormatSync.java:126)
        at io.onetable.spi.sync.TableFormatSync.syncSnapshot(TableFormatSync.java:55)
        at io.onetable.client.OneTableClient.lambda$syncSnapshot$5(OneTableClient.java:164)
        at java.base/java.util.HashMap.forEach(HashMap.java:1336)
        at io.onetable.client.OneTableClient.syncSnapshot(OneTableClient.java:162)
        at io.onetable.client.OneTableClient.sync(OneTableClient.java:115)
        at io.onetable.utilities.RunSync.main(RunSync.java:162)
Caused by: java.lang.NoClassDefFoundError: Could not initialize class org.apache.hudi.avro.model.DecimalWrapper
        at org.apache.hudi.avro.HoodieAvroUtils.wrapValueIntoAvro(HoodieAvroUtils.java:1224)
        at org.apache.hudi.metadata.HoodieMetadataPayload.lambda$createColumnStatsRecords$9(HoodieMetadataPayload.java:631)
        at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
        at java.base/java.util.HashMap$ValueSpliterator.tryAdvance(HashMap.java:1696)
        at java.base/java.util.stream.StreamSpliterators$WrappingSpliterator.lambda$initPartialTraversalState$0(StreamSpliterators.java:294)
        at java.base/java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.fillBuffer(StreamSpliterators.java:206)
        at java.base/java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.doAdvance(StreamSpliterators.java:169)
        at java.base/java.util.stream.StreamSpliterators$WrappingSpliterator.tryAdvance(StreamSpliterators.java:300)
        at java.base/java.util.Spliterators$1Adapter.hasNext(Spliterators.java:681)
        at java.base/java.util.Iterator.forEachRemaining(Iterator.java:132)
        at java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
        at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:658)
        at java.base/java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:274)
        at java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1655)
        at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
        at java.base/java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:952)
        at java.base/java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:926)
        at java.base/java.util.stream.AbstractTask.compute(AbstractTask.java:327)
        at java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:746)
        at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
        at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
        at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
        at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
        at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

For time being, I proceeded by removing the decimal field.

sagarlakshmipathy avatar Nov 09 '23 23:11 sagarlakshmipathy