ChoETL icon indicating copy to clipboard operation
ChoETL copied to clipboard

DB Null Exception

Open RKvign01 opened this issue 1 year ago • 3 comments

Hi,

            SqlConnection conn = new SqlConnection(StreamExtensions.getConnectionString());
            conn.Open();
            SqlCommand cmd = new SqlCommand(queryDB, conn);

            using (var reader = cmd.ExecuteReader())
            {

                using (var parser = new ChoParquetWriter("test.parquet")
                .Configure(c => c.CompressionMethod = CompressionMethod.Snappy)
                .Configure(c => c.LiteParsing = false)
                .Configure(c => c.RowGroupSize = 20)
                .NotifyAfter(1000)
                .OnRowsWritten((o, e) => $"Rows: {e.RowsWritten} <--- {DateTime.Now}".Print()))
                {
                    if (reader.HasRows)
                    {
                        parser.Write(reader);
                    }
                }
            }

            conn.Close();
            
            
           I am using the above code to perform a dbraeder to parquet conversion but I am facing below cast issue,
           
           
image

RKvign01 avatar Jul 27 '23 15:07 RKvign01

Please find the version I am using,

image

RKvign01 avatar Jul 27 '23 15:07 RKvign01

Here is one way to handle DbNull values by subscribing to BeforeRecordFieldWrite event and process such values.

        using (var r = command.ExecuteReader(CommandBehavior.CloseConnection))
        {
            using (var parser = new ChoParquetWriter(filePath)
                .Configure(c => c.CompressionMethod = Parquet.CompressionMethod.Gzip)
                .Configure(c => c.RowGroupSize = 1000)
                .NotifyAfter(1000)
                .OnRowsWritten((o, e) => $"Rows: {e.RowsWritten} <--- {DateTime.Now}".Print())
                .Setup(s => s.BeforeRecordFieldWrite += (o, e) =>
                {
                    if (e.Source == DBNull.Value)
                        e.Source = null;
                })
                )
            {
                if (r.HasRows)
                {
                    parser.Write(r);
                }
            }
        }

Cinchoo avatar Aug 04 '23 15:08 Cinchoo

With the latest release v1.2.1.61, this will be handled by the framework automatically, no need to use BeforeRecordFieldWrite event.

Cinchoo avatar Aug 08 '23 12:08 Cinchoo