ChoETL
ChoETL copied to clipboard
DB Null Exception
Hi,
SqlConnection conn = new SqlConnection(StreamExtensions.getConnectionString());
conn.Open();
SqlCommand cmd = new SqlCommand(queryDB, conn);
using (var reader = cmd.ExecuteReader())
{
using (var parser = new ChoParquetWriter("test.parquet")
.Configure(c => c.CompressionMethod = CompressionMethod.Snappy)
.Configure(c => c.LiteParsing = false)
.Configure(c => c.RowGroupSize = 20)
.NotifyAfter(1000)
.OnRowsWritten((o, e) => $"Rows: {e.RowsWritten} <--- {DateTime.Now}".Print()))
{
if (reader.HasRows)
{
parser.Write(reader);
}
}
}
conn.Close();
I am using the above code to perform a dbraeder to parquet conversion but I am facing below cast issue,
Please find the version I am using,
Here is one way to handle DbNull values by subscribing to BeforeRecordFieldWrite
event and process such values.
using (var r = command.ExecuteReader(CommandBehavior.CloseConnection))
{
using (var parser = new ChoParquetWriter(filePath)
.Configure(c => c.CompressionMethod = Parquet.CompressionMethod.Gzip)
.Configure(c => c.RowGroupSize = 1000)
.NotifyAfter(1000)
.OnRowsWritten((o, e) => $"Rows: {e.RowsWritten} <--- {DateTime.Now}".Print())
.Setup(s => s.BeforeRecordFieldWrite += (o, e) =>
{
if (e.Source == DBNull.Value)
e.Source = null;
})
)
{
if (r.HasRows)
{
parser.Write(r);
}
}
}
With the latest release v1.2.1.61, this will be handled by the framework automatically, no need to use BeforeRecordFieldWrite
event.