qbeast-spark
qbeast-spark copied to clipboard
"SET TBLPROPERTIES" does not persist changes in the _delta_log
What went wrong?
Running ALTER TABLE table_name SET TBLPROPERTIES ()
does not persist the changes in the _delta_log/.
How to reproduce?
import org.apache.spark.sql.delta.actions.Action
import org.apache.spark.sql.delta.DeltaLog
import org.apache.spark.sql.delta.actions.CommitInfo
import org.apache.spark.sql.delta.util.FileNames
val df = loadTestData(spark)
val tmpDir = "/tmp/test"
df.write.format("qbeast").option("columnsToIndex", "user_id,price").save(tmpDir)
spark.sql(s"CREATE TABLE t1 USING QBEAST LOCATION $tmpDir")
spark.sql("ALTER TABLE t1 SET TBLPROPERTIES ('k' = 'v')")
val deltaLog = DeltaLog.forTable(spark, tablePath)
val conf = deltaLog.newDeltaHadoopConf()
val snapshot = deltaLog.update()
val v = snapshot.version
val isCommitted = (deltaLog.store
.read(FileNames.deltaFile(deltaLog.logPath, v), conf)
.map(Action.fromJson)
.exists {
case c: CommitInfo =>
println(c)
c.operation == "SET TBLPROPERTIES" &&
c.operationParameters == Map("properties" -> """{"k":"v"}""")
case _ => false
}
)
isCommitted shouldBe true