qbeast-spark icon indicating copy to clipboard operation
qbeast-spark copied to clipboard

"SET TBLPROPERTIES" does not persist changes in the _delta_log

Open Jiaweihu08 opened this issue 7 months ago • 0 comments

What went wrong?

Running ALTER TABLE table_name SET TBLPROPERTIES () does not persist the changes in the _delta_log/.

How to reproduce?

import org.apache.spark.sql.delta.actions.Action
import org.apache.spark.sql.delta.DeltaLog
import org.apache.spark.sql.delta.actions.CommitInfo
import org.apache.spark.sql.delta.util.FileNames

val df = loadTestData(spark)
val tmpDir = "/tmp/test"
df.write.format("qbeast").option("columnsToIndex", "user_id,price").save(tmpDir)
spark.sql(s"CREATE TABLE t1 USING QBEAST LOCATION $tmpDir")
spark.sql("ALTER TABLE t1 SET TBLPROPERTIES ('k' = 'v')")

val deltaLog = DeltaLog.forTable(spark, tablePath)
val conf = deltaLog.newDeltaHadoopConf()
val snapshot = deltaLog.update()
val v = snapshot.version

val isCommitted = (deltaLog.store
	.read(FileNames.deltaFile(deltaLog.logPath, v), conf)
	.map(Action.fromJson)
	.exists {
		case c: CommitInfo =>
			println(c)
			c.operation == "SET TBLPROPERTIES" &&
				c.operationParameters == Map("properties" -> """{"k":"v"}""")
		case _ => false
	}
)

isCommitted shouldBe true

Jiaweihu08 avatar Jul 02 '24 13:07 Jiaweihu08