Path validation inconsistent making already written files unreadable

[lambdaupb]

Version 1.3.2

Apparently device and measurement names may not contain - characters. However this is not really checked when writing data as the Path(String) is used.

It is then checked when trying to read data, which became a problem for me as those files are now unreadable without patching the library.

This seems connected with the expression language parsing somehow, but I don't see why the path constructor should care about that.

Which leads me to a more specific question, how are device or measurement names saved when using the escaping BQUOTA_SRTING ? - That suggests arbitrary device names are possible, but not using the java API.

https://github.com/apache/tsfile/blob/d4a11413a89ac14be605eea899554308a7944a9c/java/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathLexer.g4#L181-L184

What would be great is either removing the check=true from MetadataQuerierByFileImpl.loadChunkMetaDatas or provide some way of escaping.

Here is a reproduction test:

 
 
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TsFileReader;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DoubleDataPoint;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.jupiter.api.Test;
 
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
 
class Test {
 
 
    @Test
    void reproduce() throws IOException, WriteProcessException {
 
        String filename = "test.tsfile";
        String device = "test-device";
        String seriesName = "test_series";
 
        Files.deleteIfExists(Paths.get(filename));
 
        // WRITE succeeds
        try(TsFileWriter tsFileWriter = new TsFileWriter(Paths.get(filename).toFile())) {
            MeasurementSchema measurementSchema = new MeasurementSchema(
                    seriesName,
                    TSDataType.DOUBLE,
                    TSEncoding.GORILLA,
                    CompressionType.ZSTD
            );
            tsFileWriter.registerTimeseries(new org.apache.iotdb.tsfile.read.common.Path(device), measurementSchema);
 
 
            for (int i = 0; i < 10; i++) {
                tsFileWriter.write(new TSRecord(i, new PlainDeviceID(device)).addTuple(new DoubleDataPoint(seriesName, i)));
            }
 
            tsFileWriter.flushAllChunkGroups();
            System.out.println("Device "+device+ " and measurement "+seriesName+" written to "+filename);
        }
 
 
        // READ fails
        try(TsFileSequenceReader sequenceReader = new TsFileSequenceReader(filename)) {
            try {
                for (Path p : sequenceReader.getAllPaths()) {
                    System.out.println(p);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
            try (TsFileReader tsFileReader = new TsFileReader(sequenceReader)) {
                Path path = new Path(device, seriesName, false);
 
                QueryExpression queryExpression = QueryExpression.create()
                        .addSelectedPath(path);
                QueryDataSet query = tsFileReader.query(queryExpression);
                while (query.hasNext()) {
                    RowRecord record = query.next();
                    System.out.println(record);
                }
            }
        }
        /*
 
        OK:
        Device test:device and measurement test_series written to test.tsfile
        test:device.test_series
        0   0.0
        1   1.0
        2   2.0
        3   3.0
        4   4.0
        5   5.0
        6   6.0
        7   7.0
        8   8.0
        9   9.0
 
        #################################################################################################################
        NOT OK:
 
 
        Device test-device and measurement test_series written to test.tsfile
        org.apache.iotdb.tsfile.exception.PathParseException: test-device.test_series is not a legal path.
            at org.apache.iotdb.tsfile.read.common.parser.PathNodesGenerator.splitPathToNodes(PathNodesGenerator.java:46)
            at org.apache.iotdb.tsfile.read.common.Path.<init>(Path.java:123)
            at org.apache.iotdb.tsfile.read.common.Path.<init>(Path.java:100)
            at org.apache.iotdb.tsfile.read.TsFileSequenceReader.getAllPaths(TsFileSequenceReader.java:848)
            at Test.reproduce(Test.java:60)
        
        
        org.apache.iotdb.tsfile.exception.PathParseException: test-device.test_series is not a legal path.
        
            at org.apache.iotdb.tsfile.read.common.parser.PathNodesGenerator.splitPathToNodes(PathNodesGenerator.java:46)
            at org.apache.iotdb.tsfile.read.common.Path.<init>(Path.java:123)
            at org.apache.iotdb.tsfile.read.common.Path.<init>(Path.java:100)
            at org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl.loadChunkMetaDatas(MetadataQuerierByFileImpl.java:135)
            at org.apache.iotdb.tsfile.read.query.executor.TsFileExecutor.execute(TsFileExecutor.java:71)
            at org.apache.iotdb.tsfile.read.TsFileReader.query(TsFileReader.java:48)
            at Test.reproduce(Test.java:71)
         */
    }
}

[github-actions[bot]]

Hi, this is your first issue in the Apache TsFile project. Thanks for your report. Welcome to join the community!