fdb-record-layer icon indicating copy to clipboard operation
fdb-record-layer copied to clipboard

Make PointsFormat Codec Lazy to only read when needed for queries

Open jleach4 opened this issue 1 year ago • 0 comments

The PointsReader does this for each segment open...

`public Lucene86PointsReader(SegmentReadState readState) throws IOException { this.readState = readState;

String metaFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name,
    readState.segmentSuffix,
    Lucene86PointsFormat.META_EXTENSION);
String indexFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name,
    readState.segmentSuffix,
    Lucene86PointsFormat.INDEX_EXTENSION);
String dataFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name,
    readState.segmentSuffix,
    Lucene86PointsFormat.DATA_EXTENSION);

boolean success = false;
try {
  indexIn = readState.directory.openInput(indexFileName, readState.context);
  CodecUtil.checkIndexHeader(indexIn,
      Lucene86PointsFormat.INDEX_CODEC_NAME,
      Lucene86PointsFormat.VERSION_START,
      Lucene86PointsFormat.VERSION_CURRENT,
      readState.segmentInfo.getId(),
      readState.segmentSuffix);

  dataIn = readState.directory.openInput(dataFileName, readState.context);
  CodecUtil.checkIndexHeader(dataIn,
      Lucene86PointsFormat.DATA_CODEC_NAME,
      Lucene86PointsFormat.VERSION_START,
      Lucene86PointsFormat.VERSION_CURRENT,
      readState.segmentInfo.getId(),
      readState.segmentSuffix);

  long indexLength = -1, dataLength = -1;
  try (ChecksumIndexInput metaIn = readState.directory.openChecksumInput(metaFileName, readState.context)) {
    Throwable priorE = null;
    try {
      CodecUtil.checkIndexHeader(metaIn,
          Lucene86PointsFormat.META_CODEC_NAME,
          Lucene86PointsFormat.VERSION_START,
          Lucene86PointsFormat.VERSION_CURRENT,
          readState.segmentInfo.getId(),
          readState.segmentSuffix);

      while (true) {
        int fieldNumber = metaIn.readInt();
        if (fieldNumber == -1) {
          break;
        } else if (fieldNumber < 0) {
          throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn);
        }
        BKDReader reader = new BKDReader(metaIn, indexIn, dataIn);
        readers.put(fieldNumber, reader);
      }
      indexLength = metaIn.readLong();
      dataLength = metaIn.readLong();
    } catch (Throwable t) {
      priorE = t;
    } finally {
      CodecUtil.checkFooter(metaIn, priorE);
    }
  }
  // At this point, checksums of the meta file have been validated so we
  // know that indexLength and dataLength are very likely correct.
  CodecUtil.retrieveChecksum(indexIn, indexLength);
  CodecUtil.retrieveChecksum(dataIn, dataLength);
  success = true;
} finally {
  if (success == false) {
    IOUtils.closeWhileHandlingException(this);
  }
}

}`

jleach4 avatar Jun 08 '23 23:06 jleach4