bioformats
bioformats copied to clipboard
Published
20 hours ago •
ome
ome
Flex: Merge IDR changes and add new group plate option
This PR attempts to merge the IDR specific Flex changes from https://github.com/IDR/bioformats/commits/master/components/formats-gpl/src/loci/formats/in/FlexReader.java
It also adds a new option flex.group.plate which can be used to configure the IDR behaviour. Based on initial feedback I have defaulted the option to be on.
This will require a minor release and will be memo file breaking.
Swapped the default behaviour due to large number of repo test failures: https://merge-ci.openmicroscopy.org/jenkins/job/BIOFORMATS-test-folder/55446/console
Moving to 6.6.0 as this PR is memo breaking and not suitable for a patch release
Initial testing using a copy of this plate from idr0001 with an options file setting flex.group.plate=true seems to reveal:
- Bio-Formats
showinfcorrectly includes all measurement folders as part of the filesets and creates a plate with multiple acquisitions - OMERO
omero import -ffails to select all the files as part of the fileset.
I will keep investigating tomorrow.
Performed some additional testing using the first plate acquisition of the sample idr0001 fileset with the command-line tools with and without the flex.group.plate option.
Without the option, showinf detect 96 series and lists all files under Meas_01(2011-03-31_11-53-03) in the used files list. With the grouping option turned on, showinf -option flex.group.plates true detects 480 series (5 x 96 wells?) and only lists the files under Meas_01(2011-03-31_11-53-03) in the used files lists.
In both cases, the initial steps of the reader initialization parses all flex files across measurements.
Looking more closely at the diff between this PR and the state of the reader on https://github.com/IDR/bioformats
diff --git a/components/formats-gpl/src/loci/formats/in/FlexReader.java b/components/formats-gpl/src/loci/formats/in/FlexReader.java
index f2beb13f26..9a50150ee2 100644
--- a/components/formats-gpl/src/loci/formats/in/FlexReader.java
+++ b/components/formats-gpl/src/loci/formats/in/FlexReader.java
@@ -99,6 +99,9 @@ public class FlexReader extends FormatReader {
// -- Fields --
+ public static final String GROUP_PLATES = "flex.group.plate";
+ public static final boolean GROUP_PLATES_DEFAULT = false;
+
/** Camera binning values. */
private int binX, binY;
@@ -269,10 +272,11 @@ public class FlexReader extends FormatReader {
int nBytes = 0;
int bpp = 0;
double factor = 0;
+
try (RandomAccessInputStream s =
new RandomAccessInputStream(getFileHandle(file.file))) {
- IFD ifd;
TiffParser tp = new TiffParser(s);
+ IFD ifd;
if (file.offsets == null) {
if (imageNumber < file.ifds.size()) {
ifd = file.ifds.get(imageNumber);
@@ -305,10 +309,13 @@ public class FlexReader extends FormatReader {
offsets[i] += offset;
}
ifd.putIFDValue(tag, offsets);
- } else if (file.offsets != null) {
+ }
+ else if (file.offsets != null) {
ifd = tp.getIFD(file.offsets[imageNumber]);
}
else {
+ tp.getStream().close();
+ s.close();
return buf;
}
nBytes = ifd.getBitsPerSample()[0] / 8;
@@ -386,6 +393,25 @@ public class FlexReader extends FormatReader {
measurementFiles = new ArrayList<String>();
acquisitionDates = new HashMap<Integer, Timestamp>();
+ Location currentFile = new Location(id).getAbsoluteFile();
+ Location dir = currentFile.getParentFile();
+ runDirs = new ArrayList<Location>();
+ if (!dir.getName().startsWith("Meas_") || !groupPlates()) {
+ runDirs.add(dir);
+ }
+ else {
+ // look for other acquisitions of the same plate
+ dir = dir.getParentFile();
+ String[] parentDirs = dir.list(true);
+ Arrays.sort(parentDirs);
+ for (String d : parentDirs) {
+ Location f = new Location(dir.getAbsoluteFile(), d);
+ if (f.isDirectory() && d.startsWith("Meas_")) {
+ runDirs.add(f);
+ }
+ }
+ }
+
if (checkSuffix(id, FLEX_SUFFIX)) {
initFlexFile(id);
}
@@ -394,6 +420,7 @@ public class FlexReader extends FormatReader {
}
else initMeaFile(id);
+ if (runCount == 0) runCount = 1;
if (plateCount == flexFiles.size() / runCount) {
plateCount /= wellCount;
if ((plateCount % fieldCount) == 0) {
@@ -531,24 +558,6 @@ public class FlexReader extends FormatReader {
if (doGrouping) {
// group together .flex files that are in the same directory
- Location dir = currentFile.getParentFile();
- runDirs = new ArrayList<Location>();
- if (!dir.getName().startsWith("Meas_")) {
- runDirs.add(dir);
- }
- else {
- // look for other acquisitions of the same plate
- dir = dir.getParentFile();
- String[] parentDirs = dir.list(true);
- Arrays.sort(parentDirs);
- for (String d : parentDirs) {
- Location f = new Location(dir.getAbsoluteFile(), d);
- if (f.isDirectory() && d.startsWith("Meas_")) {
- runDirs.add(f);
- }
- }
- }
-
for (Location runDir : runDirs) {
String[] files = runDir.list(true);
@@ -557,7 +566,7 @@ public class FlexReader extends FormatReader {
LOGGER.debug("Checking if {} belongs in the same dataset.", file);
if (file.endsWith(".flex") && file.length() == 14) {
flex.add(new Location(runDir, file).getAbsolutePath());
- LOGGER.debug("Added {} to dataset.", flex.get(flex.size() - 1));
+ //LOGGER.debug("Added {} to dataset.", flex.get(flex.size() - 1));
}
}
}
@@ -576,8 +585,8 @@ public class FlexReader extends FormatReader {
LOGGER.debug("Determined that {} .flex files belong together.",
files.length);
- runCount = runDirs.size();
-
+ runCount = 1;
+ if (runCount == 0) runCount = 1;
groupFiles(files, store);
populateMetadataStore(store);
}
@@ -661,6 +670,7 @@ public class FlexReader extends FormatReader {
store.setWellSampleIndex(new NonNegativeInteger(i), pos[2], well, sampleIndex);
store.setWellSampleImageRef(imageID, pos[2], well, sampleIndex);
+ if (runCount == 0) runCount = 1;
store.setPlateAcquisitionWellSampleRef(wellSample, 0, pos[3], i % (getSeriesCount() / runCount));
}
@@ -1325,10 +1335,8 @@ public class FlexReader extends FormatReader {
if (firstBarcode == null) {
firstBarcode = barcode;
}
-
if (compressed == firstCompressed && barcode.equals(firstBarcode) &&
- ifdCount <= firstIFDCount)
- {
+ ifdCount == firstIFDCount) {
int[] well = getWell(file);
int field = getField(file);
if (well[0] > nRows) nRows = well[0];
@@ -1394,14 +1402,21 @@ public class FlexReader extends FormatReader {
for (String f : sortedFiles) {
files.add(f);
}
+ if (runCount == 0) runCount = 1;
for (int field=0; field<nFiles; field++) {
FlexFile file = new FlexFile();
file.row = row;
file.column = col;
+ LOGGER.info("FlexTest: RunCount = " + runCount + " NFiles = " + nFiles);
+ if (runCount == 0) runCount = 1;
+ if (nFiles == 0) nFiles = 1;
+ if (runCount > nFiles) {
+ runCount = 1;
+ }
file.field = field % (nFiles / runCount);
file.file = files.get(field);
- file.acquisition = runDirs.size() == 0 ? 0:
+ file.acquisition = (runDirs == null || runDirs.size() == 0) ? 0:
runDirs.indexOf(new Location(file.file).getParentFile());
if (file.file == null) {
@@ -1846,6 +1861,9 @@ public class FlexReader extends FormatReader {
if (y != null) binY = Integer.parseInt(y);
}
}
+ else if (qName.equals("Plate")) {
+ plateCount++;
+ }
else if (qName.equals("WellCoordinate")) {
if (wellNumber.length == 1) {
wellNumber[0][0] = Integer.parseInt(attributes.getValue("Row")) - 1;
@@ -2137,4 +2155,12 @@ public class FlexReader extends FormatReader {
}
}
+ public boolean groupPlates() {
+ MetadataOptions options = getMetadataOptions();
+ if (options instanceof DynamicMetadataOptions) {
+ return ((DynamicMetadataOptions) options).getBoolean(
+ GROUP_PLATES, GROUP_PLATES_DEFAULT);
+ }
+ return GROUP_PLATES_DEFAULT;
+ }
}
There are a few places hardcoding runCount=1. Is this the reason why the extra files are excluded?
With the last changes, the non-regression tests are successfully passing with the default option behavior esp. using the representative plate from idr0001.
Turning the plate grouping option on either via passing it in the command line or using an options file with a copy of the fileset leads to a NullPointerException though
build@5bf9d954f85a:~$ /bio-formats-build/bioformats/tools/showinf -nopix -option flex.group.plate true /uod/idr/repos/curated/flex/public/idr0001/X_110331_S13/Meas_01\(2011-03-31_11-53-03\)/001001000.flex
Checking file format [Evotec Flex]
Initializing reader
FlexReader initializing /uod/idr/repos/curated/flex/public/idr0001/X_110331_S13/Meas_01(2011-03-31_11-53-03)/001001000.flex
Storing well indices
Looking for other .flex files
Looking for files that are in the same dataset as /uod/idr/repos/curated/flex/public/idr0001/X_110331_S13/Meas_01(2011-03-31_11-53-03)/001001000.flex
Measurement files not found.
Making sure that all .flex files are valid
Grouping together files in the same dataset
parsing /uod/idr/repos/curated/flex/public/idr0001/X_110331_S13/Meas_01(2011-03-31_11-53-03)/006002000.flex
...
Exception in thread "main" java.lang.ArithmeticException: / by zero
at loci.formats.in.FlexReader.groupFiles(FlexReader.java:1409)
at loci.formats.in.FlexReader.initFlexFile(FlexReader.java:590)
at loci.formats.in.FlexReader.initFile(FlexReader.java:419)
at loci.formats.FormatReader.setId(FormatReader.java:1421)
at loci.formats.ImageReader.setId(ImageReader.java:849)
at loci.formats.ReaderWrapper.setId(ReaderWrapper.java:650)
at loci.formats.tools.ImageInfo.testRead(ImageInfo.java:1035)
at loci.formats.tools.ImageInfo.main(ImageInfo.java:1121)
Conflicting PR. Removed from build BIOFORMATS-push#1070. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
--conflicts
Conflicting PR. Removed from build BIOFORMATS-push#7. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
--conflicts
Conflicting PR. Removed from build BIOFORMATS-push#8. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
--conflicts
Conflicting PR. Removed from build BIOFORMATS-push#9. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
--conflicts
Conflicting PR. Removed from build BIOFORMATS-push#1. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
--conflicts
Conflicting PR. Removed from build BIOFORMATS-push#4. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
--conflicts
Conflicting PR. Removed from build BIOFORMATS-push#1120. See the console output for more details. Possible conflicts:
- PR #3347 melissalinkert 'Add FormatTools methods for creating well names'
- components/formats-gpl/src/loci/formats/in/FlexReader.java
- Upstream changes
- components/formats-gpl/src/loci/formats/in/FlexReader.java
~~--conflicts~~ Conflict resolved in build BIOFORMATS-push#745. See the console output for more details.
Closing this as we might be exploring other options than trying to backport the IDR specific changes to the mainline. Effectively, this feature has only ever been used in the context of one studies (idr0001) so we might also want to collect additional use cases.
Testing on idr-testing as omero-server... with today's build 699 (https://merge-ci.openmicroscopy.org/jenkins/job/BIOFORMATS-build/669/default/artifact/bio-formats-build/bioformats/components/formats-gpl/target/)
$ cd /opt/omero/server
$ wget https://merge-ci.openmicroscopy.org/jenkins/job/BIOFORMATS-build/669/default/artifact/bio-formats-build/bioformats/components/formats-gpl/target/formats-gpl-7.2.0-SNAPSHOT.jar
$ mv formats-gpl-7.2.0-SNAPSHOT.jar formats-gpl-7.2.0-SNAPSHOT_b669.jar
$ rm OMERO.server/lib/server/formats-gpl.jar
$ rm OMERO.server/lib/client/formats-gpl.jar
$ cp formats-gpl-7.2.0-SNAPSHOT_b669.jar OMERO.server/lib/server/
$ cp formats-gpl-7.2.0-SNAPSHOT_b669.jar OMERO.server/lib/client/
Restarted server sudo service omero-server restart
Viewing images from first acquisition of first plate from idr0001 worked OK (after short wait for memo file to be created). Then tried viewing Image from 2nd acquisition of the plate...
2024-01-18 17:20:14,218 INFO [ ome.security.basic.BasicEventContext] (l.Server-1) cctx: group=3
2024-01-18 17:20:14,228 DEBUG [ loci.formats.Memoizer] (l.Server-2) loaded memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267401 bytes)
2024-01-18 17:20:14,228 DEBUG [ loci.formats.Memoizer] (l.Server-2) start[1705598414162] time[65] tag[loci.formats.Memoizer.loadMemo]
2024-01-18 17:20:14,229 DEBUG [ loci.formats.Memoizer] (l.Server-2) start[1705598414159] time[69] tag[loci.formats.Memoizer.setId]
2024-01-18 17:20:14,230 ERROR [ ome.io.nio.PixelsService] (l.Server-2) Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
java.lang.IllegalArgumentException: Invalid series: 96 index=96
at loci.formats.FormatReader.seriesToCoreIndex(FormatReader.java:1341)
at loci.formats.FormatReader.setSeries(FormatReader.java:972)
at loci.formats.ImageReader.setSeries(ImageReader.java:491)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:375)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:375)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:375)
at ome.io.bioformats.BfPixelBuffer.setSeries(BfPixelBuffer.java:125)
at ome.io.nio.PixelsService.createBfPixelBuffer(PixelsService.java:898)
at ome.io.nio.PixelsService._getPixelBuffer(PixelsService.java:653)
at ome.io.nio.PixelsService.getPixelBuffer(PixelsService.java:571)
at ome.services.RenderingBean$12.doWork(RenderingBean.java:2205)
at jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
...
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy123.load_async(Unknown Source)
at omero.api._RenderingEngineTie.load_async(_RenderingEngineTie.java:248)
at omero.api._RenderingEngineDisp.___load(_RenderingEngineDisp.java:1223)
at omero.api._RenderingEngineDisp.__dispatch(_RenderingEngineDisp.java:2405)
at IceInternal.Incoming.invoke(Incoming.java:221)
at Ice.ConnectionI.invokeAll(ConnectionI.java:2536)
at Ice.ConnectionI.dispatch(ConnectionI.java:1145)
at Ice.ConnectionI.message(ConnectionI.java:1056)
at IceInternal.ThreadPool.run(ThreadPool.java:395)
at IceInternal.ThreadPool.access$300(ThreadPool.java:12)
at IceInternal.ThreadPool$EventHandlerThread.run(ThreadPool.java:832)
at java.base/java.lang.Thread.run(Thread.java:829)
2024-01-18 17:20:14,232 INFO [ org.perf4j.TimingLogger] (l.Server-2) start[1705598414149] time[82] tag[omero.call.exception]
2024-01-18 17:20:14,232 INFO [ ome.services.util.ServiceHandler] (l.Server-2) Excp: ome.conditions.ResourceError: Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:14,232 INFO [ org.perf4j.TimingLogger] (l.Server-2) start[1705598414136] time[96] tag[omero.call.exception]
2024-01-18 17:20:14,232 INFO [ ome.services.util.ServiceHandler] (l.Server-2) Excp: ome.conditions.ResourceError: Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:14,237 INFO [ ome.security.basic.EventHandler] (l.Server-1) Auth: user=52,group=3,event=367837690(User),sess=b9658bb2-4668-4159-a1f0-fca8feebc30f
2024-01-18 17:20:14,238 INFO [ ome.services.util.ServiceHandler] (l.Server-3) Meth: interface ome.api.StatefulServiceInterface.close
2024-01-18 17:20:14,238 INFO [ ome.services.util.ServiceHandler] (l.Server-3) Args: ()
2024-01-18 17:20:14,238 INFO [ org.perf4j.TimingLogger] (l.Server-3) start[1705598414238] time[0] tag[omero.call.success.ome.services.RenderingBean.close]
2024-01-18 17:20:14,238 INFO [ ome.services.util.ServiceHandler] (l.Server-3) Rslt: null
2024-01-18 17:20:14,239 INFO [ omero.cmd.SessionI] (l.Server-3) Unregistered servant:b9658bb2-4668-4159-a1f0-fca8feebc30f/3afc97df-ae11-42fb-adc2-32c10e690959omero.api.RenderingEngine(omero.api._RenderingEngineTie@503df9dc)
2024-01-18 17:20:14,240 INFO [o.services.sessions.SessionContext$Count] (l.Server-7) -Reference count: b9658bb2-4668-4159-a1f0-fca8feebc30f=1
2024-01-18 17:20:14,240 INFO [ omero.cmd.SessionI] (l.Server-7) cleanupSelf(ServiceFactoryI(session-3fce9345-9599-4ed7-97cf-2275fe718582/b9658bb2-4668-4159-a1
Relevant memo file logs, creation loading etc
$ grep "2015-11/23/16-25-15.579" /opt/omero/server/OMERO.server/var/log/Blitz-0.log
2024-01-18 17:20:02,524 WARN [ loci.formats.FormatHandler] (l.Server-0) parsing /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/008012001.flex
2024-01-18 17:20:04,213 DEBUG [ loci.formats.Memoizer] (l.Server-0) saved to temp file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo16987755357726547062
2024-01-18 17:20:04,213 DEBUG [ loci.formats.Memoizer] (l.Server-0) saved memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267401 bytes)
2024-01-18 17:20:04,213 INFO [ ome.io.nio.PixelsService] (l.Server-0) Creating BfPixelBuffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex Series: 0
2024-01-18 17:20:05,266 INFO [ ome.services.OmeroFilePathResolver] (l.Server-0) Metadata only file, resulting path: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:05,434 DEBUG [ loci.formats.Memoizer] (l.Server-0) loaded memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267401 bytes)
2024-01-18 17:20:05,436 INFO [ ome.io.nio.PixelsService] (l.Server-0) Creating BfPixelBuffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex Series: 0
2024-01-18 17:20:05,598 INFO [ ome.services.OmeroFilePathResolver] (l.Server-5) Metadata only file, resulting path: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:05,695 DEBUG [ loci.formats.Memoizer] (l.Server-5) loaded memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267401 bytes)
2024-01-18 17:20:05,697 INFO [ ome.io.nio.PixelsService] (l.Server-5) Creating BfPixelBuffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex Series: 0
2024-01-18 17:20:06,285 INFO [ ome.services.OmeroFilePathResolver] (l.Server-5) Metadata only file, resulting path: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:06,367 DEBUG [ loci.formats.Memoizer] (l.Server-5) loaded memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267401 bytes)
2024-01-18 17:20:06,369 INFO [ ome.io.nio.PixelsService] (l.Server-5) Creating BfPixelBuffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex Series: 0
2024-01-18 17:20:14,156 INFO [ ome.services.OmeroFilePathResolver] (l.Server-2) Metadata only file, resulting path: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:14,228 DEBUG [ loci.formats.Memoizer] (l.Server-2) loaded memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267401 bytes)
2024-01-18 17:20:14,230 ERROR [ ome.io.nio.PixelsService] (l.Server-2) Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:14,232 INFO [ ome.services.util.ServiceHandler] (l.Server-2) Excp: ome.conditions.ResourceError: Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-01-18 17:20:14,232 INFO [ ome.services.util.ServiceHandler] (l.Server-2) Excp: ome.conditions.ResourceError: Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
Excluding due to failures in jdk17 tests when saving memo files: https://merge-ci.openmicroscopy.org/jenkins/job/BIOFORMATS-test-folder/47958/console
I was reminded of this when cleaning up old tabs I have open.... It would be great if we could get this working in time for the NGFF upgrade since it requires memo file regeneration (best to combine the changes so we only have to regenerate memo files once).
Since this PR is getting merged into builds now (e.g. today's https://merge-ci.openmicroscopy.org/jenkins/job/BIOFORMATS-push/45/console) tried testing again...
Testing on pilot_idrngff... Update from today's build of formats-gpl-8.0.0-SNAPSHOT.jar as described above... (Also updated ZarrReader to today's build).
As omero-server
wget https://merge-ci.openmicroscopy.org/jenkins/job/BIOFORMATS-build/45/label=testintegration/artifact/bio-formats-build/bioformats/components/formats-gpl/target/formats-gpl-8.0.0-SNAPSHOT.jar
mv formats-gpl-8.0.0-SNAPSHOT.jar formats-gpl-8.0.0-SNAPSHOT_b45.jar
rm OMERO.server/lib/server/formats-gpl.jar
rm OMERO.server/lib/client/formats-gpl.jar
cp formats-gpl-8.0.0-SNAPSHOT_b45.jar OMERO.server/lib/server/
cp formats-gpl-8.0.0-SNAPSHOT_b45.jar OMERO.server/lib/client/
Restarted server...
Tried viewing Preview panel of first Well from 2nd Acquisition of the first idr0001 Plate... Looks like same error as above...
2024-04-22 12:29:57,155 INFO [ o.s.b.repo.ManagedReaderSecurityCheck] (l.Server-5) noted managed repository cdf35825-def1-4580-8d0b-9c349b8f78d6
2024-04-22 12:29:57,501 DEBUG [ loci.formats.Memoizer] (l.Server-5) saved to temp file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo14747644009559314075
2024-04-22 12:29:57,501 DEBUG [ loci.formats.Memoizer] (l.Server-5) start[1713788997266] time[234] tag[loci.formats.Memoizer.saveMemo]
2024-04-22 12:29:57,501 DEBUG [ loci.formats.Memoizer] (l.Server-5) saved memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267369 bytes)
2024-04-22 12:29:57,501 DEBUG [ loci.formats.Memoizer] (l.Server-5) start[1713788925239] time[72261] tag[loci.formats.Memoizer.setId]
2024-04-22 12:29:57,505 ERROR [ ome.io.nio.PixelsService] (l.Server-5) Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
java.lang.IllegalArgumentException: Invalid series: 96 index=96
at loci.formats.FormatReader.seriesToCoreIndex(FormatReader.java:1341)
at loci.formats.FormatReader.setSeries(FormatReader.java:972)
at loci.formats.ImageReader.setSeries(ImageReader.java:491)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:375)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:375)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:375)
at ome.io.bioformats.BfPixelBuffer.setSeries(BfPixelBuffer.java:125)
at ome.io.nio.PixelsService.createBfPixelBuffer(PixelsService.java:898)
As discussed this morning, we need to be sure that the memo file is deleted, and we're not using a memo file generated before the fix was applied...
First plate:
rm -rf "/data/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo"
Tried viewing image from 3rd Acquisition of first plate Meas03...
Web returned:
serverExceptionClass = ome.conditions.ResourceError
message = Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
bash-4.2$ grep "16-25-15.579" /opt/omero/server/OMERO.server/var/log/Blitz-0.log
...
2024-04-23 09:39:12,690 INFO [ ome.services.util.ServiceHandler] (.Server-14) Rslt: ([demo_2/2015-11/23/16-25-15.579/Meas_03(2012-07-31_13-12-10)/, 005010001.flex, idr/filesets/idr0001-graml-sysgro/20151116-verified/JL_120731_S6A/Meas_01(2012-07-31_10-41-12)/00501], [demo_2/2015-11/23/16-25-15.579/Meas_03(2012-07-31_13-12-10)/, 005011001.flex, idr/filesets/idr0001-graml-sysgro/20151116-verified/JL_120731_S6A/Meas_01(2012-07-31_10-41-12)/00501], [demo_2/2015-11/23/16-25-15.579/Meas_03(2012-07-31_13-12-10)/, 005012001.flex, idr/filesets/idr0001-graml-sysgro/20151116-verified/JL_120731_S6A/Meas_01(2012-07-31_10-41-12)/00501], ... 573 more)
2024-04-23 09:42:04,461 INFO [ ome.services.OmeroFilePathResolver] (l.Server-7) Metadata only file, resulting path: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-04-23 09:42:05,577 INFO [ loci.formats.ImageReader] (l.Server-7) FlexReader initializing /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-04-23 09:42:05,578 INFO [ loci.formats.FormatHandler] (l.Server-7) Looking for files that are in the same dataset as /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-04-23 09:42:05,735 WARN [ loci.formats.FormatHandler] (l.Server-7) parsing /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-04-23 09:42:05,739 WARN [ loci.formats.FormatHandler] (l.Server-7) parsing /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001002001.flex
...
2024-04-23 09:42:11,207 WARN [ loci.formats.FormatHandler] (l.Server-7) parsing /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/008011001.flex
2024-04-23 09:42:11,254 WARN [ loci.formats.FormatHandler] (l.Server-7) parsing /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/008012001.flex
2024-04-23 09:42:11,845 DEBUG [ loci.formats.Memoizer] (l.Server-7) saved to temp file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo17264536529646091535
2024-04-23 09:42:11,845 DEBUG [ loci.formats.Memoizer] (l.Server-7) saved memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267077 bytes)
2024-04-23 09:42:11,845 ERROR [ ome.io.nio.PixelsService] (l.Server-7) Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-04-23 09:42:11,848 INFO [ ome.services.util.ServiceHandler] (l.Server-7) Excp: ome.conditions.ResourceError: Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
2024-04-23 09:42:11,848 INFO [ ome.services.util.ServiceHandler] (l.Server-7) Excp: ome.conditions.ResourceError: Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
...
2024-04-23 09:42:11,666 INFO [ loci.formats.FormatHandler] (l.Server-7) Parsing IFDs for well H12
2024-04-23 09:42:11,667 INFO [ loci.formats.FormatHandler] (l.Server-7) Parsing .flex file (well H12, field -1)
2024-04-23 09:42:11,667 INFO [ loci.formats.FormatHandler] (l.Server-7) Parsing XML in .flex file
2024-04-23 09:42:11,669 INFO [ loci.formats.FormatHandler] (l.Server-7) Populating pixel scaling factors
2024-04-23 09:42:11,669 WARN [ loci.formats.FormatHandler] (l.Server-7) mismatch between image count, names and factors (count=3072, names=32, factors=32)
2024-04-23 09:42:11,669 INFO [ loci.formats.FormatHandler] (l.Server-7) Populating MetadataStore
2024-04-23 09:42:11,845 DEBUG [ loci.formats.Memoizer] (l.Server-7) saved to temp file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo17264536529646091535
2024-04-23 09:42:11,845 DEBUG [ loci.formats.Memoizer] (l.Server-7) start[1713865331760] time[84] tag[loci.formats.Memoizer.saveMemo]
2024-04-23 09:42:11,845 DEBUG [ loci.formats.Memoizer] (l.Server-7) saved memo file: /data/OMERO/BioFormatsCache/data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/.001001001.flex.bfmemo (7267077 bytes)
2024-04-23 09:42:11,845 DEBUG [ loci.formats.Memoizer] (l.Server-7) start[1713865324466] time[7378] tag[loci.formats.Memoizer.setId]
2024-04-23 09:42:11,845 ERROR [ ome.io.nio.PixelsService] (l.Server-7) Error instantiating pixel buffer: /data/OMERO/ManagedRepository/demo_2/2015-11/23/16-25-15.579/Meas_01(2012-07-31_10-41-12)/001001001.flex
java.lang.IllegalArgumentException: Invalid series: 192 index=96
at loci.formats.FormatReader.seriesToCoreIndex(FormatReader.java:1355)
at loci.formats.FormatReader.setSeries(FormatReader.java:972)
at loci.formats.ImageReader.setSeries(ImageReader.java:493)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:407)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:407)
at loci.formats.ReaderWrapper.setSeries(ReaderWrapper.java:407)
at ome.io.bioformats.BfPixelBuffer.setSeries(BfPixelBuffer.java:125)
at ome.io.nio.PixelsService.createBfPixelBuffer(PixelsService.java:898)
at ome.io.nio.PixelsService._getPixelBuffer(PixelsService.java:653)