hail
hail copied to clipboard
Published
20 hours ago •
hail-is
hail-is
[query] filter intervals causing a failed partitioner assertion
What happened?
A user reported this bug on discuss: https://discuss.hail.is/t/illegalargumentexception/3808.
This looks like it may be related to #14420.
Version
0.2.130
Relevant log output
In [13]: mtf = hl.filter_intervals(mt, [hl.parse_locus_interval('chr19:50927163-50927163', reference_genome=rg)])
In [14]: mtf.show()
---------------------------------------------------------------------------
FatalError Traceback (most recent call last)
File /usr/local/lib/python3.11/dist-packages/IPython/core/formatters.py:711, in PlainTextFormatter.__call__(self, obj)
704 stream = StringIO()
705 printer = pretty.RepresentationPrinter(stream, self.verbose,
706 self.max_width, self.newline,
707 max_seq_length=self.max_seq_length,
708 singleton_pprinters=self.singleton_printers,
709 type_pprinters=self.type_printers,
710 deferred_pprinters=self.deferred_printers)
--> 711 printer.pretty(obj)
712 printer.flush()
713 return stream.getvalue()
File /usr/local/lib/python3.11/dist-packages/IPython/lib/pretty.py:411, in RepresentationPrinter.pretty(self, obj)
408 return meth(obj, self, cycle)
409 if cls is not object \
410 and callable(cls.__dict__.get('__repr__')):
--> 411 return _repr_pprint(obj, self, cycle)
413 return _default_pprint(obj, self, cycle)
414 finally:
File /usr/local/lib/python3.11/dist-packages/IPython/lib/pretty.py:779, in _repr_pprint(obj, p, cycle)
777 """A pprint that just redirects to the normal repr function."""
778 # Find newlines and replace them with p.break_()
--> 779 output = repr(obj)
780 lines = output.splitlines()
781 with p.group():
File /usr/local/lib/python3.11/dist-packages/hail/matrixtable.py:2825, in MatrixTable._Show.__repr__(self)
2824 def __repr__(self):
-> 2825 return self.__str__()
File /usr/local/lib/python3.11/dist-packages/hail/matrixtable.py:2819, in MatrixTable._Show.__str__(self)
2818 def __str__(self):
-> 2819 s = self.table_show.__str__()
2820 if self.displayed_n_cols != self.actual_n_cols:
2821 s += f"showing the first { self.displayed_n_cols } of { self.actual_n_cols } columns"
File /usr/local/lib/python3.11/dist-packages/hail/table.py:2154, in Table._Show.__str__(self)
2153 def __str__(self):
-> 2154 return self._ascii_str()
File /usr/local/lib/python3.11/dist-packages/hail/table.py:2180, in Table._Show._ascii_str(self)
2177 return s[: truncate - 3] + "..."
2178 return s
-> 2180 rows, has_more, dtype = self.data()
2181 fields = list(dtype)
2182 trunc_fields = [trunc(f) for f in fields]
File /usr/local/lib/python3.11/dist-packages/hail/table.py:2164, in Table._Show.data(self)
2162 row_dtype = t.row.dtype
2163 t = t.select(**{k: hl._showstr(v) for (k, v) in t.row.items()})
-> 2164 rows, has_more = t._take_n(self.n)
2165 self._data = (rows, has_more, row_dtype)
2166 return self._data
File /usr/local/lib/python3.11/dist-packages/hail/table.py:2310, in Table._take_n(self, n)
2308 has_more = False
2309 else:
-> 2310 rows = self.take(n + 1)
2311 has_more = len(rows) > n
2312 rows = rows[:n]
File <decorator-gen-1250>:2, in take(self, n, _localize)
File /usr/local/lib/python3.11/dist-packages/hail/typecheck/check.py:585, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
582 @decorator
583 def wrapper(__original_func: Callable[..., T], *args, **kwargs) -> T:
584 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 585 return __original_func(*args_, **kwargs_)
File /usr/local/lib/python3.11/dist-packages/hail/table.py:3027, in Table.take(self, n, _localize)
2993 @typecheck_method(n=int, _localize=bool)
2994 def take(self, n, _localize=True):
2995 """Collect the first `n` rows of the table into a local list.
2996
2997 Examples
(...)
3024 List of row structs.
3025 """
-> 3027 return self.head(n).collect(_localize)
File <decorator-gen-1244>:2, in collect(self, _localize, _timed)
File /usr/local/lib/python3.11/dist-packages/hail/typecheck/check.py:585, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
582 @decorator
583 def wrapper(__original_func: Callable[..., T], *args, **kwargs) -> T:
584 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 585 return __original_func(*args_, **kwargs_)
File /usr/local/lib/python3.11/dist-packages/hail/table.py:2814, in Table.collect(self, _localize, _timed)
2812 e = construct_expr(rows_ir, hl.tarray(t.row.dtype))
2813 if _localize:
-> 2814 return Env.backend().execute(e._ir, timed=_timed)
2815 else:
2816 return e
File /usr/local/lib/python3.11/dist-packages/hail/backend/spark_backend.py:226, in SparkBackend.execute(self, ir, timed)
223 except Exception as fatal:
224 raise err from fatal
--> 226 raise err
File /usr/local/lib/python3.11/dist-packages/hail/backend/spark_backend.py:218, in SparkBackend.execute(self, ir, timed)
216 def execute(self, ir: BaseIR, timed: bool = False) -> Any:
217 try:
--> 218 return super().execute(ir, timed)
219 except Exception as err:
220 if self._copy_log_on_error:
File /usr/local/lib/python3.11/dist-packages/hail/backend/backend.py:190, in Backend.execute(self, ir, timed)
188 result, timings = self._rpc(ActionTag.EXECUTE, payload)
189 except FatalError as e:
--> 190 raise e.maybe_user_error(ir) from None
191 if ir.typ == tvoid:
192 value = None
File /usr/local/lib/python3.11/dist-packages/hail/backend/backend.py:188, in Backend.execute(self, ir, timed)
186 payload = ExecutePayload(self._render_ir(ir), '{"name":"StreamBufferSpec"}', timed)
187 try:
--> 188 result, timings = self._rpc(ActionTag.EXECUTE, payload)
189 except FatalError as e:
190 raise e.maybe_user_error(ir) from None
File /usr/local/lib/python3.11/dist-packages/hail/backend/py4j_backend.py:221, in Py4JBackend._rpc(self, action, payload)
219 if resp.status_code >= 400:
220 error_json = orjson.loads(resp.content)
--> 221 raise fatal_error_from_java_error_triplet(
222 error_json['short'], error_json['expanded'], error_json['error_id']
223 )
224 return resp.content, resp.headers.get('X-Hail-Timings', '')
FatalError: IllegalArgumentException: requirement failed
Java stack trace:
java.lang.IllegalArgumentException: requirement failed
at scala.Predef$.require(Predef.scala:268)
at is.hail.expr.ir.lowering.TableStage.repartitionNoShuffle(LowerTableIR.scala:357)
at is.hail.rvd.AbstractRVDSpec$.$anonfun$readZippedLowered$12(AbstractRVDSpec.scala:207)
at is.hail.expr.ir.TableNativeZippedReader.lower(TableIR.scala:2042)
at is.hail.expr.ir.TableReader.lower(TableIR.scala:663)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:1061)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$2(LowerTableIR.scala:1050)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:2242)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$2(LowerTableIR.scala:1050)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:2192)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$2(LowerTableIR.scala:1050)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:1654)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$2(LowerTableIR.scala:1050)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:2192)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$2(LowerTableIR.scala:1050)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:1361)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$2(LowerTableIR.scala:1050)
at is.hail.expr.ir.lowering.LowerTableIR$.applyTable(LowerTableIR.scala:1654)
at is.hail.expr.ir.lowering.LowerTableIR$.lower$1(LowerTableIR.scala:728)
at is.hail.expr.ir.lowering.LowerTableIR$.apply(LowerTableIR.scala:823)
at is.hail.expr.ir.lowering.LowerToCDA$.lower(LowerToCDA.scala:27)
at is.hail.expr.ir.lowering.LowerToCDA$.apply(LowerToCDA.scala:11)
at is.hail.expr.ir.lowering.LowerToDistributedArrayPass.transform(LoweringPass.scala:91)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.evaluate$1(LowerOrInterpretNonCompilable.scala:27)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:59)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.apply(LowerOrInterpretNonCompilable.scala:64)
at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.transform(LoweringPass.scala:83)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:32)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:84)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:32)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:84)
at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:30)
at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:29)
at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.apply(LoweringPass.scala:78)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:21)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:19)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:19)
at is.hail.expr.ir.lowering.EvalRelationalLets$.execute$1(EvalRelationalLets.scala:13)
at is.hail.expr.ir.lowering.EvalRelationalLets$.lower$1(EvalRelationalLets.scala:21)
at is.hail.expr.ir.lowering.EvalRelationalLets$.apply(EvalRelationalLets.scala:35)
at is.hail.expr.ir.lowering.EvalRelationalLetsPass.transform(LoweringPass.scala:168)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:32)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:84)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:32)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:84)
at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:30)
at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:29)
at is.hail.expr.ir.lowering.EvalRelationalLetsPass.apply(LoweringPass.scala:162)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:21)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:19)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:19)
at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:45)
at is.hail.backend.spark.SparkBackend._execute(SparkBackend.scala:600)
at is.hail.backend.spark.SparkBackend.$anonfun$execute$4(SparkBackend.scala:636)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:84)
at is.hail.backend.spark.SparkBackend.$anonfun$execute$3(SparkBackend.scala:631)
at is.hail.backend.spark.SparkBackend.$anonfun$execute$3$adapted(SparkBackend.scala:630)
at is.hail.backend.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:78)
at is.hail.utils.package$.using(package.scala:664)
at is.hail.backend.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:78)
at is.hail.utils.package$.using(package.scala:664)
at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:13)
at is.hail.backend.ExecuteContext$.scoped(ExecuteContext.scala:65)
at is.hail.backend.spark.SparkBackend.$anonfun$withExecuteContext$2(SparkBackend.scala:407)
at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:55)
at is.hail.utils.ExecutionTimer$.logTime(ExecutionTimer.scala:62)
at is.hail.backend.spark.SparkBackend.withExecuteContext(SparkBackend.scala:393)
at is.hail.backend.spark.SparkBackend.execute(SparkBackend.scala:630)
at is.hail.backend.BackendHttpHandler.handle(BackendServer.scala:88)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:822)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:794)
at sun.net.httpserver.ServerImpl$DefaultExecutor.execute(ServerImpl.java:199)
at sun.net.httpserver.ServerImpl$Dispatcher.handle(ServerImpl.java:544)
at sun.net.httpserver.ServerImpl$Dispatcher.run(ServerImpl.java:509)
at java.lang.Thread.run(Thread.java:750)
Hail version: 0.2.130-bea04d9c79b5
Error summary: IllegalArgumentException: requirement failed
