JohnSnowLabs
Cannot run embeddings model or any model below 5.1.1
I cannot run any model with Spark NLP below 5.0.0 version. How can I solve this?
Steps to Reproduce
1.[ — ]
An error occurred while calling z:com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel.
: java.lang.NoClassDefFoundError: Could not initialize class org.tensorflow.Graph
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.readGraph(TensorflowWrapper.scala:415)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.unpackWithoutBundle(TensorflowWrapper.scala:330)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.read(TensorflowWrapper.scala:484)
at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel(TensorflowSerializeModel.scala:154)
at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel$(TensorflowSerializeModel.scala:123)
at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readTensorflowModel(BertSentenceEmbeddings.scala:560)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel(BertSentenceEmbeddings.scala:469)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel$(BertSentenceEmbeddings.scala:464)
at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readModel(BertSentenceEmbeddings.scala:560)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1(BertSentenceEmbeddings.scala:494)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1$adapted(BertSentenceEmbeddings.scala:494)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1(ParamsAndFeaturesReadable.scala:50)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1$adapted(ParamsAndFeaturesReadable.scala:49)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.onRead(ParamsAndFeaturesReadable.scala:49)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1(ParamsAndFeaturesReadable.scala:61)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1$adapted(ParamsAndFeaturesReadable.scala:61)
at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:38)
at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:24)
at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:515)
at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:507)
at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader$.downloadModel(ResourceDownloader.scala:714)
at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel(ResourceDownloader.scala)
at sun.reflect.GeneratedMethodAccessor181.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.lang.ExceptionInInitializerError: Exception java.lang.UnsatisfiedLinkError: no jnitensorflow in java.library.path [in thread "Thread-4"]
at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1863)
at java.lang.Runtime.loadLibrary0(Runtime.java:843)
at java.lang.System.loadLibrary(System.java:1134)
at org.bytedeco.javacpp.Loader.loadLibrary(Loader.java:1738)
at org.bytedeco.javacpp.Loader.load(Loader.java:1345)
at org.bytedeco.javacpp.Loader.load(Loader.java:1157)
at org.bytedeco.javacpp.Loader.load(Loader.java:1133)
at org.tensorflow.internal.c_api.global.tensorflow.(tensorflow.java:12)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.bytedeco.javacpp.Loader.load(Loader.java:1212)
at org.bytedeco.javacpp.Loader.load(Loader.java:1157)
at org.bytedeco.javacpp.Loader.load(Loader.java:1149)
at org.tensorflow.NativeLibrary.load(NativeLibrary.java:64)
at org.tensorflow.TensorFlow.(TensorFlow.java:156)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at org.tensorflow.Graph.(Graph.java:1341)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.readGraph(TensorflowWrapper.scala:415)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.unpackWithoutBundle(TensorflowWrapper.scala:330)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.read(TensorflowWrapper.scala:484)
at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel(TensorflowSerializeModel.scala:154)
at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel$(TensorflowSerializeModel.scala:123)
at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readTensorflowModel(BertSentenceEmbeddings.scala:560)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel(BertSentenceEmbeddings.scala:469)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel$(BertSentenceEmbeddings.scala:464)
at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readModel(BertSentenceEmbeddings.scala:560)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1(BertSentenceEmbeddings.scala:494)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1$adapted(BertSentenceEmbeddings.scala:494)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1(ParamsAndFeaturesReadable.scala:50)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1$adapted(ParamsAndFeaturesReadable.scala:49)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.onRead(ParamsAndFeaturesReadable.scala:49)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1(ParamsAndFeaturesReadable.scala:61)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1$adapted(ParamsAndFeaturesReadable.scala:61)
at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:38)
at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:24)
at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:515)
at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:507)
at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader$.downloadModel(ResourceDownloader.scala:714)
at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel(ResourceDownloader.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
... 10 more
[OK!]
Py4JJavaError Traceback (most recent call last)
Cell In[53], line 5
1 document_assembler = nlp.DocumentAssembler()
2 .setInputCol("text")
3 .setOutputCol("document")
----> 5 embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_uncased_legal", "en")
6 .setInputCols("document")
7 .setOutputCol("document_embeddings")
File ~/anaconda3/lib/python3.11/site-packages/sparknlp/annotator/embeddings/bert_sentence_embeddings.py:224, in BertSentenceEmbeddings.pretrained(name, lang, remote_loc) 206 """Downloads and loads a pretrained model. 207 208 Parameters (...) 221 The restored model 222 """ 223 from sparknlp.pretrained import ResourceDownloader --> 224 return ResourceDownloader.downloadModel(BertSentenceEmbeddings, name, lang, remote_loc)
File ~/anaconda3/lib/python3.11/site-packages/sparknlp/pretrained/resource_downloader.py:98, in ResourceDownloader.downloadModel(reader, name, language, remote_loc, j_dwn) 96 except Py4JJavaError as e: 97 sys.stdout.write("\n" + str(e)) ---> 98 raise e 99 finally: 100 stop_threads = True
File ~/anaconda3/lib/python3.11/site-packages/sparknlp/pretrained/resource_downloader.py:95, in ResourceDownloader.downloadModel(reader, name, language, remote_loc, j_dwn) 93 t1.start() 94 try: ---> 95 j_obj = _internal._DownloadModel(reader.name, name, language, remote_loc, j_dwn).apply() 96 except Py4JJavaError as e: 97 sys.stdout.write("\n" + str(e))
File ~/anaconda3/lib/python3.11/site-packages/sparknlp/internal/init.py:537, in _DownloadModel.init(self, reader, name, language, remote_loc, validator) 536 def init(self, reader, name, language, remote_loc, validator): --> 537 super(_DownloadModel, self).init( 538 "com.johnsnowlabs.nlp.pretrained." + validator + ".downloadModel", 539 reader, 540 name, 541 language, 542 remote_loc, 543 )
File ~/anaconda3/lib/python3.11/site-packages/sparknlp/internal/extended_java_wrapper.py:27, in ExtendedJavaWrapper.init(self, java_obj, *args) 25 super(ExtendedJavaWrapper, self).init(java_obj) 26 self.sc = SparkContext._active_spark_context ---> 27 self._java_obj = self.new_java_obj(java_obj, *args) 28 self.java_obj = self._java_obj
File ~/anaconda3/lib/python3.11/site-packages/sparknlp/internal/extended_java_wrapper.py:37, in ExtendedJavaWrapper.new_java_obj(self, java_class, *args) 36 def new_java_obj(self, java_class, *args): ---> 37 return self._new_java_obj(java_class, *args)
File ~/anaconda3/lib/python3.11/site-packages/pyspark/ml/wrapper.py:86, in JavaWrapper._new_java_obj(java_class, *args) 84 java_obj = getattr(java_obj, name) 85 java_args = [_py2java(sc, arg) for arg in args] ---> 86 return java_obj(*java_args)
File ~/anaconda3/lib/python3.11/site-packages/py4j/java_gateway.py:1322, in JavaMember.call(self, *args)
1316 command = proto.CALL_COMMAND_NAME +
1317 self.command_header +
1318 args_command +
1319 proto.END_COMMAND_PART
1321 answer = self.gateway_client.send_command(command)
-> 1322 return_value = get_return_value(
1323 answer, self.gateway_client, self.target_id, self.name)
1325 for temp_arg in temp_args:
1326 if hasattr(temp_arg, "_detach"):
File ~/anaconda3/lib/python3.11/site-packages/pyspark/errors/exceptions/captured.py:169, in capture_sql_exception.
File ~/anaconda3/lib/python3.11/site-packages/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name) 324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client) 325 if answer[1] == REFERENCE_TYPE: --> 326 raise Py4JJavaError( 327 "An error occurred while calling {0}{1}{2}.\n". 328 format(target_id, ".", name), value) 329 else: 330 raise Py4JError( 331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n". 332 format(target_id, ".", name, value))
Py4JJavaError: An error occurred while calling z:com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel.
: java.lang.NoClassDefFoundError: Could not initialize class org.tensorflow.Graph
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.readGraph(TensorflowWrapper.scala:415)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.unpackWithoutBundle(TensorflowWrapper.scala:330)
at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.read(TensorflowWrapper.scala:484)
at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel(TensorflowSerializeModel.scala:154)
at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel$(TensorflowSerializeModel.scala:123)
at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readTensorflowModel(BertSentenceEmbeddings.scala:560)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel(BertSentenceEmbeddings.scala:469)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel$(BertSentenceEmbeddings.scala:464)
at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readModel(BertSentenceEmbeddings.scala:560)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1(BertSentenceEmbeddings.scala:494)
at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1$adapted(BertSentenceEmbeddings.scala:494)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1(ParamsAndFeaturesReadable.scala:50)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1$adapted(ParamsAndFeaturesReadable.scala:49)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.onRead(ParamsAndFeaturesReadable.scala:49)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1(ParamsAndFeaturesReadable.scala:61)
at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1$adapted(ParamsAndFeaturesReadable.scala:61)
at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:38)
at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:24)
at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:515)
at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:507)
at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader$.downloadModel(ResourceDownloader.scala:714)
at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel(ResourceDownloader.scala)
at sun.reflect.GeneratedMethodAccessor181.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.lang.ExceptionInInitializerError: Exception java.lang.UnsatisfiedLinkError: no jnitensorflow in java.library.path [in thread "Thread-4"]
at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1863)
at java.lang.Runtime.loadLibrary0(Runtime.java:843)
at java.lang.System.loadLibrary(System.java:1134)
at org.bytedeco.javacpp.Loader.loadLibrary(Loader.java:1738)
at org.bytedeco.javacpp.Loader.load(Loader.java:1345)
at org.bytedeco.javacpp.Loader.load(Loader.java:1157)
at org.bytedeco.javacpp.Loader.load(Loader.java:1133)
at org.tensorflow.internal.c_api.global.tensorflow.
Your Environment
- Spark-NLP version:
- SparkSession - in-memory SparkContext Spark UI Version v3.4.0 Master local[*] AppName Spark NLP
SparkSession - in-memory SparkContext Spark UI Version v3.4.0 Master local[*] AppName Spark NLP
Im using Jupyter notebook on Mac
Hi @valentinapremolipep , Is this still an issue? If so, could you share your notebook so I can take a closer look?
