I use this project on my work server spark cluster which version is 2.3.0 and python version is 3.6.1 . I must install this offline , but your introduction is an online setup .what should I do with offline install on offline use.
I install spark-knn0.3.0 on my cluster class path, and move your /python/pyspark-knn on my code.
use /python/test.py to test .
my console info
Traceback (most recent call last):
File "", line 2, in
File "/usr/hdp/current/spark2-client/python/pyspark/init.py", line 105, in wrapper
return func(self, **kwargs)
File "/home/wsk/ttrms_local/src/jobs/holiday_forecast/knn/classification.py", line 19, in init
"org.apache.spark.ml.classification.KNNClassifier", self.uid)
File "/usr/hdp/current/spark2-client/python/pyspark/ml/wrapper.py", line 63, in _new_java_obj
return java_obj(*java_args)
File "/opt/anaconda2/envs/py36/lib/python3.6/site-packages/py4j/java_gateway.py", line 1428, in call
answer, self._gateway_client, None, self._fqn)
File "/usr/hdp/current/spark2-client/python/pyspark/sql/utils.py", line 63, in deco
return f(*a, **kw)
File "/opt/anaconda2/envs/py36/lib/python3.6/site-packages/py4j/protocol.py", line 320, in get_return_value
format(target_id, ".", name), value)
py4j.protocol.Py4JJavaError: An error occurred while calling None.org.apache.spark.ml.classification.KNNClassifier.
: java.lang.NoSuchMethodError: org.apache.spark.ml.param.shared.HasInputCols.$init$(Lorg/apache/spark/ml/param/shared/HasInputCols;)V
at org.apache.spark.ml.classification.KNNClassifier.(KNNClassifier.scala:23)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:238)
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
at py4j.GatewayConnection.run(GatewayConnection.java:214)
at java.lang.Thread.run(Thread.java:745)
spark-knn copied to clipboard
saurfang
