dask-sql
dask-sql copied to clipboard
dask-contrib
[BUG] Can't create a table from file using double quotes for arguments
When I try to create a table from a file using double quotes rather than single quotes, I get a parsing error.
from dask.distributed import Client
from dask_sql import Context
from dask.distributed import LocalCluster
import dask.dataframe as dd
import pandas as pd
cluster = LocalCluster()
client = Client(cluster)
c = Context()
c.ipython_magic()
df = pd.DataFrame({"a":range(50000), "b":range(50000)})
df.to_json("temp.jsonl", lines=True, orient="records")
%%sql
CREATE TABLE reviews WITH (
format='json',
location='temp.jsonl',
gpu=False,
lines=True
)
WARNING:dask_sql.context:No tables are registered.
Execution time: 1.05s
| a | b
-- | -- | --
49995 | 49995
49996 | 49996
49997 | 49997
49998 | 49998
49999 | 49999
%%sql
CREATE TABLE reviews WITH (
format="json",
location="temp.jsonl",
gpu=False,
lines=True
)
WARNING:dask_sql.context:No tables are registered.
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Input In [14], in <cell line: 1>()
----> 1 get_ipython().run_cell_magic('sql', '', '\nCREATE TABLE reviews WITH (\n format="json",\n location="temp.jsonl",\n gpu=False,\n lines=True\n)\n')
File ~/miniconda3/envs/rapids-22.10-dasksql/lib/python3.9/site-packages/IPython/core/interactiveshell.py:2358, in InteractiveShell.run_cell_magic(self, magic_name, line, cell)
2356 with self.builtin_trap:
2357 args = (magic_arg_s, cell)
-> 2358 result = fn(*args, **kwargs)
2359 return result
File ~/miniconda3/envs/rapids-22.10-dasksql/lib/python3.9/site-packages/dask_sql/integrations/ipython.py:83, in _register_ipython_magic.<locals>.sql(line, cell, local_ns)
80 dataframes = c._get_tables_from_stack()
82 t0 = time.time()
---> 83 res = c.sql(sql_statement, return_futures=False, dataframes=dataframes)
84 if (
85 "CREATE OR REPLACE TABLE" in sql_statement
86 or "CREATE OR REPLACE VIEW" in sql_statement
87 ):
88 table = sql_statement.split("CREATE OR REPLACE")[1]
File ~/miniconda3/envs/rapids-22.10-dasksql/lib/python3.9/site-packages/dask_sql/context.py:501, in Context.sql(self, sql, return_futures, dataframes, gpu, config_options)
496 else:
497 raise RuntimeError(
498 f"Encountered unsupported `LogicalPlan` sql type: {type(sql)}"
499 )
--> 501 return self._compute_table_from_rel(rel, return_futures)
File ~/miniconda3/envs/rapids-22.10-dasksql/lib/python3.9/site-packages/dask_sql/context.py:830, in Context._compute_table_from_rel(self, rel, return_futures)
829 def _compute_table_from_rel(self, rel: "LogicalPlan", return_futures: bool = True):
--> 830 dc = RelConverter.convert(rel, context=self)
832 # Optimization might remove some alias projects. Make sure to keep them here.
833 select_names = [field for field in rel.getRowType().getFieldList()]
File ~/miniconda3/envs/rapids-22.10-dasksql/lib/python3.9/site-packages/dask_sql/physical/rel/convert.py:61, in RelConverter.convert(cls, rel, context)
55 raise NotImplementedError(
56 f"No relational conversion for node type {node_type} available (yet)."
57 )
58 logger.debug(
59 f"Processing REL {rel} using {plugin_instance.__class__.__name__}..."
60 )
---> 61 df = plugin_instance.convert(rel, context=context)
62 logger.debug(f"Processed REL {rel} into {LoggableDataFrame(df)}")
63 return df
File ~/miniconda3/envs/rapids-22.10-dasksql/lib/python3.9/site-packages/dask_sql/physical/rel/custom/create_table.py:55, in CreateTablePlugin.convert(self, rel, context)
50 elif not create_table.getOrReplace():
51 raise RuntimeError(
52 f"A table with the name {table_name} is already present."
53 )
---> 55 kwargs = convert_sql_kwargs(create_table.getSQLWithOptions())
57 logger.debug(
58 f"Creating new table with name {table_name} and parameters {kwargs}"
59 )
61 format = kwargs.pop("format", None)
TypeError: "encountered unexpected Expr type: Identifier(Ident { value: \"json\", quote_style: Some('\"') })"
# packages in environment at /home/nicholasb/miniconda3/envs/rapids-22.10-dasksql:
cucim 22.10.00a220930 cuda_11_py39_gf4229e3_51 rapidsai-nightly
cudf 22.10.00a220929 cuda_11_py39_g5fad28942e_286 rapidsai-nightly
cudf_kafka 22.10.00a220929 py39_g920b58f948_288 rapidsai-nightly
cugraph 22.10.00a220930 cuda11_py39_g91598080_88 rapidsai-nightly
cuml 22.10.00a220930 cuda11_py39_g96da84cc1_50 rapidsai-nightly
cusignal 22.10.00a220930 py39_gd075e87_12 rapidsai-nightly
cuspatial 22.10.00a220930 py39_g6922ef5_55 rapidsai-nightly
custreamz 22.10.00a220929 py39_g920b58f948_288 rapidsai-nightly
cuxfilter 22.10.00a220930 py39_ge1aa0b2_17 rapidsai-nightly
dask 2022.7.1 pyhd8ed1ab_0 conda-forge
dask-core 2022.7.1 pyhd8ed1ab_0 conda-forge
dask-cuda 22.10.00a220930 py39_gc0ae66c_20 rapidsai-nightly
dask-cudf 22.10.00a220929 cuda_11_py39_g920b58f948_288 rapidsai-nightly
dask-sql 2022.9.1a220928 py39_ga7583b5_13 dask/label/dev
datashader 0.13.1a py_0 rapidsai-nightly
libcucim 22.10.00a220930 cuda11_gf4229e3_51 rapidsai-nightly
libcudf 22.10.00a220929 cuda11_g920b58f948_288 rapidsai-nightly
libcudf_kafka 22.10.00a220929 g920b58f948_288 rapidsai-nightly
libcugraph 22.10.00a220930 cuda11_g91598080_88 rapidsai-nightly
libcugraph_etl 22.10.00a220930 cuda11_g91598080_88 rapidsai-nightly
libcugraphops 22.10.00a220930 cuda11_g553bacf_29 rapidsai-nightly
libcuml 22.10.00a220930 cuda11_g96da84cc1_50 rapidsai-nightly
libcumlprims 22.10.00a220804 cuda11_g2adfe69_0 rapidsai-nightly
libcuspatial 22.10.00a220930 cuda11_g6922ef5_55 rapidsai-nightly
libraft-distance 22.10.00a220930 cuda11_g2e98138c_57 rapidsai-nightly
libraft-headers 22.10.00a220930 cuda11_g2e98138c_57 rapidsai-nightly
libraft-nn 22.10.00a220930 cuda11_g2e98138c_57 rapidsai-nightly
librmm 22.10.00a220929 cuda11_g8a3a552e_28 rapidsai-nightly
libxgboost 1.6.2dev.rapidsai22.10 cuda_11_0 rapidsai-nightly
ptxcompiler 0.6.0 cuda_11_py39_g455bc7f_2 rapidsai-nightly
py-xgboost 1.6.2dev.rapidsai22.10 cuda_11_py39_0 rapidsai-nightly
pylibcugraph 22.10.00a220930 cuda11_py39_g91598080_88 rapidsai-nightly
pylibraft 22.10.00a220930 cuda11_py39_g2e98138c_57 rapidsai-nightly
raft-dask 22.10.00a220930 cuda11_py39_g2e98138c_57 rapidsai-nightly
rapids 22.10.00a220930 cuda11_py39_gbce77c5_69 rapidsai-nightly
rapids-xgboost 22.10.00a220930 cuda11_py39_gbce77c5_69 rapidsai-nightly
rmm 22.10.00a220929 cuda11_py39_g8a3a552e_28 rapidsai-nightly
ucx-proc 1.0.0 gpu rapidsai-nightly
ucx-py 0.28.00a220927 py39_g2ab6070_27 rapidsai-nightly
xgboost 1.6.2dev.rapidsai22.10 cuda_11_py39_0 rapidsai-nightly
