seafowl
seafowl copied to clipboard
Can't create an external table with a URL with an asterisk in it
curl -i -XPOST -H "Content-Type: application/json"
-H "Authorization: Bearer $SEAFOWL_PASSWORD"
https://demo.seafowl.io/q -d@- <<EOF
{"query": "
CREATE EXTERNAL TABLE ebola (
location VARCHAR,
iso_code VARCHAR,
date TIMESTAMP,
total_cases INTEGER,
new_cases INTEGER,
total_hospitalized INTEGER,
new_hospitalized INTEGER,
total_icu INTEGER NULL,
new_icu INTEGER NULL,
total_deaths INTEGER NULL,
new_deaths INTEGER NULL,
total_recovered INTEGER NULL,
new_recovered INTEGER NULL,
fatality_rate NUMERIC NULL,
new_cases_smoothed NUMERIC NULL,
new_deaths_smoothed NUMERIC NULL,
new_cases_per_million NUMERIC NULL,
new_deaths_per_million NUMERIC NULL,
total_deaths_per_million NUMERIC NULL,
new_deaths_smoothed_per_million NUMERIC NULL)
STORED AS CSV
LOCATION 'https://doc-10-7k-sheets.googleusercontent.com/pub/3sm34aofsvmt5ehut1q6bddr3o/v8o220b27nrul867591pgvgleo/1668018305000/106161642273204794399/*/e@2PACX-1vQelcvFosb_CZfKBlXr4C-n8xXGb7oPalH7tPJwLWbQynuu5vY2UW9sADUTPXoodmTq3eF6fvjbBLnG?gid=0&single=true&output=csv';
"}
EOF
leads to
2022-11-09T18:37:36.164 app[977e017f] lhr [info] thread 'tokio-runtime-worker' panicked at 'should be URL safe: BadSegment { path: "/doc-10-7k-sheets.googleusercontent.com/pub/3sm34aofsvmt5ehut1q6bddr3o/v8o220b27nrul867591pgvgleo/1668018305000/106161642273204794399/*/e@2PACX-1vQelcvFosb_CZfKBlXr4C-n8xXGb7oPalH7tPJwLWbQynuu5vY2UW9sADUTPXoodmTq3eF6fvjbBLnG", source: InvalidPart { segment: "*", illegal: "*" } }', /home/runner/.cargo/registry/src/github.com-1ecc6299db9ec823/datafusion-12.0.0/src/datasource/listing/url.rs:111:46
Looks like this is now working
$ curl -i -XPOST -H "Content-Type: application/json" -H "Authorization: Bearer write_password" http://127.0.0.1:8080/q -d@- <<EOF
{"query": "CREATE EXTERNAL TABLE fruits \
STORED AS CSV \
LOCATION 'https://test-seafowl-asterisk-url.s3.eu-west-3.amazonaws.com/*/fruits.csv';\
SELECT * FROM staging.fruits"}
EOF
HTTP/1.1 200 OK
content-type: application/octet-stream
vary: Content-Type, Origin, X-Seafowl-Query
content-length: 115
date: Mon, 03 Apr 2023 10:55:52 GMT
{"column_1":"fruit_id","column_2":"name"}
{"column_1":"1","column_2":"apple"}
{"column_1":"2","column_2":"orange"}