csvlint.rb
csvlint.rb copied to clipboard
Data-Liberation-Front
Cannot validate "number" data type using CSVW
Expected Behaviour
When CSVW schema contains "datatype": "number" in column definition, the values should be validated as numbers
Current Behaviour (for problems)
csvlint crashes with:
NOTE: Csvlint::Schema.load_from_json is deprecated; use load_from_uri instead. It will be removed on or after 2018-01-01.
Csvlint::Schema.load_from_json called from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:59.
./var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/number_format.rb:178:in `parse': undefined method `gsub!' for 1.0:Float (NoMethodError)
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/column.rb:265:in `block in <class:Column>'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/column.rb:105:in `block in validate'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/column.rb:103:in `each'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/column.rb:103:in `validate'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/table.rb:64:in `block in validate_row'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/table.rb:64:in `map'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/table.rb:64:in `validate_row'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/csvw/table_group.rb:38:in `validate_row'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:200:in `parse_contents'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:166:in `validate_line'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:146:in `parse_line'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:110:in `block in validate_stream'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:108:in `each_line'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:108:in `validate_stream'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:101:in `validate'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/validate.rb:86:in `initialize'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:141:in `new'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:141:in `validate_csv'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:86:in `block in fetch_schema_tables'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:79:in `each'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:79:in `fetch_schema_tables'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/lib/csvlint/cli.rb:22:in `validate'
from /var/lib/gems/2.3.0/gems/thor-0.20.0/lib/thor/command.rb:27:in `run'
from /var/lib/gems/2.3.0/gems/thor-0.20.0/lib/thor/invocation.rb:126:in `invoke_command'
from /var/lib/gems/2.3.0/gems/thor-0.20.0/lib/thor.rb:387:in `dispatch'
from /var/lib/gems/2.3.0/gems/thor-0.20.0/lib/thor/base.rb:466:in `start'
from /var/lib/gems/2.3.0/gems/csvlint-0.4.0/bin/csvlint:9:in `<top (required)>'
from /usr/local/bin/csvlint:23:in `load'
from /usr/local/bin/csvlint:23:in `<main>'
Steps to Reproduce (for problems)
CSVW schema:
{
"@context": ["http://www.w3.org/ns/csvw",{"@language": "cs"}],
"url": "test.csv",
"dc:description": "Registr kandidátů",
"tableSchema": {
"aboutUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/kandidát/{VOLKRAJ}/{KSTRANA}/{PORCISLO}",
"columns": [{
"name": "VOLKRAJ",
"titles": "VOLKRAJ",
"dc:description": "Číslo volebního kraje",
"required": false,
"datatype": "number",
"aboutUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/kandidátní-listina/{VOLKRAJ}/{KSTRANA}",
"propertyUrl": ":volebníKraj",
"valueUrl": "https://data.czso.cz/zdroj/volební-kraj/{VOLKRAJ}"
},{
"name": "KSTRANA",
"titles": "KSTRANA",
"dc:description": "Vylosované číslo strany (vazba na RKL)",
"required": false,
"datatype": "string",
"propertyUrl": ":subjekt",
"valueUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/politický-subjekt/{KSTRANA}"
},{
"name": "PORCISLO",
"titles": "PORCISLO",
"dc:description": "Pořadí na KL (hlasovacím lístku)",
"required": false,
"datatype": "string",
"propertyUrl": ":pořadí"
},{
"name": "JMENO",
"titles": "JMENO",
"dc:description": "Jméno, příp. i další jména",
"required": true,
"datatype": "string",
"propertyUrl": "foaf:givenName"
},{
"name": "PRIJMENI",
"titles": "PRIJMENI",
"dc:description": "Příjmení",
"required": true,
"datatype": "string",
"propertyUrl": "foaf:familyName"
},{
"name": "TITULPRED",
"titles": "TITULPRED",
"dc:description": "Tituly před jménem",
"required": true,
"datatype": "string"
},{
"name": "TITULZA",
"titles": "TITULZA",
"dc:description": "Tituly za příjmením",
"required": false,
"datatype": "string"
},{
"name": "VEK",
"titles": "VEK",
"dc:description": "Věk",
"required": true,
"datatype": "string",
"propertyUrl": "foaf:age"
},{
"name": "POVOLANI",
"titles": "POVOLANI",
"dc:description": "Povolání",
"required": true,
"datatype": "string",
"propertyUrl": "s:jobTitle"
},{
"name": "BYDLISTEN",
"titles": "BYDLISTEN",
"dc:description": "Název obce - bydliště",
"required": true,
"datatype": "string",
"suppressOutput": true
},{
"name": "BYDLISTEK",
"titles": "BYDLISTEK",
"dc:description": "Kód obce - bydliště",
"required": true,
"datatype": "string",
"propertyUrl": ":obecTrvaléhoPobytu",
"valueUrl": "https://data.czso.cz/zdroj/obec/{BYDLISTEK}"
},{
"name": "PSTRANA",
"titles": "PSTRANA",
"dc:description": "Kód politické příslušnosti",
"required": true,
"datatype": "string",
"propertyUrl": ":jeČlenem",
"valueUrl": "https://data.czso.cz/zdroj/politický-subjekt/{PSTRANA}"
},{
"name": "NSTRANA",
"titles": "NSTRANA",
"dc:description": "Kód navrhující strany",
"required": true,
"datatype": "string",
"propertyUrl": ":navrhl",
"aboutUrl": "https://data.czso.cz/zdroj/politický-subjekt/{PSTRANA}",
"valueUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/kandidát/{VOLKRAJ}/{KSTRANA}/{PORCISLO}"
},{
"name": "PLATNOST",
"titles": "PLATNOST",
"dc:description": "Platnost (A = platný, N = neplatný)",
"required": true,
"datatype": "string"
},{
"name": "POCHLASU",
"titles": "POCHLASU",
"dc:description": "Počet přednostních hlasů",
"required": false,
"datatype": "string"
},{
"name": "POCPROC",
"titles": "POCPROC",
"dc:description": "Procento přednostních hlasů, je-li větší nebo rovno 5",
"required": false,
"datatype": "string"
},{
"name": "POCPROCVSE",
"titles": "POCPROCVSE",
"dc:description": "Procento přednostních hlasů",
"required": false,
"datatype": "string"
},{
"name": "MANDAT",
"titles": "MANDAT",
"dc:description": "A = zvolen, N = nezvolen",
"required": false,
"datatype": "string"
},{
"name": "PORADIMAND",
"titles": "PORADIMAND",
"dc:description": "Pořadí přidělení mandátu (1-36, 0=strana nezískala žádný mandát)",
"required": false,
"datatype": "string"
},{
"name": "PORADINAHR",
"titles": "PORADINAHR",
"dc:description": "Pořadí náhradníka (1-35, 0=strana nezískala žádný mandát)",
"required": false,
"datatype": "string"
},{
"virtual": true,
"aboutUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/kandidátní-listina/{VOLKRAJ}/{KSTRANA}",
"propertyUrl": "rdf:type",
"valueUrl": ":KandidátníListina"
},{
"virtual": true,
"aboutUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/kandidátní-listina/{VOLKRAJ}/{KSTRANA}",
"propertyUrl": ":kandidát",
"valueUrl": "https://data.czso.cz/zdroj/volby-do-poslanecké-sněmovny-parlamentu-čr-2017/kandidát/{VOLKRAJ}/{KSTRANA}/{PORCISLO}"
},{
"virtual": true,
"propertyUrl": "rdf:type",
"valueUrl": ":Kandidát"
}
],
"primaryKey": ["VOLKRAJ","KSTRANA","PORCISLO"]
}
}
CSV:
VOLKRAJ,KSTRANA,PORCISLO,JMENO,PRIJMENI,TITULPRED,TITULZA,VEK,POVOLANI,BYDLISTEN,BYDLISTEK,PSTRANA,NSTRANA,PLATNOST,POCHLASU,POCPROC,POCPROCVSE,MANDAT,PORADIMAND,PORADINAHR
1,1,1,"Jana","Černochová","Mgr.","",43,"starostka MČ Praha 2","Praha",1,53,53,"A",,,,"",,
- Download above to
test.jsonandtest.csv - Run
csvlint -s test.json
Your Environment
Bash on Ubuntu on Windows, csvlint installed using gem install csvlint
