d3-dsv
d3-dsv copied to clipboard
Streaming parse?
Like the new shapefile.
This is fine for Node: https://github.com/mafintosh/csv-parser
Would still be nice to have streaming CSV parsing in the browser.
you probably mean something else, but I wrote a little class that "streams" csv rows as they are loaded through the progress event:
import { request, csvParse, csvParseRows } from 'd3'
import EventEmitter from 'eventemitter3'
export default class CsvLoader extends EventEmitter {
constructor (url) {
super()
this.loadedChars = 0
this.header = undefined
this.data = []
this.xhr = undefined
this.request = request(url)
.mimeType('text')
.on('progress', this.onProgress)
.on('load', () => this.emit('loaded'))
return this
}
get () {
this.request.get()
return this
}
onProgress = (p) => {
const progress = p.total / p.loaded
const responseText = p.target.responseText
// console.log(responseText.length)
const lineBreak = responseText.lastIndexOf('\n') + 1
const char = responseText.slice(this.loadedChars, lineBreak)
let parsed = csvParseRows(char)
if (!this.header) {
this.header = parsed.shift()
}
const obj = parsed.map(p =>
this.header.reduce((a, b, i) => {
a[b] = p[i]
return a
}, {})
)
this.loadedChars = lineBreak
this.emit('row', obj)
this.emit('progress', progress)
}
}
https://observablehq.com/@mbostock/streaming-csv