midi-dataset
midi-dataset copied to clipboard
Switch to mpk (?)
Loading seems to be 10-50x faster than deepdish/h5, size is comparable to compressed h5, and types are preserved completely:
import msgpack
import msgpack_numpy
msgpack_numpy.patch()
import numpy as np
with open('temp/blah.mpk', 'w') as f:
d = {'hey': np.array([1, 2, 3]),
'cool': np.array(0., dtype=np.float32),
'me': 10.,
'you': 10,
'stuff': ['a', [1, 2, 3], {'cool': 10}, np.array([1., 3.])],
'string': 'boom'}
for k, v in d.items():
print k, type(v), v
if isinstance(v, np.ndarray):
print v.dtype
f.write(msgpack.packb(d))
me <type 'float'> 10.0
string <type 'str'> boom
hey <type 'numpy.ndarray'> [1 2 3]
int64
stuff <type 'list'> ['a', [1, 2, 3], {'cool': 10}, array([ 1., 3.])]
you <type 'int'> 10
cool <type 'numpy.ndarray'> 0.0
float32
with open('temp/blah.mpk') as g:
d = msgpack.load(g)
for k, v in d.items():
print k, type(v), v
if isinstance(v, np.ndarray):
print v.dtype
me <type 'float'> 10.0
string <type 'str'> boom
hey <type 'numpy.ndarray'> [1 2 3]
int64
stuff <type 'list'> ['a', [1, 2, 3], {'cool': 10}, array([ 1., 3.])]
you <type 'int'> 10
cool <type 'numpy.ndarray'> 0.0
float32