midi-dataset icon indicating copy to clipboard operation
midi-dataset copied to clipboard

Switch to mpk (?)

Open craffel opened this issue 9 years ago • 0 comments

Loading seems to be 10-50x faster than deepdish/h5, size is comparable to compressed h5, and types are preserved completely:

import msgpack
import msgpack_numpy
msgpack_numpy.patch()
import numpy as np

with open('temp/blah.mpk', 'w') as f:
    d = {'hey': np.array([1, 2, 3]),
         'cool': np.array(0., dtype=np.float32),
         'me': 10.,
         'you': 10,
         'stuff': ['a', [1, 2, 3], {'cool': 10}, np.array([1., 3.])],
         'string': 'boom'}
    for k, v in d.items():
        print k, type(v), v
        if isinstance(v, np.ndarray):
            print v.dtype
    f.write(msgpack.packb(d))
me <type 'float'> 10.0
string <type 'str'> boom
hey <type 'numpy.ndarray'> [1 2 3]
int64
stuff <type 'list'> ['a', [1, 2, 3], {'cool': 10}, array([ 1.,  3.])]
you <type 'int'> 10
cool <type 'numpy.ndarray'> 0.0
float32
with open('temp/blah.mpk') as g:
    d = msgpack.load(g)
    for k, v in d.items():
        print k, type(v), v
        if isinstance(v, np.ndarray):
            print v.dtype
me <type 'float'> 10.0
string <type 'str'> boom
hey <type 'numpy.ndarray'> [1 2 3]
int64
stuff <type 'list'> ['a', [1, 2, 3], {'cool': 10}, array([ 1.,  3.])]
you <type 'int'> 10
cool <type 'numpy.ndarray'> 0.0
float32

craffel avatar Apr 09 '16 16:04 craffel