msgpack
msgpack copied to clipboard
msgpack marshal longer than json marshal for the same map[string]interface{}
the following is my code, and output is
=== RUN TestMsgPack
json len: 3474
<nil>
msgpack len: 3831
base64 len: 5108
--- PASS: TestMsgPack (0.00s)
PASS
package traceinfo_lib
import (
"encoding/base64"
"encoding/json"
"fmt"
"testing"
"github.com/vmihailenco/msgpack/v5"
)
const try = `{
"action_feature": {
"DocFeature.ClcFeature": [
303348,
60157,
46515,
21939,
15041,
76858,
12453,
9222,
3795,
2409,
303348,
60157,
46515,
21939,
15041
],
"DocFeature.DetailActionFeature": [
107862,
3117,
11801317376,
808511040,
771776,
33189,
16145,
474,
1831473920,
148402880,
143090,
6430,
107862,
3117,
11801317376,
808511040,
771776,
33189
],
"QueryDocFeature.ClcFeature": [
110412,
39225,
35243,
12865,
9992,
25595,
8083,
7174,
2159,
1641,
110412,
39225,
35243,
12865,
9992
],
"QueryDocFeature.DetailActionFeature": [
68289,
1784,
7537300480,
559157440,
531392,
22070,
9726,
250,
1160727680,
100144848,
96229,
4192,
68289,
1784,
7537300480,
559157440,
531392,
22070
],
"QueryFeature.ClcFeature": [
26706556,
2373118,
718925,
720777,
227012,
951498,
87680,
25452,
25522,
8215,
5712187,
523152,
159002,
159485,
53010
],
"QueryFeature.DetailActionFeature": [
1689341,
65599,
215091314688,
18717943808,
13701883,
1082489,
68257,
2558,
8379656192,
642240576,
508541,
43689,
433616,
16328,
54671339520,
4073900800,
3165243,
270308
]
},
"cqr": 0,
"cqr_alias_book_name": 1,
"cqr_alias_book_name_whitespace_fine": 0,
"cqr_alias_names": 0,
"cqr_author": 0,
"cqr_author_whitespace_fine": 0,
"cqr_category": 0,
"cqr_description_whitespace_fine": 1,
"cqr_resource": 0,
"cqr_tags": 0,
"cqr_title": 1,
"cqr_title_whitespace_fine": 1,
"ctr": 0,
"ctr_alias_book_name": 0.4,
"ctr_alias_book_name_whitespace_fine": 0,
"ctr_alias_names": 0.4,
"ctr_author": 0.4,
"ctr_author_whitespace_fine": 0.4,
"ctr_category": 0.4,
"ctr_description_whitespace_fine": 0.5,
"ctr_resource": 0.4,
"ctr_tags": 0.4,
"ctr_title": 0.4,
"ctr_title_whitespace_fine": 0.5,
"es": 2086.4413713627196,
"featureArray": [
0.6083000004291534,
0.296900000423193,
0.017500000074505806,
4.884299999015639,
1.9012000103248283,
1.7088999831685214,
0.12454189966868286,
0.1561645270412482,
0.010240505732850756,
39225,
110412,
35243,
12865,
9992,
0.35244236131251616,
0.31645199482677505,
0.11463892690789367,
0.08881936316236232,
68289,
1784,
68265.22914175995,
531392,
22070,
5064.281418686374,
0.1975392153232293,
0.01543061660368332,
2373118,
26706556,
718925,
720777,
227012,
0.08875114719482814,
0.02685810421604392,
0.026927373650930165,
0.008465487146780219,
1689341,
65599,
8053.876909025634,
13701883,
1082489,
700.8744896945904,
0.040457979018845125,
0.0024375858703508902,
60157,
303348,
46515,
21939,
15041,
0.19689508523441182,
0.1520608992737178,
0.07140651434749136,
0.04881648365004469,
107862,
3117,
38903.56084760737,
771776,
33189,
2665.292139720717,
0.10830309887952567,
0.00992260503819213,
0,
0,
0,
1,
1,
0,
1,
0,
1,
0,
2086.4413713627196,
1,
1,
0,
1,
0.4,
0,
1,
0.4,
1,
1,
1,
1,
0,
1,
1,
0.4,
0.4,
0.4,
1,
0,
0.4,
1,
0,
0.4,
0,
0.9147285857933416
],
"omit": 0,
"omit_alias_book_name": 0,
"omit_alias_book_name_whitespace_fine": 0,
"omit_alias_names": 0,
"omit_author": 0,
"omit_author_whitespace_fine": 0,
"omit_category": 0,
"omit_description_whitespace_fine": 1,
"omit_resource": 0,
"omit_tags": 0,
"omit_title": 0,
"omit_title_whitespace_fine": 1,
"proximity": 0,
"proximity_alias_book_name": 1,
"proximity_alias_book_name_whitespace_fine": 1,
"proximity_alias_names": 1,
"proximity_author": 1,
"proximity_author_whitespace_fine": 1,
"proximity_category": 1,
"proximity_description_whitespace_fine": 1,
"proximity_resource": 1,
"proximity_tags": 1,
"proximity_title": 1,
"proximity_title_whitespace_fine": 1,
"tightness_alias_book_name": 1,
"tightness_alias_book_name_whitespace_fine": 1,
"tightness_alias_names": 1,
"tightness_author": 1,
"tightness_author_whitespace_fine": 1,
"tightness_category": 1,
"tightness_description_whitespace_fine": 1,
"tightness_resource": 1,
"tightness_tags": 1,
"tightness_title": 1,
"tightness_title_whitespace_fine": 1,
"timeDuration_model_score": 0.6556097269058228,
"vector_similarity": 0.9147285857933416
}`
func TestMsgPack(t *testing.T) {
var c2 map[string]interface{}
err := json.Unmarshal([]byte(try), &c2)
v12, _ := json.Marshal(c2)
fmt.Printf("json len: %d\n", len([]byte(v12)))
//fmt.Println(string(v12))
fmt.Println(err)
v1, _ := msgpack.Marshal(c2)
fmt.Printf("msgpack len: %d\n", len([]byte(v1)))
b64 := make([]byte, base64.StdEncoding.EncodedLen(len(v1)))
base64.StdEncoding.Encode(b64, v1)
fmt.Printf("base64 len: %d\n", len([]byte(b64)))
//s := time.Now()
//for i := 0; i < 40000; i++ {
// json.Marshal(c2)
//}
//fmt.Println(time.Now().Sub(s).String())
//s = time.Now()
//fmt.Println(string(v1))
//for i := 0; i < 40000; i++ {
// v1, _ := msgpack.Marshal(c2)
// b64 := make([]byte, base64.StdEncoding.EncodedLen(len(v1)))
// base64.StdEncoding.Encode(b64, v1)
//}
//fmt.Println(time.Now().Sub(s).String())
}
Perhaps you are looking for https://pkg.go.dev/github.com/vmihailenco/msgpack/v5#Encoder.UseCompactInts