msgpack icon indicating copy to clipboard operation
msgpack copied to clipboard

msgpack marshal longer than json marshal for the same map[string]interface{}

Open cityblack1 opened this issue 4 years ago • 1 comments

the following is my code, and output is

=== RUN   TestMsgPack
json len: 3474
<nil>
msgpack len: 3831
base64 len: 5108
--- PASS: TestMsgPack (0.00s)
PASS
package traceinfo_lib

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"testing"

	"github.com/vmihailenco/msgpack/v5"
)

const try = `{
    "action_feature": {
        "DocFeature.ClcFeature": [
            303348,
            60157,
            46515,
            21939,
            15041,
            76858,
            12453,
            9222,
            3795,
            2409,
            303348,
            60157,
            46515,
            21939,
            15041
        ],
        "DocFeature.DetailActionFeature": [
            107862,
            3117,
            11801317376,
            808511040,
            771776,
            33189,
            16145,
            474,
            1831473920,
            148402880,
            143090,
            6430,
            107862,
            3117,
            11801317376,
            808511040,
            771776,
            33189
        ],
        "QueryDocFeature.ClcFeature": [
            110412,
            39225,
            35243,
            12865,
            9992,
            25595,
            8083,
            7174,
            2159,
            1641,
            110412,
            39225,
            35243,
            12865,
            9992
        ],
        "QueryDocFeature.DetailActionFeature": [
            68289,
            1784,
            7537300480,
            559157440,
            531392,
            22070,
            9726,
            250,
            1160727680,
            100144848,
            96229,
            4192,
            68289,
            1784,
            7537300480,
            559157440,
            531392,
            22070
        ],
        "QueryFeature.ClcFeature": [
            26706556,
            2373118,
            718925,
            720777,
            227012,
            951498,
            87680,
            25452,
            25522,
            8215,
            5712187,
            523152,
            159002,
            159485,
            53010
        ],
        "QueryFeature.DetailActionFeature": [
            1689341,
            65599,
            215091314688,
            18717943808,
            13701883,
            1082489,
            68257,
            2558,
            8379656192,
            642240576,
            508541,
            43689,
            433616,
            16328,
            54671339520,
            4073900800,
            3165243,
            270308
        ]
    },
    "cqr": 0,
    "cqr_alias_book_name": 1,
    "cqr_alias_book_name_whitespace_fine": 0,
    "cqr_alias_names": 0,
    "cqr_author": 0,
    "cqr_author_whitespace_fine": 0,
    "cqr_category": 0,
    "cqr_description_whitespace_fine": 1,
    "cqr_resource": 0,
    "cqr_tags": 0,
    "cqr_title": 1,
    "cqr_title_whitespace_fine": 1,
    "ctr": 0,
    "ctr_alias_book_name": 0.4,
    "ctr_alias_book_name_whitespace_fine": 0,
    "ctr_alias_names": 0.4,
    "ctr_author": 0.4,
    "ctr_author_whitespace_fine": 0.4,
    "ctr_category": 0.4,
    "ctr_description_whitespace_fine": 0.5,
    "ctr_resource": 0.4,
    "ctr_tags": 0.4,
    "ctr_title": 0.4,
    "ctr_title_whitespace_fine": 0.5,
    "es": 2086.4413713627196,
    "featureArray": [
        0.6083000004291534,
        0.296900000423193,
        0.017500000074505806,
        4.884299999015639,
        1.9012000103248283,
        1.7088999831685214,
        0.12454189966868286,
        0.1561645270412482,
        0.010240505732850756,
        39225,
        110412,
        35243,
        12865,
        9992,
        0.35244236131251616,
        0.31645199482677505,
        0.11463892690789367,
        0.08881936316236232,
        68289,
        1784,
        68265.22914175995,
        531392,
        22070,
        5064.281418686374,
        0.1975392153232293,
        0.01543061660368332,
        2373118,
        26706556,
        718925,
        720777,
        227012,
        0.08875114719482814,
        0.02685810421604392,
        0.026927373650930165,
        0.008465487146780219,
        1689341,
        65599,
        8053.876909025634,
        13701883,
        1082489,
        700.8744896945904,
        0.040457979018845125,
        0.0024375858703508902,
        60157,
        303348,
        46515,
        21939,
        15041,
        0.19689508523441182,
        0.1520608992737178,
        0.07140651434749136,
        0.04881648365004469,
        107862,
        3117,
        38903.56084760737,
        771776,
        33189,
        2665.292139720717,
        0.10830309887952567,
        0.00992260503819213,
        0,
        0,
        0,
        1,
        1,
        0,
        1,
        0,
        1,
        0,
        2086.4413713627196,
        1,
        1,
        0,
        1,
        0.4,
        0,
        1,
        0.4,
        1,
        1,
        1,
        1,
        0,
        1,
        1,
        0.4,
        0.4,
        0.4,
        1,
        0,
        0.4,
        1,
        0,
        0.4,
        0,
        0.9147285857933416
    ],
    "omit": 0,
    "omit_alias_book_name": 0,
    "omit_alias_book_name_whitespace_fine": 0,
    "omit_alias_names": 0,
    "omit_author": 0,
    "omit_author_whitespace_fine": 0,
    "omit_category": 0,
    "omit_description_whitespace_fine": 1,
    "omit_resource": 0,
    "omit_tags": 0,
    "omit_title": 0,
    "omit_title_whitespace_fine": 1,
    "proximity": 0,
    "proximity_alias_book_name": 1,
    "proximity_alias_book_name_whitespace_fine": 1,
    "proximity_alias_names": 1,
    "proximity_author": 1,
    "proximity_author_whitespace_fine": 1,
    "proximity_category": 1,
    "proximity_description_whitespace_fine": 1,
    "proximity_resource": 1,
    "proximity_tags": 1,
    "proximity_title": 1,
    "proximity_title_whitespace_fine": 1,
    "tightness_alias_book_name": 1,
    "tightness_alias_book_name_whitespace_fine": 1,
    "tightness_alias_names": 1,
    "tightness_author": 1,
    "tightness_author_whitespace_fine": 1,
    "tightness_category": 1,
    "tightness_description_whitespace_fine": 1,
    "tightness_resource": 1,
    "tightness_tags": 1,
    "tightness_title": 1,
    "tightness_title_whitespace_fine": 1,
    "timeDuration_model_score": 0.6556097269058228,
    "vector_similarity": 0.9147285857933416
}`

func TestMsgPack(t *testing.T) {
	var c2 map[string]interface{}
	err := json.Unmarshal([]byte(try), &c2)
	v12, _ := json.Marshal(c2)
	fmt.Printf("json len: %d\n", len([]byte(v12)))
	//fmt.Println(string(v12))
	fmt.Println(err)
	v1, _ := msgpack.Marshal(c2)
	fmt.Printf("msgpack len: %d\n", len([]byte(v1)))
	b64 := make([]byte, base64.StdEncoding.EncodedLen(len(v1)))
	base64.StdEncoding.Encode(b64, v1)
	fmt.Printf("base64 len: %d\n", len([]byte(b64)))
	//s := time.Now()
	//for i := 0; i < 40000; i++ {
	//	json.Marshal(c2)
	//}
	//fmt.Println(time.Now().Sub(s).String())
	//s = time.Now()
	//fmt.Println(string(v1))
	//for i := 0; i < 40000; i++ {
	//	v1, _ := msgpack.Marshal(c2)
	//	b64 := make([]byte, base64.StdEncoding.EncodedLen(len(v1)))
	//	base64.StdEncoding.Encode(b64, v1)
	//}
	//fmt.Println(time.Now().Sub(s).String())
}

cityblack1 avatar Jan 04 '21 05:01 cityblack1

Perhaps you are looking for https://pkg.go.dev/github.com/vmihailenco/msgpack/v5#Encoder.UseCompactInts

vmihailenco avatar Jan 04 '21 07:01 vmihailenco