provide benchmark for insert/append log speed

[calvin2021y]

provide benchmark for insert/append log speed.

[nitisht]

Hey @calvin2021y we'll get back with this in a few days.

[KlavsKlavsen]

and it would be really nice with a comparison to f.ex. Loki and opensearch/elasticsearch - also on parameters such as cpu and memory use during such a performance test (and in normal operations with f.ex. X amount of log data) - and how much space it takes on disk - vs. source size. elasticsearch storage, often ends up taking about 4x the uncompressed logfiles size - which is very costly in disk cost in the cloud :)

[nitisht]

closing this for now because Performance of self hosted software gets quite subjective to use cases, load type, and of course the testing environment. We recommend you try this testing within your internal environment with K6.io. Here is a sample setup we use internally :

First save the below file as test.js. Change to the actual IP of parseable server and replace with actual basic auth header.

import http from 'k6/http';
import { sleep } from 'k6';
import { randomString, randomItem, randomIntBetween } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'

export const options = {
    discardResponseBodies: true,
    scenarios: {
        contacts: {
            executor: 'constant-vus',
            vus: 5000,
            duration: '600s',
        },
    },
};

function current_time() {
	let event = new Date();
	return event.toISOString();
}

function payload(host) {
    return JSON.stringify([{
        "version": "1.1",
        "host": host,
        "short_message": randomString(16),
        "full_message": randomString(128),
        "time": current_time(), 
        "level": 1,
        "user_id": randomIntBetween(5000, 8000),
        "status": randomItem([200,500]) 
    }])
}

export default function() {
    const url1 = 'http://<server-url>/api/v1/logstream/sixstreamtest1';
    const url2 = 'http://<server-url>/api/v1/logstream/sixstreamtest2';
    const url3 = 'http://<server-url>/api/v1/logstream/sixstreamtest3';
    const url4 = 'http://<server-url>/api/v1/logstream/sixstreamtest4';
    const url5= 'http://<server-url>/api/v1/logstream/sixstreamtest5';
    const url6= 'http://<server-url>/api/v1/logstream/sixstreamtest6';

    const params = {
        headers: {
            'Content-Type': 'application/json',
            'Authorization': '<auth-header-value>',
            'X-P-META-Host': '10.116.0.3',
            'X-P-META-PodLabels': 'app=go-app,pod-template-hash=6c87bc9cc9'
        }
    }
    http.post(url1, payload("sixstreamtest1"), params);
    http.post(url2, payload("sixstreamtest2"), params);
    http.post(url3, payload("sixstreamtest3"), params);
    http.post(url4, payload("sixstreamtest4"), params);
    http.post(url5, payload("sixstreamtest5"), params);
    http.post(url6, payload("sixstreamtest6"), params);
}

Then save this as a shell script in the same directory.

#!/bin/bash

curl --location --request PUT 'http://<server-url>/api/v1/logstream/sixstreamtest1' --header 'Authorization: <auth-header-value>',
curl --location --request PUT 'http://<server-url>/api/v1/logstream/sixstreamtest2' --header 'Authorization: <auth-header-value>',
curl --location --request PUT 'http://<server-url>/api/v1/logstream/sixstreamtest3' --header 'Authorization: <auth-header-value>',
curl --location --request PUT 'http://<server-url>/api/v1/logstream/sixstreamtest4' --header 'Authorization: <auth-header-value>',
curl --location --request PUT 'http://<server-url>/api/v1/logstream/sixstreamtest5' --header 'Authorization: <auth-header-value>',
curl --location --request PUT 'http://<server-url>/api/v1/logstream/sixstreamtest6' --header 'Authorization: <auth-header-value>',

k6 run ./test.js