DB Client consumes way more memory than needed

[Saremox]

The DB Client is consuming a huge amount of memory if used with many Operations in the JBatch data structure. At 100.000 Operations around 4 GiB are used to store the JBatch. The KV backend for example has a memory footprint of only 300 MiB with 200.000 Operations. If one looks at the internal data structures associated with a db operation it becomes obvious why the db client is consuming so much memory

(gdb) print sizeof(JBackendOperation); 
$1 = 20992

JBackendOperation needs a flat 20.5 KiB with and without any content. Most of the memory is consumed by creating 40! JBackendOperationParam structs which result into also having 40 bson_t structs. Since bson_t structs get aligned to fit into exact 2 cachelines we get a very wasteful memory layout of the struct in its current version. Here is a dump of pahole for both of the structs.

struct JBackendOperationParam {
	JBackendOperationParamType type;                 /*     0     4 */

	/* XXX 124 bytes hole, try to pack */

	/* --- cacheline 2 boundary (128 bytes) --- */
	union {
		struct {
			gboolean   bson_initialized;     /*   128     4 */

			/* XXX 124 bytes hole, try to pack */

			/* --- cacheline 4 boundary (256 bytes) --- */
			bson_t     bson __attribute__((__aligned__(128))); /*   256   128 */
		} __attribute__((__aligned__(128))) __attribute__((__aligned__(128)));     /*   128   256 */
		struct {
			const gchar  * error_quark_string; /*   128     8 */
			GError     error;                /*   136    16 */
			GError *   error_ptr;            /*   152     8 */
		};                                       /*   128    32 */
	} __attribute__((__aligned__(128)));             /*   128   256 */
	/* --- cacheline 6 boundary (384 bytes) --- */
	union {
		gconstpointer      ptr_const;            /*   384     8 */
		gpointer           ptr;                  /*   384     8 */
	};                                               /*   384     8 */
	gint                       len;                  /*   392     4 */

	/* size: 512, cachelines: 8, members: 4 */
	/* sum members: 272, holes: 1, sum holes: 124 */
	/* padding: 116 */
	/* forced alignments: 1, forced holes: 1, sum forced holes: 124 */
} __attribute__((__aligned__(128)));

struct JBackendOperation {
	gboolean                   (*backend_func)(struct JBackend *, gpointer, struct JBackendOperation *); /*     0     8 */
	guint                      in_param_count;       /*     8     4 */
	guint                      out_param_count;      /*    12     4 */

	/* XXX 112 bytes hole, try to pack */

	/* --- cacheline 2 boundary (128 bytes) --- */
	JBackendOperationParam     in_param[20] __attribute__((__aligned__(128))); /*   128 10240 */
	/* --- cacheline 162 boundary (10368 bytes) --- */
	JBackendOperationParam     out_param[20] __attribute__((__aligned__(128))); /* 10368 10240 */
	/* --- cacheline 322 boundary (20608 bytes) --- */
	guint                      unref_func_count;     /* 20608     4 */

	/* XXX 4 bytes hole, try to pack */

	GDestroyNotify             unref_funcs[20];      /* 20616   160 */
	/* --- cacheline 324 boundary (20736 bytes) was 40 bytes ago --- */
	gpointer                   unref_values[20];     /* 20776   160 */

	/* size: 20992, cachelines: 328, members: 8 */
	/* sum members: 20820, holes: 2, sum holes: 116 */
	/* padding: 56 */
	/* forced alignments: 2, forced holes: 1, sum forced holes: 112 */
} __attribute__((__aligned__(128)));

A quickfix for this Issue is to reduce the in params to 4 and the out to 2 to reduce the memory inpact to only around 4 KiB in include/core/jbackend-operation.h

Another possible solution would be to rearrange the struct like this:

struct JBackendOperationParam
{
	// Only for temporary static storage
	union
	{
		struct
		{
			bson_t bson;		
		};
		struct
		{
			const gchar* error_quark_string;
			GError error;
			GError* error_ptr;
		};
	};
	gboolean bson_initialized;
	JBackendOperationParamType type;
	union
	{
		gconstpointer ptr_const;
		gpointer ptr;
	};

	// Length of ptr data
	gint len;
};

This rearrangement would cut the memory impact of JBackendOperationParam in half and further reduce the overall consumption to only 2 KiB.

struct JBackendOperationParam {
	union {
		struct {
			bson_t     bson __attribute__((__aligned__(128))); /*     0   128 */
		} __attribute__((__aligned__(128))) __attribute__((__aligned__(128)));     /*     0   128 */
		struct {
			const gchar  * error_quark_string; /*     0     8 */
			GError     error;                /*     8    16 */
			GError *   error_ptr;            /*    24     8 */
		};                                       /*     0    32 */
	} __attribute__((__aligned__(128)));             /*     0   128 */
	/* --- cacheline 2 boundary (128 bytes) --- */
	gboolean                   bson_initialized;     /*   128     4 */
	JBackendOperationParamType type;                 /*   132     4 */
	union {
		gconstpointer      ptr_const;            /*   136     8 */
		gpointer           ptr;                  /*   136     8 */
	};                                               /*   136     8 */
	gint                       len;                  /*   144     4 */

	/* size: 256, cachelines: 4, members: 5 */
	/* padding: 108 */
	/* forced alignments: 1 */
} __attribute__((__aligned__(128)));

[michaelkuhn]

Thanks for your suggestions! I have just pushed 841f99878c86ed48f1518a4e9ef060d77b4ad611, which should reduce memory consumption significantly.

I'll leave the issue open for now as a reminder to come back to this. It might make more sense to allocate the parameter arrays dynamically.