abnormal disk usage: keep growing until service restart

[bovy89]

After a server reinstall we tried to migrate to tsi1 but, due to an high memory usage, we reverted it to inmem (this solved our problem). While the memory issue has been resolved, the amount of data under /var/lib/influxdb/data is keep growing until a service restart (before that is was almost constant).

Influxdb version: influxdb-1.8.4-1.x86_64 OS: CentOS 7

influxdb.conf

reporting-disabled = true
bind-address = ":8088"

[meta]
  dir = "/var/lib/influxdb/meta"

[data]
  dir = "/var/lib/influxdb/data"
  wal-dir = "/var/lib/influxdb/wal"
  index-version = "inmem"
  series-id-set-cache-size = 100
  max-series-per-database = 0
  max-values-per-tag = 0

[coordinator]
  query-timeout = "600s"
  log-queries-after = "30s"

[retention]

[shard-precreation]

[monitor]
  store-interval = "180s"

[http]
  bind-address = ":8086"
  auth-enabled = true
  pprof-enabled = false
  max-row-limit = 100000

[logging]
  suppress-logo = true

[subscriber]
  enabled = false

[[graphite]]

[[collectd]]

[[opentsdb]]

[[udp]]

[continuous_queries]
  query-stats-enabled = true
  run-interval = "1m"

[tls]

Disk usage drop ---> influxdb service restart

/var/lib/influxdb/data post service restart: 15GB /var/lib/influxdb/data pre service restart: 33GB

post service restart:

# date
Wed Jan 19 13:35:26 CET 2022

# du -sh /var/lib/influxdb/*
13G    /var/lib/influxdb/data
24K    /var/lib/influxdb/meta
52M    /var/lib/influxdb/wal

# du -sh /var/lib/influxdb/data/*
119M    /var/lib/influxdb/data/_internal
4.5M    /var/lib/influxdb/data/extra_metrics
13G    /var/lib/influxdb/data/sensu_metrics

# du -sh /var/lib/influxdb/data/sensu_metrics/*
1.5G    /var/lib/influxdb/data/sensu_metrics/one_month
1.2G    /var/lib/influxdb/data/sensu_metrics/one_week
9.3M    /var/lib/influxdb/data/sensu_metrics/_series
5.3G    /var/lib/influxdb/data/sensu_metrics/six_months
4.6G    /var/lib/influxdb/data/sensu_metrics/thirteen_months

# du -sh /var/lib/influxdb/data/sensu_metrics/one_week/*
141M    /var/lib/influxdb/data/sensu_metrics/one_week/6251
142M    /var/lib/influxdb/data/sensu_metrics/one_week/6255
142M    /var/lib/influxdb/data/sensu_metrics/one_week/6259
142M    /var/lib/influxdb/data/sensu_metrics/one_week/6263
141M    /var/lib/influxdb/data/sensu_metrics/one_week/6267
142M    /var/lib/influxdb/data/sensu_metrics/one_week/6271
142M    /var/lib/influxdb/data/sensu_metrics/one_week/6277
169M    /var/lib/influxdb/data/sensu_metrics/one_week/6281

# du -sh /var/lib/influxdb/data/sensu_metrics/one_month/*
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6162
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6166
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6170
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6174
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6178
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6182
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6188
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6192
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6196
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6200
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6204
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6208
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6212
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6218
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6222
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6226
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6230
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6234
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6238
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6242
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6248
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6252
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6256
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6260
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6264
51M    /var/lib/influxdb/data/sensu_metrics/one_month/6268
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6272
52M    /var/lib/influxdb/data/sensu_metrics/one_month/6278
95M    /var/lib/influxdb/data/sensu_metrics/one_month/6282

# du -sh /var/lib/influxdb/data/sensu_metrics/one_week/6251/*
140M    /var/lib/influxdb/data/sensu_metrics/one_week/6251/000000029-000000002.tsm
896K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/fields.idx

pre service restart:

# date
Wed Jan 19 13:22:47 CET 2022

# du -sh /var/lib/influxdb/*
31G    /var/lib/influxdb/data
24K    /var/lib/influxdb/meta
50M    /var/lib/influxdb/wal

# du -sh /var/lib/influxdb/data/*
376M    /var/lib/influxdb/data/_internal
86M    /var/lib/influxdb/data/extra_metrics
31G    /var/lib/influxdb/data/sensu_metrics

# du -sh /var/lib/influxdb/data/sensu_metrics/*
6.1G    /var/lib/influxdb/data/sensu_metrics/one_month
2.5G    /var/lib/influxdb/data/sensu_metrics/one_week
9.3M    /var/lib/influxdb/data/sensu_metrics/_series
11G    /var/lib/influxdb/data/sensu_metrics/six_months
12G    /var/lib/influxdb/data/sensu_metrics/thirteen_months

# du -sh /var/lib/influxdb/data/sensu_metrics/one_week/*
331M    /var/lib/influxdb/data/sensu_metrics/one_week/6251
332M    /var/lib/influxdb/data/sensu_metrics/one_week/6255
332M    /var/lib/influxdb/data/sensu_metrics/one_week/6259
330M    /var/lib/influxdb/data/sensu_metrics/one_week/6263
330M    /var/lib/influxdb/data/sensu_metrics/one_week/6267
330M    /var/lib/influxdb/data/sensu_metrics/one_week/6271
331M    /var/lib/influxdb/data/sensu_metrics/one_week/6277
169M    /var/lib/influxdb/data/sensu_metrics/one_week/6281

# du -sh /var/lib/influxdb/data/sensu_metrics/one_month/*
271M    /var/lib/influxdb/data/sensu_metrics/one_month/6162
209M    /var/lib/influxdb/data/sensu_metrics/one_month/6166
190M    /var/lib/influxdb/data/sensu_metrics/one_month/6170
194M    /var/lib/influxdb/data/sensu_metrics/one_month/6174
210M    /var/lib/influxdb/data/sensu_metrics/one_month/6178
178M    /var/lib/influxdb/data/sensu_metrics/one_month/6182
162M    /var/lib/influxdb/data/sensu_metrics/one_month/6188
193M    /var/lib/influxdb/data/sensu_metrics/one_month/6192
162M    /var/lib/influxdb/data/sensu_metrics/one_month/6196
271M    /var/lib/influxdb/data/sensu_metrics/one_month/6200
194M    /var/lib/influxdb/data/sensu_metrics/one_month/6204
255M    /var/lib/influxdb/data/sensu_metrics/one_month/6208
193M    /var/lib/influxdb/data/sensu_metrics/one_month/6212
255M    /var/lib/influxdb/data/sensu_metrics/one_month/6218
256M    /var/lib/influxdb/data/sensu_metrics/one_month/6222
272M    /var/lib/influxdb/data/sensu_metrics/one_month/6226
272M    /var/lib/influxdb/data/sensu_metrics/one_month/6230
256M    /var/lib/influxdb/data/sensu_metrics/one_month/6234
178M    /var/lib/influxdb/data/sensu_metrics/one_month/6238
257M    /var/lib/influxdb/data/sensu_metrics/one_month/6242
273M    /var/lib/influxdb/data/sensu_metrics/one_month/6248
195M    /var/lib/influxdb/data/sensu_metrics/one_month/6252
274M    /var/lib/influxdb/data/sensu_metrics/one_month/6256
179M    /var/lib/influxdb/data/sensu_metrics/one_month/6260
179M    /var/lib/influxdb/data/sensu_metrics/one_month/6264
163M    /var/lib/influxdb/data/sensu_metrics/one_month/6268
257M    /var/lib/influxdb/data/sensu_metrics/one_month/6272
179M    /var/lib/influxdb/data/sensu_metrics/one_month/6278
95M    /var/lib/influxdb/data/sensu_metrics/one_month/6282

# du -sh /var/lib/influxdb/data/sensu_metrics/one_week/6251/*
140M    /var/lib/influxdb/data/sensu_metrics/one_week/6251/000000029-000000002.tsm
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/10.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/11.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/13.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/14.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/16.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/17.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/19.tmp
190M    /var/lib/influxdb/data/sensu_metrics/one_week/6251/1.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/20.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/2.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/4.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/5.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/7.tmp
4.0K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/8.tmp
896K    /var/lib/influxdb/data/sensu_metrics/one_week/6251/fields.idx

After comparing this output with an another installation, it seems that those .tmp should not be there.

Any idea?

[norbertvannobelen]

Also observed in 1.8.3-debian-10-r88

[norbertvannobelen]

1.8.5 seems to solve the issue: 2 weeks test run with 1.8.5 did not show the disk space issue.

Other observed effect of 1.8.5 seems to be that memory usage drops, in our case from 40GB to 6GB.

[jeffreyssmith2nd]

Closing on the basis that 1.8.5 appears to resolve the issue.