cstor-operators
cstor-operators copied to clipboard
Published
20 hours ago •
openebs-archive
openebs-archive
cstor-pool : core dump
I have a CSPC that started to crash in loop this weekend
cspc-iep-mirror-hr8z-66b67d79c-xtkrx 3/3 Running 0 3d22h
cspc-iep-mirror-t46v-6d65b46d55-s5hgp 2/3 CrashLoopBackOff 35 3d22h
kubectl logs -n openebs cspc-iep-mirror-t46v-6d65b46d55-s5hgp cstor-pool
root@test-pcl109:~# kubectl logs -n openebs cspc-iep-mirror-t46v-6d65b46d55-s5hgp cstor-pool
Disabling dumping core
sleeping for 2 sec
2020-12-21/14:28:07.678 disabled auto import (reading of zpool.cache)
physmem = 6167776 pages (23.53 GB)
2020-12-21/14:28:34.695 zvol cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-dd5be511-db06-4b4e-a362-e0e5456e0477 status change: DEGRADED -> DEGRADED
2020-12-21/14:28:34.695 zvol cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-dd5be511-db06-4b4e-a362-e0e5456e0477 rebuild status change: INIT -> INIT
2020-12-21/14:28:34.696 Instantiating zvol cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-dd5be511-db06-4b4e-a362-e0e5456e0477
2020-12-21/14:28:34.696 [tgt 10.103.111.95:6060:23]: Connected
2020-12-21/14:28:34.696 [tgt 10.103.111.95:6060:23]: Handshake command for zvol pvc-dd5be511-db06-4b4e-a362-e0e5456e0477
2020-12-21/14:28:34.699 zvol cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-540876e9-f4fb-4494-adaf-9418565234c0 status change: DEGRADED -> DEGRADED
2020-12-21/14:28:34.699 zvol cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-540876e9-f4fb-4494-adaf-9418565234c0 rebuild status change: INIT -> INIT
pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt) == 0 (0x1c == 0x0)
ASSERT at kernel.c:198:zk_thread_create()Fatal signal received: 6
Stack trace:
/usr/local/bin/zrepl(+0x1e82)[0x55f68d4c9e82]
/lib/x86_64-linux-gnu/libc.so.6(+0x3f040)[0x7f514f101040]
/lib/x86_64-linux-gnu/libc.so.6(gsignal+0xc7)[0x7f514f100fb7]
/lib/x86_64-linux-gnu/libc.so.6(abort+0x141)[0x7f514f102921]
/usr/lib/libzpool.so.2(+0x3c7ae)[0x7f514fb857ae]
/usr/lib/libzpool.so.2(zk_thread_create+0x2c8)[0x7f514fb85e38]
/usr/lib/libzpool.so.2(taskq_create+0x161)[0x7f514fb88f11]
/usr/lib/libcstor.so.2(uzfs_zinfo_init+0xa0)[0x7f514f6e8ef0]
/usr/lib/libcstor.so.2(uzfs_zvol_create_cb+0x98)[0x7f514f6e5e38]
/usr/lib/libzpool.so.2(+0x601b0)[0x7f514fba91b0]
/usr/lib/libzpool.so.2(+0x60253)[0x7f514fba9253]
/usr/lib/libzpool.so.2(dmu_objset_find+0x51)[0x7f514fbacdb1]
/usr/lib/libcstor.so.2(uzfs_zvol_create_minors+0x65)[0x7f514f6e5c75]
/usr/lib/libzpool.so.2(spa_import+0x4b4)[0x7f514fbf3964]
/usr/lib/libzfs.so.2(uzfs_handle_ioctl+0x24df)[0x7f514f92efdf]
/usr/lib/libcstor.so.2(+0xd72e)[0x7f514f6df72e]
/usr/lib/libzpool.so.2(zk_thread_helper+0x12c)[0x7f514fb858dc]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x76db)[0x7f514f4ba6db]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x3f)[0x7f514f1e371f]
Aborted
root@test-pcl109:~#
root@test-pcl109:~# kubectl version
Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.2", GitCommit:"52c56ce7a8272c798dbc29846288d7cd9fbae032", GitTreeState:"clean", BuildDate:"2020-04-16T11:56:40Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.2", GitCommit:"52c56ce7a8272c798dbc29846288d7cd9fbae032", GitTreeState:"clean", BuildDate:"2020-04-16T11:48:36Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}
root@test-pcl109:~#
root@test-pcl109:~# kubectl get nodes
NAME STATUS ROLES AGE VERSION
test-pcl109 Ready master 303d v1.18.2
test-pcl110 Ready <none> 299d v1.18.2
test-pcl111 Ready <none> 299d v1.18.2
test-pcl112 Ready <none> 299d v1.18.2
test-pcl113 Ready <none> 115d v1.18.2
root@test-pcl109:~#
root@test-pcl109:/tmp/openebs-setup# cat cspc-iep-mirror.yaml
apiVersion: cstor.openebs.io/v1
kind: CStorPoolCluster
metadata:
name: cspc-iep-mirror
namespace: openebs
spec:
pools:
- nodeSelector:
kubernetes.io/hostname: "test-pcl110"
dataRaidGroups:
- blockDevices:
- blockDeviceName: "blockdevice-f0dd8711cd492380429590024356ffa7"
- blockDeviceName: "blockdevice-e5a986b230344a58bec8901acb1b7b0f"
- blockDeviceName: "blockdevice-8e0ec85f552fa3ce491b78df5803d3b4"
- blockDeviceName: "blockdevice-58c88616b9c3f10002912346620cf14f"
poolConfig:
dataRaidGroupType: "mirror"
- nodeSelector:
kubernetes.io/hostname: "test-pcl112"
dataRaidGroups:
- blockDevices:
- blockDeviceName: "blockdevice-3c913b4adbd6edcf14324c8830430392"
- blockDeviceName: "blockdevice-3ee6fa76ebfce9c0003cc54df6818a4f"
- blockDeviceName: "blockdevice-4e4c0d6274874b994017af7116d0235e"
- blockDeviceName: "blockdevice-542729527e349da1e505c64e1d856940"
poolConfig:
dataRaidGroupType: "mirror"
root@test-pcl109:/tmp/openebs-setup#
kubernetes.io/hostname: "test-pcl110
root@test-pcl109:/tmp/openebs-setup# kubectl get bd -n openebs blockdevice-f0dd8711cd492380429590024356ffa7 -o yaml apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:18:33Z"
finalizers:
- openebs.io/bd-protection
generation: 18
labels:
kubernetes.io/hostname: test-pcl110
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:18Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:47Z"
name: blockdevice-f0dd8711cd492380429590024356ffa7
namespace: openebs
resourceVersion: "80135210"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-f0dd8711cd492380429590024356ffa7
uid: a035e214-f86b-47f5-b2be-0f65ef40624c
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-a035e214-f86b-47f5-b2be-0f65ef40624c
namespace: openebs
resourceVersion: "67026571"
uid: a9815003-82e4-4edf-9984-b10aef0ac924
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: TN04
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0055-1V4104
physicalBlockSize: 512
serial: ZC22V1E4
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/wwn-0x5000c500b47473ce-part3
- /dev/disk/by-id/ata-ST2000NM0055-1V4104_ZC22V1E4-part3
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-3-part3
filesystem: {}
nodeAttributes:
nodeName: test-pcl110
partitioned: "No"
path: /dev/sdc3
status:
claimState: Claimed
state: Active
root@test-pcl109:/tmp/openebs-setup#
root@test-pcl109:/tmp/openebs-setup# kubectl get bd -n openebs blockdevice-e5a986b230344a58bec8901acb1b7b0f -o yaml
apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:17:50Z"
finalizers:
- openebs.io/bd-protection
generation: 19
labels:
kubernetes.io/hostname: test-pcl110
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:18Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:47Z"
name: blockdevice-e5a986b230344a58bec8901acb1b7b0f
namespace: openebs
resourceVersion: "80135195"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-e5a986b230344a58bec8901acb1b7b0f
uid: 3187c7e0-924f-4459-85a1-61efe7373fe4
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-3187c7e0-924f-4459-85a1-61efe7373fe4
namespace: openebs
resourceVersion: "67026573"
uid: b3dd8384-12c3-4375-bb84-e7a97fc3f1ef
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: TN04
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0055-1V4104
physicalBlockSize: 512
serial: ZC22V26W
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/wwn-0x5000c500b4744599-part3
- /dev/disk/by-id/ata-ST2000NM0055-1V4104_ZC22V26W-part3
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-2-part3
filesystem: {}
nodeAttributes:
nodeName: test-pcl110
partitioned: "No"
path: /dev/sdb3
status:
claimState: Claimed
state: Active
root@test-pcl109:/tmp/openebs-setup#
root@test-pcl109:/tmp/openebs-setup# kubectl get bd -n openebs blockdevice-8e0ec85f552fa3ce491b78df5803d3b4 -o yaml
apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:18:33Z"
finalizers:
- openebs.io/bd-protection
generation: 19
labels:
kubernetes.io/hostname: test-pcl110
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:18Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:47Z"
name: blockdevice-8e0ec85f552fa3ce491b78df5803d3b4
namespace: openebs
resourceVersion: "80135201"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-8e0ec85f552fa3ce491b78df5803d3b4
uid: 738ad6a2-6507-4dac-85b7-e061515da25d
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-738ad6a2-6507-4dac-85b7-e061515da25d
namespace: openebs
resourceVersion: "67026578"
uid: a3ae1c87-f6b4-4524-a5ce-d6c98cb03c2a
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: TN04
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0055-1V4104
physicalBlockSize: 512
serial: ZC22V1E4
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/ata-ST2000NM0055-1V4104_ZC22V1E4-part1
- /dev/disk/by-id/wwn-0x5000c500b47473ce-part1
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-3-part1
filesystem: {}
nodeAttributes:
nodeName: test-pcl110
partitioned: "No"
path: /dev/sdc1
status:
claimState: Claimed
state: Active
root@test-pcl109:/tmp/openebs-setup#
root@test-pcl109:/tmp/openebs-setup# kubectl get bd -n openebs blockdevice-58c88616b9c3f10002912346620cf14f -o yaml
apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:18:33Z"
finalizers:
- openebs.io/bd-protection
generation: 19
labels:
kubernetes.io/hostname: test-pcl110
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:18Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:47Z"
name: blockdevice-58c88616b9c3f10002912346620cf14f
namespace: openebs
resourceVersion: "80135202"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-58c88616b9c3f10002912346620cf14f
uid: 149273b7-45d8-4558-95da-9b17bb44c30a
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-149273b7-45d8-4558-95da-9b17bb44c30a
namespace: openebs
resourceVersion: "67026584"
uid: e27f0f25-9195-4683-b95e-ac090280582b
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: TN04
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0055-1V4104
physicalBlockSize: 512
serial: ZC22V1E4
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/wwn-0x5000c500b47473ce-part2
- /dev/disk/by-id/ata-ST2000NM0055-1V4104_ZC22V1E4-part2
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-3-part2
filesystem: {}
nodeAttributes:
nodeName: test-pcl110
partitioned: "No"
path: /dev/sdc2
status:
claimState: Claimed
state: Active
root@test-pcl109:/tmp/openebs-setup#
on test-pcl112
root@test-pcl109:/tmp/openebs-setup# kubectl get bd -n openebs blockdevice-3c913b4adbd6edcf14324c8830430392 blockdevice-3ee6fa76ebfce9c0003cc54df6818a4f blockdevice-4e4c0d6274874b994017af7116d0235e blockdevice-542729527e349da1e505c64e1d856940 -o yaml
apiVersion: v1
items:
- apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:23:06Z"
finalizers:
- openebs.io/bd-protection
generation: 18
labels:
kubernetes.io/hostname: test-pcl112
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:19Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:42Z"
name: blockdevice-3c913b4adbd6edcf14324c8830430392
namespace: openebs
resourceVersion: "80135060"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-3c913b4adbd6edcf14324c8830430392
uid: dd067adb-2583-4dc1-b8b8-8efe85d618cd
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-dd067adb-2583-4dc1-b8b8-8efe85d618cd
namespace: openebs
resourceVersion: "67026594"
uid: 9071a4af-b29a-4374-aaac-4152794cc6e5
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: SN02
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0011
physicalBlockSize: 512
serial: Z1P3YAN8
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/ata-ST2000NM0011_Z1P3YAN8-part3
- /dev/disk/by-id/wwn-0x5000c5004e8b428b-part3
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-3-part3
filesystem: {}
nodeAttributes:
nodeName: test-pcl112
partitioned: "No"
path: /dev/sdc3
status:
claimState: Claimed
state: Active
- apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:20:54Z"
finalizers:
- openebs.io/bd-protection
generation: 71
labels:
kubernetes.io/hostname: test-pcl112
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:19Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:42Z"
name: blockdevice-3ee6fa76ebfce9c0003cc54df6818a4f
namespace: openebs
resourceVersion: "80135034"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-3ee6fa76ebfce9c0003cc54df6818a4f
uid: 4b37a5ef-becd-41c5-aed6-8889babda414
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-4b37a5ef-becd-41c5-aed6-8889babda414
namespace: openebs
resourceVersion: "67026603"
uid: 56d95c00-e3cb-45b0-89d2-397090e2d680
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: SN12
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST32000644NS
physicalBlockSize: 512
serial: 9WM89NYP
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/ata-ST32000644NS_9WM89NYP-part3
- /dev/disk/by-id/wwn-0x5000c5004d0f7090-part3
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-2-part3
filesystem: {}
nodeAttributes:
nodeName: test-pcl112
partitioned: "No"
path: /dev/sdb3
status:
claimState: Claimed
state: Active
- apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:23:06Z"
finalizers:
- openebs.io/bd-protection
generation: 19
labels:
kubernetes.io/hostname: test-pcl112
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:19Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:42Z"
name: blockdevice-4e4c0d6274874b994017af7116d0235e
namespace: openebs
resourceVersion: "80135054"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-4e4c0d6274874b994017af7116d0235e
uid: 8fdd9075-36b4-467b-9b78-a81ddec6cf4e
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-8fdd9075-36b4-467b-9b78-a81ddec6cf4e
namespace: openebs
resourceVersion: "67026611"
uid: aba83cd2-bd39-486c-ae75-eccaa558a442
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: SN02
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0011
physicalBlockSize: 512
serial: Z1P3YAN8
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/wwn-0x5000c5004e8b428b-part1
- /dev/disk/by-id/ata-ST2000NM0011_Z1P3YAN8-part1
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-3-part1
filesystem: {}
nodeAttributes:
nodeName: test-pcl112
partitioned: "No"
path: /dev/sdc1
status:
claimState: Claimed
state: Active
- apiVersion: openebs.io/v1alpha1
kind: BlockDevice
metadata:
annotations:
internal.openebs.io/uuid-scheme: gpt
creationTimestamp: "2020-11-27T13:23:06Z"
finalizers:
- openebs.io/bd-protection
generation: 19
labels:
kubernetes.io/hostname: test-pcl112
ndm.io/blockdevice-type: blockdevice
ndm.io/managed: "true"
managedFields:
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:status:
f:claimState: {}
manager: ndo
operation: Update
time: "2020-11-27T19:29:20Z"
- apiVersion: openebs.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:internal.openebs.io/uuid-scheme: {}
f:labels:
.: {}
f:kubernetes.io/hostname: {}
f:ndm.io/blockdevice-type: {}
f:ndm.io/managed: {}
f:spec:
.: {}
f:capacity:
.: {}
f:logicalSectorSize: {}
f:physicalSectorSize: {}
f:storage: {}
f:details:
.: {}
f:compliance: {}
f:deviceType: {}
f:driveType: {}
f:firmwareRevision: {}
f:hardwareSectorSize: {}
f:logicalBlockSize: {}
f:model: {}
f:physicalBlockSize: {}
f:serial: {}
f:vendor: {}
f:devlinks: {}
f:filesystem: {}
f:nodeAttributes:
.: {}
f:nodeName: {}
f:partitioned: {}
f:path: {}
f:status:
.: {}
f:state: {}
manager: ndm
operation: Update
time: "2020-12-17T16:21:42Z"
name: blockdevice-542729527e349da1e505c64e1d856940
namespace: openebs
resourceVersion: "80135057"
selfLink: /apis/openebs.io/v1alpha1/namespaces/openebs/blockdevices/blockdevice-542729527e349da1e505c64e1d856940
uid: 227e553d-999e-4f54-b678-26d28af4f25e
spec:
capacity:
logicalSectorSize: 512
physicalSectorSize: 512
storage: 536870912000
claimRef:
apiVersion: openebs.io/v1alpha1
kind: BlockDeviceClaim
name: bdc-cstor-227e553d-999e-4f54-b678-26d28af4f25e
namespace: openebs
resourceVersion: "67026617"
uid: dc53d294-1b9e-4f02-ab9d-9dd414f4df9b
details:
compliance: SPC-3
deviceType: partition
driveType: ""
firmwareRevision: SN02
hardwareSectorSize: 0
logicalBlockSize: 512
model: ST2000NM0011
physicalBlockSize: 512
serial: Z1P3YAN8
vendor: ""
devlinks:
- kind: by-id
links:
- /dev/disk/by-id/wwn-0x5000c5004e8b428b-part2
- /dev/disk/by-id/ata-ST2000NM0011_Z1P3YAN8-part2
- kind: by-path
links:
- /dev/disk/by-path/pci-0000:00:1f.2-ata-3-part2
filesystem: {}
nodeAttributes:
nodeName: test-pcl112
partitioned: "No"
path: /dev/sdc2
status:
claimState: Claimed
state: Active
kind: List
metadata:
resourceVersion: ""
selfLink: ""
root@test-pcl109:/tmp/openebs-setup#
root@test-pcl110:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 55.9G 0 disk
├─sda1 8:1 0 1M 0 part
└─sda2 8:2 0 55.9G 0 part /
sdb 8:16 0 1.8T 0 disk
├─sdb1 8:17 0 500G 0 part
├─sdb2 8:18 0 500G 0 part
├─sdb3 8:19 0 500G 0 part
└─sdb4 8:20 0 363G 0 part
sdc 8:32 0 1.8T 0 disk
├─sdc1 8:33 0 500G 0 part
├─sdc2 8:34 0 500G 0 part
├─sdc3 8:35 0 500G 0 part
└─sdc4 8:36 0 363G 0 part
sde 8:64 0 1G 0 disk /var/lib/kubelet/pods/f409acde-d623-4df7-9f2e-54
sdf 8:80 0 1G 0 disk /var/lib/kubelet/pods/c2239e59-dd6c-4b3e-8401-59
sdg 8:96 0 1G 0 disk /var/lib/kubelet/pods/ded2773e-0658-4c18-8b7f-45
sdh 8:112 0 2G 0 disk /var/lib/kubelet/pods/54a0dde1-0a64-4cca-992a-b9
root@test-pcl110:~#
root@test-pcl112:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 29.8G 0 disk
└─sda1 8:1 0 29.8G 0 part /
sdb 8:16 0 1.8T 0 disk
├─sdb1 8:17 0 500G 0 part
├─sdb2 8:18 0 500G 0 part
├─sdb3 8:19 0 500G 0 part
└─sdb4 8:20 0 363G 0 part
sdc 8:32 0 1.8T 0 disk
├─sdc1 8:33 0 500G 0 part
├─sdc2 8:34 0 500G 0 part
├─sdc3 8:35 0 500G 0 part
└─sdc4 8:36 0 363G 0 part
sdd 8:48 0 2G 0 disk /var/lib/kubelet/pods/e6196390-5d89-4989-a575-a7
sde 8:64 0 2G 0 disk /var/lib/kubelet/pods/9dd8ccc2-1bef-489e-9798-7f
sdf 8:80 0 2G 0 disk /var/lib/kubelet/pods/43a564e5-9441-4fe1-a283-ef
sdg 8:96 0 2G 0 disk /var/lib/kubelet/pods/902158b0-aa08-4b56-9026-7b
root@test-pcl112:~#
Hi @survivant, can you share output of ‘kubectl get cspi -A‘ looks like pool is full
NAME HEALTHYINSTANCES PROVISIONEDINSTANCES DESIREDINSTANCES AGE
cstorpoolcluster.cstor.openebs.io/cspc-iep-localpv 2 2 2 23d
cstorpoolcluster.cstor.openebs.io/cspc-iep-mirror 2 2 2 23d
NAME HOSTNAME FREE CAPACITY READONLY PROVISIONEDREPLICAS HEALTHYREPLICAS STATUS AGE
cstorpoolinstance.cstor.openebs.io/cspc-iep-localpv-bx2h test-pcl112 1280G 1280404M false 6 6 ONLINE 23d
cstorpoolinstance.cstor.openebs.io/cspc-iep-localpv-vlj9 test-pcl110 1240G 1282200M false 8 8 ONLINE 23d
cstorpoolinstance.cstor.openebs.io/cspc-iep-mirror-hr8z test-pcl112 473G 480740M false 10 10 ONLINE 23d
cstorpoolinstance.cstor.openebs.io/cspc-iep-mirror-t46v test-pcl110 474G 480010M false 9 9 ONLINE 23d
root@test-pcl109:/tmp/openebs-setup#
disk space on node : test-pcl110
root@test-pcl110:~# df -k
Filesystem 1K-blocks Used Available Use% Mounted on
udev 12305564 0 12305564 0% /dev
tmpfs 2467112 9532 2457580 1% /run
/dev/sda2 57429088 24081940 30400128 45% /
tmpfs 12335552 0 12335552 0% /dev/shm
tmpfs 5120 0 5120 0% /run/lock
tmpfs 12335552 0 12335552 0% /sys/fs/cgroup
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/4fb57469-eac2-42a9-ae8f-05e2591bd50f/volumes/kubern etes.io~secret/calico-node-token-2fjtj
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/3485fecd-eead-4a8a-a685-a8df6c896ec9/volumes/kubern etes.io~secret/velero-server-token-nlxnk
tmpfs 12335552 4 12335548 1% /var/lib/kubelet/pods/3485fecd-eead-4a8a-a685-a8df6c896ec9/volumes/kubern etes.io~secret/cloud-credentials
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/46f8b307-7e2d-4ff2-864d-6f41b3f3be3c/volumes/kubern etes.io~secret/kube-proxy-token-sf82w
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/6de0c0f557b6038fbe2f37a97d0242363d2717af02b3e4bb 61836ab6e1d73ac6/merged
shm 65536 0 65536 0% /var/lib/docker/containers/a43c3fe918b7308a5e3e7f62034b2e6c84b2589079ae19 6d80b3f131b4c4cf2e/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1654748a0618f3576670b5c57a08b2abe0a067c9d44693db 36a781723c2a728e/merged
shm 65536 0 65536 0% /var/lib/docker/containers/b95066d6d22e715852821c1bc7d312463e0ab8bb63aba2 b7bdba6627f02f5f3e/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/55fe55aa65e71d19d06445d47414674522fd0ae18c5a82d0 fba234e4f55326a6/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/2bc09e68f536474a5ed2932a03ae0b3379622f41ecccd509 e6606b30d42ede5c/merged
shm 65536 0 65536 0% /var/lib/docker/containers/be2d1c593bcb4d7cfa9019126450f0a988931c4fe6f07d e7f37033d58c7709a8/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/742a855e85df193fada9dc55606633adadec68a7008e3847 63d162942f0ecc5c/merged
tmpfs 2467108 0 2467108 0% /run/user/1000
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/68e5f382-c84e-4897-8af5-54cc987284eb/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/783bda50-c1c7-4428-876f-4a0e50c75b97/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7cff18e846106b77ffc4b929f2f8b2f00e46982ee7dbb061 223396da10f1674d/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/255f02288290547b23dacbcd5660a1a28655216391bcc0fa 8acc82fecbeeadd8/merged
shm 65536 0 65536 0% /var/lib/docker/containers/c74b0fda7da0b27216020e77fadc6cafa00527e0a902f2 8ad8b3b976803f1873/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/80e521bfb8fc730bce3eb06ab9399d8ced155eb949dd79 e2fdc1a69a92de16ff/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/82e2e8f7-bd35-44d1-9579-1b1edea79cc3/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/b2fba53a-e64a-4878-bcdc-776ef8d76ecd/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/d2c3d59c-ef75-43ee-8249-f10c43b0465a/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7ec7c9a348817128f0424ce6ea664be813b9db3dbd7cb518 a79f26d392e6a956/merged
shm 65536 0 65536 0% /var/lib/docker/containers/667418daf2a9ab333cb49c581f6449386c2ead8a29fd9f 95b8f4c95e6738233b/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/c3b2f19ea73567515b7d9ed37bdbcfde6520c329f7215a7a 6caf860a7382a4d0/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/bfa2203b-9a9d-4267-a0d6-302b5d0f4a06/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
shm 65536 0 65536 0% /var/lib/docker/containers/1c1e52419571373790ea610d6e27b232c491a273e65026 6195fa4067063a5a38/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/373d1f4c-0186-4c8a-96c9-c6b64fe3650f/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/5b982dbaad25100f3fff4b771ae7191e7f4da6d916314eb8 fa931c8080cbbe28/merged
shm 65536 0 65536 0% /var/lib/docker/containers/20e92e7b3af6ba2a0928053b60ea0d2efd816b2b8ec8be e92bad98cd9f953474/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/9afb4e0c8c18c78ac22691242097aa28dfbc193a70d262e0 1dbe2910e23352f8/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/195c15fa-5437-4a3d-9d88-19d37b152572/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/cf8df485bfe6c5d40e8c4d0ebbbbe80888f90afda85a5bcc 6da62fba535afc95/merged
shm 65536 0 65536 0% /var/lib/docker/containers/606735dbb677f90edd93ff6fa7da55b865fe271c8a700f a95cf23aa83b03057b/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/8b4c177b-1fd8-40fb-ac0b-98963e879600/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/db8e71897ae768b738d2090ddb38fbcb76d9c447ffa95341 17cb3731bcad2964/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/90f650a9bf4ac4eb94b758a533446c9e6bcab7fd3724f233 21a4a82e1e36e01e/merged
shm 65536 0 65536 0% /var/lib/docker/containers/5375f7cf8e90ec3f5ab4a7cb121b9cdec3480add71a13c 3e13b38409cbc1420a/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/bac6e260-61ac-4bbd-876d-b37fcb627f47/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/c4030cc8-e196-49e6-80d0-30f5945d473d/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/e47ecb2b79d521cd25aac780673f7a272ef5974eaf8276a1 c46b52c5efd49a0b/merged
shm 65536 0 65536 0% /var/lib/docker/containers/9fce2469078dc17c524a0683268be5ad690194101a5a32 92e2814dd935a8fd41/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/9d31a3647224001a23e5f84e31d820d6a92c2034ff0e2ea4 7114ce8c6e301aac/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/0ca78b0a-f893-4a15-a955-0557a74c84e6/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
shm 65536 0 65536 0% /var/lib/docker/containers/9825996d2a07670f6dfe0d4c0690db0fd700e5f076a63c 12db402f09abcf0b89/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/8fc9c9819e05fd194d875eace74c3464f62763e5a28fbc7f 239bd78b0cf64758/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/11a4e359-3593-4a27-9c48-95639aea3728/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
shm 65536 0 65536 0% /var/lib/docker/containers/d6490e621d44c7bc4c23d0c0fe6900e6294a4113bc9ddf 19ef5ac8089b77bca7/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/2a3c18458aae9ac8fe252cbb86a0c97d1d7ec234143badf3 50821d8d96a6c63d/merged
shm 65536 0 65536 0% /var/lib/docker/containers/2a2874b9f469f36dc30d32e0757c0885f960c9db73015a 9deecce7ec8a3ba408/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/e718d572-d4f4-4692-ac5c-045ad5779e95/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/acb7be93-5d13-4295-8c41-002ef4bb2e86/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/667179ee416488220f43a35a8ecfea0fca72c25c98273a4f d7c1b925f1161495/merged
shm 65536 0 65536 0% /var/lib/docker/containers/e6b9079df1f131d401a7502a2cd4cdcbb1107d4cc37246 f7a9470d891a4610bb/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/945ccc7ea0692910e99957e3f527079ad7762ba89c3f15ef 62cd455d16ca7819/merged
shm 65536 0 65536 0% /var/lib/docker/containers/17e8aba3ac9c667eec115cedea99dd76cabb5d2fbf7474 9109f7ccb192709b47/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/151c3b43-bd8a-4f9e-8c4a-437c51afbea5/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/d3b7230f-9533-4c33-bfaa-6f897f75a08a/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/e63f4eedbcf71d1803e4460e3676b9fdd5b49a4aadfe3a19 8e61226c082518cb/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/9d1707de1144244c72e5c31a2a573f8658e07773485b3803 52d874c562327e62/merged
shm 65536 0 65536 0% /var/lib/docker/containers/c32bbd4b2a119fc2f29361fe91a59e7ee5fa30cdccfac0 2f4467b2cf22ce0123/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/b1a0453e64f890afb39d084eba5cd4f67a0dbe21ccbf56f6 b88d0c4ce65488c5/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/79beb7a8b9614be54facf5b65e94ffb156020744235b7c64 b870cd5dbfd9eaf6/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1ff8a4ea39cff1a70fd6df56f77e2c476cf51c8dab6a1d32 8d97a06c81ebfee3/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/2818c17f-48ad-47b2-a172-dbfbb6fec5e4/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
shm 65536 0 65536 0% /var/lib/docker/containers/3ab71d81ef25634f9d4a8dcf93292c53a6d94bb5b3adb6 b7a2c47be6eb91d5c7/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/2f9fb44b4d855eb04716f46e6b27b6cf00713013149a627c f27519090b1654ab/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/e1ef8e16-b9c2-4802-bef3-a60a22e5ada5/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/ada73f0d-f173-4bc6-905e-eb886b4bcf10/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
shm 65536 0 65536 0% /var/lib/docker/containers/ae8c9ea19a242869963f4742e966ddbfa665ac2ab41b91 231340a85cacffeb24/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/873b9b31dac72313eb14e1d055cf9925af92058280e9e8 d78ab75e7d855e3c52/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/381f1ddd-4527-435d-a1af-1fff67d1870b/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/6acb0690e281b1d09045f3e7e1a1bc679d41b940a2b1f433 9c9f4a7e0cdf2e75/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/bb183e2a34ad7c32853b1c32524ba7527a6f44fc84f24655 39fa53297f02e97d/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/b1f5abc83c0fc5127c9b5d965c1cd6438aee9276db5ccdbf 7510c9fd1431c57c/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/77ce0bac9f4b1078feab0dd80118a11cbfddb6a8fd8903f2 4926b21706b009ff/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/0fe13da209d40983703e8fc63592695ffbf170f78a37a36a 0adf04e7ab527e63/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/3f38f13c4f91752c1657967bea88b559c250cbe046c3cf02 fbb2f285dbe31041/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/a508d3ae8fdd1bc7512134a638d59af6a46f984b7e12dae2 620499de8cbb354b/merged
shm 65536 0 65536 0% /var/lib/docker/containers/b2e24fa34d8820fd5e50929c0956d83dba622d4b37639a 6ea8649b20a0dce7d6/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/8c673c8822836b615b6a0cf592f8d914f5d2c595d1f5fd 9595c755979a274555/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/95d622d59ef2577af6055c1789520288aa7a4df36d8592 18b903a1d34a7aeca2/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/ff35acdc888df4d7f4747f6cfc360d094eb415819cae3f 30601dd9abf99c8e4d/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/24bd199375b14fc6ae9e4cd2babc489e8bf2b49a4c47a400 e7b7c1da6b5a188d/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/3c94d5e26f3b052cc1673af5fc0b1eda469c1c25cb4d06a1 ed0eab6254f3b563/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/ae89136c2177f932f39ee6bf39385445e46f979de397903f fee110ae2e2b7bc3/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/208cc89dd898a0cc1ba883bb5481ac05425a7c8e8c9764b8 b1881e01a6a88914/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/0c2f47c6e0a33e5c5ba92b5f4e46e620d8a5777e5cf44ff8 93200c1836623c9f/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/8579f5d65fd9b658ee57b3e456d5618f169228c700f996ed fb353f97178e1419/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/48dd6f5afb9d2fe053e6d4b4af9e936e6ef593b8ca5daccb 8fb5b0a45f47c366/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/aaca7701427078bf43c67a8b61cc64e9da7a00f11ca15ae9 7849db1235821a23/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/25aabcf2e282d94fbaaf5f6eb9ca07ea23475131e8de40ad da10e803600c8247/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/82f7f0a627660a0a55b8ce336a09bb44e5ffb67db1fc379a 7f33d8dd537c30dc/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/158efa25a404b4acb647c5943962c4f33599b1fb165df735 1f4f6a9363803ba2/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7e9ac428d742be11aca96b88b0d6be34d9e7dac8fb331e21 e924007f1e4f44f9/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1ebb2de6fe9a5154646088b2a6835161a345c3f2b8c5552c 01443e3401fd69fe/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/c7152f5447673ee85a9707585536a3772dc9c475f2976ecb c64e448b6dc307e4/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/d9271079a55885344726746748d066a41b94013843f81348 9c0e6557a02df2c2/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/ee1c2840544b7ec04f3cc9b6b6dc76be149db80c6c2f004a c489d4757a91cb52/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/e97c490aba3cdb86e318ca6343da0f5d72ad2fbe9e4e22c7 2586e1e2c9076a2a/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/ca1911b0dffec58e5277871bc5af6672de13122e6e50b950 dd2dfedbefc88f5a/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/e12e2606451b67d8c63da82d8a53026d0f4af1b33ce657a6 3a009d588a006247/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1b9b24a503ce99860caa34ef2fdadc9a2d5b65e200353139 3bd65bb125619c51/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/65c5939983debbc257a6906ee89b002f4deb81c8aeb0dfb6 8a45b572cff31fe6/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/5af2881b728824b219aba8dfd2d5e830db2c040dccff924c 30f4540c4bc4f96b/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/5a63f4e5b54783de93dd74f07fe8a07384b031d190b81608 5df59ed234320875/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1e894b0d006ee183a353f74e5d1e24cd91f30085f2da9004 f73fc5cd09d5697e/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/03a9d52794d4adabe4f33f6d12669580bca71149d01f923a 489378849104fd34/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/dd277861c656a487d975ac3f7f43393f749f6e58adf513f2 33b1444cc643a95c/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/6ac62646cf28e896aaf9e58b79ae738a0e7d1592e3207340 a75b641ca064c165/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/720828f062b8dcc289a497a1497b0cc41c4416ee0ebdb89c 37d33db09350a6fa/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/a30efff0dcb9839551c58defa99c9c8e43b6be2fd86a9dc2 3bb4a600d3e9b849/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/eafc995c5e44d0ddfa7537617979c408e0ce47212072037f 9f5dc75923e31daf/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/8eff5f6232b20a38481e1ad8b291796c8e2e59c9e21359c5 b84908b836d6a54d/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/82344e72451d6d533fe21d23e078a153d8336c2e6c0d94ca c49e2c2d55efbb26/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/557c6320aa0d2a6e40d4041704f8d237d30468244fe3fb8a 2e954d7bfc5c1e34/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/6f1a5fdb8dd6b6726cb4e3a1313dd9d8631e9c5245314e5e 4d719c4af12453f1/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/f9ceb147bbd96bfdb47667feed629e8bc130c7ee577a2829 f80d894137e82d89/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/0391e89ac19a10276f57b1f1245170b2d9a572da2fa9e443 fef4a9d3a3870368/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/0fbc2883c2c6154bba3357ecf3ba09258872341a13c61860 4a61fbd1e9dac985/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/dce0b4e19e60ac28d87b542087e9498c692dbb20be9912f9 6a203cfcedc555ad/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/14e499a83117af8d152fd7312c1ed2442ef1938199697012 7e089cfa4420543d/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/61cca63454dc8c869e2730aee7a1f753800cf97ba77c2576 8b97d6546d3bf9ea/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/695839f2a915df22eef6a19b4a2f6717f350551448d6a873 e2493c8093a16991/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/dca98d05587ad7320c15faadd339e65ed5aca6abf34c22f2 6ea337baf36b9ae6/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/2b13842a327e5d3ddfcad22e9c449e44a9bbd74c9f4930d7 f3f351811b736c19/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/744ccfbafa82937671ec534bc728a5a9ba6689b54c1d5582 0d2e44652dca2cd9/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/29521e0f4563da9870b719f51a615874e83a46a1116739f4 62412bf57bbe3e00/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/2f07dc9020599806283dfc9b37961e6fd0e03336545fb928 155f2cd567f43186/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7e9c7f5c9c9fb0533b2c3cd71f610e21a8a8318653861bc5 ce3cf92a94a39226/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1f15928dd61827e0e2d54ff9340738a593d6c6287354654b 60dc97253d0c32c5/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/3a8317e86a1b5ab8aae0a10dff40843f1b4e981f0f39321c 2db405d0bfcbd207/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/201b418f1ff4bfda08e62aea0f19a7dba96977dd36baba99 04e31eff285f05d3/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1b44312ed96b37414aab9def2c44f70906958e995a70e79b ef057ffa87926dd3/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/e9623f1e-ef3b-40d7-aeb2-e608252ca514/volumes/kubern etes.io~secret/openebs-cstor-csi-node-sa-token-m5nlt
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/9e08e1685ea6607277a5e9fc1faef2e41aff2ba5f9fba49a 81dfd162f72afdd2/merged
shm 65536 0 65536 0% /var/lib/docker/containers/6a989b0766b5cab5ecdfe2e486bbc81b4bb8aeb2572b7e de1b9ba226df4275e1/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7aa3962208ad78769e9764deda0890fddba4b94a1399d1e7 e3384108773e613b/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/8f4f1fc5faf25fb20dd51cd2a3d419c62ca768c0d0adf9a6 fa336c97d343313c/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/60dd8e6d-d58c-42e9-8171-922327ed2d13/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/f3e9773864a644a54e4d2f843798dbacbf37486e8f945b1d 15e24534a68f070a/merged
shm 65536 0 65536 0% /var/lib/docker/containers/ebf4eb2d548f6b217b36f12e2379b006400f516c2a5130 f5d1034b46c2194dc1/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/f65c11f9817ebe575d1a144574bf23fd91df28882bbdbd3a e2a14e22fd817bce/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/fbe8651d-d692-45b2-b6fa-6b2b577c591a/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7ed9904da4c76138bcbfe18212f8ec66b437429f55030a94 400cab3600085c45/merged
shm 65536 0 65536 0% /var/lib/docker/containers/d7f7a08872dbe7e960fcb2bf1f1495bb78b270cb43c4ab 77e58504c39963d217/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/a46da5f8-2af0-4ced-be3c-52b925f97d38/volumes/kubern etes.io~secret/iep-prometheus-node-exporter-token-n84hd
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/7219d6bb-d0ea-4599-af5c-f86ab1555802/volumes/kubern etes.io~secret/iep-filebeat-token-zvsq2
tmpfs 12335552 8 12335544 1% /var/lib/kubelet/pods/ded2773e-0658-4c18-8b7f-45078f14fd86/volumes/kubern etes.io~downward-api/podinfo
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/be30fd90f17af4d743e7c59413bd016f84f2f733c615e07f 375474632b979ef1/merged
shm 65536 0 65536 0% /var/lib/docker/containers/98c9d9b858e4dd3a50a3744e1d5cddc1b887091eedf832 7b28d754b98b184e4e/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/5f5d4f344f073397f68caa5ab330cdf1a5282295f879b591 c38a6c44d36b6b48/merged
shm 65536 0 65536 0% /var/lib/docker/containers/09aa32c44fffdf74d5a2558ce3722e9508aa7201826dec 477456699ea60c9123/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/861ea56d90523adc3c10f523bad0e113d31234cca00a9f8c 591cb7b5e667f6e3/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/7bba15e1af8f92b9d69f7e7a1832c9c37cec130fe0d53b5e 2493de528b084b3b/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/73c09e6b356b177918d6a19846ca9df428fba493cb1ccd3d 2ed00309f25840b6/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/f96499e7-ae39-4608-9dcb-cb28e6532abc/volumes/kubern etes.io~secret/iep-nginx-backend-token-b2swj
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/ded2773e-0658-4c18-8b7f-45078f14fd86/volumes/kubern etes.io~secret/sa-twin-wui-token-lxdlv
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/4938e137-3acb-45de-b5cb-6a0b712421b4/volumes/kubern etes.io~secret/iep-kube-state-metrics-token-kr7qx
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/4a194773-a4e3-454f-b205-60f403b6c9e0/volumes/kubern etes.io~secret/kps-operator-token-zvp24
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/4a194773-a4e3-454f-b205-60f403b6c9e0/volumes/kubern etes.io~secret/tls-secret
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/a7e10edd9d7402a3428a00b90462addf6adc8133d6e64f99 af85ddb7e715808f/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/d1c81b902b6de42c5b945e0aa173b9e887998eb9ac8465a0 195f2a227dee5734/merged
shm 65536 0 65536 0% /var/lib/docker/containers/eb134647228873a92f1740107ec974bc5fe5ee53a70fd9 6a84ae89903583f732/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/ec5b88fe93d43e0f67716e4e2c9024d3f8ae3ffdc04e02 3b03d73fe595734643/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/08e23c986d81dc4a2eea54f75b4b6a2ca96b99e28fb76896 df39c52d30a5b76c/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/bfee6c714901f1612af7c8f90fd73dfa65a53ecbd34026f1 af2f6ab77ac64815/merged
shm 65536 0 65536 0% /var/lib/docker/containers/0a7da77fe6998a4fbf6a678148f69cfcb1fac63226a448 df66419577b59a62ed/mounts/shm
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/0e4ecf1d-13e4-4f69-b537-f6178950bef0/volumes/kubern etes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/b9eece70-1abc-4b66-bfe4-cc785d0ea38a/volumes/kubern etes.io~secret/iep-nginx-token-v9948
tmpfs 12335552 8 12335544 1% /var/lib/kubelet/pods/c2239e59-dd6c-4b3e-8401-599c39b12077/volumes/kubern etes.io~downward-api/podinfo
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/33a64b6c-2653-4049-a0a2-ec2ea4a5c364/volumes/kubern etes.io~secret/iep-metallb-speaker-token-wnd2g
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/c2239e59-dd6c-4b3e-8401-599c39b12077/volumes/kubern etes.io~secret/sa-twin-api-service-token-qs4nm
tmpfs 12335552 8 12335544 1% /var/lib/kubelet/pods/f409acde-d623-4df7-9f2e-54653ff360aa/volumes/kubern etes.io~downward-api/podinfo
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/8fe25acb-8ada-46d4-8163-e3bb08ab2d2d/volumes/kubern etes.io~secret/default-token-jcc69
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/6e268ba8efb92bc3df1f80ec2be0555b12a64411717181d5 a191140df2b44caf/merged
shm 65536 0 65536 0% /var/lib/docker/containers/95434f43c43feb8e099b3f4ce3872ee96f92e2f471dd13 2471aca24edc690bf2/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/c73f341301f30a0a30e3c45cd9d4af1f8b7f7edeb3316767 abb28354c274f5a9/merged
shm 65536 0 65536 0% /var/lib/docker/containers/60f827c476b9b7e3efa918ebea333aa97be7eedfdfaaee b4a6db35bbef89ca4d/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/4af29db174f8277713d85301a79480499f41b46213e103c5 20ca1866c55b658c/merged
shm 65536 0 65536 0% /var/lib/docker/containers/5a8c64fd2ec07094748e2a33e6469b631cb9d32d6b0493 c8c5b711ae442f7c24/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/a8e3a4b9ef86a384dddd748376b81b70782250d84873d810 77c8d594a20bd4f9/merged
shm 65536 0 65536 0% /var/lib/docker/containers/e4d6eca79e19aae0700becf2d782fe301b17bb59d0d66a 38054d10eb326af619/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/aeb7512c4c92ec66c11c6f4e0fe13841e8363b6ab20155ff 85c8ee6da029bb22/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/3a34e6c356c70d787448f7011635ce268ae1f5e147538bd0 3cc03cf18234e7db/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/110ef601be610a3845286773e43e11cbb2f61bd6da075aeb 3e5c0ac9d1ede2d5/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/01c02e216505895f68f783930645e80d1b8a4277a360026e ae2d91edafd1f2c7/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/380529bdf967e2e2c155f86622a154bf1cd1781204f79630 fa7278b7faeaf9d3/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/f409acde-d623-4df7-9f2e-54653ff360aa/volumes/kubern etes.io~secret/sa-gateway-api-token-jst6k
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/2c7bebbc7efdef6e9690f2a1b6cabacf7ac5a521915d0796 17c1bda6e8618332/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/9ceec2c5aa1801b210460601be912e90eb0e3e6e59d72ebb 6ae75fa07533bdb5/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/ecf56be3825679bc687d7efd3f331d73e5e8ebea160e8113 c359574f86ead11c/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/651382d0c32c26c1629a4fb64f9cef1660ea011f0da11924 b55b6b9fcacb1091/merged
/dev/sde 999320 1284 981652 1% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-2aac64e4-eb8d-4e1b-8d2d -a339e4aefef5/globalmount
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/5dc0f837cbc8334b96d2e165e19b3f9d3281393f288112cc ca33049bba2b570c/merged
shm 65536 0 65536 0% /var/lib/docker/containers/5f29d961ef6c8304b7a99f740b1340857a8b39ba5df3bd cddb32cf2ba14f1ae0/mounts/shm
/dev/sdf 999320 1284 981652 1% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-75c003f8-3a2b-43d8-927a -f6b5be972303/globalmount
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/ec5a786e002026ccb743ed0f082ca8cdb944ede72243e09f a07348d9eb604641/merged
shm 65536 0 65536 0% /var/lib/docker/containers/d15f16d324355c97a5bc9f0bacd6729c679c508e1191a8 4c6eec6d395848cec1/mounts/shm
/dev/sdg 999320 1284 981652 1% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-68ed88c0-235c-44de-bd1c -4ea6c09c7288/globalmount
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/af61f46dbd9e91e6216d82632db1794ff33365fa46bbf160 ae8b217e722e8962/merged
shm 65536 0 65536 0% /var/lib/docker/containers/9acded55734a5da74f50b68b1877eb49586045e6ae32f5 aa4973569cbfa8d32e/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/212a8df827df3f3e30e8dcebc5118de06922b20cae26b42a fd6278160fe965f0/merged
tmpfs 12335552 12 12335540 1% /var/lib/kubelet/pods/54a0dde1-0a64-4cca-992a-b9dfdee1b5c0/volumes/kubern etes.io~secret/default-token-jcc69
/dev/sdh 1998672 55608 1926680 3% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-706f2979-de43-4aa9-b8ac -80246b7032b6/globalmount
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/a3126b9dd399615f594a42b5e97771d6e11c259cc7073c21 fe655c2eb4d0a58e/merged
shm 65536 0 65536 0% /var/lib/docker/containers/38071dd8ab3b901f674e138989bcaacbbe1b0a043af996 5256c43289055ee4e9/mounts/shm
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/52669b10fff369dee5b14b688c151c9b91a0f4509421f4aa 5c28ce2bd66c02a1/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/130b1daa57fd235e8ee35b8aa7d3cf14d38d64aa2a8376f3 99186d112fa97192/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/15101e1ac04f42422adc6a8de7502842f4eaa673e63a93f8 bab024e9801a7736/merged
tmpfs 2467108 0 2467108 0% /run/user/0
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/30e8a83b28746b2cee023759d78d6d167b20041bb637459d d9058564bba669ed/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/74a09581f7b03d22ec27a81be3e49bd4c8d1f0142f3d7b0e 0efe3216ae111c9c/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/595b5b3ce6134fd9b1214a40c54337d066478bbc1602aeed b1d12d470bdaf029/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/64135af5abfb95142b9b7646f14d63a99b8e8a5bf4dc3ba5 c0c30393549a5d4d/merged
overlay 57429088 24081940 30400128 45% /var/lib/docker/overlay2/1f2ff425600c68cdcc18acc3fb32b4a4de61e199e79c29fe 4626dc0b998cd622/merged
root@test-pcl110:~#
on node : test-pcl112
root@test-pcl112:~# df -k
Filesystem 1K-blocks Used Available Use% Mounted on
udev 6112520 0 6112520 0% /dev
tmpfs 1228500 7484 1221016 1% /run
/dev/sda1 30626664 17364464 11683400 60% /
tmpfs 6142488 0 6142488 0% /dev/shm
tmpfs 5120 0 5120 0% /run/lock
tmpfs 6142488 0 6142488 0% /sys/fs/cgroup
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/db995bd5-9bcf-450b-a2b5-b7e40e4d54d5/volumes/kubernetes.io~secret/calico-node-token-2fjtj
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/e7e0d1d1-184b-4c2e-add9-5f1c9b72d5f3/volumes/kubernetes.io~secret/kube-proxy-token-sf82w
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/3c5dbd2238bd7723e428d78d8c9ff2ae76af75a3f9888493430af36d3a0e767a/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/5a9b93acad4bf85d07b3140806937e45247f29f2372762a15341952861bd9371/merged
shm 65536 0 65536 0% /var/lib/docker/containers/4ad149a6c02b26eab10de4754814fe65cc4ed99bbb8914b077bed5b46faaabc2/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/1d364a9e06286bde16437464c0f0b6d5c6fe715f7a0c74e4574cdbf9eb40763f/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/ced4431ba4a469b9db35603842a22c608fa49264f6efeb910d6cc83232719443/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/036cecf63841c670598924f87391aa3b8d910ef6102f125290c27937e1f3e1c3/merged
tmpfs 6142488 4 6142484 1% /var/lib/kubelet/pods/04444101-705f-4a2f-9c3c-8ea1392a0857/volumes/kubernetes.io~secret/cloud-credentials
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/04444101-705f-4a2f-9c3c-8ea1392a0857/volumes/kubernetes.io~secret/velero-server-token-nlxnk
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/5712ef162cfd2455ac190b4dd29524eec7b4affe6bf513e04c7afd11f436cf17/merged
shm 65536 0 65536 0% /var/lib/docker/containers/e7f2f56083671e82233096a41fdd1538fe7e883bb8527d5122376c9b2a89bf45/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/81a937c3d7bf2a71d749d3811bc9a1ae90bcc18459c53c863f2128c008b99ef3/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/25726556-ce96-4ffa-a9d8-bb92ff2098f1/volumes/kubernetes.io~secret/velero-server-token-nlxnk
tmpfs 6142488 4 6142484 1% /var/lib/kubelet/pods/25726556-ce96-4ffa-a9d8-bb92ff2098f1/volumes/kubernetes.io~secret/cloud-credentials
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/7c818e35ece6a45571766e98fb67b5950e45559541e486bbedd5d2b387419a04/merged
shm 65536 0 65536 0% /var/lib/docker/containers/1d83c1f1387908f18af5b8aca6509de5a42022a995db410cfe646f29fc955d8e/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/86bfcef8edeb49a17074f964e2b8f3e97ead6ad19339cf11ed142b3e0e649261/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/0b778f08-d1ef-45a1-b6f7-c6760a376033/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/1ce25d0e-7847-458d-a940-96ec95cd158d/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/880da3bf8efc060cd088ab0e066252e13dec053fcfcd61540c5c949de27a9ea1/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/fef71477c7db04175a4b2412e66abb776af37230f79b4b8b1b2c5d911638b951/merged
shm 65536 0 65536 0% /var/lib/docker/containers/a22a89df3ab7e315b402ea4a43b49d098f26ceaadc9fe6eeed392edbd2bbc9bb/mounts/shm
shm 65536 0 65536 0% /var/lib/docker/containers/531f7e220865e1a3c4e8b88ad9b4c9a9cc078f8a934c441d392ac50bdf2f1735/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/469ce33d8617b41ec6c92af1a039fbbb71e91a24d158545f4a176b87f38ce173/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/a0e10150e7688b5b85ac80c55a4ecc02d60c6dc5602258f8bc76c7b415379787/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/7bede5ebec995cedc9c6690e1cba93c313c237f6a7ee69f62681616159263985/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/f4dc57bcc1d0dd7898147981bbb8cd2d513dc1970de84b29ea0d066fecb4f7e0/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/62d46af5c5dcd50de5e59cf5b1f21bef9720fb43df033fcd67474168ed89e617/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/939bd162d8a9e26194cdc7312813c3eb62d47891033de8ed38e3942e51651b71/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/6118205a-b19e-44d3-9e4c-50a48c33dc90/volumes/kubernetes.io~secret/openebs-cstor-csi-node-sa-token-m5nlt
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/0ae524db-fc95-44d3-b553-3df288a067b2/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/2514e069e2a159f0a16cafa1fba078ddeb678bd297fca30cf3655ccff87243a4/merged
shm 65536 0 65536 0% /var/lib/docker/containers/64bf9d4e8e307620c718906b27d73e9a740bd91c111ce7962afc306b466f2241/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/6137a0772d1643fa5897d4ba25c2dd2433f3125653bbaaac063f356af4b1a8c1/merged
shm 65536 0 65536 0% /var/lib/docker/containers/5807f128b4994716ad4c83b31479c06f65d2a0c03c839dc27b9df8f5e8ba7de3/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/712d0f77a536aa1eafe34d6275d33316979810e1f18b3bf262e54e2a64dec70b/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/338f5d973ce09353cbe59ef61d566cefe941f0807ea80b5df5c753f71bb83f0b/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/f20eddcacf217a10d8ebd2f5393717b9de706b55f7b99c3274f5076f29369137/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/ba773f00-eb0a-4863-8120-5ccb45dcbece/volumes/kubernetes.io~secret/iep-prometheus-node-exporter-token-n84hd
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/b2624865-860c-47ea-9eb6-2929bab1ed65/volumes/kubernetes.io~secret/iep-filebeat-token-zvsq2
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/e11699f0-f8ff-4099-a4aa-0db129327091/volumes/kubernetes.io~secret/iep-metallb-speaker-token-wnd2g
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/33a079bc6f620c84768b143163d8e6587d389c71ec0775a1878564cf8e466653/merged
shm 65536 0 65536 0% /var/lib/docker/containers/48916c6bf62d1d306cabdd1703210fc68329e7c4c8843996ed235b33e9d6b374/mounts/shm
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/1a2da268-340b-4684-b155-4d9e67de8f7d/volumes/kubernetes.io~secret/iep-nginx-token-v9948
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/ba0912076146de1614e8c8f1f3625aabba2f40dce9431ac305387e89c8411f6b/merged
shm 65536 0 65536 0% /var/lib/docker/containers/29abd4a650f58a7725443de3b82df8fb6f053b734eb22a016c817b0d7d8d05f0/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/6031ae5ea827c92362c7ec0e6440c95d11403b10698690e38b9fe9ae99fd029e/merged
shm 65536 0 65536 0% /var/lib/docker/containers/cb1c5be91e23dbb6cf0d9919804d5431cf093bbb84f87f92f3ca3c2e876e117c/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/5ad00feea85b7d93d4f92c665a87b6a01a0a8dca9e30ae81fa81f498eb1b41e5/merged
shm 65536 0 65536 0% /var/lib/docker/containers/5ee930c04391246e007a037456c4621db69f02ea848a691bd355e876805bafa1/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/08aa944126ed52fe729b1bd99c7065396cdeb69452b3d3cfff9bf29c5b6d077b/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/2421f63feb33d5c74f9caa74701927586a5a951d43b00db0383e7c42f6b74026/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/034dbf70a93eea68773818b42b3ff36a60e5d5be214bb4b062a6670cbd84d4d3/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/81275df24e1058e50a973f1b1ed72373ccb12ea872d6265a0df7f8a4b086ba9d/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/1e1ac355-5af2-47b0-b238-c145bfb35d22/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/78ad103ab71fc0708339659c6090e38e01f249a32d39e014cca5f36d0a11cf36/merged
shm 65536 0 65536 0% /var/lib/docker/containers/d267943837a880010d62b4a136e3b9edcbf7c9b80f45e894c6e5ccb7642904da/mounts/shm
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/f4614c79-721d-4929-93d6-d96b4e215e38/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/f12817ec-1e9b-45b0-8333-4d5ce74be5ec/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/d5475845d7d4feedf1cc09bf4654bea78312b0e3379e76838859b08e79369f3a/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/e55f1fa25b66a129a472bfb94fbc0bf5c7bc35310075d742929a65ddceb4398f/merged
shm 65536 0 65536 0% /var/lib/docker/containers/ea84acd970580356bf2931b83e51b9904cdb588d876b54d1e3e57cc98dd4b468/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/ab49e800b877fcf2a0587462716f0587cca23a4a9a060112e1e95d9b3107f5c1/merged
shm 65536 0 65536 0% /var/lib/docker/containers/1449ddec8282cfbc013d55f908344ac6547ffda7e52a012993491ffc508f1c97/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/bdb49ccdd298c60c597fa5904b1a904cef1b88dc1fbbb7658bbc32847e01d0ad/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/8198570f55cca16e15e2947478822717e311e1dd6213ebfb79f6cd1c0d27d04d/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/32bc5567c1bb3613d6a6d2385c3c287a9bd6de759157e3e10d0bbd17624ffc8f/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/8dddac1f9e9fa230c3df0356752dd2daae5dbec491a0deef03b747f77925a5b0/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/4abdcf6c3d45e3a874efc16d30dc2ba5dd1fe2dabcc3c6f0f8e0a5f63c8a0e65/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/e52adf416caf570c6a8cf93800d758a3caf54c4eb38704f918b07399abb51cfe/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/7d87410375b16e73680b2a527da3765b21567dca439d8ff7c8e4404f199e01ed/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/30bc5cbd8263e69f4de9d987abaca254f3819bfbbb27a9c046ed1ba91f64fdae/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/4d971d73-7e82-4a76-8b6a-8dd359b88347/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/869071b556035135bec203e01a6f818b3b1673e406ec8b3fdb52cc1f00957f17/merged
shm 65536 0 65536 0% /var/lib/docker/containers/4b25700c4558203e0a91b06cc21227d69693d3d6853d0effbc85f816e3babb23/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/fb4471e7a83db79855749713197598833fcab2fe1ee154d01cbf8f42d8f58834/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/86c0a4d4f96466b9a9a5415f68f3ecec4626f6140e3c03196ded147eaaa913be/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/ff76c8c19788cacf82235324e9bbfbdde78aefd70900c4c60e187d96ff7c1040/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/e741af7b-5871-4b65-943b-8c4849d4ef73/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/d21762b2-4a43-4235-a358-d97941c86a00/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/33fbaef32397592bee2f906e076a22a776305bae2d84d779d806588328a3c162/merged
shm 65536 0 65536 0% /var/lib/docker/containers/7f0a35238571e526cd2128010b851a410fac562f93ed554885af5b0160a48a40/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/d93d10f676a3c1868c31b41d0e28938a9c14a900f7027d06d956b280b624b2f8/merged
shm 65536 0 65536 0% /var/lib/docker/containers/0f55fb72030a66a0d4e06bb6e46260f0e40a563b5f69a437cee6b9808de572b2/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/bf754436f83d466c27bd766652f13a84f6488168f38dbfebdb690580c00e9413/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/3aea3079d608f408e614b316a10e8a882b0062c5b29f0b9e3d0f7e741083eb3d/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/067856e78117e6a167d972c92a9134848b494817582bcde9e44803c8ebaead7f/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/3e6782545f3b61c597d569ae8221322d5901141fc210ce8726297589a9455cfe/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/edc0b7d3d83d390dd015c22dc24e8b49e748773d8a0a774b567fea7143a291b3/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/567cdc30-d9ac-413e-98c0-04c3014b2caf/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/59732c957324bbddb65f71b4a9783ec0c9d26a0d048aaa29a201f8cd671f2cd8/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/a78fa9daf451c7cdada117c134408440bc36dff23f5d68fb75cc2c835a157fbe/merged
shm 65536 0 65536 0% /var/lib/docker/containers/b64a9d8d25e80c16c0f6931e9f6cd296fb7f41b9a97b88ba57e6608467808e82/mounts/shm
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/9feeb17e-bc64-45d4-abb0-f8f282fea6f3/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/692a6c7c1da09157a603a335937889e5e54019566a3c5af64828b65e7f5253ca/merged
shm 65536 0 65536 0% /var/lib/docker/containers/548ebe171c44a38263db3ee503a206dc18ee46b6b6319fcb3a52db6ce690ce3a/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/e2bc1c4a85159f0b18b53a92a284eab919b7b4dfba36a2299ec8b9e08088c7da/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/9231be0bf5f28eb89b37ed5b5cc6805d950bbfcb106311bea34fd34b849b90eb/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/c22dbb70daddf16485fcc6d437a0ab28b8c43f11446cd3c049472cf4073ed67f/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/fca62c5ece92dae99a49ad42abc54a63d59f5cab7de9596ca64ffe5bc7c8c411/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/c3f444126f3c8825f0e4da5b699784f2a3cb2513647e2ecd21be1c6e1684616c/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/e3ecaba4164d40ec0342b91439bc0d5f0d08f485a737ba1023cdf031ac6f15a2/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/5d8f5a26-e80a-443a-89a9-dc37eccd18ae/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/187b44e6f1bd2db37e24303ee551abf4c442cad44c7c99d4d007ee1a89573cbe/merged
shm 65536 0 65536 0% /var/lib/docker/containers/636d5d2fe42610c9a1e4e1953770a2ee40a0c0d6b45f645bab853f60f8d10b07/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/99ff830103e1f6bb0f2b4b5b05ee69d504203a36ca34019ca3cc9d40e43d0b14/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/f5cf119ffb000234a60265fc54e78fe81a2291852e876841701fc8fa63896440/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/dbeca9c8d8c4b73bd49f0c310ab984ed442bb75d089bce7ae4e2f1e9b709c6d8/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/e3d91bd5-6b4a-443a-bfa7-1652d1375715/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/da5634f15854277ac3638d8a88888abd9397ac77fab954addb86e93d38dabc7a/merged
shm 65536 0 65536 0% /var/lib/docker/containers/d491b6e298368ab59d25a4352688ebeb05282c0f4ad9227cadd9255e787dca6c/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/14c59b613e2fef49ff01724ac39c22780139297d1ce30d0c9daef7f7396f999c/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/2c5e09e75e56f771a48fc8b44440e0405eb08d9fc67ba176a870dee8f7de4cee/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/facf7169c771b798ecdb3356994bb680c185f023dabd22d358b81a4d54fbbc9e/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/50216d2c-0736-4541-af73-3a2b61918ebd/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/4d4d1e8fd9777ee8c2d1dbccc03decee5994022bcb9f28fa38651b53c62ff212/merged
shm 65536 0 65536 0% /var/lib/docker/containers/00e8d46fe170fcef7ebee056dbe7fa98a32ed099be56ca54402fe142249018c4/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/4a75c084d07b006da5518f6b675f11206a7095472f8c8abf805811857391551c/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/479b3bfd858c37d44be646545dcec5134a0e87c4edef32194ff266a64396b3f7/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/a4826353555c14ea7ad0f7d5d8941245d6fa617527f5c6ce7580e19944618503/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/9dd8ccc2-1bef-489e-9798-7f0357363b4e/volumes/kubernetes.io~secret/production-manager-influxdb-token-6cf7d
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/e6196390-5d89-4989-a575-a71579b8db42/volumes/kubernetes.io~secret/default-token-jcc69
/dev/sdd 1998672 210008 1772280 11% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-21ce8f31-c6e8-4b00-be38-adc213a6c254/globalmount
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/98ed238ec27ab3e4114a732d151fcb03570b1911aec3a8b85a4ab3718eade0b4/merged
shm 65536 0 65536 0% /var/lib/docker/containers/a62e812d2dc8e293370247bd22fcf82e6f1e96d7a0ab17c59ca249f0e2e86593/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/89a5d1026e98b0749f8e2a68eaa8babcfb688f169464967fdce8f9f6e12c6bdd/merged
/dev/sde 1998672 23096 1959192 2% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-4a1907b6-f802-42b8-aeda-a2904620bb9d/globalmount
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/d166241140a3db17b2b8a727a684dd6f6b1bb526ced1e01db79c1c606f97a45f/merged
shm 65536 0 65536 0% /var/lib/docker/containers/aef621c4d3cec5c491222bd3f4a3b71e6ca3d3a90f370cf705aea062e51f8774/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/252076aff2e7f58dfd3a90ddaf461369a877a090756609eda188ea65a4044290/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/bd876544-d12c-4d25-be96-d38787221011/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/8d7dcbb9e0246e64f8e20124e6fa5c90268b3bc8f0350f3562afbeac0f2ee001/merged
shm 65536 0 65536 0% /var/lib/docker/containers/1e0d277d914426f3271b8ab19b5c854d989f96e9763d5727db8c8f61960064e0/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/e1336966bc5cffee68550880961cf941fb64f4fb3747545aa42751b6f77fd021/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/2ac4df58ed879f0c47305de9b6655d5b490394aeb8c844efe2307d0e1ac1629a/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/fafbaf9086504390ad0247c8c399d128e4717952910d95f68b2e2a70fe74fcbd/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/43a564e5-9441-4fe1-a283-ef3aa3b4275e/volumes/kubernetes.io~secret/default-token-jcc69
/dev/sdf 1998672 213664 1768624 11% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-c2bf25e0-0c7c-4027-bafd-c37afea47c0f/globalmount
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/14ae8db782ec8c989631a0858e8c23c90078709840d3880cbf906c15c22f9a59/merged
shm 65536 0 65536 0% /var/lib/docker/containers/ccacc04f9e0e8f3a8f0317ffd3de4b954e2f140982aeb73b01c6d32b086f4438/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/7138eee2e4be92c5d10fa7cc3f59e79a97e97bb7cfad7292aebbda8bbfd7fbcb/merged
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/902158b0-aa08-4b56-9026-7b126d912eed/volumes/kubernetes.io~secret/default-token-jcc69
tmpfs 6142488 12 6142476 1% /var/lib/kubelet/pods/3919439a-8953-4efb-9ae8-8911d342838c/volumes/kubernetes.io~secret/openebs-maya-operator-token-69f5s
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/0b87ce1871e800d3b9f7e5aa97eab0cbddaa7f1ea7b9aadbde979c2c41ec5951/merged
shm 65536 0 65536 0% /var/lib/docker/containers/4ecfc225c9904b4a3f13c7df6d85cecb22e7a053a9c86bb4e1e817cea5bc3220/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/77e4de58d3aa907b412fbed99328de8ad9834b2694f49b46f3458cebcbf3cf20/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/9b3d0538a540690c23f931f9c27448eed5d73cd1bbdb2ce52b88349de3ff3faf/merged
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/36ede26d0adff2ea79dadf99f82b61c156bcc572df5e0dc362a806e400b277f5/merged
/dev/sdg 1998672 3108 1979180 1% /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-a6bd8693-5d65-4bb9-b980-57088dc0e184/globalmount
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/14bdc6be3b6450574725f45c7eae86299d210aadeace2946fb4c8f03e9679435/merged
shm 65536 0 65536 0% /var/lib/docker/containers/5ca3d67015125d32d3151914bb814abaa1713e2df598055159b42310c616b2e0/mounts/shm
overlay 30626664 17364464 11683400 60% /var/lib/docker/overlay2/d1b569083e107e15251e18752e37d3c7877ef9c2ee9793fd564fb040b2f7dbe4/merged
tmpfs 1228496 0 1228496 0% /run/user/0
root@test-pcl112:~#
Thanks for the information!!
pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt) == 0 (0x1c == 0x0) ASSERT at kernel.c:198:zk_thread_create()Fatal signal received: 6 Stack trace: /usr/local/bin/zrepl(+0x1e82)[0x55f68d4c9e82] ... ... NAME HOSTNAME FREE CAPACITY READONLY PROVISIONEDREPLICAS HEALTHYREPLICAS STATUS AGE cstorpoolinstance.cstor.openebs.io/cspc-iep-mirror-t46v test-pcl110 474G 480010M false 9 9 ONLINE 23d
The above error says that there no space(0x1c = ENOSPC) on the disk to perform writes but from the above outputs, CSPI is showing still it has474G as available.
root@test-pcl110:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sda 8:0 0 55.9G 0 disk ├─sda1 8:1 0 1M 0 part └─sda2 8:2 0 55.9G 0 part / sdb 8:16 0 1.8T 0 disk ├─sdb1 8:17 0 500G 0 part ├─sdb2 8:18 0 500G 0 part ├─sdb3 8:19 0 500G 0 part └─sdb4 8:20 0 363G 0 part sdc 8:32 0 1.8T 0 disk ├─sdc1 8:33 0 500G 0 part ├─sdc2 8:34 0 500G 0 part ├─sdc3 8:35 0 500G 0 part └─sdc4 8:36 0 363G 0 part
To understand the issue more clear. Can I know how the disks are provisioned? Are these disks are backed by vmdk files? (Or) Is these disks are directly attached to the node? If these disks are backed by vmdk files can we there is there an outage in storage?
We are on premise. It's 2 Hard Drive (2T) SATA plugged directly in the computer. We poweroff the PC, plug the hard drives, reboot the PC, I used fdisk to create 4 partitions (500G,500G,500G,3xxG). OpenEBS created the blockdevice and I assigned them to a cstor pool.
they are not virtual devices
Looks like there is something more needs to understand. Can you provide the output of zfs list by executing the below command?
kubectl exec -it cspc-iep-mirror-t46v-6d65b46d55-s5hgp -n openebs -c cstor-pool -- zfs list
the pods changed name since I filled the issue. Here the output of the 2 new pods (which are not in error anymore :( ). We did a cleanup yesterday. Removed all our applications, PV, PVC (but didn't touch to OpenEBS)
root@test-pcl109:~# kubectl exec -it cspc-iep-mirror-hr8z-66b67d79c-769cw -n openebs -c cstor-pool -- zfs list
NAME USED AVAIL REFER MOUNTPOINT
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87 7.29G 473G 512B /cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-2066749a-bb10-46dd-b4d1-906b9b6a2d4e 1.14M 473G 1.14M -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-5e1d38ec-0939-4962-9e57-84e49c4afa02 17.7M 473G 17.7M -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-743e2fa0-e00f-4b26-b7fc-0206c00258a1 3.82M 473G 3.82M -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-8b848f31-64d3-40d6-ac26-42735c23ce9a 227K 473G 227K -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-a348afa0-d238-4b96-98ab-b11210ba7880 1.03M 473G 1.03M -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-a7680839-90fd-4548-a197-fb6df2d73a45 129K 473G 129K -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-d5b0a77e-82f9-4584-a42b-e3ef14125115 7.21G 473G 3.77G -
root@test-pcl109:~# kubectl exec -it cspc-iep-mirror-t46v-6d65b46d55-qq4v8 -n openebs -c cstor-pool -- zfs list
NAME USED AVAIL REFER MOUNTPOINT
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87 2.58G 478G 512B /cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-2889d20c-36b3-4aad-80ec-85d860c82218 610K 478G 610K -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-2ac75bac-332c-43c1-82ba-3996429ffbd7 226K 478G 226K -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-38aa9e68-1d14-4132-b5e4-59422a638bb2 13.7M 478G 13.7M -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-9e5053fd-cc87-4bc6-897f-07bd3b61610b 1.78M 478G 1.78M -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-c70a0b92-d6bc-41aa-89ce-ed94b54af4f6 670K 478G 670K -
cstor-48e9ac96-24ff-42c0-9a14-2531c8118a87/pvc-f40e89a7-5e5e-47ea-959b-913a985db25b 2.52G 478G 2.52G -
root@test-pcl109:~#
Removed all our applications, PV, PVC
So did you deleted the PVC and application which are on top of OpenEBS?
all the applications that were in our cluster in the default namespace were removed and reinstalled. I couldn't keep the cluster not working longer.
Here the output of the 2 new pods (which are not in error anymore :( ). We did a cleanup yesterday. Removed all our applications, PV, PVC (but didn't touch to OpenEBS)
I am trying to understand how the OpenEBS issue was fixed by cleaning up default namespace applications. Could you elaborate how the default applications are getting PVC? Are those PVCs given by OpenEBS?
they got there PVC because I created a PVC in each application using a storageclass : sc-iep-mirror
a quick example
{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "app.fullname" . }}
labels:
{{- include "app.labels" . | nindent 4 }}
spec:
storageClassName: {{ .Values.persistence.storageClass }}
accessModes:
- {{ .Values.persistence.accessMode | quote }}
resources:
requests:
storage: {{ .Values.persistence.size | quote }}
{{- end }}
persistence:
enabled: true
# The path the volume will be mounted
mountPath: /localpv-test
## A manually managed Persistent Volume and Claim
## Requires persistence.enabled: true
## If defined, PVC must be created manually before volume will be bound
#existingClaim: ""
## Storage class of PV to bind. By default it looks for standard storage class.
## If the PV uses a different storage class, specify that here.
storageClass: sc-iep-localpv
accessMode: ReadWriteOnce
size: 50Mi
#elasticsearch
database:
persistence:
enabled: true
volumeClaimTemplate:
accessModes: [ "ReadWriteOnce" ]
storageClassName: sc-iep-mirror
resources:
requests:
storage: 2Gi