simulate test harbor warehouse object storage S3 failure, node cache image cannot be download

[mansoncui]

problem: simulate test harbor warehouse object storage S3 failure, node cache image cannot be download?

before closing minio s3, image can be download normally

root@ops-k8s-testwxnode8:~# /opt/kubernetes/bin/crictl pull  harbor.test.wifi.com/library/clickhouse-server:20.3.9.70
Image is up to date for sha256:abe55fc6544d42937af1eccad0a639a039bded091b0972124d3069eba82089b8

close minio s3

download image
## /opt/kubernetes/bin/crictl pull  harbor.test.wifi.com/library/clickhouse-server:20.3.9.70 
E0606 19:12:26.429217   30123 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": failed to resolve reference \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": pulling from host 127.0.0.1:65001 failed with status code [manifests 20.3.9.70]: 500 Internal Server Error" image="harbor.test.wifi.com/library/clickhouse-server:20.3.9.70"
FATA[0048] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "harbor.test.wifi.com/library/clickhouse-server:20.3.9.70": failed to resolve reference "harbor.test.wifi.com/library/clickhouse-server:20.3.9.70": pulling from host 127.0.0.1:65001 failed with status code [manifests 20.3.9.70]: 500 Internal Server Error 

scheduler core.log

{"level":"info","ts":"2024-06-06 10:57:05.004","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc000771ab0), BeginTime:0x17d6650f7601702f, EndTime:0x17d6650f783332f5, Success:true, Code:200, FinishedCount:18, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:27, RangeStart:0x6c00000, RangeSize:0x400000, PieceMd5:\"d034a86c17a0bf7bc11c4a91255579db\", PieceOffset:0x6c00000, PieceStyle:0, DownloadCost:0x7854}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.006","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.87.184-27447-66351329-ab97-46b0-a570-f25f70ebe0ba\", PieceInfo:(*common.PieceInfo)(0xc000771b90), BeginTime:0x17d6650f76ef7065, EndTime:0x17d6650f7858ef2c, Success:true, Code:200, FinishedCount:19, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:20, RangeStart:0x5000000, RangeSize:0x400000, PieceMd5:\"0d276af7efa73351b6eae501b1cd43a3\", PieceOffset:0x5000000, PieceStyle:0, DownloadCost:0x14d66}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.006","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc000771ce0), BeginTime:0x17d6650f760a8a1c, EndTime:0x17d6650f78589c10, Success:true, Code:200, FinishedCount:19, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:16, RangeStart:0x4000000, RangeSize:0x400000, PieceMd5:\"88c03447b108c0d5bcdec1c5a325050f\", PieceOffset:0x4000000, PieceStyle:0, DownloadCost:0x714c}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.011","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.30.40-19280-55e0c0a5-4222-48db-8a47-49a71449464c\", PieceInfo:(*common.PieceInfo)(0xc000771dc0), BeginTime:0x17d6650f77448431, EndTime:0x17d6650f78a2db16, Success:true, Code:200, FinishedCount:21, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:23, RangeStart:0x5c00000, RangeSize:0x400000, PieceMd5:\"4c7a7320f8bfe002ae45109fb0effa28\", PieceOffset:0x5c00000, PieceStyle:0, DownloadCost:0x17638}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.068","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc000771f10), BeginTime:0x17d6650f7834faff, EndTime:0x17d6650f7c01e6a6, Success:true, Code:200, FinishedCount:22, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:22, RangeStart:0x5800000, RangeSize:0x400000, PieceMd5:\"922a3f4ccdc6a8f0e56706557ddf711b\", PieceOffset:0x5800000, PieceStyle:0, DownloadCost:0x704e}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.071","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc00070e460), BeginTime:0x17d6650f785a4e2d, EndTime:0x17d6650f7c36d6df, Success:true, Code:200, FinishedCount:23, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:4, RangeStart:0x1000000, RangeSize:0x400000, PieceMd5:\"75de8b002f98510d975eaac52636ee0e\", PieceOffset:0x1000000, PieceStyle:0, DownloadCost:0x719d}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.074","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc00070e620), BeginTime:0x17d6650f785aba3f, EndTime:0x17d6650f7c60e656, Success:true, Code:200, FinishedCount:24, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:9, RangeStart:0x2400000, RangeSize:0x400000, PieceMd5:\"8340b0b5f8a90d55c528a19e3e5acc4e\", PieceOffset:0x2400000, PieceStyle:0, DownloadCost:0x71d8}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.077","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.30.40-19280-55e0c0a5-4222-48db-8a47-49a71449464c\", PieceInfo:(*common.PieceInfo)(0xc00070e7e0), BeginTime:0x17d6650f78a44cb1, EndTime:0x17d6650f7c8ee5a4, Success:true, Code:200, FinishedCount:25, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:10, RangeStart:0x2800000, RangeSize:0x400000, PieceMd5:\"63ade6173630f10623366a87aab09461\", PieceOffset:0x2800000, PieceStyle:0, DownloadCost:0x19c62}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
  

seed peer core.log not log

helm install value.yaml 删除 injectConfigPath: true这个参数，会在/etc/containerd/config.toml文件新增以下配置

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.test.wifi.com"]
   endpoint = ["http://127.0.0.1:65001", "https://harbor.test.wifi.com"]

Dragonfly version: 2.1.0 OS: ubuntu 18.04 Kernel (e.g. uname -a): 5.4.193-0504193-generic Others: k8s 1.23.6 containerd 1.6.6