jib
jib copied to clipboard
Published
20 hours ago •
GoogleContainerTools
GoogleContainerTools
[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content","detail":"digest mismatch"}] while uploading image
Environment:
- Jib version: 3.3.1
- Build tool: Gradle 8.1
- OS: Ubuntu 22.04.2 LTS
Description of the issue: Sometimes upload to Sonatype Nexus Repository ManagerOSS 3.49.0-02 fails with the following error:
Caused by: com.google.api.client.http.HttpResponseException: 400 Bad Request
PUT http://docker-registry.mydomain.com:5000/v2/myapplication/my-service/blobs/uploads/faa17442-17fb-421e-8501-0adee7348cd3?digest=sha256:bcc3effc72b1ba2f2c6d3ebb19855a1d4eb1d23172d9b95dc01619bc50c7ff16
{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content","detail":"digest mismatch"}]}
at com.google.api.client.http.HttpResponseException$Builder.build(HttpResponseException.java:293)
at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1118)
at com.google.cloud.tools.jib.http.FailoverHttpClient.call(FailoverHttpClient.java:349)
Expected behavior: Upload to the repository will always succeed
Steps to reproduce:
- Run
jibGradle task - Upload will fail randomly
jib-gradle-plugin Configuration:
jib {
from {
image = "$docker_registry_target/openjdk-17:version_02"
}
to {
image = "${docker_registry_target}${(project.projectDir.toString() - project.rootDir.toString())}${tag}"
auth {
username = docker_registry_username
password = docker_registry_password
}
}
container {
entrypoint = 'INHERIT'
ports = ['8080']
}
}
Log output:
org.gradle.api.tasks.TaskExecutionException: Execution failed for task ':my-service:jib'.
at org.gradle.api.internal.tasks.execution.ExecuteActionsTaskExecuter.lambda$executeIfValid$1(ExecuteActionsTaskExecuter.java:149)
at org.gradle.internal.Try$Failure.ifSuccessfulOrElse(Try.java:282)
at org.gradle.api.internal.tasks.execution.ExecuteActionsTaskExecuter.executeIfValid(ExecuteActionsTaskExecuter.java:147)
at org.gradle.api.internal.tasks.execution.ExecuteActionsTaskExecuter.execute(ExecuteActionsTaskExecuter.java:135)
at org.gradle.api.internal.tasks.execution.FinalizePropertiesTaskExecuter.execute(FinalizePropertiesTaskExecuter.java:46)
at org.gradle.api.internal.tasks.execution.ResolveTaskExecutionModeExecuter.execute(ResolveTaskExecutionModeExecuter.java:51)
at org.gradle.api.internal.tasks.execution.SkipTaskWithNoActionsExecuter.execute(SkipTaskWithNoActionsExecuter.java:57)
at org.gradle.api.internal.tasks.execution.SkipOnlyIfTaskExecuter.execute(SkipOnlyIfTaskExecuter.java:74)
at org.gradle.api.internal.tasks.execution.CatchExceptionTaskExecuter.execute(CatchExceptionTaskExecuter.java:36)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter$1.executeTask(EventFiringTaskExecuter.java:77)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter$1.call(EventFiringTaskExecuter.java:55)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter$1.call(EventFiringTaskExecuter.java:52)
at org.gradle.internal.operations.DefaultBuildOperationRunner$CallableBuildOperationWorker.execute(DefaultBuildOperationRunner.java:204)
at org.gradle.internal.operations.DefaultBuildOperationRunner$CallableBuildOperationWorker.execute(DefaultBuildOperationRunner.java:199)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:66)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:157)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.call(DefaultBuildOperationRunner.java:53)
at org.gradle.internal.operations.DefaultBuildOperationExecutor.call(DefaultBuildOperationExecutor.java:73)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter.execute(EventFiringTaskExecuter.java:52)
at org.gradle.execution.plan.LocalTaskNodeExecutor.execute(LocalTaskNodeExecutor.java:42)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$InvokeNodeExecutorsAction.execute(DefaultTaskExecutionGraph.java:337)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$InvokeNodeExecutorsAction.execute(DefaultTaskExecutionGraph.java:324)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$BuildOperationAwareExecutionAction.execute(DefaultTaskExecutionGraph.java:317)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$BuildOperationAwareExecutionAction.execute(DefaultTaskExecutionGraph.java:303)
at org.gradle.execution.plan.DefaultPlanExecutor$ExecutorWorker.execute(DefaultPlanExecutor.java:463)
at org.gradle.execution.plan.DefaultPlanExecutor$ExecutorWorker.run(DefaultPlanExecutor.java:380)
at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:49)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: org.gradle.internal.UncheckedException: com.google.cloud.tools.jib.plugins.common.BuildStepsExecutionException: Tried to push BLOB for docker-registry.mydomain.com:5000/myapplication/my-service with digest sha256:bcc3effc72b1ba2f2c6d3ebb19855a1d4eb1d23172d9b95dc01619bc50c7ff16 but failed because: other: provided digest did not match uploaded content
at org.gradle.internal.UncheckedException.throwAsUncheckedException(UncheckedException.java:68)
at org.gradle.internal.UncheckedException.throwAsUncheckedException(UncheckedException.java:41)
at org.gradle.internal.reflect.JavaMethod.invoke(JavaMethod.java:128)
at org.gradle.api.internal.project.taskfactory.StandardTaskAction.doExecute(StandardTaskAction.java:58)
at org.gradle.api.internal.project.taskfactory.StandardTaskAction.execute(StandardTaskAction.java:51)
at org.gradle.api.internal.project.taskfactory.StandardTaskAction.execute(StandardTaskAction.java:29)
at org.gradle.api.internal.tasks.execution.TaskExecution$3.run(TaskExecution.java:242)
at org.gradle.internal.operations.DefaultBuildOperationRunner$1.execute(DefaultBuildOperationRunner.java:29)
at org.gradle.internal.operations.DefaultBuildOperationRunner$1.execute(DefaultBuildOperationRunner.java:26)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:66)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:157)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.run(DefaultBuildOperationRunner.java:47)
at org.gradle.internal.operations.DefaultBuildOperationExecutor.run(DefaultBuildOperationExecutor.java:68)
at org.gradle.api.internal.tasks.execution.TaskExecution.executeAction(TaskExecution.java:227)
at org.gradle.api.internal.tasks.execution.TaskExecution.executeActions(TaskExecution.java:210)
at org.gradle.api.internal.tasks.execution.TaskExecution.executeWithPreviousOutputFiles(TaskExecution.java:193)
at org.gradle.api.internal.tasks.execution.TaskExecution.execute(TaskExecution.java:166)
at org.gradle.internal.execution.steps.ExecuteStep.executeInternal(ExecuteStep.java:105)
at org.gradle.internal.execution.steps.ExecuteStep.access$000(ExecuteStep.java:44)
at org.gradle.internal.execution.steps.ExecuteStep$1.call(ExecuteStep.java:59)
at org.gradle.internal.execution.steps.ExecuteStep$1.call(ExecuteStep.java:56)
at org.gradle.internal.operations.DefaultBuildOperationRunner$CallableBuildOperationWorker.execute(DefaultBuildOperationRunner.java:204)
at org.gradle.internal.operations.DefaultBuildOperationRunner$CallableBuildOperationWorker.execute(DefaultBuildOperationRunner.java:199)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:66)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:157)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.call(DefaultBuildOperationRunner.java:53)
at org.gradle.internal.operations.DefaultBuildOperationExecutor.call(DefaultBuildOperationExecutor.java:73)
at org.gradle.internal.execution.steps.ExecuteStep.execute(ExecuteStep.java:56)
at org.gradle.internal.execution.steps.ExecuteStep.execute(ExecuteStep.java:44)
at org.gradle.internal.execution.steps.RemovePreviousOutputsStep.execute(RemovePreviousOutputsStep.java:67)
at org.gradle.internal.execution.steps.RemovePreviousOutputsStep.execute(RemovePreviousOutputsStep.java:37)
at org.gradle.internal.execution.steps.CancelExecutionStep.execute(CancelExecutionStep.java:41)
at org.gradle.internal.execution.steps.TimeoutStep.executeWithoutTimeout(TimeoutStep.java:74)
at org.gradle.internal.execution.steps.TimeoutStep.execute(TimeoutStep.java:55)
at org.gradle.internal.execution.steps.CreateOutputsStep.execute(CreateOutputsStep.java:50)
at org.gradle.internal.execution.steps.CreateOutputsStep.execute(CreateOutputsStep.java:28)
at org.gradle.internal.execution.steps.CaptureStateAfterExecutionStep.executeDelegateBroadcastingChanges(CaptureStateAfterExecutionStep.java:100)
at org.gradle.internal.execution.steps.CaptureStateAfterExecutionStep.execute(CaptureStateAfterExecutionStep.java:72)
at org.gradle.internal.execution.steps.CaptureStateAfterExecutionStep.execute(CaptureStateAfterExecutionStep.java:50)
at org.gradle.internal.execution.steps.ResolveInputChangesStep.execute(ResolveInputChangesStep.java:40)
at org.gradle.internal.execution.steps.ResolveInputChangesStep.execute(ResolveInputChangesStep.java:29)
at org.gradle.internal.execution.steps.BuildCacheStep.executeWithoutCache(BuildCacheStep.java:166)
at org.gradle.internal.execution.steps.BuildCacheStep.lambda$execute$1(BuildCacheStep.java:70)
at org.gradle.internal.Either$Right.fold(Either.java:175)
at org.gradle.internal.execution.caching.CachingState.fold(CachingState.java:59)
at org.gradle.internal.execution.steps.BuildCacheStep.execute(BuildCacheStep.java:68)
at org.gradle.internal.execution.steps.BuildCacheStep.execute(BuildCacheStep.java:46)
at org.gradle.internal.execution.steps.StoreExecutionStateStep.execute(StoreExecutionStateStep.java:36)
at org.gradle.internal.execution.steps.StoreExecutionStateStep.execute(StoreExecutionStateStep.java:25)
at org.gradle.internal.execution.steps.RecordOutputsStep.execute(RecordOutputsStep.java:36)
at org.gradle.internal.execution.steps.RecordOutputsStep.execute(RecordOutputsStep.java:22)
at org.gradle.internal.execution.steps.SkipUpToDateStep.executeBecause(SkipUpToDateStep.java:91)
at org.gradle.internal.execution.steps.SkipUpToDateStep.lambda$execute$2(SkipUpToDateStep.java:55)
at java.base/java.util.Optional.orElseGet(Optional.java:364)
at org.gradle.internal.execution.steps.SkipUpToDateStep.execute(SkipUpToDateStep.java:55)
at org.gradle.internal.execution.steps.SkipUpToDateStep.execute(SkipUpToDateStep.java:37)
at org.gradle.internal.execution.steps.ResolveChangesStep.execute(ResolveChangesStep.java:65)
at org.gradle.internal.execution.steps.ResolveChangesStep.execute(ResolveChangesStep.java:36)
at org.gradle.internal.execution.steps.legacy.MarkSnapshottingInputsFinishedStep.execute(MarkSnapshottingInputsFinishedStep.java:37)
at org.gradle.internal.execution.steps.legacy.MarkSnapshottingInputsFinishedStep.execute(MarkSnapshottingInputsFinishedStep.java:27)
at org.gradle.internal.execution.steps.ResolveCachingStateStep.execute(ResolveCachingStateStep.java:76)
at org.gradle.internal.execution.steps.ResolveCachingStateStep.execute(ResolveCachingStateStep.java:37)
at org.gradle.internal.execution.steps.ValidateStep.execute(ValidateStep.java:94)
at org.gradle.internal.execution.steps.ValidateStep.execute(ValidateStep.java:49)
at org.gradle.internal.execution.steps.CaptureStateBeforeExecutionStep.execute(CaptureStateBeforeExecutionStep.java:71)
at org.gradle.internal.execution.steps.CaptureStateBeforeExecutionStep.execute(CaptureStateBeforeExecutionStep.java:45)
at org.gradle.internal.execution.steps.SkipEmptyWorkStep.executeWithNonEmptySources(SkipEmptyWorkStep.java:177)
at org.gradle.internal.execution.steps.SkipEmptyWorkStep.execute(SkipEmptyWorkStep.java:81)
at org.gradle.internal.execution.steps.SkipEmptyWorkStep.execute(SkipEmptyWorkStep.java:53)
at org.gradle.internal.execution.steps.RemoveUntrackedExecutionStateStep.execute(RemoveUntrackedExecutionStateStep.java:32)
at org.gradle.internal.execution.steps.RemoveUntrackedExecutionStateStep.execute(RemoveUntrackedExecutionStateStep.java:21)
at org.gradle.internal.execution.steps.legacy.MarkSnapshottingInputsStartedStep.execute(MarkSnapshottingInputsStartedStep.java:38)
at org.gradle.internal.execution.steps.LoadPreviousExecutionStateStep.execute(LoadPreviousExecutionStateStep.java:36)
at org.gradle.internal.execution.steps.LoadPreviousExecutionStateStep.execute(LoadPreviousExecutionStateStep.java:23)
at org.gradle.internal.execution.steps.CleanupStaleOutputsStep.execute(CleanupStaleOutputsStep.java:75)
at org.gradle.internal.execution.steps.CleanupStaleOutputsStep.execute(CleanupStaleOutputsStep.java:41)
at org.gradle.internal.execution.steps.AssignWorkspaceStep.lambda$execute$0(AssignWorkspaceStep.java:32)
at org.gradle.api.internal.tasks.execution.TaskExecution$4.withWorkspace(TaskExecution.java:287)
at org.gradle.internal.execution.steps.AssignWorkspaceStep.execute(AssignWorkspaceStep.java:30)
at org.gradle.internal.execution.steps.AssignWorkspaceStep.execute(AssignWorkspaceStep.java:21)
at org.gradle.internal.execution.steps.IdentityCacheStep.execute(IdentityCacheStep.java:37)
at org.gradle.internal.execution.steps.IdentityCacheStep.execute(IdentityCacheStep.java:27)
at org.gradle.internal.execution.steps.IdentifyStep.execute(IdentifyStep.java:47)
at org.gradle.internal.execution.steps.IdentifyStep.execute(IdentifyStep.java:34)
at org.gradle.internal.execution.impl.DefaultExecutionEngine$1.execute(DefaultExecutionEngine.java:64)
at org.gradle.api.internal.tasks.execution.ExecuteActionsTaskExecuter.executeIfValid(ExecuteActionsTaskExecuter.java:146)
at org.gradle.api.internal.tasks.execution.ExecuteActionsTaskExecuter.execute(ExecuteActionsTaskExecuter.java:135)
at org.gradle.api.internal.tasks.execution.FinalizePropertiesTaskExecuter.execute(FinalizePropertiesTaskExecuter.java:46)
at org.gradle.api.internal.tasks.execution.ResolveTaskExecutionModeExecuter.execute(ResolveTaskExecutionModeExecuter.java:51)
at org.gradle.api.internal.tasks.execution.SkipTaskWithNoActionsExecuter.execute(SkipTaskWithNoActionsExecuter.java:57)
at org.gradle.api.internal.tasks.execution.SkipOnlyIfTaskExecuter.execute(SkipOnlyIfTaskExecuter.java:74)
at org.gradle.api.internal.tasks.execution.CatchExceptionTaskExecuter.execute(CatchExceptionTaskExecuter.java:36)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter$1.executeTask(EventFiringTaskExecuter.java:77)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter$1.call(EventFiringTaskExecuter.java:55)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter$1.call(EventFiringTaskExecuter.java:52)
at org.gradle.internal.operations.DefaultBuildOperationRunner$CallableBuildOperationWorker.execute(DefaultBuildOperationRunner.java:204)
at org.gradle.internal.operations.DefaultBuildOperationRunner$CallableBuildOperationWorker.execute(DefaultBuildOperationRunner.java:199)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:66)
at org.gradle.internal.operations.DefaultBuildOperationRunner$2.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:157)
at org.gradle.internal.operations.DefaultBuildOperationRunner.execute(DefaultBuildOperationRunner.java:59)
at org.gradle.internal.operations.DefaultBuildOperationRunner.call(DefaultBuildOperationRunner.java:53)
at org.gradle.internal.operations.DefaultBuildOperationExecutor.call(DefaultBuildOperationExecutor.java:73)
at org.gradle.api.internal.tasks.execution.EventFiringTaskExecuter.execute(EventFiringTaskExecuter.java:52)
at org.gradle.execution.plan.LocalTaskNodeExecutor.execute(LocalTaskNodeExecutor.java:42)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$InvokeNodeExecutorsAction.execute(DefaultTaskExecutionGraph.java:337)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$InvokeNodeExecutorsAction.execute(DefaultTaskExecutionGraph.java:324)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$BuildOperationAwareExecutionAction.execute(DefaultTaskExecutionGraph.java:317)
at org.gradle.execution.taskgraph.DefaultTaskExecutionGraph$BuildOperationAwareExecutionAction.execute(DefaultTaskExecutionGraph.java:303)
at org.gradle.execution.plan.DefaultPlanExecutor$ExecutorWorker.execute(DefaultPlanExecutor.java:463)
at org.gradle.execution.plan.DefaultPlanExecutor$ExecutorWorker.run(DefaultPlanExecutor.java:380)
at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:49)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: com.google.cloud.tools.jib.plugins.common.BuildStepsExecutionException: Tried to push BLOB for docker-registry.mydomain.com:5000/myapplication/my-service with digest sha256:bcc3effc72b1ba2f2c6d3ebb19855a1d4eb1d23172d9b95dc01619bc50c7ff16 but failed because: other: provided digest did not match uploaded content
at com.google.cloud.tools.jib.plugins.common.JibBuildRunner.runBuild(JibBuildRunner.java:280)
at com.google.cloud.tools.jib.gradle.BuildImageTask.buildImage(BuildImageTask.java:122)
at jdk.internal.reflect.GeneratedMethodAccessor881.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at org.gradle.internal.reflect.JavaMethod.invoke(JavaMethod.java:125)
... 117 more
Caused by: com.google.cloud.tools.jib.registry.RegistryErrorException: Tried to push BLOB for docker-registry.mydomain.com:5000/myapplication/my-service with digest sha256:bcc3effc72b1ba2f2c6d3ebb19855a1d4eb1d23172d9b95dc01619bc50c7ff16 but failed because: other: provided digest did not match uploaded content
at com.google.cloud.tools.jib.registry.RegistryErrorExceptionBuilder.build(RegistryErrorExceptionBuilder.java:101)
at com.google.cloud.tools.jib.registry.RegistryEndpointCaller.newRegistryErrorException(RegistryEndpointCaller.java:212)
at com.google.cloud.tools.jib.registry.RegistryEndpointCaller.call(RegistryEndpointCaller.java:153)
at com.google.cloud.tools.jib.registry.RegistryEndpointCaller.call(RegistryEndpointCaller.java:114)
at com.google.cloud.tools.jib.registry.RegistryClient.callRegistryEndpoint(RegistryClient.java:623)
at com.google.cloud.tools.jib.registry.RegistryClient.pushBlob(RegistryClient.java:564)
at com.google.cloud.tools.jib.builder.steps.PushBlobStep.call(PushBlobStep.java:89)
at com.google.cloud.tools.jib.builder.steps.PushContainerConfigurationStep.call(PushContainerConfigurationStep.java:71)
at com.google.cloud.tools.jib.builder.steps.StepsRunner.lambda$pushContainerConfiguration$12(StepsRunner.java:500)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:74)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
... 3 more
Caused by: com.google.cloud.tools.jib.http.ResponseException: 400 Bad Request
PUT http://docker-registry.mydomain.com:5000/v2/myapplication/my-service/blobs/uploads/faa17442-17fb-421e-8501-0adee7348cd3?digest=sha256:bcc3effc72b1ba2f2c6d3ebb19855a1d4eb1d23172d9b95dc01619bc50c7ff16
{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content","detail":"digest mismatch"}]}
at com.google.cloud.tools.jib.http.FailoverHttpClient.call(FailoverHttpClient.java:355)
at com.google.cloud.tools.jib.http.FailoverHttpClient.call(FailoverHttpClient.java:255)
at com.google.cloud.tools.jib.registry.RegistryEndpointCaller.call(RegistryEndpointCaller.java:138)
... 12 more
Caused by: com.google.api.client.http.HttpResponseException: 400 Bad Request
PUT http://docker-registry.mydomain.com:5000/v2/myapplication/my-service/blobs/uploads/faa17442-17fb-421e-8501-0adee7348cd3?digest=sha256:bcc3effc72b1ba2f2c6d3ebb19855a1d4eb1d23172d9b95dc01619bc50c7ff16
{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content","detail":"digest mismatch"}]}
at com.google.api.client.http.HttpResponseException$Builder.build(HttpResponseException.java:293)
at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1118)
at com.google.cloud.tools.jib.http.FailoverHttpClient.call(FailoverHttpClient.java:349)
... 14 more
Additional Information:
Hi @sergeykad, could you provide more information on this issue? For example, the network traffic log is helpful for debugging such an issue. Please refer here for how to examine network traffic.
Also, given a previous issue, this maybe a concurrent bug in the registry, you could try -Djib.serialize=true to disable concurrency (comment).
I added the -Djava.util.logging.config.file=path/to/logging.properties -Djib.serialize=true but I am not sure if I see any change at all. There is no additional logging, but it is possible that it solved the original issue. I will monitor the build to see if it happens again.
On the other hand, after I added the flags, I saw the upload get stuck for about an hour until it was killed due to timeout. Edit: the build of the same image got stuck again at the same 91.7%. It uses the same Jib configuration as many other images we have.
Executing tasks:
[=============== ] 51.4% complete
> launching layer pushers
> pushing blob sha256:52cdd9f8700d9cb489a9528af...
Executing tasks:
[=============== ] 51.7% complete
> launching layer pushers
> pushing blob sha256:52cdd9f8700d9cb489a9528af...
Executing tasks:
[================ ] 51.7% complete
> launching layer pushers
> launching layer pushers
Executing tasks:
[================ ] 51.7% complete
> launching layer pushers
> pushing blob sha256:4e15725c22da6eed9e81a957b...
Executing tasks:
[================ ] 53.3% complete
> launching layer pushers
> launching layer pushers
Executing tasks:
[================ ] 53.3% complete
> launching layer pushers
> pushing blob sha256:416d5add0de096256e8ab9570...
Executing tasks:
[================= ] 55.0% complete
> launching layer pushers
> launching layer pushers
Executing tasks:
[================= ] 55.0% complete
> launching layer pushers
> pushing blob sha256:5508d90493dc104092cc77bf4...
Executing tasks:
[================= ] 56.7% complete
> launching layer pushers
> launching layer pushers
Executing tasks:
[================= ] 56.7% complete
> launching layer pushers
> pushing blob sha256:0b2303eef1b4f0ae437e1effc...
Executing tasks:
[================== ] 58.3% complete
> launching layer pushers
Executing tasks:
[================== ] 58.3% complete
> launching layer pushers
> scheduling pushing container configurations
Executing tasks:
[================== ] 58.3% complete
> launching layer pushers
> pushing container configuration
Executing tasks:
[================== ] 58.3% complete
> launching layer pushers
> pushing blob sha256:0ff59cbfde2a1789cb63d2bf1...
Executing tasks:
[==================== ] 66.7% complete
> launching layer pushers
Executing tasks:
[==================== ] 66.7% complete
> launching layer pushers
> checking existence of manifest for sha256:088...
Executing tasks:
[======================= ] 75.0% complete
> launching layer pushers
Executing tasks:
[======================= ] 75.0% complete
> launching layer pushers
> scheduling pushing manifests
Executing tasks:
[======================= ] 75.0% complete
> launching layer pushers
> launching manifest pushers
Executing tasks:
[======================= ] 75.0% complete
> launching layer pushers
> pushing manifest for MR-7982
Executing tasks:
[========================= ] 83.3% complete
> launching layer pushers
Executing tasks:
[========================= ] 83.3% complete
> launching layer pushers
> launching manifest list pushers
Executing tasks:
[============================ ] 91.7% complete
> launching layer pushers
Executing tasks:
[============================ ] 91.7% complete
> launching layer pushers
Built and pushed image as docker-registry.domain.com:5000/services/export/upload:MR-7982
Executing tasks:
[============================ ] 91.7% complete
> launching layer pushers
Cancelling nested steps due to timeout
Sending interrupt signal to process
@sergeykad, We may need to gain a bit more context on why exactly the digest isn't being accepted by the registry. There have been cases in the past (#2372, #1986) where this was caused by unexpected behavior of the registry. To help investigate this further, here are few suggestions:
- As a step to unblock, please try running
-Djib.serializewithout the HTTP traffic collection first. To overcome the timeout issue, you could try increasing the timeout value with-Djib.httpTimeout=... - Is this happening intermittently? If yes, as @JoeWang1127 mentioned, please verify if there are any differences in HTTP traffic of the successful and unsuccessful runs. Please follow these instructions without the
-Djib.serialize=true. If configured correctly, they should look like this:
Mar 31, 2020 9:55:52 AM com.google.api.client.http.HttpResponse <init>
CONFIG: -------------- RESPONSE --------------
HTTP/1.1 202 Accepted
Content-Length: 0
Docker-Distribution-Api-Version: registry/2.0
Docker-Upload-Uuid: 6292f0d7-93cb-4a8e-8336-78a1bf7febd2
Location: https://registry-1.docker.io/v2/...
Range: 0-657292
Date: Tue, 31 Mar 2020 13:55:52 GMT
Strict-Transport-Security: max-age=31536000
- Once you've attained the logs, this example comparative analysis is a great reference to verify the layer SHAs being pushed by jib and understand how the registry responds in each run.
I double-checked my configuration and even enabled Gradle debug log, but I still do not see these HTTP log messages. Where should they be printed to?
With the Jib logging enabled it seems that the issues I am experiencing correlate with timeout errors. I did not find the root cause yet, since everything else seems normal. If I retry failed jib execution it passes on the second attempt.
PUT http://docker-registry.server.com:5000/v2/best-services/authentication/blobs/uploads/a4837341-06ae-47db-aace-9c5630bb67b5?digest=sha256:de27c5eb92e0d4f02f5e69ee68d3cdbfb0d9e4514fcf4d88bfcc952d9718e460 failed and will be retried
exception thrown while executing request
java.net.SocketTimeoutException: Read timed out
at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at com.google.api.client.http.apache.v2.ApacheHttpRequest.execute(ApacheHttpRequest.java:73)
at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1012)
at com.google.cloud.tools.jib.http.FailoverHttpClient.call(FailoverHttpClient.java:349)
at com.google.cloud.tools.jib.http.FailoverHttpClient.call(FailoverHttpClient.java:255)
at com.google.cloud.tools.jib.registry.RegistryEndpointCaller.call(RegistryEndpointCaller.java:138)
at com.google.cloud.tools.jib.registry.RegistryEndpointCaller.call(RegistryEndpointCaller.java:114)
at com.google.cloud.tools.jib.registry.RegistryClient.callRegistryEndpoint(RegistryClient.java:623)
at com.google.cloud.tools.jib.registry.RegistryClient.pushBlob(RegistryClient.java:564)
at com.google.cloud.tools.jib.builder.steps.PushBlobStep.call(PushBlobStep.java:89)
at com.google.cloud.tools.jib.builder.steps.PushLayerStep.call(PushLayerStep.java:92)
at com.google.cloud.tools.jib.builder.steps.PushLayerStep.call(PushLayerStep.java:33)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:74)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)