pulsar
pulsar copied to clipboard
[fix] [broker] fix deadlock when disable topic level Geo-Replication
Motivation
Fix deadlock when disabling topic level Geo-Replication, which is using the metadata store thread to call another metadata reading(clusters list).
You can reproduce the issue by OneWayReplicatorUsingGlobalZKTest.testDeleteNonPartitionedTopic
logs
2024-05-17T18:50:58,304 - ERROR - [configuration-metadata-store-147-1:PersistentTopicsBase] - [null] Failed to perform setReplicationClusters on topic persistent://public/default/tp_-8fb2b1c4-fb11-4e64-ba7f-7f4d36a0b184
org.apache.pulsar.broker.web.RestException: HTTP 500 Failed to get children of /admin/clusters
at org.apache.pulsar.broker.admin.AdminResource.clusters(AdminResource.java:460) ~[classes/:?]
at org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.lambda$internalSetReplicationClusters$315(PersistentTopicsBase.java:3272) ~[classes/:?]
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
at org.apache.pulsar.metadata.impl.ZKMetadataStore.lambda$existsFromStore$11(ZKMetadataStore.java:364) ~[classes/:?]
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.108.Final.jar:4.1.108.Final]
at java.base/java.lang.Thread.run(Thread.java:833) [?:?]
2024-05-17T18:50:58,311 - INFO - [configuration-metadata-store-147-1:RequestLog] - 127.0.0.1 - - [17/May/2024:18:50:28 +0800] "POST /admin/v2/persistent/public/default/tp_-8fb2b1c4-fb11-4e64-ba7f-7f4d36a0b184/replication HTTP/1.1" 500 1997 "-" "Pulsar-Java-v3.3.0-SNAPSHOT" 30020
2024-05-17T18:50:58,313 - WARN - [AsyncHttpClient-217-1:BaseResource] - [http://localhost:51788/admin/v2/persistent/public/default/tp_-8fb2b1c4-fb11-4e64-ba7f-7f4d36a0b184/replication] Failed to perform http post request: javax.ws.rs.InternalServerErrorException: HTTP 500 {"reason":"\n --- An unexpected error occurred in the server ---\n\nMessage: Failed to get children of /admin/clusters\n\nStacktrace:\n\norg.apache.pulsar.metadata.api.MetadataStoreException: Failed to get children of /admin/clusters\n\tat org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:78)\n\tat org.apache.pulsar.broker.resources.ClusterResources.list(ClusterResources.java:63)\n\tat org.apache.pulsar.broker.admin.AdminResource.clusters(AdminResource.java:456)\n\tat org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.lambda$internalSetReplicationClusters$315(PersistentTopicsBase.java:3272)\n\tat java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.apache.pulsar.metadata.impl.ZKMetadataStore.lambda$existsFromStore$11(ZKMetadataStore.java:364)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: java.util.concurrent.TimeoutException\n\tat java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1960)\n\tat java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2095)\n\tat org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:73)\n\t... 14 more\n"}
org.apache.pulsar.client.admin.PulsarAdminException$ServerSideErrorException:
--- An unexpected error occurred in the server ---
Message: Failed to get children of /admin/clusters
Stacktrace:
org.apache.pulsar.metadata.api.MetadataStoreException: Failed to get children of /admin/clusters
at org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:78)
at org.apache.pulsar.broker.resources.ClusterResources.list(ClusterResources.java:63)
at org.apache.pulsar.broker.admin.AdminResource.clusters(AdminResource.java:456)
at org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.lambda$internalSetReplicationClusters$315(PersistentTopicsBase.java:3272)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at org.apache.pulsar.metadata.impl.ZKMetadataStore.lambda$existsFromStore$11(ZKMetadataStore.java:364)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.util.concurrent.TimeoutException
at java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1960)
at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2095)
at org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:73)
... 14 more
at org.apache.pulsar.client.admin.PulsarAdminException.wrap(PulsarAdminException.java:252)
at org.apache.pulsar.client.admin.internal.BaseResource.sync(BaseResource.java:352)
at org.apache.pulsar.client.admin.internal.TopicsImpl.setReplicationClusters(TopicsImpl.java:2686)
at org.apache.pulsar.broker.service.OneWayReplicatorTestBase.setTopicLevelClusters(OneWayReplicatorTestBase.java:337)
at org.apache.pulsar.broker.service.OneWayReplicatorTest.testDeleteNonPartitionedTopic(OneWayReplicatorTest.java:712)
at org.apache.pulsar.broker.service.OneWayReplicatorUsingGlobalZKTest.testDeleteNonPartitionedTopic(OneWayReplicatorUsingGlobalZKTest.java:45)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at org.testng.internal.invokers.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:139)
at org.testng.internal.invokers.TestInvoker.invokeMethod(TestInvoker.java:677)
at org.testng.internal.invokers.TestInvoker.invokeTestMethod(TestInvoker.java:221)
at org.testng.internal.invokers.MethodRunner.runInSequence(MethodRunner.java:50)
at org.testng.internal.invokers.TestInvoker$MethodInvocationAgent.invoke(TestInvoker.java:969)
at org.testng.internal.invokers.TestInvoker.invokeTestMethods(TestInvoker.java:194)
at org.testng.internal.invokers.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:148)
at org.testng.internal.invokers.TestMethodWorker.run(TestMethodWorker.java:128)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
at org.testng.TestRunner.privateRun(TestRunner.java:829)
at org.testng.TestRunner.run(TestRunner.java:602)
at org.testng.SuiteRunner.runTest(SuiteRunner.java:437)
at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:431)
at org.testng.SuiteRunner.privateRun(SuiteRunner.java:391)
at org.testng.SuiteRunner.run(SuiteRunner.java:330)
at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:95)
at org.testng.TestNG.runSuitesSequentially(TestNG.java:1256)
at org.testng.TestNG.runSuitesLocally(TestNG.java:1176)
at org.testng.TestNG.runSuites(TestNG.java:1099)
at org.testng.TestNG.run(TestNG.java:1067)
at com.intellij.rt.testng.IDEARemoteTestNG.run(IDEARemoteTestNG.java:66)
at com.intellij.rt.testng.RemoteTestNGStarter.main(RemoteTestNGStarter.java:105)
Suppressed: org.apache.pulsar.client.admin.PulsarAdminException$ServerSideErrorException:
--- An unexpected error occurred in the server ---
Message: Failed to get children of /admin/clusters
Stacktrace:
org.apache.pulsar.metadata.api.MetadataStoreException: Failed to get children of /admin/clusters
at org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:78)
at org.apache.pulsar.broker.resources.ClusterResources.list(ClusterResources.java:63)
at org.apache.pulsar.broker.admin.AdminResource.clusters(AdminResource.java:456)
at org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.lambda$internalSetReplicationClusters$315(PersistentTopicsBase.java:3272)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at org.apache.pulsar.metadata.impl.ZKMetadataStore.lambda$existsFromStore$11(ZKMetadataStore.java:364)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.util.concurrent.TimeoutException
at java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1960)
at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2095)
at org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:73)
... 14 more
at org.apache.pulsar.client.admin.internal.BaseResource.getApiException(BaseResource.java:272)
at org.apache.pulsar.client.admin.internal.BaseResource$2.failed(BaseResource.java:168)
at org.glassfish.jersey.client.JerseyInvocation$1.failed(JerseyInvocation.java:882)
at org.glassfish.jersey.client.JerseyInvocation$1.completed(JerseyInvocation.java:863)
at org.glassfish.jersey.client.ClientRuntime.processResponse(ClientRuntime.java:232)
at org.glassfish.jersey.client.ClientRuntime.access$200(ClientRuntime.java:62)
at org.glassfish.jersey.client.ClientRuntime$2.lambda$response$0(ClientRuntime.java:176)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244)
at org.glassfish.jersey.internal.Errors.process(Errors.java:292)
at org.glassfish.jersey.internal.Errors.process(Errors.java:274)
at org.glassfish.jersey.internal.Errors.process(Errors.java:244)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:288)
at org.glassfish.jersey.client.ClientRuntime$2.response(ClientRuntime.java:176)
at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector.lambda$apply$1(AsyncHttpConnector.java:258)
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector.lambda$retryOperation$4(AsyncHttpConnector.java:300)
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at org.asynchttpclient.netty.NettyResponseFuture.loadContent(NettyResponseFuture.java:222)
at org.asynchttpclient.netty.NettyResponseFuture.done(NettyResponseFuture.java:257)
at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.finishUpdate(AsyncHttpClientHandler.java:241)
at org.asynchttpclient.netty.handler.HttpHandler.handleChunk(HttpHandler.java:114)
at org.asynchttpclient.netty.handler.HttpHandler.handleRead(HttpHandler.java:143)
at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.channelRead(AsyncHttpClientHandler.java:78)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: javax.ws.rs.InternalServerErrorException: HTTP 500 {"reason":"\n --- An unexpected error occurred in the server ---\n\nMessage: Failed to get children of /admin/clusters\n\nStacktrace:\n\norg.apache.pulsar.metadata.api.MetadataStoreException: Failed to get children of /admin/clusters\n\tat org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:78)\n\tat org.apache.pulsar.broker.resources.ClusterResources.list(ClusterResources.java:63)\n\tat org.apache.pulsar.broker.admin.AdminResource.clusters(AdminResource.java:456)\n\tat org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.lambda$internalSetReplicationClusters$315(PersistentTopicsBase.java:3272)\n\tat java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.apache.pulsar.metadata.impl.ZKMetadataStore.lambda$existsFromStore$11(ZKMetadataStore.java:364)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: java.util.concurrent.TimeoutException\n\tat java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1960)\n\tat java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2095)\n\tat org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:73)\n\t... 14 more\n"}
at org.glassfish.jersey.client.JerseyInvocation.convertToException(JerseyInvocation.java:960)
at org.glassfish.jersey.client.JerseyInvocation.access$700(JerseyInvocation.java:82)
... 54 more
Caused by: [CIRCULAR REFERENCE: javax.ws.rs.InternalServerErrorException: HTTP 500 {"reason":"\n --- An unexpected error occurred in the server ---\n\nMessage: Failed to get children of /admin/clusters\n\nStacktrace:\n\norg.apache.pulsar.metadata.api.MetadataStoreException: Failed to get children of /admin/clusters\n\tat org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:78)\n\tat org.apache.pulsar.broker.resources.ClusterResources.list(ClusterResources.java:63)\n\tat org.apache.pulsar.broker.admin.AdminResource.clusters(AdminResource.java:456)\n\tat org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.lambda$internalSetReplicationClusters$315(PersistentTopicsBase.java:3272)\n\tat java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)\n\tat org.apache.pulsar.metadata.impl.ZKMetadataStore.lambda$existsFromStore$11(ZKMetadataStore.java:364)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: java.util.concurrent.TimeoutException\n\tat java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1960)\n\tat java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2095)\n\tat org.apache.pulsar.broker.resources.BaseResources.getChildren(BaseResources.java:73)\n\t... 14 more\n"}]
Modifications
Change the sync method clusters() to async method clustersAsync() when calling disable topic level Geo-Replication
Documentation
- [ ]
doc - [ ]
doc-required - [x]
doc-not-needed - [ ]
doc-complete
Matching PR in forked repository
PR in forked repository: x