BQ live sink: com.google.common.util.concurrent.UncheckedExecutionException: com.google.cloud.bigquery.BigQueryException: Remote host closed connection during handshake

[whd]

Stack trace:

{
    "attributeMap": {
        "args": "",
        "client_id": "n/a",
        "content_length": "1143",
        "document_id": "264ae3b7-5746-42c0-b4c7-55dea61f63f3",
        "document_namespace": "activity-stream",
        "document_type": "impression-stats",
        "document_version": "1",
        "error_message": "com.google.common.util.concurrent.UncheckedExecutionException: com.google.cloud.bigquery.BigQueryException: Remote host closed connection during handshake",
        "error_type": "KeyByBigQueryTableDestination",
        "exception_class": "com.google.common.util.concurrent.UncheckedExecutionException",
        "geo_city": "Federal Way",
        "geo_country": "US",
        "geo_db_version": "2019-09-30T16:25:20Z",
        "geo_subdivision1": "WA",
        "host": "upstream_docker",
        "method": "POST",
        "normalized_country_code": "US",
        "protocol": "HTTP/1.0",
        "proxy_timestamp": "2019-10-03T07:39:23.578127Z",
        "sample_id": "51",
        "stack_trace": "[com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2051), com.google.common.cache.LocalCache.get(LocalCache.java:3953), com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4873), com.mozilla.telemetry.transforms.KeyByBigQueryTableDestination.getTableDestination(KeyByBigQueryTableDestination.java:92), com.mozilla.telemetry.transforms.KeyByBigQueryTableDestination.processElement(KeyByBigQueryTableDestination.java:122), com.mozilla.telemetry.transforms.KeyByBigQueryTableDestination.processElement(KeyByBigQueryTableDestination.java:35), com.mozilla.telemetry.transforms.MapElementsWithErrors$DoFnWithErrors.processElementOrError(MapElementsWithErrors.java:85), com.mozilla.telemetry.transforms.MapElementsWithErrors$DoFnWithErrors$DoFnInvoker.invokeProcessElement(Unknown Source), org.apache.beam.runners.dataflow.worker.repackaged.org.apache.beam.runners.core.SimpleDoFnRunner.invokeProcessElement(SimpleDoFnRunner.java:214), org.apache.beam.runners.dataflow.worker.repackaged.org.apac...",
        "stack_trace_cause_1": "[com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.translate(HttpBigQueryRpc.java:100), com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.getDataset(HttpBigQueryRpc.java:112), com.google.cloud.bigquery.BigQueryImpl$5.call(BigQueryImpl.java:289), com.google.cloud.bigquery.BigQueryImpl$5.call(BigQueryImpl.java:286), com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105), com.google.cloud.RetryHelper.run(RetryHelper.java:76), com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50), com.google.cloud.bigquery.BigQueryImpl.getDataset(BigQueryImpl.java:285), com.google.cloud.bigquery.BigQueryImpl.getDataset(BigQueryImpl.java:276), com.mozilla.telemetry.transforms.KeyByBigQueryTableDestination.lambda$getTableDestination$2(KeyByBigQueryTableDestination.java:94), com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4878), com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3529), com.google.common.cache.LocalCache$Segmen...",
        "stack_trace_cause_2": "[sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1002), sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385), sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413), sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397), sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:559), sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:185), sun.net.www.protocol.https.HttpsURLConnectionImpl.connect(HttpsURLConnectionImpl.java:153), com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:142), com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:84), com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1011), com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:499), com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnpa...",
        "stack_trace_cause_3": "[sun.security.ssl.InputRecord.read(InputRecord.java:505), sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983), sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385), sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413), sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397), sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:559), sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:185), sun.net.www.protocol.https.HttpsURLConnectionImpl.connect(HttpsURLConnectionImpl.java:153), com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:142), com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:84), com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1011), com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:499), com.google.api.client.goo...",
        "submission_timestamp": "2019-10-03T07:39:23.010083Z",
        "uri": "/submit/activity-stream/impression-stats/1/264ae3b7-5746-42c0-b4c7-55dea61f63f3",
        "user_agent_browser": "Firefox",
        "user_agent_os": "Windows 10",
        "user_agent_version": "69"
    },
    "payload": "at gs://moz-fx-data-prod-data/structured-decoded_bq-sink/error/2019-10-03/07/error-2019-10-03T07-30-00.000Z-2019-10-03T07-40-00.000Z-0-00058-of-00060.ndjson.gz"
}

SQL:

select * from `moz-fx-data-shared-prod.payload_bytes_decoded`.structured_activity_stream__impression_stats_v1 where document_id = '264ae3b7-5746-42c0-b4c7-55dea61f63f3' and date(submission_timestamp) = '2019-10-03'
1
select count(*) from `moz-fx-data-shared-prod.activity_stream_live`.impression_stats_v1 where document_id = '264ae3b7-5746-42c0-b4c7-55dea61f63f3' and date(submission_timestamp) = '2019-10-03'
0

This is a very rare case apparently but I haven't trolled the bq sink logs in a while. It's perhaps not worth investigating too much with https://github.com/mozilla/gcp-ingestion/pull/681 on the horizon, but we may want to reprocess these data at some point in the future.