Clock skew prevents logs from being uploaded

[smayberry]

The following error was the last log uploaded from the affected container. time="2016-05-04T00:56:33Z" level=error msg="Log upload failed - length: 2, error: InvalidSignatureException: Signature expired: 20160504T004903Z is now earlier than 20160504T005133Z (20160504T005633Z - 5 min.)\n\tstatus code: 400, request id: 0b7795b1-1193-11e6-a4e2-07618ccc7f7f"

I verified that the clock was correct in the ECS instance. Here are all the 'docker inspect' details:

[
  {
    "Id": "ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25",
    "Created": "2016-04-29T21:50:03.783665928Z",
    "Path": "/bin/logspout",
    "Args": [
      "cloudwatch://janus-ecs-instances-dev-LogGroup-1GG41D36E77TN"
    ],
    "State": {
      "Status": "running",
      "Running": true,
      "Paused": false,
      "Restarting": false,
      "OOMKilled": false,
      "Dead": false,
      "Pid": 3011,
      "ExitCode": 0,
      "Error": "",
      "StartedAt": "2016-04-29T21:50:05.006200631Z",
      "FinishedAt": "0001-01-01T00:00:00Z"
    },
    "Image": "8f76d798c2f7743f2b20da57dbbbe5b9fa2b607386e63f21b5ebcb7d42583361",
    "ResolvConfPath": "/var/lib/docker/containers/ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25/resolv.conf",
    "HostnamePath": "/var/lib/docker/containers/ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25/hostname",
    "HostsPath": "/var/lib/docker/containers/ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25/hosts",
    "LogPath": "/var/lib/docker/containers/ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25/ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25-json.log",
    "Name": "/logspout",
    "RestartCount": 0,
    "Driver": "devicemapper",
    "ExecDriver": "native-0.2",
    "MountLabel": "",
    "ProcessLabel": "",
    "AppArmorProfile": "",
    "ExecIDs": null,
    "HostConfig": {
      "Binds": [
        "/var/run/docker.sock:/tmp/docker.sock"
      ],
      "ContainerIDFile": "",
      "LxcConf": [],
      "Memory": 33554432,
      "MemoryReservation": 0,
      "MemorySwap": 67108864,
      "KernelMemory": 0,
      "CpuShares": 0,
      "CpuPeriod": 0,
      "CpusetCpus": "",
      "CpusetMems": "",
      "CpuQuota": 0,
      "BlkioWeight": 0,
      "OomKillDisable": false,
      "MemorySwappiness": -1,
      "Privileged": false,
      "PortBindings": {},
      "Links": null,
      "PublishAllPorts": false,
      "Dns": [],
      "DnsOptions": [],
      "DnsSearch": [],
      "ExtraHosts": null,
      "VolumesFrom": null,
      "Devices": [],
      "NetworkMode": "default",
      "IpcMode": "",
      "PidMode": "",
      "UTSMode": "",
      "CapAdd": null,
      "CapDrop": null,
      "GroupAdd": null,
      "RestartPolicy": {
        "Name": "always",
        "MaximumRetryCount": 0
      },
      "SecurityOpt": null,
      "ReadonlyRootfs": false,
      "Ulimits": null,
      "LogConfig": {
        "Type": "json-file",
        "Config": {}
      },
      "CgroupParent": "",
      "ConsoleSize": [
        0,
        0
      ],
      "VolumeDriver": ""
    },
    "GraphDriver": {
      "Name": "devicemapper",
      "Data": {
        "DeviceId": "153",
        "DeviceName": "docker-202:1-263241-ec5722eb67bc94fa13a08f04be4546e1c2c1d81196aa443546008dbb28744c25",
        "DeviceSize": "107374182400"
      }
    },
    "Mounts": [
      {
        "Source": "/var/run/docker.sock",
        "Destination": "/tmp/docker.sock",
        "Mode": "",
        "RW": true
      },
      {
        "Name": "0ba0f63bc3177879a8578509fedacb1b181e84f1452dcf3e0102daa864e17b0a",
        "Source": "/var/lib/docker/volumes/0ba0f63bc3177879a8578509fedacb1b181e84f1452dcf3e0102daa864e17b0a/_data",
        "Destination": "/mnt/routes",
        "Driver": "local",
        "Mode": "",
        "RW": true
      }
    ],
    "Config": {
      "Hostname": "ip-10-0-3-82",
      "Domainname": "",
      "User": "",
      "AttachStdin": false,
      "AttachStdout": false,
      "AttachStderr": false,
      "ExposedPorts": {
        "80/tcp": {}
      },
      "Tty": false,
      "OpenStdin": false,
      "StdinOnce": false,
      "Env": [
        "AWS_REGION=us-west-2",
        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
      ],
      "Cmd": [
        "cloudwatch://janus-ecs-instances-dev-LogGroup-1GG41D36E77TN"
      ],
      "Image": "bradgignac/logspout-cloudwatch:v0.1.2",
      "Volumes": {
        "/mnt/routes": {}
      },
      "WorkingDir": "",
      "Entrypoint": [
        "/bin/logspout"
      ],
      "OnBuild": null,
      "Labels": {},
      "StopSignal": "SIGTERM"
    },
    "NetworkSettings": {
      "Bridge": "",
      "SandboxID": "e4af89c04792fccc7952321ca1c00deca2cb9d32c031f7a524c93c3d147cf3d5",
      "HairpinMode": false,
      "LinkLocalIPv6Address": "",
      "LinkLocalIPv6PrefixLen": 0,
      "Ports": {
        "80/tcp": null
      },
      "SandboxKey": "/var/run/docker/netns/e4af89c04792",
      "SecondaryIPAddresses": null,
      "SecondaryIPv6Addresses": null,
      "EndpointID": "b9bf3b449225a345e2b3eeff7091db194f3a283a352e521dd36989ce2ec48fd8",
      "Gateway": "172.17.0.1",
      "GlobalIPv6Address": "",
      "GlobalIPv6PrefixLen": 0,
      "IPAddress": "172.17.0.2",
      "IPPrefixLen": 16,
      "IPv6Gateway": "",
      "MacAddress": "02:42:ac:11:00:02",
      "Networks": {
        "bridge": {
          "EndpointID": "b9bf3b449225a345e2b3eeff7091db194f3a283a352e521dd36989ce2ec48fd8",
          "Gateway": "172.17.0.1",
          "IPAddress": "172.17.0.2",
          "IPPrefixLen": 16,
          "IPv6Gateway": "",
          "GlobalIPv6Address": "",
          "GlobalIPv6PrefixLen": 0,
          "MacAddress": "02:42:ac:11:00:02"
        }
      }
    }
  }
]