create a xdsIR per gateway
create a infraIR per gateway
use the gateway namespace-name as the key for above IRs
populate the envoy bootstrap config with a node id that matches the IR key
populate the xds server snapshot for each xds request based on the node id

Fixes: https://github.com/envoyproxy/gateway/issues/349

Signed-off-by: Arko Dasgupta [email protected]

Sep 21 '22 23:09 arkodg

attaching some test logs done with 2 gateways

2 Gateway resources

$ kubectl get gateway/eg -o yaml
apiVersion: gateway.networking.k8s.io/v1beta1
kind: Gateway
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"gateway.networking.k8s.io/v1alpha2","kind":"Gateway","metadata":{"annotations":{},"name":"eg","namespace":"default"},"spec":{"gatewayClassName":"eg","listeners":[{"name":"http","port":8080,"protocol":"HTTP"}]}}
  creationTimestamp: "2022-09-23T02:00:18Z"
  generation: 1
  name: eg
  namespace: default
  resourceVersion: "1007"
  uid: c1b84856-c1f8-4126-892d-f8762ed9da44
spec:
  gatewayClassName: eg
  listeners:
  - allowedRoutes:
      namespaces:
        from: Same
    name: http
    port: 8080
    protocol: HTTP
status:
  conditions:
  - lastTransitionTime: "2022-09-23T02:00:18Z"
    message: The Gateway has been scheduled by Envoy Gateway
    observedGeneration: 1
    reason: Scheduled
    status: "True"
    type: Scheduled
  - lastTransitionTime: "2022-09-23T02:00:18Z"
    message: No addresses have been assigned to the Gateway
    observedGeneration: 1
    reason: AddressNotAssigned
    status: "False"
    type: Ready
  listeners:
  - attachedRoutes: 1
    conditions:
    - lastTransitionTime: "2022-09-23T02:00:22Z"
      message: Listener is ready
      observedGeneration: 1
      reason: Ready
      status: "True"
      type: Ready
    name: http
    supportedKinds:
    - group: gateway.networking.k8s.io
      kind: HTTPRoute

$ kubectl get gateway/eg-2 -o yaml
apiVersion: gateway.networking.k8s.io/v1beta1
kind: Gateway
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"gateway.networking.k8s.io/v1alpha2","kind":"Gateway","metadata":{"annotations":{},"name":"eg-2","namespace":"default"},"spec":{"gatewayClassName":"eg","listeners":[{"hostname":"www.example-2.com","name":"http-2","port":8080,"protocol":"HTTP"}]}}
  creationTimestamp: "2022-09-23T02:00:53Z"
  generation: 1
  name: eg-2
  namespace: default
  resourceVersion: "1008"
  uid: c6eb611a-09cc-4880-8b33-e8535344e344
spec:
  gatewayClassName: eg
  listeners:
  - allowedRoutes:
      namespaces:
        from: Same
    hostname: www.example-2.com
    name: http-2
    port: 8080
    protocol: HTTP
status:
  conditions:
  - lastTransitionTime: "2022-09-23T02:00:53Z"
    message: The Gateway has been scheduled by Envoy Gateway
    observedGeneration: 1
    reason: Scheduled
    status: "True"
    type: Scheduled
  - lastTransitionTime: "2022-09-23T02:00:53Z"
    message: No addresses have been assigned to the Gateway
    observedGeneration: 1
    reason: AddressNotAssigned
    status: "False"
    type: Ready
  listeners:
  - attachedRoutes: 1
    conditions:
    - lastTransitionTime: "2022-09-23T02:00:53Z"
      message: Listener is ready
      observedGeneration: 1
      reason: Ready
      status: "True"
      type: Ready
    name: http-2
    supportedKinds:
    - group: gateway.networking.k8s.io
      kind: HTTPRoute

2 HTTPRoute resources

$ kubectl get httproute/httpbin -o yaml
apiVersion: gateway.networking.k8s.io/v1beta1
kind: HTTPRoute
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"gateway.networking.k8s.io/v1alpha2","kind":"HTTPRoute","metadata":{"annotations":{},"name":"httpbin","namespace":"default"},"spec":{"hostnames":["www.example.com"],"parentRefs":[{"name":"eg"}],"rules":[{"backendRefs":[{"group":"","kind":"Service","name":"httpbin","port":80,"weight":1}],"matches":[{"path":{"type":"PathPrefix","value":"/"}}]}]}}
  creationTimestamp: "2022-09-23T02:00:22Z"
  generation: 3
  name: httpbin
  namespace: default
  resourceVersion: "1006"
  uid: aafe12cc-9ff7-489c-a9e8-8129aa2c4d80
spec:
  hostnames:
  - www.example.com
  parentRefs:
  - group: gateway.networking.k8s.io
    kind: Gateway
    name: eg
  rules:
  - backendRefs:
    - group: ""
      kind: Service
      name: httpbin
      port: 80
      weight: 1
    matches:
    - path:
        type: PathPrefix
        value: /
status:
  parents:
  - conditions:
    - lastTransitionTime: "2022-09-23T02:00:22Z"
      message: Route is accepted
      observedGeneration: 1
      reason: Accepted
      status: "True"
      type: Accepted
    controllerName: gateway.envoyproxy.io/gatewayclass-controller
    parentRef:
      group: gateway.networking.k8s.io
      kind: Gateway
      name: eg
  - conditions:
    - lastTransitionTime: "2022-09-23T02:01:00Z"
      message: Route is accepted
      observedGeneration: 2
      reason: Accepted
      status: "True"
      type: Accepted
    controllerName: gateway.envoyproxy.io/gatewayclass-controller
    parentRef:
      group: gateway.networking.k8s.io
      kind: Gateway
      name: eg-2      
$ kubectl get httproute/httpbin-2 -o yaml
apiVersion: gateway.networking.k8s.io/v1beta1
kind: HTTPRoute
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"gateway.networking.k8s.io/v1alpha2","kind":"HTTPRoute","metadata":{"annotations":{},"name":"httpbin-2","namespace":"default"},"spec":{"hostnames":["www.example-2.com"],"parentRefs":[{"name":"eg-2"}],"rules":[{"backendRefs":[{"group":"","kind":"Service","name":"httpbin","port":80,"weight":1}],"matches":[{"path":{"type":"PathPrefix","value":"/"}}]}]}}
  creationTimestamp: "2022-09-23T02:01:22Z"
  generation: 1
  name: httpbin-2
  namespace: default
  resourceVersion: "993"
  uid: 47ff8327-7f01-45b7-8f9b-6236fe2b8a9a
spec:
  hostnames:
  - www.example-2.com
  parentRefs:
  - group: gateway.networking.k8s.io
    kind: Gateway
    name: eg-2
  rules:
  - backendRefs:
    - group: ""
      kind: Service
      name: httpbin
      port: 80
      weight: 1
    matches:
    - path:
        type: PathPrefix
        value: /
status:
  parents:
  - conditions:
    - lastTransitionTime: "2022-09-23T02:01:22Z"
      message: Route is accepted
      observedGeneration: 1
      reason: Accepted
      status: "True"
      type: Accepted
    controllerName: gateway.envoyproxy.io/gatewayclass-controller
    parentRef:
      group: gateway.networking.k8s.io
      kind: Gateway
      name: eg-2

Port forward to the 2 envoy services

$ kubectl -n envoy-gateway-system port-forward service/envoy-default-eg 8888:8080 &
$ kubectl -n envoy-gateway-system port-forward service/envoy-default-eg-2 8889:8080 &

Can reach www.example.com from service/envoy-default-eg but not www.example-2.com

$ curl --verbose --header "Host: www.example.com" http://localhost:8888/get
*   Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 8888 (#0)
> GET /get HTTP/1.1
> Host: www.example.com
> User-Agent: curl/7.64.1
> Accept: */*
> 
Handling connection for 8888
< HTTP/1.1 200 OK
< server: envoy
< date: Fri, 23 Sep 2022 02:06:09 GMT
< content-type: application/json
< content-length: 238
< access-control-allow-origin: *
< access-control-allow-credentials: true
< x-envoy-upstream-service-time: 1
< 
{
  "args": {}, 
  "headers": {
    "Accept": "*/*", 
    "Host": "www.example.com", 
    "User-Agent": "curl/7.64.1", 
    "X-Envoy-Expected-Rq-Timeout-Ms": "15000"
  }, 
  "origin": "10.1.0.12", 
  "url": "http://www.example.com/get"
}
* Connection #0 to host localhost left intact
* Closing connection 0

$ curl --verbose --header "Host: www.example-2.com" http://localhost:8888/get
*   Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 8888 (#0)
> GET /get HTTP/1.1
> Host: www.example-2.com
> User-Agent: curl/7.64.1
> Accept: */*
> 
Handling connection for 8888
< HTTP/1.1 404 Not Found
< date: Fri, 23 Sep 2022 02:06:28 GMT
< server: envoy
< content-length: 0
< 
* Connection #0 to host localhost left intact
* Closing connection 0

Can reach www.example-2.com from service/envoy-default-eg-2 but not www.example.com

$ curl --verbose --header "Host: www.example.com" http://localhost:8889/get
*   Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 8889 (#0)
> GET /get HTTP/1.1
> Host: www.example.com
> User-Agent: curl/7.64.1
> Accept: */*
> 
Handling connection for 8889
< HTTP/1.1 404 Not Found
< date: Fri, 23 Sep 2022 02:06:35 GMT
< server: envoy
< content-length: 0
< 
* Connection #0 to host localhost left intact
* Closing connection 0
$ curl --verbose --header "Host: www.example-2.com" http://localhost:8889/get
*   Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 8889 (#0)
> GET /get HTTP/1.1
> Host: www.example-2.com
> User-Agent: curl/7.64.1
> Accept: */*
> 
Handling connection for 8889
< HTTP/1.1 200 OK
< server: envoy
< date: Fri, 23 Sep 2022 02:06:40 GMT
< content-type: application/json
< content-length: 242
< access-control-allow-origin: *
< access-control-allow-credentials: true
< x-envoy-upstream-service-time: 1
< 
{
  "args": {}, 
  "headers": {
    "Accept": "*/*", 
    "Host": "www.example-2.com", 
    "User-Agent": "curl/7.64.1", 
    "X-Envoy-Expected-Rq-Timeout-Ms": "15000"
  }, 
  "origin": "10.1.0.13", 
  "url": "http://www.example-2.com/get"
}
* Connection #0 to host localhost left intact
* Closing connection 0

Sep 23 '22 02:09 arkodg

reviewers please note that this PR doesnt handle delete of Gateway resources, will raise an issue to track it once this PR is in, and will fix be fixed in a follow up commit.

Sep 23 '22 23:09 arkodg

Codecov Report

Merging #404 (5f5f281) into main (96e7f54) will decrease coverage by 0.31%. The diff coverage is 70.22%.

@@            Coverage Diff             @@
##             main     #404      +/-   ##
==========================================
- Coverage   60.46%   60.15%   -0.32%     
==========================================
  Files          40       40              
  Lines        4323     4327       +4     
==========================================
- Hits         2614     2603      -11     
- Misses       1560     1574      +14     
- Partials      149      150       +1

Impacted Files	Coverage Δ
internal/cmd/xdstest.go	`3.65% <0.00%> (ø)`
internal/envoygateway/config/config.go	`0.00% <ø> (ø)`
internal/gatewayapi/runner/runner.go	`55.78% <0.00%> (-5.92%)`	:arrow_down:
internal/infrastructure/kubernetes/infra.go	`68.49% <0.00%> (ø)`
internal/message/types.go	`92.45% <0.00%> (-7.55%)`	:arrow_down:
internal/xds/server/runner/runner.go	`24.71% <0.00%> (ø)`
internal/xds/translator/runner/runner.go	`72.00% <45.45%> (-13.19%)`	:arrow_down:
internal/infrastructure/kubernetes/service.go	`72.15% <75.00%> (+0.72%)`	:arrow_up:
internal/crypto/certgen.go	`77.21% <100.00%> (ø)`
internal/gatewayapi/translator.go	`86.07% <100.00%> (+0.14%)`	:arrow_up:
... and 8 more

Help us with your feedback. Take ten seconds to tell us how you rate us. Have a feature suggestion? Share it here.

Sep 26 '22 23:09 codecov-commenter

xref: https://github.com/envoyproxy/gateway/issues/380

Sep 27 '22 18:09 danehans

xref: https://github.com/envoyproxy/gateway/issues/382

Sep 27 '22 18:09 danehans

@danehans I cherry-picked https://github.com/envoyproxy/gateway/pull/435 on top of this PR and CI now passes

Sep 27 '22 19:09 arkodg

I tested this PR with https://github.com/envoyproxy/gateway/pull/438 and the additional conformance tests pass.

Sep 28 '22 00:09 danehans

gateway
gateway copied to clipboard

envoy per gateway

Codecov Report

gateway gateway copied to clipboard

envoy per gateway

Codecov Report

gateway
gateway copied to clipboard