mirrord
mirrord copied to clipboard
Published
20 hours ago •
metalbear-co
metalbear-co
connect + getaddrinfo fail with curl
Bug Description
getaddrinfo returns a linkedlist of resolved addresses and seems like when the request is sent to curl localhost:80, it is resolved by the agent into both ipv4 and ipv6 addresses, and then the layer tries to create a socket based on the socket type of the resolved address, which for this context is ipv6, socket creation succeeds since locally ipv6 is supported but fails later when connect is called on the created ipv6 socket and the ipv6 address.
possible issue:
- ipv6 is not enabled on the remote/agent pod and hence we should check the network stack of the remote and fail with -1 if the requested socket type is not supported on the remote.
mehula@mehul-machine:~/mirrord-demo$ kubectl get svc user-service -o yaml
apiVersion: v1
kind: Service
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"metalbear-users"},"name":"user-service","namespace":"default"},"spec":{"ports":[{"nodePort":32000,"port":80,"protocol":"TCP","targetPort":80}],"selector":{"app":"metalbear-users"},"sessionAffinity":"None","type":"NodePort"}}
creationTimestamp: "2022-09-02T04:28:52Z"
labels:
app: metalbear-users
name: user-service
namespace: default
resourceVersion: "2836549"
uid: 82a8c73e-ed54-449e-b9c5-75e7fbe76f7a
spec:
clusterIP: 10.106.158.180
clusterIPs:
- 10.106.158.180
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- nodePort: 32000
port: 80
protocol: TCP
targetPort: 80
selector:
app: metalbear-users
sessionAffinity: None
type: NodePort
status:
loadBalancer: {}
inside the pod:
root@metalbear-deployment-85c754c75f-6k7mg:/app# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.244.0.45 netmask 255.255.255.0 broadcast 10.244.0.255
inet6 fe80::1cad:11ff:fe4a:c713 prefixlen 64 scopeid 0x20<link>
ether 1e:ad:11:4a:c7:13 txqueuelen 0 (Ethernet)
RX packets 797 bytes 11299843 (10.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 729 bytes 61844 (60.3 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 136 bytes 11051 (10.7 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 136 bytes 11051 (10.7 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
Steps to Reproduce
IRRORD_AGENT_IMAGE=test RUST_LOG=mirrord=trace ../mirrord/target/debug/mirrord exec -c --pod-name metalbear-deployment-85c754c75f-6k7mg curl localhost:80/users
Backtrace
2022-09-12T04:26:53.178473Z TRACE mirrord_layer::file::hooks: replace -> hooked "openat" true
2022-09-12T04:26:53.178490Z TRACE mirrord_layer::file::hooks: replace -> hooking "fopen"
2022-09-12T04:26:53.178530Z TRACE mirrord_layer::file::hooks: replace -> hooked "fopen" true
2022-09-12T04:26:53.178549Z TRACE mirrord_layer::file::hooks: replace -> hooking "fdopen"
2022-09-12T04:26:53.178582Z TRACE mirrord_layer::file::hooks: replace -> hooked "fdopen" true
2022-09-12T04:26:53.178600Z TRACE mirrord_layer::file::hooks: replace -> hooking "read"
2022-09-12T04:26:53.178634Z TRACE mirrord_layer::file::hooks: replace -> hooked "read" true
2022-09-12T04:26:53.178651Z TRACE mirrord_layer::file::hooks: replace -> hooking "fread"
2022-09-12T04:26:53.178684Z TRACE mirrord_layer::file::hooks: replace -> hooked "fread" true
2022-09-12T04:26:53.178713Z TRACE mirrord_layer::file::hooks: replace -> hooking "fileno"
2022-09-12T04:26:53.178759Z TRACE mirrord_layer::file::hooks: replace -> hooked "fileno" true
2022-09-12T04:26:53.178789Z TRACE mirrord_layer::file::hooks: replace -> hooking "lseek"
2022-09-12T04:26:53.178834Z TRACE mirrord_layer::file::hooks: replace -> hooked "lseek" true
2022-09-12T04:26:53.178863Z TRACE mirrord_layer::file::hooks: replace -> hooking "write"
2022-09-12T04:26:53.178907Z TRACE mirrord_layer::file::hooks: replace -> hooked "write" true
2022-09-12T04:26:53.178936Z TRACE mirrord_layer::file::hooks: replace -> hooking "access"
2022-09-12T04:26:53.178980Z TRACE mirrord_layer::file::hooks: replace -> hooked "access" true
2022-09-12T04:26:53.179007Z TRACE mirrord_layer::file::hooks: replace -> hooking "faccessat"
2022-09-12T04:26:53.179047Z TRACE mirrord_layer::file::hooks: replace -> hooked "faccessat" true
2022-09-12T04:26:53.180842Z INFO mirrord_layer::go_env: replace -> hooking "runtime.goenvs_unix"
2022-09-12T04:26:53.185342Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse {
"PYTHON_SETUPTOOLS_VERSION": "58.1.0",
"USER_SERVICE_PORT": "tcp://10.106.158.180:80",
"GPG_KEY": "E3FF2839C048B25C084DEBE9B26995E310250568",
"KUBERNETES_SERVICE_HOST": "10.96.0.1",
"USER_SERVICE_SERVICE_PORT": "80",
"BFF_USERS_SERVICE_PORT_3000_TCP_PROTO": "tcp",
"USER_SERVICE_PORT_80_TCP_PORT": "80",
"KUBERNETES_PORT_443_TCP_ADDR": "10.96.0.1",
"BFF_USERS_SERVICE_PORT_3000_TCP_PORT": "3000",
"BFF_USERS_SERVICE_PORT_3000_TCP": "tcp://10.106.233.7:3000",
"KUBERNETES_PORT": "tcp://10.96.0.1:443",
"KUBERNETES_PORT_443_TCP_PORT": "443",
"HOSTNAME": "metalbear-deployment-85c754c75f-6k7mg",
"USER_SERVICE_PORT_80_TCP_PROTO": "tcp",
"PYTHON_VERSION": "3.9.13",
"KUBERNETES_PORT_443_TCP": "tcp://10.96.0.1:443",
"BFF_USERS_SERVICE_SERVICE_HOST": "10.106.233.7",
"KUBERNETES_SERVICE_PORT": "443",
"PYTHON_GET_PIP_SHA256": "5aefe6ade911d997af080b315ebcb7f882212d070465df544e1175ac2be519b4",
"USER_SERVICE_PORT_80_TCP_ADDR": "10.106.158.180",
"USER_SERVICE_SERVICE_HOST": "10.106.158.180",
"PYTHON_PIP_VERSION": "22.0.4",
"PYTHON_GET_PIP_URL": "https://github.com/pypa/get-pip/raw/5eaac1050023df1f5c98b173b248c260023f2278/public/get-pip.py",
"USER_SERVICE_PORT_80_TCP": "tcp://10.106.158.180:80",
"BFF_USERS_SERVICE_PORT_3000_TCP_ADDR": "10.106.233.7",
"BFF_USERS_SERVICE_PORT": "tcp://10.106.233.7:3000",
"KUBERNETES_SERVICE_PORT_HTTPS": "443",
"BFF_USERS_SERVICE_SERVICE_PORT": "3000",
"LANG": "C.UTF-8",
"KUBERNETES_PORT_443_TCP_PROTO": "tcp",
}!
2022-09-12T04:26:53.185462Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_SETUPTOOLS_VERSION" value "58.1.0"
2022-09-12T04:26:53.185498Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT" value "tcp://10.106.158.180:80"
2022-09-12T04:26:53.185526Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "GPG_KEY" value "E3FF2839C048B25C084DEBE9B26995E310250568"
2022-09-12T04:26:53.185568Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_SERVICE_HOST" value "10.96.0.1"
2022-09-12T04:26:53.185595Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_SERVICE_PORT" value "80"
2022-09-12T04:26:53.185622Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP_PROTO" value "tcp"
2022-09-12T04:26:53.185648Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP_PORT" value "80"
2022-09-12T04:26:53.185675Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP_ADDR" value "10.96.0.1"
2022-09-12T04:26:53.185700Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP_PORT" value "3000"
2022-09-12T04:26:53.185726Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP" value "tcp://10.106.233.7:3000"
2022-09-12T04:26:53.185752Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT" value "tcp://10.96.0.1:443"
2022-09-12T04:26:53.185779Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP_PORT" value "443"
2022-09-12T04:26:53.185806Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "HOSTNAME" value "metalbear-deployment-85c754c75f-6k7mg"
2022-09-12T04:26:53.185832Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP_PROTO" value "tcp"
2022-09-12T04:26:53.185857Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_VERSION" value "3.9.13"
2022-09-12T04:26:53.185881Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP" value "tcp://10.96.0.1:443"
2022-09-12T04:26:53.185908Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_SERVICE_HOST" value "10.106.233.7"
2022-09-12T04:26:53.185932Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_SERVICE_PORT" value "443"
2022-09-12T04:26:53.185957Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_GET_PIP_SHA256" value "5aefe6ade911d997af080b315ebcb7f882212d070465df544e1175ac2be519b4"
2022-09-12T04:26:53.185984Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP_ADDR" value "10.106.158.180"
2022-09-12T04:26:53.186009Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_SERVICE_HOST" value "10.106.158.180"
2022-09-12T04:26:53.186036Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_PIP_VERSION" value "22.0.4"
2022-09-12T04:26:53.186061Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_GET_PIP_URL" value "https://github.com/pypa/get-pip/raw/5eaac1050023df1f5c98b173b248c260023f2278/public/get-pip.py"
2022-09-12T04:26:53.186090Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP" value "tcp://10.106.158.180:80"
2022-09-12T04:26:53.186115Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP_ADDR" value "10.106.233.7"
2022-09-12T04:26:53.186139Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT" value "tcp://10.106.233.7:3000"
2022-09-12T04:26:53.186166Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_SERVICE_PORT_HTTPS" value "443"
2022-09-12T04:26:53.186192Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_SERVICE_PORT" value "3000"
2022-09-12T04:26:53.186217Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "LANG" value "C.UTF-8"
2022-09-12T04:26:53.186239Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP_PROTO" value "tcp"
2022-09-12T04:26:53.186506Z TRACE mirrord_layer: close_detour -> fd 10
2022-09-12T04:26:53.186558Z TRACE mirrord_layer: close_detour -> fd 11
2022-09-12T04:26:53.187750Z TRACE mirrord_layer::file::hooks: fopen_detour ->
2022-09-12T04:26:53.190990Z TRACE mirrord_layer::socket::hooks: socket_detour -> domain 10 | type:2 | protocol 0
2022-09-12T04:26:53.191872Z TRACE mirrord_layer::socket::ops: socket -> domain 10 | type:2 | protocol 0
2022-09-12T04:26:53.192103Z DEBUG mirrord_layer::socket::ops: socket -> socket_fd 10 | new_socket UserSocket {
domain: 10,
type_: 2,
protocol: 0,
state: Initialized,
kind: Udp(
2,
),
}
2022-09-12T04:26:53.193502Z TRACE mirrord_layer: close_detour -> fd 10
2022-09-12T04:26:53.193861Z TRACE mirrord_layer::file::hooks: fopen_detour ->
2022-09-12T04:26:53.194368Z TRACE mirrord_layer::file::hooks: fopen_detour ->
2022-09-12T04:26:53.196240Z TRACE mirrord_layer::file::ops: fopen -> path "/home/mehula/.curlrc" | open_options OpenOptionsInternal {
read: true,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
}
2022-09-12T04:26:53.197759Z TRACE mirrord_layer::file::ops: open -> path "/home/mehula/.curlrc" | open_options OpenOptionsInternal {
read: true,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
}
2022-09-12T04:26:53.199181Z TRACE mirrord_layer: Layer::handle_hook_message -> hook_message File(Open(Open { path: "/home/mehula/.curlrc", file_channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) }, open_options: OpenOptionsInternal { read: true, write: false, append: false, truncate: false, create: false, create_new: false } }))
2022-09-12T04:26:53.199229Z DEBUG mirrord_layer::file: HookMessage::OpenFileHook path "/home/mehula/.curlrc" | options OpenOptionsInternal {
read: true,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
}
2022-09-12T04:26:53.200504Z DEBUG mirrord_layer::file: DaemonMessage::OpenFileResponse Err(
RemoteIO(
RemoteIOError {
raw_os_error: Some(
2,
),
kind: NotFound,
},
),
)!
2022-09-12T04:26:53.200784Z TRACE mirrord_layer::error: Error occured in Layer >> ResponseError(RemoteIO(RemoteIOError { raw_os_error: Some(2), kind: NotFound }))
2022-09-12T04:26:53.200865Z TRACE mirrord_layer::file::hooks: fileno_detour ->
2022-09-12T04:26:53.200965Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 10 | cmd 3
2022-09-12T04:26:53.201005Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:26:53.201031Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 10 | cmd 4
2022-09-12T04:26:53.201050Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:26:53.201069Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 11 | cmd 3
2022-09-12T04:26:53.201090Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:26:53.201110Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 11 | cmd 4
2022-09-12T04:26:53.201127Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:26:53.202147Z TRACE mirrord_layer::socket::hooks: getaddrinfo_detour -> raw_node 0x0000562ebb2e92e0 | raw_service 0x00007fc92cf8d9dc | raw_hints addrinfo {
ai_flags: 0,
ai_family: 0,
ai_socktype: 1,
ai_protocol: 0,
ai_addrlen: 0,
ai_addr: 0x0000000000000000,
ai_canonname: 0x0000000000000000,
ai_next: 0x0000000000000000,
} | out? false
2022-09-12T04:26:53.202199Z DEBUG mirrord_layer::socket::ops: getaddrinfo -> node Some(
"localhost",
) | service Some(
"80",
) | hints Some(
AddrInfoHint {
ai_family: 0,
ai_socktype: 1,
ai_protocol: 0,
ai_flags: 0,
},
)
2022-09-12T04:26:53.202306Z TRACE mirrord_layer: Layer::handle_hook_message -> hook_message GetAddrInfoHook(GetAddrInfoHook { node: Some("localhost"), service: Some("80"), hints: Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 }), hook_channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: false, is_tx_task_set: false } }) } })
2022-09-12T04:26:53.202356Z TRACE mirrord_layer: HookMessage::GetAddrInfo
2022-09-12T04:26:53.205307Z TRACE mirrord_layer: DaemonMessage::GetAddrInfoResponse Ok(
[
AddrInfoInternal {
socktype: 1,
protocol: 6,
address: 10,
sockaddr: [::1]:80,
canonname: None,
flags: 0,
},
AddrInfoInternal {
socktype: 1,
protocol: 6,
address: 2,
sockaddr: 127.0.0.1:80,
canonname: None,
flags: 0,
},
],
)
2022-09-12T04:26:53.205805Z INFO mirrord_layer::socket::ops: getaddrinfo -> result Ok(
0x00007fc918000d70,
)
2022-09-12T04:26:53.205845Z TRACE mirrord_layer::socket::hooks: result: 0
2022-09-12T04:26:53.205869Z TRACE mirrord_layer::socket::hooks: freeaddrinfo_detour -> addrinfo addrinfo {
ai_flags: 0,
ai_family: 10,
ai_socktype: 1,
ai_protocol: 6,
ai_addrlen: 28,
ai_addr: 0x00007fc918000d50,
ai_canonname: 0x0000000000000000,
ai_next: 0x00007fc918000d10,
}
2022-09-12T04:26:53.205980Z TRACE mirrord_layer: close_detour -> fd 13
2022-09-12T04:26:53.206022Z TRACE mirrord_layer: close_detour -> fd 12
2022-09-12T04:26:53.206052Z TRACE mirrord_layer::socket::hooks: socket_detour -> domain 10 | type:1 | protocol 6
2022-09-12T04:26:53.206074Z TRACE mirrord_layer::socket::ops: socket -> domain 10 | type:1 | protocol 6
2022-09-12T04:26:53.206103Z DEBUG mirrord_layer::socket::ops: socket -> socket_fd 12 | new_socket UserSocket {
domain: 10,
type_: 1,
protocol: 6,
state: Initialized,
kind: Tcp(
1,
),
}
2022-09-12T04:26:53.206145Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 12 | cmd 3
2022-09-12T04:26:53.206171Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:26:53.206191Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 12 | cmd 4
2022-09-12T04:26:53.206207Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:26:53.206227Z DEBUG mirrord_layer::socket::hooks: connect_detour -> sockfd 12
2022-09-12T04:26:53.206242Z DEBUG mirrord_layer::socket::ops: connect -> sockfd 12 | remote_address [::%69]:80
2022-09-12T04:26:53.206264Z TRACE mirrord_layer::socket::ops: connect -> SocketState::Initialized UserSocket {
domain: 10,
type_: 1,
protocol: 6,
state: Initialized,
kind: Tcp(
1,
),
}
2022-09-12T04:26:53.206391Z TRACE mirrord_layer: Layer::handle_hook_message -> hook_message TcpOutgoing(Connect(Connect { remote_address: [::%69]:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }))
2022-09-12T04:26:53.206519Z TRACE mirrord_layer::outgoing::tcp: handle_hook_message -> message Connect(Connect { remote_address: [::%69]:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
2022-09-12T04:26:53.206560Z TRACE mirrord_layer::outgoing::tcp: Connect -> remote_address [::%69]:80
2022-09-12T04:26:53.207606Z TRACE mirrord_layer::outgoing::tcp: handle_daemon_message -> message Connect(Err(RemoteIO(RemoteIOError { raw_os_error: Some(111), kind: ConnectionRefused })))
2022-09-12T04:26:53.207650Z TRACE mirrord_layer::outgoing::tcp: Connect -> connect Err(
RemoteIO(
RemoteIOError {
raw_os_error: Some(
111,
),
kind: ConnectionRefused,
},
),
)
2022-09-12T04:26:53.208137Z ERROR mirrord_layer: Error handling daemon message: ResponseError(RemoteIO(RemoteIOError { raw_os_error: Some(111), kind: ConnectionRefused }))
2022-09-12T04:26:53.208320Z ERROR mirrord_layer::error: Error occured in Layer >> RecvError(RecvError(()))
2022-09-12T04:26:53.208394Z TRACE mirrord_layer: close_detour -> fd 12
2022-09-12T04:26:53.208434Z TRACE mirrord_layer::socket::hooks: socket_detour -> domain 2 | type:1 | protocol 6
2022-09-12T04:26:53.208457Z TRACE mirrord_layer::socket::ops: socket -> domain 2 | type:1 | protocol 6
2022-09-12T04:26:53.208486Z DEBUG mirrord_layer::socket::ops: socket -> socket_fd 12 | new_socket UserSocket {
domain: 2,
type_: 1,
protocol: 6,
state: Initialized,
kind: Tcp(
1,
),
}
2022-09-12T04:26:53.208526Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 12 | cmd 3
2022-09-12T04:26:53.208551Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:26:53.208565Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 12 | cmd 4
2022-09-12T04:26:53.208584Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:26:53.208603Z DEBUG mirrord_layer::socket::hooks: connect_detour -> sockfd 12
2022-09-12T04:26:53.208621Z DEBUG mirrord_layer::socket::ops: connect -> sockfd 12 | remote_address 127.0.0.1:80
2022-09-12T04:26:53.208640Z TRACE mirrord_layer::socket::ops: connect -> SocketState::Initialized UserSocket {
domain: 2,
type_: 1,
protocol: 6,
state: Initialized,
kind: Tcp(
1,
),
}
2022-09-12T04:26:53.208754Z ERROR mirrord_layer::error: Error occured in Layer >> SendErrorHookMessage(SendError(TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: true, is_rx_task_set: false, is_tx_task_set: false } }) } }))))
2022-09-12T04:26:53.208796Z TRACE mirrord_layer: close_detour -> fd 12
curl: (7) Couldn't connect to server
2022-09-12T04:26:53.208922Z TRACE mirrord_layer: close_detour -> fd 10
2022-09-12T04:26:53.208954Z TRACE mirrord_layer: close_detour -> fd 11
### Relevant Logs
```shell
Without reversing the iterator for the vector returned by the agent: [Call succeeds]
MIRRORD_AGENT_IMAGE=test RUST_LOG=mirrord=trace ../mirrord/target/debug/mirrord exec -c --pod-name metalbear-deployment-85c754c75f-6k7mg curl localhost:80/users
2022-09-12T04:30:20.501251Z INFO mirrord: Launching "curl" with arguments ["localhost:80/users"]
2022-09-12T04:30:20.952659Z DEBUG mirrord: Extracted library file to "/tmp/fstmgqejcc-libmirrord_layer.so"
2022-09-12T04:30:21.008922Z INFO mirrord_layer: Initializing mirrord-layer!
2022-09-12T04:30:21.009141Z INFO mirrord_layer: Using port `52843` for communication
2022-09-12T04:30:21.013534Z WARN mirrord_layer::pod_api: Accepting invalid certificates
2022-09-12T04:30:21.024691Z INFO mirrord_layer::pod_api: No container name specified, defaulting to first container found
2022-09-12T04:30:21.111927Z DEBUG mirrord_layer::pod_api: Pod Phase = "Pending"
2022-09-12T04:30:21.164023Z DEBUG mirrord_layer::pod_api: Pod Phase = "Pending"
2022-09-12T04:30:25.938030Z DEBUG mirrord_layer::pod_api: Pod Phase = "Running"
2022-09-12T04:30:25.966090Z TRACE mirrord_layer: replace -> hooking "close"
2022-09-12T04:30:25.966236Z TRACE mirrord_layer: replace -> hooked "close" true
2022-09-12T04:30:25.966268Z TRACE mirrord_layer::socket::hooks: replace -> hooking "socket"
2022-09-12T04:30:25.966310Z TRACE mirrord_layer::socket::hooks: replace -> hooked "socket" true
2022-09-12T04:30:25.966327Z TRACE mirrord_layer::socket::hooks: replace -> hooking "bind"
2022-09-12T04:30:25.966357Z TRACE mirrord_layer::socket::hooks: replace -> hooked "bind" true
2022-09-12T04:30:25.966375Z TRACE mirrord_layer::socket::hooks: replace -> hooking "listen"
2022-09-12T04:30:25.966406Z TRACE mirrord_layer::socket::hooks: replace -> hooked "listen" true
2022-09-12T04:30:25.966441Z TRACE mirrord_layer::socket::hooks: replace -> hooking "connect"
2022-09-12T04:30:25.966477Z TRACE mirrord_layer::socket::hooks: replace -> hooked "connect" true
2022-09-12T04:30:25.966499Z TRACE mirrord_layer::socket::hooks: replace -> hooking "fcntl"
2022-09-12T04:30:25.966551Z TRACE mirrord_layer::socket::hooks: replace -> hooked "fcntl" true
2022-09-12T04:30:25.966573Z TRACE mirrord_layer::socket::hooks: replace -> hooking "dup"
2022-09-12T04:30:25.966606Z TRACE mirrord_layer::socket::hooks: replace -> hooked "dup" true
2022-09-12T04:30:25.966623Z TRACE mirrord_layer::socket::hooks: replace -> hooking "dup2"
2022-09-12T04:30:25.966672Z TRACE mirrord_layer::socket::hooks: replace -> hooked "dup2" true
2022-09-12T04:30:25.966691Z TRACE mirrord_layer::socket::hooks: replace -> hooking "getpeername"
2022-09-12T04:30:25.966734Z TRACE mirrord_layer::socket::hooks: replace -> hooked "getpeername" true
2022-09-12T04:30:25.966752Z TRACE mirrord_layer::socket::hooks: replace -> hooking "getsockname"
2022-09-12T04:30:25.966783Z TRACE mirrord_layer::socket::hooks: replace -> hooked "getsockname" true
2022-09-12T04:30:25.966797Z TRACE mirrord_layer::socket::hooks: replace -> hooking "uv__accept4"
2022-09-12T04:30:25.966821Z TRACE mirrord_layer::socket::hooks: replace -> hooking "accept4"
2022-09-12T04:30:25.966855Z TRACE mirrord_layer::socket::hooks: replace -> hooked "accept4" true
2022-09-12T04:30:25.966874Z TRACE mirrord_layer::socket::hooks: replace -> hooking "dup3"
2022-09-12T04:30:25.966901Z TRACE mirrord_layer::socket::hooks: replace -> hooked "dup3" true
2022-09-12T04:30:25.966936Z TRACE mirrord_layer::socket::hooks: replace -> hooking "accept"
2022-09-12T04:30:25.966972Z TRACE mirrord_layer::socket::hooks: replace -> hooked "accept" true
2022-09-12T04:30:25.967009Z TRACE mirrord_layer::socket::hooks: replace -> hooking "getaddrinfo"
2022-09-12T04:30:25.967045Z TRACE mirrord_layer::socket::hooks: replace -> hooked "getaddrinfo" true
2022-09-12T04:30:25.967063Z TRACE mirrord_layer::socket::hooks: replace -> hooking "freeaddrinfo"
2022-09-12T04:30:25.967111Z TRACE mirrord_layer::socket::hooks: replace -> hooked "freeaddrinfo" true
2022-09-12T04:30:25.967132Z TRACE mirrord_layer::file::hooks: replace -> hooking "open"
2022-09-12T04:30:25.967177Z TRACE mirrord_layer::file::hooks: replace -> hooked "open" true
2022-09-12T04:30:25.967195Z TRACE mirrord_layer::file::hooks: replace -> hooking "openat"
2022-09-12T04:30:25.967228Z TRACE mirrord_layer::file::hooks: replace -> hooked "openat" true
2022-09-12T04:30:25.967244Z TRACE mirrord_layer::file::hooks: replace -> hooking "fopen"
2022-09-12T04:30:25.967302Z TRACE mirrord_layer::file::hooks: replace -> hooked "fopen" true
2022-09-12T04:30:25.967332Z TRACE mirrord_layer::file::hooks: replace -> hooking "fdopen"
2022-09-12T04:30:25.967382Z TRACE mirrord_layer::file::hooks: replace -> hooked "fdopen" true
2022-09-12T04:30:25.967402Z TRACE mirrord_layer::file::hooks: replace -> hooking "read"
2022-09-12T04:30:25.967435Z TRACE mirrord_layer::file::hooks: replace -> hooked "read" true
2022-09-12T04:30:25.967453Z TRACE mirrord_layer::file::hooks: replace -> hooking "fread"
2022-09-12T04:30:25.967482Z TRACE mirrord_layer::file::hooks: replace -> hooked "fread" true
2022-09-12T04:30:25.967497Z TRACE mirrord_layer::file::hooks: replace -> hooking "fileno"
2022-09-12T04:30:25.967532Z TRACE mirrord_layer::file::hooks: replace -> hooked "fileno" true
2022-09-12T04:30:25.967549Z TRACE mirrord_layer::file::hooks: replace -> hooking "lseek"
2022-09-12T04:30:25.967575Z TRACE mirrord_layer::file::hooks: replace -> hooked "lseek" true
2022-09-12T04:30:25.967587Z TRACE mirrord_layer::file::hooks: replace -> hooking "write"
2022-09-12T04:30:25.967613Z TRACE mirrord_layer::file::hooks: replace -> hooked "write" true
2022-09-12T04:30:25.967628Z TRACE mirrord_layer::file::hooks: replace -> hooking "access"
2022-09-12T04:30:25.967652Z TRACE mirrord_layer::file::hooks: replace -> hooked "access" true
2022-09-12T04:30:25.967664Z TRACE mirrord_layer::file::hooks: replace -> hooking "faccessat"
2022-09-12T04:30:25.967687Z TRACE mirrord_layer::file::hooks: replace -> hooked "faccessat" true
2022-09-12T04:30:25.969283Z INFO mirrord_layer::go_env: replace -> hooking "runtime.goenvs_unix"
2022-09-12T04:30:25.974911Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse {
"USER_SERVICE_PORT_80_TCP_PROTO": "tcp",
"BFF_USERS_SERVICE_PORT_3000_TCP_ADDR": "10.106.233.7",
"USER_SERVICE_PORT_80_TCP_PORT": "80",
"KUBERNETES_PORT": "tcp://10.96.0.1:443",
"KUBERNETES_SERVICE_PORT": "443",
"KUBERNETES_SERVICE_PORT_HTTPS": "443",
"LANG": "C.UTF-8",
"BFF_USERS_SERVICE_PORT_3000_TCP_PORT": "3000",
"USER_SERVICE_PORT_80_TCP_ADDR": "10.106.158.180",
"BFF_USERS_SERVICE_PORT_3000_TCP": "tcp://10.106.233.7:3000",
"USER_SERVICE_PORT": "tcp://10.106.158.180:80",
"GPG_KEY": "E3FF2839C048B25C084DEBE9B26995E310250568",
"KUBERNETES_PORT_443_TCP_ADDR": "10.96.0.1",
"BFF_USERS_SERVICE_SERVICE_PORT": "3000",
"USER_SERVICE_SERVICE_PORT": "80",
"KUBERNETES_PORT_443_TCP_PORT": "443",
"PYTHON_PIP_VERSION": "22.0.4",
"PYTHON_GET_PIP_SHA256": "5aefe6ade911d997af080b315ebcb7f882212d070465df544e1175ac2be519b4",
"USER_SERVICE_PORT_80_TCP": "tcp://10.106.158.180:80",
"USER_SERVICE_SERVICE_HOST": "10.106.158.180",
"BFF_USERS_SERVICE_PORT_3000_TCP_PROTO": "tcp",
"KUBERNETES_PORT_443_TCP": "tcp://10.96.0.1:443",
"HOSTNAME": "metalbear-deployment-85c754c75f-6k7mg",
"KUBERNETES_PORT_443_TCP_PROTO": "tcp",
"PYTHON_VERSION": "3.9.13",
"BFF_USERS_SERVICE_PORT": "tcp://10.106.233.7:3000",
"PYTHON_GET_PIP_URL": "https://github.com/pypa/get-pip/raw/5eaac1050023df1f5c98b173b248c260023f2278/public/get-pip.py",
"KUBERNETES_SERVICE_HOST": "10.96.0.1",
"PYTHON_SETUPTOOLS_VERSION": "58.1.0",
"BFF_USERS_SERVICE_SERVICE_HOST": "10.106.233.7",
}!
2022-09-12T04:30:25.975026Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP_PROTO" value "tcp"
2022-09-12T04:30:25.975064Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP_ADDR" value "10.106.233.7"
2022-09-12T04:30:25.975093Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP_PORT" value "80"
2022-09-12T04:30:25.975119Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT" value "tcp://10.96.0.1:443"
2022-09-12T04:30:25.975143Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_SERVICE_PORT" value "443"
2022-09-12T04:30:25.975166Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_SERVICE_PORT_HTTPS" value "443"
2022-09-12T04:30:25.975192Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "LANG" value "C.UTF-8"
2022-09-12T04:30:25.975218Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP_PORT" value "3000"
2022-09-12T04:30:25.975245Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP_ADDR" value "10.106.158.180"
2022-09-12T04:30:25.975269Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP" value "tcp://10.106.233.7:3000"
2022-09-12T04:30:25.975295Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT" value "tcp://10.106.158.180:80"
2022-09-12T04:30:25.975321Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "GPG_KEY" value "E3FF2839C048B25C084DEBE9B26995E310250568"
2022-09-12T04:30:25.975353Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP_ADDR" value "10.96.0.1"
2022-09-12T04:30:25.975381Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_SERVICE_PORT" value "3000"
2022-09-12T04:30:25.975407Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_SERVICE_PORT" value "80"
2022-09-12T04:30:25.975432Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP_PORT" value "443"
2022-09-12T04:30:25.975453Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_PIP_VERSION" value "22.0.4"
2022-09-12T04:30:25.975472Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_GET_PIP_SHA256" value "5aefe6ade911d997af080b315ebcb7f882212d070465df544e1175ac2be519b4"
2022-09-12T04:30:25.975493Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_PORT_80_TCP" value "tcp://10.106.158.180:80"
2022-09-12T04:30:25.975513Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "USER_SERVICE_SERVICE_HOST" value "10.106.158.180"
2022-09-12T04:30:25.975533Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT_3000_TCP_PROTO" value "tcp"
2022-09-12T04:30:25.975554Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP" value "tcp://10.96.0.1:443"
2022-09-12T04:30:25.975578Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "HOSTNAME" value "metalbear-deployment-85c754c75f-6k7mg"
2022-09-12T04:30:25.975602Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_PORT_443_TCP_PROTO" value "tcp"
2022-09-12T04:30:25.975624Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_VERSION" value "3.9.13"
2022-09-12T04:30:25.975646Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_PORT" value "tcp://10.106.233.7:3000"
2022-09-12T04:30:25.975668Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_GET_PIP_URL" value "https://github.com/pypa/get-pip/raw/5eaac1050023df1f5c98b173b248c260023f2278/public/get-pip.py"
2022-09-12T04:30:25.975693Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "KUBERNETES_SERVICE_HOST" value "10.96.0.1"
2022-09-12T04:30:25.975715Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "PYTHON_SETUPTOOLS_VERSION" value "58.1.0"
2022-09-12T04:30:25.975737Z DEBUG mirrord_layer: DaemonMessage::GetEnvVarsResponse set key "BFF_USERS_SERVICE_SERVICE_HOST" value "10.106.233.7"
2022-09-12T04:30:25.975827Z TRACE mirrord_layer: close_detour -> fd 10
2022-09-12T04:30:25.975868Z TRACE mirrord_layer: close_detour -> fd 11
2022-09-12T04:30:25.976788Z TRACE mirrord_layer::file::hooks: fopen_detour ->
2022-09-12T04:30:25.979578Z TRACE mirrord_layer::socket::hooks: socket_detour -> domain 10 | type:2 | protocol 0
2022-09-12T04:30:25.979958Z TRACE mirrord_layer::socket::ops: socket -> domain 10 | type:2 | protocol 0
2022-09-12T04:30:25.980264Z DEBUG mirrord_layer::socket::ops: socket -> socket_fd 10 | new_socket UserSocket {
domain: 10,
type_: 2,
protocol: 0,
state: Initialized,
kind: Udp(
2,
),
}
2022-09-12T04:30:25.984601Z TRACE mirrord_layer: close_detour -> fd 10
2022-09-12T04:30:25.985377Z TRACE mirrord_layer::file::hooks: fopen_detour ->
2022-09-12T04:30:25.986202Z TRACE mirrord_layer::file::hooks: fopen_detour ->
2022-09-12T04:30:25.988468Z TRACE mirrord_layer::file::ops: fopen -> path "/home/mehula/.curlrc" | open_options OpenOptionsInternal {
read: true,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
}
2022-09-12T04:30:25.998987Z TRACE mirrord_layer::file::ops: open -> path "/home/mehula/.curlrc" | open_options OpenOptionsInternal {
read: true,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
}
2022-09-12T04:30:26.000795Z TRACE mirrord_layer: Layer::handle_hook_message -> hook_message File(Open(Open { path: "/home/mehula/.curlrc", file_channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: false, is_tx_task_set: false } }) }, open_options: OpenOptionsInternal { read: true, write: false, append: false, truncate: false, create: false, create_new: false } }))
2022-09-12T04:30:26.000854Z DEBUG mirrord_layer::file: HookMessage::OpenFileHook path "/home/mehula/.curlrc" | options OpenOptionsInternal {
read: true,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
}
2022-09-12T04:30:26.002168Z DEBUG mirrord_layer::file: DaemonMessage::OpenFileResponse Err(
RemoteIO(
RemoteIOError {
raw_os_error: Some(
2,
),
kind: NotFound,
},
),
)!
2022-09-12T04:30:26.002488Z TRACE mirrord_layer::error: Error occured in Layer >> ResponseError(RemoteIO(RemoteIOError { raw_os_error: Some(2), kind: NotFound }))
2022-09-12T04:30:26.003052Z TRACE mirrord_layer::file::hooks: fileno_detour ->
2022-09-12T04:30:26.003291Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 10 | cmd 3
2022-09-12T04:30:26.003500Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:30:26.003706Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 10 | cmd 4
2022-09-12T04:30:26.004102Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:30:26.004399Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 11 | cmd 3
2022-09-12T04:30:26.004539Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:30:26.006012Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 11 | cmd 4
2022-09-12T04:30:26.006047Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:30:26.006253Z TRACE mirrord_layer::socket::hooks: getaddrinfo_detour -> raw_node 0x000055fa1d946e20 | raw_service 0x00007f5fe9c229dc | raw_hints addrinfo {
ai_flags: 0,
ai_family: 0,
ai_socktype: 1,
ai_protocol: 0,
ai_addrlen: 0,
ai_addr: 0x0000000000000000,
ai_canonname: 0x0000000000000000,
ai_next: 0x0000000000000000,
} | out? false
2022-09-12T04:30:26.006310Z DEBUG mirrord_layer::socket::ops: getaddrinfo -> node Some(
"localhost",
) | service Some(
"80",
) | hints Some(
AddrInfoHint {
ai_family: 0,
ai_socktype: 1,
ai_protocol: 0,
ai_flags: 0,
},
)
2022-09-12T04:30:26.006448Z TRACE mirrord_layer: Layer::handle_hook_message -> hook_message GetAddrInfoHook(GetAddrInfoHook { node: Some("localhost"), service: Some("80"), hints: Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 }), hook_channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
2022-09-12T04:30:26.006683Z TRACE mirrord_layer: HookMessage::GetAddrInfo
2022-09-12T04:30:26.008254Z TRACE mirrord_layer: DaemonMessage::GetAddrInfoResponse Ok(
[
AddrInfoInternal {
socktype: 1,
protocol: 6,
address: 10,
sockaddr: [::1]:80,
canonname: None,
flags: 0,
},
AddrInfoInternal {
socktype: 1,
protocol: 6,
address: 2,
sockaddr: 127.0.0.1:80,
canonname: None,
flags: 0,
},
],
)
2022-09-12T04:30:26.008552Z INFO mirrord_layer::socket::ops: getaddrinfo -> result Ok(
0x00007f5fd4000d70,
)
2022-09-12T04:30:26.008592Z TRACE mirrord_layer::socket::hooks: result: 0
2022-09-12T04:30:26.008619Z TRACE mirrord_layer::socket::hooks: freeaddrinfo_detour -> addrinfo addrinfo {
ai_flags: 0,
ai_family: 2,
ai_socktype: 1,
ai_protocol: 6,
ai_addrlen: 16,
ai_addr: 0x00007f5fd4000d50,
ai_canonname: 0x0000000000000000,
ai_next: 0x00007f5fd4000d10,
}
2022-09-12T04:30:26.009087Z TRACE mirrord_layer: close_detour -> fd 13
2022-09-12T04:30:26.009129Z TRACE mirrord_layer: close_detour -> fd 12
2022-09-12T04:30:26.009158Z TRACE mirrord_layer::socket::hooks: socket_detour -> domain 2 | type:1 | protocol 6
2022-09-12T04:30:26.009176Z TRACE mirrord_layer::socket::ops: socket -> domain 2 | type:1 | protocol 6
2022-09-12T04:30:26.009221Z DEBUG mirrord_layer::socket::ops: socket -> socket_fd 12 | new_socket UserSocket {
domain: 2,
type_: 1,
protocol: 6,
state: Initialized,
kind: Tcp(
1,
),
}
2022-09-12T04:30:26.009300Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 12 | cmd 3
2022-09-12T04:30:26.009320Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 2
2022-09-12T04:30:26.009336Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> fd 12 | cmd 4
2022-09-12T04:30:26.009353Z TRACE mirrord_layer::socket::hooks: fcntl_detour -> result 0
2022-09-12T04:30:26.009376Z DEBUG mirrord_layer::socket::hooks: connect_detour -> sockfd 12
2022-09-12T04:30:26.009395Z DEBUG mirrord_layer::socket::ops: connect -> sockfd 12 | remote_address 127.0.0.1:80
2022-09-12T04:30:26.009418Z TRACE mirrord_layer::socket::ops: connect -> SocketState::Initialized UserSocket {
domain: 2,
type_: 1,
protocol: 6,
state: Initialized,
kind: Tcp(
1,
),
}
2022-09-12T04:30:26.009567Z TRACE mirrord_layer: Layer::handle_hook_message -> hook_message TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: false, is_tx_task_set: false } }) } }))
2022-09-12T04:30:26.009626Z TRACE mirrord_layer::outgoing::tcp: handle_hook_message -> message Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: false, is_tx_task_set: false } }) } })
2022-09-12T04:30:26.009652Z TRACE mirrord_layer::outgoing::tcp: Connect -> remote_address 127.0.0.1:80
2022-09-12T04:30:26.010689Z TRACE mirrord_layer::outgoing::tcp: handle_daemon_message -> message Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))
2022-09-12T04:30:26.010750Z TRACE mirrord_layer::outgoing::tcp: Connect -> connect Ok(
DaemonConnect {
connection_id: 0,
remote_address: 127.0.0.1:80,
},
)
2022-09-12T04:30:26.011255Z DEBUG mirrord_layer::socket::ops: connect -> connect_result -1
2022-09-12T04:30:26.011323Z TRACE mirrord_layer::socket::hooks: getpeername_detour -> sockfd 12
2022-09-12T04:30:26.011350Z TRACE mirrord_layer::socket::ops: getpeername -> sockfd 12
2022-09-12T04:30:26.011441Z TRACE mirrord_layer::socket::hooks: getsockname_detour -> sockfd 12
2022-09-12T04:30:26.011568Z TRACE mirrord_layer::socket::ops: getsockname -> sockfd 12
2022-09-12T04:30:26.031464Z TRACE mirrord_layer::outgoing::tcp: handle_daemon_message -> message Read(Ok(DaemonRead { connection_id: 0, bytes (length): 296 }))
2022-09-12T04:30:26.031863Z TRACE mirrord_layer::outgoing::tcp: Read -> read Ok(DaemonRead { connection_id: 0, bytes (length): 296 })
2022-09-12T04:30:26.031950Z TRACE mirrord_layer::outgoing::tcp: handle_daemon_message -> message Close(0)
2022-09-12T04:30:26.031978Z TRACE mirrord_layer::outgoing::tcp: Close -> connection_id 0
2022-09-12T04:30:26.032065Z WARN mirrord_layer::outgoing::tcp: interceptor_task -> exiting due to remote stream closed!
2022-09-12T04:30:26.032093Z TRACE mirrord_layer::outgoing::tcp: interceptor_task done -> connection_id 0
[{"Last":"Arora","Name":"Mehul"},{"Last":"Arora","Name":"Mehul"},{"Last":"Arora","Name":"Mehul"},{"Last":"Arora","Name":"Mehul"}]
2022-09-12T04:30:26.032215Z TRACE mirrord_layer: close_detour -> fd 12
2022-09-12T04:30:26.032283Z TRACE mirrord_layer: close_detour -> fd 10
2022-09-12T04:30:26.032310Z TRACE mirrord_layer: close_detour -> fd 11
Your operating system and version
mehula@mehul-machine:~/mirrord-demo$ uname -a Linux mehul-machine 5.15.0-1019-azure #24~20.04.1-Ubuntu SMP Tue Aug 23 15:52:52 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
Local process
IRRORD_AGENT_IMAGE=test RUST_LOG=mirrord=trace ../mirrord/target/debug/mirrord exec -c --pod-name metalbear-deployment-85c754c75f-6k7mg curl localhost:80/users
Local process version
No response
Additional Info
No response
Can we fallback to IPv4 socket creation if IPv6 fails?
I think that's what it tries to do and #377 should solve this bug as well I believe ( we can't do the fallback, as the app/user provided a specific IP and family type..)
@infiniteregrets how do you know IPv6 is not enabled on the pod? What about the IPv6 addresses in the ifconfig?
inet6 fe80::1cad:11ff:fe4a:c713 prefixlen 64 scopeid 0x20<link>
inet6 ::1 prefixlen 128 scopeid 0x10<host>
@infiniteregrets how do you know IPv6 is not enabled on the pod? What about the IPv6 addresses in the
ifconfig?inet6 fe80::1cad:11ff:fe4a:c713 prefixlen 64 scopeid 0x20<link>inet6 ::1 prefixlen 128 scopeid 0x10<host>
That's link address (you can see it by <link> in the end of it
@aviramha the example is all local, and the request is to localhost, how do you know [::1] is disabled?
@aviramha the example is all local, and the request is to localhost, how do you know
[::1]is disabled?
Fair point
@infiniteregrets how do you know IPv6 is not enabled on the pod? What about the IPv6 addresses in the
ifconfig?inet6 fe80::1cad:11ff:fe4a:c713 prefixlen 64 scopeid 0x20<link>inet6 ::1 prefixlen 128 scopeid 0x10<host>
Just speculation, inspecting the service it's only ipv4 stack inside the pod if i check its ipv6 as well (if i check catting some Linux files) but inside the pod curl to ipv6 localhost fails
Right, IPv6 is not enabled for the service:
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
Do you think the agent should check that?
Is the deployed application listening on IPv6 btw?
Looks like curl localhost:80 works (slightly edited logs to reduce noise):
2022-09-26T18:12:04.467610Z INFO ThreadId(01) mirrord_layer: Initializing mirrord-layer!
(...)
2022-09-26T18:12:07.811635Z TRACE ThreadId(01) enable_hooks{enabled_file_ops=true enabled_remote_dns=true}: mirrord_layer: enter
(...)
2022-09-26T18:12:07.842714Z TRACE ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}:getaddrinfo{node=Some("localhost") service=Some("80")
hints=Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 })}: mirrord_layer::socket::ops: enter
2022-09-26T18:12:07.843104Z TRACE ThreadId(07) handle_hook_message{hook_message=GetAddrInfoHook(GetAddrInfoHook { node: Some("localhost"), service: Some("80"), hints: Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 }), hook_channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })}: mirrord_layer: enter
2022-09-26T18:12:07.843275Z TRACE ThreadId(07) handle_hook_message{hook_message=GetAddrInfoHook(GetAddrInfoHook { node: Some("localhost"), service: Some("80"), hints: Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 }), hook_channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })}: mirrord_layer: exit
2022-09-26T18:12:07.846156Z TRACE ThreadId(02) handle_daemon_message{daemon_message=GetAddrInfoResponse(Ok([AddrInfoInternal { socktype: 1, protocol: 6, address: 2, sockaddr: 127.0.0.1:80, canonname: None, flags: 0 }, AddrInfoInternal { socktype: 1, protocol: 6, address: 10, sockaddr: [::1]:80, canonname: None, flags: 0 }]))}: mirrord_layer: enter
2022-09-26T18:12:07.846257Z TRACE ThreadId(02) handle_daemon_message{daemon_message=GetAddrInfoResponse(Ok([AddrInfoInternal { socktype: 1, protocol: 6, address: 2, sockaddr: 127.0.0.1:80, canonname: None, flags: 0 }, AddrInfoInternal { socktype: 1, protocol: 6, address: 10, sockaddr: [::1]:80, canonname: None, flags: 0 }]))}: mirrord_layer: exit
2022-09-26T18:12:07.846330Z DEBUG ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}:getaddrinfo{node=Some("localhost") service=Some("80")
hints=Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 })}: mirrord_layer::socket::ops: getaddrinfo -> list [
AddrInfoInternal {
socktype: 1,
protocol: 6,
address: 2,
sockaddr: 127.0.0.1:80,
canonname: None,
flags: 0,
},
AddrInfoInternal {
socktype: 1,
protocol: 6,
address: 10,
sockaddr: [::1]:80,
canonname: None,
flags: 0,
},
]
2022-09-26T18:12:07.846558Z DEBUG ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}:getaddrinfo{node=Some("localhost") service=Some("80")
hints=Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 })}: mirrord_layer::socket::ops: getaddrinfo -> reversed addrinfo {
ai_flags: 0,
ai_family: 10,
ai_socktype: 1,
ai_protocol: 6,
ai_addrlen: 28,
ai_addr: 0x00007fc30c001ff0,
ai_canonname: 0x0000000000000000,
ai_next: 0x0000000000000000,
}
2022-09-26T18:12:07.846691Z DEBUG ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}:getaddrinfo{node=Some("localhost") service=Some("80")
hints=Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 })}: mirrord_layer::socket::ops: getaddrinfo -> reversed addrinfo {
ai_flags: 0,
ai_family: 2,
ai_socktype: 1,
ai_protocol: 6,
ai_addrlen: 16,
ai_addr: 0x00007fc30c002310,
ai_canonname: 0x0000000000000000,
ai_next: 0x0000000000000000,
}
2022-09-26T18:12:07.846793Z INFO ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}:getaddrinfo{node=Some("localhost") service=Some("80")
hints=Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 })}: mirrord_layer::socket::ops: getaddrinfo -> result Ok(
0x00007fc30c002330,
)
2022-09-26T18:12:07.846873Z TRACE ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}:getaddrinfo{node=Some("localhost") service=Some("80")
hints=Some(AddrInfoHint { ai_family: 0, ai_socktype: 1, ai_protocol: 0, ai_flags: 0 })}: mirrord_layer::socket::ops: exit
2022-09-26T18:12:07.846979Z TRACE ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}: mirrord_layer::socket::hooks: result: 0
2022-09-26T18:12:07.847042Z TRACE ThreadId(10) getaddrinfo_detour{raw_node=0x558e09326740 raw_service=0x7fc317ffe9dc}: mirrord_layer::socket::hooks: exit
2022-09-26T18:12:07.847148Z TRACE ThreadId(10) freeaddrinfo_detour: mirrord_layer::socket::hooks: enter
2022-09-26T18:12:07.847203Z TRACE ThreadId(10) freeaddrinfo_detour: mirrord_layer::socket::hooks: exit
(...)
2022-09-26T18:12:07.849496Z TRACE ThreadId(01) connect_detour{sockfd=12 address_length=16}: mirrord_layer::socket::hooks: enter
2022-09-26T18:12:07.849635Z TRACE ThreadId(01) connect_detour{sockfd=12 address_length=16}:connect{sockfd=12 remote_address=127.0.0.1:80}: mirrord_layer::socket::ops: enter
2022-09-26T18:12:07.850042Z TRACE ThreadId(02) handle_hook_message{hook_message=TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }))}: mirrord_layer: enter
2022-09-26T18:12:07.850162Z TRACE ThreadId(02) handle_hook_message{hook_message=TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }))}:handle_hook_message{Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })}: mirrord_layer::outgoing::tcp: enter
2022-09-26T18:12:07.850269Z TRACE ThreadId(02) handle_hook_message{hook_message=TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }))}:handle_hook_message{Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })}: mirrord_layer::outgoing::tcp: Connect -> remote_address 127.0.0.1:80
2022-09-26T18:12:07.850437Z TRACE ThreadId(02) handle_hook_message{hook_message=TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }))}:handle_hook_message{Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })}: mirrord_layer::outgoing::tcp: exit
2022-09-26T18:12:07.850601Z TRACE ThreadId(02) handle_hook_message{hook_message=TcpOutgoing(Connect(Connect { remote_address: 127.0.0.1:80, channel_tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }))}: mirrord_layer: exit
2022-09-26T18:12:07.853183Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}: mirrord_layer: enter
2022-09-26T18:12:07.853278Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}: mirrord_layer::outgoing::tcp: enter
2022-09-26T18:12:07.853332Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}: mirrord_layer::outgoing::tcp: Connect -> connect Ok(
DaemonConnect {
connection_id: 0,
remote_address: 127.0.0.1:80,
},
)
2022-09-26T18:12:07.853634Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:socket_detour{domain=2 type_=526337 protocol=0}: mirrord_layer::socket::hooks: enter
2022-09-26T18:12:07.853776Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:socket_detour{domain=2 type_=526337 protocol=0}: mirrord_layer::socket::hooks: exit
2022-09-26T18:12:07.853962Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:bind_detour{sockfd=13 address_length=16}: mirrord_layer::socket::hooks: enter
2022-09-26T18:12:07.854094Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:bind_detour{sockfd=13 address_length=16}: mirrord_layer::socket::hooks: exit
2022-09-26T18:12:07.854406Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:listen_detour{sockfd=13 backlog=1024}: mirrord_layer::socket::hooks: enter
2022-09-26T18:12:07.854490Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:listen_detour{sockfd=13 backlog=1024}: mirrord_layer::socket::hooks: exit
2022-09-26T18:12:07.854656Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:getsockname_detour{sockfd=13}: mirrord_layer::socket::hooks: enter
2022-09-26T18:12:07.854721Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}:getsockname_detour{sockfd=13}: mirrord_layer::socket::hooks: exit
2022-09-26T18:12:07.854934Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}:handle_daemon_message{response=Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 }))}: mirrord_layer::outgoing::tcp: exit
2022-09-26T18:12:07.855038Z TRACE ThreadId(08) handle_daemon_message{daemon_message=TcpOutgoing(Connect(Ok(DaemonConnect { connection_id: 0, remote_address: 127.0.0.1:80 })))}: mirrord_layer: exit
2022-09-26T18:12:07.855119Z TRACE ThreadId(01) connect_detour{sockfd=12 address_length=16}:connect{sockfd=12 remote_address=127.0.0.1:80}: mirrord_layer::socket::ops: exit
2022-09-26T18:12:07.855226Z TRACE ThreadId(01) connect_detour{sockfd=12 address_length=16}: mirrord_layer::socket::hooks: exit
(...)
2022-09-26T18:12:07.860805Z TRACE ThreadId(02) handle_daemon_message{daemon_message=TcpOutgoing(Read(Ok(DaemonRead { connection_id: 0, bytes (length): 270 })))}: mirrord_layer: exit
2022-09-26T18:12:07.860957Z TRACE ThreadId(02) interceptor_task{connection_id=0}: mirrord_layer::outgoing::tcp: enter
2022-09-26T18:12:07.861127Z TRACE ThreadId(02) interceptor_task{connection_id=0}: mirrord_layer::outgoing::tcp: exit
<html><body><h1>It works!</h1></body></html>
(...)
2022-09-26T18:12:07.862276Z INFO ThreadId(08) interceptor_task{connection_id=0}: mirrord_layer::outgoing::tcp: interceptor_task -> Stream 0 has no more data, closing!
(...)
-
kubectl get svc apache -o yaml:
kubectl get svc apache -o yaml
apiVersion: v1
kind: Service
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"apache"},"name":"apache","namespace":"default"},"spec":{"ports":[{"port":80,"protocol":"TCP","targetPort":80}],"selector":{"app":"apache"},"sessionAffinity":"None","type":"NodePort"}}
creationTimestamp: "2022-06-09T15:22:21Z"
labels:
app: apache
name: apache
namespace: default
resourceVersion: "111617"
uid: 973d629c-a084-4130-8882-b4e709e7f5eb
spec:
clusterIP: 10.106.72.164
clusterIPs:
- 10.106.72.164
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- nodePort: 30814
port: 80
protocol: TCP
targetPort: 80
selector:
app: apache
sessionAffinity: None
type: NodePort
status:
loadBalancer: {}
