k8s-rdma-shared-dev-plugin
k8s-rdma-shared-dev-plugin copied to clipboard
Mellanox
ubuntu server can see all GID list
I run ubuntu pod in the cluster，in the container，i can see all gid list。
root@aigc-maas-instance-leader-0:/# ibv_devinfo -d mlx5_0 -v hca_id: mlx5_0 transport: InfiniBand (0) fw_ver: 14.29.1016 node_guid: c4c0:6300:00b7:5e76 sys_image_guid: c4c0:6300:00b7:5e76 vendor_id: 0x02c9 vendor_part_id: 4117 hw_ver: 0x0 board_id: H3C0010110034 phys_port_cnt: 1 max_mr_size: 0xffffffffffffffff page_size_cap: 0xfffffffffffff000 max_qp: 262144 max_qp_wr: 32768 device_cap_flags: 0x25321c36 BAD_PKEY_CNTR BAD_QKEY_CNTR AUTO_PATH_MIG CHANGE_PHY_PORT PORT_ACTIVE_EVENT SYS_IMAGE_GUID RC_RNR_NAK_GEN MEM_WINDOW XRC MEM_MGT_EXTENSIONS MEM_WINDOW_TYPE_2B RAW_IP_CSUM MANAGED_FLOW_STEERING max_sge: 30 max_sge_rd: 30 max_cq: 16777216 max_cqe: 4194303 max_mr: 16777216 max_pd: 16777216 max_qp_rd_atom: 16 max_ee_rd_atom: 0 max_res_rd_atom: 4194304 max_qp_init_rd_atom: 16 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_HCA (1) max_ee: 0 max_rdd: 0 max_mw: 16777216 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 0 max_mcast_grp: 2097152 max_mcast_qp_attach: 240 max_total_mcast_qp_attach: 503316480 max_ah: 2147483647 max_fmr: 0 max_srq: 8388608 max_srq_wr: 32767 max_srq_sge: 31 max_pkeys: 128 local_ca_ack_delay: 16 general_odp_caps: ODP_SUPPORT ODP_SUPPORT_IMPLICIT rc_odp_caps: SUPPORT_SEND SUPPORT_RECV SUPPORT_WRITE SUPPORT_READ SUPPORT_SRQ uc_odp_caps: NO SUPPORT ud_odp_caps: SUPPORT_SEND xrc_odp_caps: SUPPORT_SEND SUPPORT_WRITE SUPPORT_READ SUPPORT_SRQ completion timestamp_mask: 0x7fffffffffffffff hca_core_clock: 156250kHZ raw packet caps: C-VLAN stripping offload Scatter FCS offload IP csum offload Delay drop device_cap_flags_ex: 0x2000001425321C36 RAW_SCATTER_FCS PCI_WRITE_END_PADDING Unknown flags: 0x2000000000000000 tso_caps: max_tso: 262144 supported_qp: SUPPORT_RAW_PACKET rss_caps: max_rwq_indirection_tables: 1048576 max_rwq_indirection_table_size: 2048 rx_hash_function: 0x1 rx_hash_fields_mask: 0x800000FF supported_qp: SUPPORT_RAW_PACKET max_wq_type_rq: 8388608 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported
cq moderation caps:
max_cq_count: 65535
max_cq_period: 4095 us
num_comp_vectors: 63
port: 1
state: PORT_ACTIVE (4)
max_mtu: 4096 (5)
active_mtu: 4096 (5)
sm_lid: 0
port_lid: 0
port_lmc: 0x00
link_layer: Ethernet
max_msg_sz: 0x40000000
port_cap_flags: 0x04010000
port_cap_flags2: 0x0000
max_vl_num: invalid value (0)
bad_pkey_cntr: 0x0
qkey_viol_cntr: 0x0
sm_sl: 0
pkey_tbl_len: 1
gid_tbl_len: 256
subnet_timeout: 0
init_type_reply: 0
active_width: 1X (1)
active_speed: 25.0 Gbps (32)
phys_state: LINK_UP (5)
**GID[ 0]: fe80:0000:0000:0000:c6c0:63ff:feb7:5e76, RoCE v1
GID[ 1]: fe80::c6c0:63ff:feb7:5e76, RoCE v2
GID[ 2]: 0000:0000:0000:0000:0000:ffff:c0a8:0603, RoCE v1
GID[ 3]: ::ffff:192.168.6.3, RoCE v2
GID[ 4]: fe80:0000:0000:0000:5c3a:85ff:fe96:3691, RoCE v1
GID[ 5]: fe80::5c3a:85ff:fe96:3691, RoCE v2
GID[ 6]: 0000:0000:0000:0000:0000:ffff:c0a8:0604, RoCE v1
GID[ 7]: ::ffff:192.168.6.4, RoCE v2
GID[ 8]: fe80:0000:0000:0000:5c1f:b7ff:fe4a:6ea7, RoCE v1
GID[ 9]: fe80::5c1f:b7ff:fe4a:6ea7, RoCE v2
GID[ 10]: 0000:0000:0000:0000:0000:ffff:c0a8:0605, RoCE v1
GID[ 11]: ::ffff:192.168.6.5, RoCE v2
GID[ 12]: fe80:0000:0000:0000:b007:89ff:fe65:cd91, RoCE v1
GID[ 13]: fe80::b007:89ff:fe65:cd91, RoCE v2
GID[ 14]: 0000:0000:0000:0000:0000:ffff:c0a8:0606, RoCE v1
GID[ 15]: ::ffff:192.168.6.6, RoCE v2
GID[ 16]: fe80:0000:0000:0000:4cbe:9aff:feeb:6b0f, RoCE v1
GID[ 17]: fe80::4cbe:9aff:feeb:6b0f, RoCE v2
GID[ 18]: 0000:0000:0000:0000:0000:ffff:0303:0303, RoCE v1
GID[ 19]: ::ffff:3.3.3.3, RoCE v2**
root@aigc-maas-instance-leader-0:/# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: net1@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default link/ether b2:07:89:65:cd:91 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 192.168.6.5/24 brd 192.168.6.255 scope global net1 valid_lft forever preferred_lft forever inet6 fe80::b007:89ff:fe65:cd91/64 scope link valid_lft forever preferred_lft forever 26696: eth0@if26697: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default link/ether aa:9a:d6:60:7b:96 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 110.64.115.181/10 brd 110.127.255.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::a89a:d6ff:fe60:7b96/64 scope link valid_lft forever preferred_lft forever
root@aigc-maas-instance-leader-0:/# cat /etc/os-release PRETTY_NAME="Ubuntu 22.04.3 LTS" NAME="Ubuntu" VERSION_ID="22.04" VERSION="22.04.3 LTS (Jammy Jellyfish)" VERSION_CODENAME=jammy ID=ubuntu ID_LIKE=debian HOME_URL="https://www.ubuntu.com/" SUPPORT_URL="https://help.ubuntu.com/" BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" UBUNTU_CODENAME=jammy
if i run the pod with image “mellanox/mofed:23.10-1.1.9.0-ubuntu22.04-amd64”，only what belongs to it will be displayed，of the two k8s yaml are the same。
[root@mellanox-my-test1-7db4744f55-vpvzn /]# ibv_devinfo -d mlx5_0 -v hca_id: mlx5_0 transport: InfiniBand (0) fw_ver: 14.29.1016 node_guid: c4c0:6300:00b7:5e76 sys_image_guid: c4c0:6300:00b7:5e76 vendor_id: 0x02c9 vendor_part_id: 4117 hw_ver: 0x0 board_id: H3C0010110034 phys_port_cnt: 1 max_mr_size: 0xffffffffffffffff page_size_cap: 0xfffffffffffff000 max_qp: 262144 max_qp_wr: 32768 device_cap_flags: 0x25321c36 BAD_PKEY_CNTR BAD_QKEY_CNTR AUTO_PATH_MIG CHANGE_PHY_PORT PORT_ACTIVE_EVENT SYS_IMAGE_GUID RC_RNR_NAK_GEN MEM_WINDOW XRC MEM_MGT_EXTENSIONS MEM_WINDOW_TYPE_2B RAW_IP_CSUM MANAGED_FLOW_STEERING max_sge: 30 max_sge_rd: 30 max_cq: 16777216 max_cqe: 4194303 max_mr: 16777216 max_pd: 16777216 max_qp_rd_atom: 16 max_ee_rd_atom: 0 max_res_rd_atom: 4194304 max_qp_init_rd_atom: 16 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_HCA (1) max_ee: 0 max_rdd: 0 max_mw: 16777216 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 0 max_mcast_grp: 2097152 max_mcast_qp_attach: 240 max_total_mcast_qp_attach: 503316480 max_ah: 2147483647 max_fmr: 0 max_srq: 8388608 max_srq_wr: 32767 max_srq_sge: 31 max_pkeys: 128 local_ca_ack_delay: 16 general_odp_caps: ODP_SUPPORT Unknown flags: 0x2 rc_odp_caps: SUPPORT_SEND SUPPORT_RECV SUPPORT_WRITE SUPPORT_READ Unknown flags: 0x20 uc_odp_caps: NO SUPPORT ud_odp_caps: SUPPORT_SEND completion timestamp_mask: 0x7fffffffffffffff hca_core_clock: 156250kHZ raw packet caps: C-VLAN stripping offload Scatter FCS offload IP csum offload Delay drop device_cap_flags_ex: 0x2000001425321C36 RAW_SCATTER_FCS PCI_WRITE_END_PADDING Unknown flags: 0x2000000000000000 tso_caps: max_tso: 262144 supported_qp: SUPPORT_RAW_PACKET rss_caps: max_rwq_indirection_tables: 1048576 max_rwq_indirection_table_size: 2048 rx_hash_function: 0x1 rx_hash_fields_mask: 0x800000FF supported_qp: SUPPORT_RAW_PACKET max_wq_type_rq: 8388608 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported
cq moderation caps:
max_cq_count: 65535
max_cq_period: 4095 us
port: 1
state: PORT_ACTIVE (4)
max_mtu: 4096 (5)
active_mtu: 4096 (5)
sm_lid: 0
port_lid: 0
port_lmc: 0x00
link_layer: Ethernet
max_msg_sz: 0x40000000
port_cap_flags: 0x04010000
port_cap_flags2: 0x0000
max_vl_num: invalid value (0)
bad_pkey_cntr: 0x0
qkey_viol_cntr: 0x0
sm_sl: 0
pkey_tbl_len: 1
gid_tbl_len: 256
subnet_timeout: 0
init_type_reply: 0
active_width: 1X (1)
active_speed: 25.0 Gbps (32)
phys_state: LINK_UP (5)
**GID[ 14]: 0000:0000:0000:0000:0000:ffff:c0a8:0606
GID[ 15]: 0000:0000:0000:0000:0000:ffff:c0a8:0606
GID[ 16]: fe80:0000:0000:0000:4cbe:9aff:feeb:6b0f
GID[ 17]: fe80:0000:0000:0000:4cbe:9aff:feeb:6b0f**
[root@mellanox-my-test1-7db4744f55-vpvzn /]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: net1@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default link/ether 4e:be:9a:eb:6b:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 192.168.6.6/24 brd 192.168.6.255 scope global net1 valid_lft forever preferred_lft forever inet6 fe80::4cbe:9aff:feeb:6b0f/64 scope link valid_lft forever preferred_lft forever 3: net2@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default link/ether 72:7a:e9:65:3a:e5 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 192.168.7.4/24 brd 192.168.7.255 scope global net2 valid_lft forever preferred_lft forever inet6 fe80::707a:e9ff:fe65:3ae5/64 scope link valid_lft forever preferred_lft forever 4: net3@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default link/ether 26:bf:21:2f:74:d6 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 192.168.8.4/24 brd 192.168.8.255 scope global net3 valid_lft forever preferred_lft forever inet6 fe80::24bf:21ff:fe2f:74d6/64 scope link valid_lft forever preferred_lft forever 5: net4@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default link/ether 3e:16:f0:d5:59:6d brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 192.168.9.4/24 brd 192.168.9.255 scope global net4 valid_lft forever preferred_lft forever inet6 fe80::3c16:f0ff:fed5:596d/64 scope link valid_lft forever preferred_lft forever 2915: eth0@if2916: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default link/ether 5a:1f:3d:f6:4b:a1 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 110.64.64.191/10 brd 110.127.255.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::581f:3dff:fef6:4ba1/64 scope link valid_lft forever preferred_lft forever [root@mellanox-my-test1-7db4744f55-vpvzn /]# cat /etc/os-release NAME="CentOS Linux" VERSION="7 (Core)" ID="centos" ID_LIKE="rhel fedora" VERSION_ID="7" PRETTY_NAME="CentOS Linux 7 (Core)" ANSI_COLOR="0;31" CPE_NAME="cpe:/o:centos:centos:7" HOME_URL="https://www.centos.org/" BUG_REPORT_URL="https://bugs.centos.org/"
CENTOS_MANTISBT_PROJECT="CentOS-7" CENTOS_MANTISBT_PROJECT_VERSION="7" REDHAT_SUPPORT_PRODUCT="centos" REDHAT_SUPPORT_PRODUCT_VERSION="7"
