[root]@[southpark][13:32:34][~]# mlxconfig -d 06:00.0 q
Device #1:
----------
Device type: ConnectX4
Name: MCX456A-FCA_Ax
Description: ConnectX-4 VPI adapter card; FDR IB (56Gb/s) and 40GbE; dual-port QSFP28; PCIe3.0 x16; ROHS R6
Device: 06:00.0
Configurations: Next Boot
PORT_OWNER True(1)
ALLOW_RD_COUNTERS True(1)
RENEG_ON_CHANGE True(1)
TRACER_ENABLE True(1)
FLEX_PARSER_PROFILE_ENABLE 0
FLEX_IPV4_OVER_VXLAN_PORT 0
SWITCH_COMPT_FEATURE_MASK 0x0(0)
NON_PREFETCHABLE_PF_BAR False(0)
VF_VPD_ENABLE False(0)
STRICT_VF_MSIX_NUM False(0)
VF_NODNIC_ENABLE False(0)
NUM_PF_MSIX_VALID True(1)
NUM_OF_VFS 8
NUM_OF_PF 2
FPP_EN True(1)
SRIOV_EN True(1)
PF_LOG_BAR_SIZE 5
VF_LOG_BAR_SIZE 1
NUM_PF_MSIX 63
NUM_VF_MSIX 11
PCIE_CREDIT_TOKEN_TIMEOUT 0
MAX_ACC_OUT_READ 0
PCI_DOWNSTREAM_PORT_OWNER Array[0..15]
PCI_WR_ORDERING per_mkey(0)
MULTI_PORT_VHCA_EN False(0)
LOG_DCR_HASH_TABLE_SIZE 14
MAX_PACKET_LIFETIME 0
DCR_LIFO_SIZE 16384
IB_PROTO_WIDTH_EN_MASK_P1 0
(same option for port 2)
KEEP_ETH_LINK_UP_P1 True(1)
KEEP_IB_LINK_UP_P1 False(0)
KEEP_LINK_UP_ON_BOOT_P1 False(0)
KEEP_LINK_UP_ON_STANDBY_P1 False(0)
DO_NOT_CLEAR_PORT_STATS_P1 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P1 False(0)
(same options for port 2)
LLDP_NB_DCBX_P1 False(0)
LLDP_NB_RX_MODE_P1 OFF(0)
LLDP_NB_TX_MODE_P1 OFF(0)
(same options for port 2)
DCBX_IEEE_P1 True(1)
DCBX_CEE_P1 True(1)
DCBX_WILLING_P1 True(1)
(same options for port 2)
MEMIC_BAR_SIZE 0
MEMIC_SIZE_LIMIT _256KB(1)
DUP_MAC_ACTION_P1 LAST_CFG(0)
SRIOV_IB_ROUTING_MODE_P1 LID(1)
IB_ROUTING_MODE_P1 LID(1)
(same options for port 2)
NUM_OF_PLANES_P1 0
LOAD_BALANCE_MODE_P1 DEVICE_DEFAULT(0)
(same options for port 2)
DYNAMIC_VF_MSIX_TABLE False(0)
EXP_ROM_UEFI_ARM_ENABLE False(0)
EXP_ROM_UEFI_x86_ENABLE True(1)
EXP_ROM_PXE_ENABLE True(1)
FORCE_ETH_PCI_SUBCLASS False(0)
ADVANCED_PCI_SETTINGS True(1)
TX_SCHEDULER_BURST 0
PARTIAL_RESET_EN False(0)
SW_RECOVERY_ON_ERRORS False(0)
RESET_WITH_HOST_ON_ERRORS False(0)
NUM_OF_VL_P1 _4_VLs(3)
NUM_OF_TC_P1 _8_TCs(0)
NUM_OF_PFC_P1 8
VL15_BUFFER_SIZE_P1 0
(same options for port 2)
ROCE_NEXT_PROTOCOL 254
ROCE_CC_PRIO_MASK_P1 255
ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0)
(same options for port 2)
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)
CLAMP_TGT_RATE_P1 False(0)
RPG_TIME_RESET_P1 300
RPG_BYTE_RESET_P1 32767
RPG_THRESHOLD_P1 1
RPG_MAX_RATE_P1 0
RPG_AI_RATE_P1 5
RPG_HAI_RATE_P1 50
RPG_GD_P1 11
RPG_MIN_DEC_FAC_P1 50
RPG_MIN_RATE_P1 1
RATE_TO_SET_ON_FIRST_CNP_P1 0
DCE_TCP_G_P1 1019
DCE_TCP_RTT_P1 1
RATE_REDUCE_MONITOR_PERIOD_P1 4
INITIAL_ALPHA_VALUE_P1 1023
MIN_TIME_BETWEEN_CNPS_P1 0
CNP_802P_PRIO_P1 6
CNP_DSCP_MODE_P1 DEVICE_DEFAULT(0)
CNP_DSCP_P1 48
(same options for port 2)
ROCE_RTT_RESP_DSCP_P1 0
ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0)
(same options for port 2)
BOOT_VLAN 1
LEGACY_BOOT_PROTOCOL ISCSI_WO_FAIL_PXE(5)
BOOT_INTERRUPT_DIS False(0)
BOOT_LACP_DIS True(1)
BOOT_VLAN_EN False(0)
IP_VER IPv4(0)
BOOT_UNDI_NETWORK_WAIT 0
BOOT_DBG_LOG False(0)
BOOT_PKEY 0
UEFI_HII_EN True(1)
UEFI_LOGS DISABLED(0)
CQE_COMPRESSION BALANCED(0)
IP_OVER_VXLAN_EN False(0)
MKEY_BY_NAME False(0)
UCTX_EN True(1)
PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
TUNNEL_ECN_COPY_DISABLE False(0)
LRO_LOG_TIMEOUT0 6
LRO_LOG_TIMEOUT1 7
LRO_LOG_TIMEOUT2 8
LRO_LOG_TIMEOUT3 13
INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)
TPH_STEERING_TAG_TABLE DEVICE_DEFAULT(0)
LINK_TYPE_P1 IB(1)
LINK_TYPE_P2 ETH(2)
SAFE_MODE_THRESHOLD 10
SAFE_MODE_ENABLE True(1)
(For info about Infiniband SR-IOV, also read the Connect-IB page.)
Create virtual functions:
[root]@[southpark][13:49:01][/sys/bus/pci/devices/0000:06:00.1]# echo 0 > sriov_drivers_autoprobe
[root]@[southpark][13:49:26][/sys/bus/pci/devices/0000:06:00.1]# echo 8 > sriov_numvfs
[root]@[southpark][13:49:44][/sys/bus/pci/devices/0000:06:00.1]# ip link
(...snip...)
8: mlx1: <BROADCAST,MULTICAST> mtu 4042 qdisc mq master green state DOWN mode DEFAULT group default qlen 1000
link/ether b8:ce:f6:4e:bc:87 brd ff:ff:ff:ff:ff:ff
vf 0 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 1 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 2 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 3 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 4 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 5 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 6 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
vf 7 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
(...snip...)
This adapter helpfully creates new PCI devices for the VFs instead of adding more functions to device 0.
[root]@[southpark][13:49:47][/sys/bus/pci/devices/0000:06:00.1]# lspci | grep Mel 06:00.0 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4] 06:00.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4] 06:01.2 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:01.3 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:01.4 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:01.5 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:01.6 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:01.7 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:02.0 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 06:02.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
Configure a VF; this one is for the Factorio server:
[root]@[southpark][14:26:26][~]# ip link set mlx1 vf 0 mac 00:02:c8:00:00:38 vlan 3
[root]@[southpark][14:26:26][~]# ip link
vf 0 link/ether 00:02:c8:00:00:38 brd ff:ff:ff:ff:ff:ff, vlan 3, spoof checking off, link-state auto, trust off, query_rss off
vf 1 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
(Notice vlan wasn't there before, but it's there now.)
The SR-IOV network pool thing in virt-manager still doesn't work. It says the PF isn't configured. I haven't dug into it at all. You have to attach the PCIe host device for the VF you want to use. Here's the working XML that virt-manager generated for factorio:
<hostdev mode="subsystem" type="pci" managed="yes">
<driver name="vfio"/>
<source>
<address domain="0x0000" bus="0x06" slot="0x01" function="0x2"/>
</source>
<alias name="hostdev0"/>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</hostdev>
There's a sysfs interface for managing driver overrides, or you can use driverctl, which is not installed by default.
Warning: `driverctl list-devices` lists all the devices on the PCI bus: the root complex, the ports, the fabric functions, anything that looks like a bridge… on southpark, it's ~2 pages of output.
Strongly recommend grepping the output for the relevant PCI devices with a start-of-line anchor and the PCI domain:
# driverctl list-devices | grep ^0000:06:0 0000:06:00.0 mlx5_core 0000:06:00.1 mlx5_core 0000:06:01.2 vfio-pci [*] 0000:06:01.3 mlx5_core [*] 0000:06:01.4 (none) 0000:06:01.5 (none) 0000:06:01.6 (none) 0000:06:01.7 (none) 0000:06:02.0 (none) 0000:06:02.1 (none)
Configure the VF as above:
[root]@[southpark][14:26:29][~]# ip link set mlx1 vf 1 mac 00:10:18:f8:02:22 vlan 3
[root]@[southpark][14:34:24][~]# ip link
8: mlx1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 4042 qdisc mq master green state UP mode DEFAULT group default qlen 1000
link/ether b8:ce:f6:4e:bc:87 brd ff:ff:ff:ff:ff:ff
vf 0 link/ether 00:02:c8:00:00:38 brd ff:ff:ff:ff:ff:ff, vlan 3, spoof checking off, link-state auto, trust off, query_rss off
vf 1 link/ether 00:10:18:f8:02:22 brd ff:ff:ff:ff:ff:ff, vlan 3, spoof checking off, link-state auto, trust off, query_rss off
vf 2 link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
Bind the driver to mlx5_core:
[root]@[southpark][14:36:01][/sys/bus/pci/devices/0000:06:00.1]# driverctl set-override 0000:06:01.3 mlx5_core [root]@[southpark][14:55:00][/sys/bus/pci/devices/0000:06:00.1]## driverctl list-devices | grep ^0000:06:0 0000:06:00.0 mlx5_core 0000:06:00.1 mlx5_core 0000:06:01.2 vfio-pci [*] 0000:06:01.3 mlx5_core [*] (...snip...)
A new network device will appear in `ip link`. Here I have already connected it to green and turned it on:
21: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 4042 qdisc mq master green state UP mode DEFAULT group default qlen 1000 link/ether 00:10:18:f8:02:22 brd ff:ff:ff:ff:ff:ff
Right now I only have one ConnectX-4: