Naptastic

This page is read only. You can view the source, but not change it. Ask your administrator if you think this is wrong.
====Firmware Configuration====
  [root]@[southpark][13:32:34][~]# mlxconfig -d 06:00.0 q
  
  Device #1:
  ----------
  
  Device type:        ConnectX4           
  Name:               MCX456A-FCA_Ax      
  Description:        ConnectX-4 VPI adapter card; FDR IB (56Gb/s) and 40GbE; dual-port QSFP28; PCIe3.0 x16; ROHS R6
  Device:             06:00.0             
  
  Configurations:                                          Next Boot
        PORT_OWNER                                  True(1)             
        ALLOW_RD_COUNTERS                           True(1)             
        RENEG_ON_CHANGE                             True(1)             
        TRACER_ENABLE                               True(1)             
        FLEX_PARSER_PROFILE_ENABLE                  0                   
        FLEX_IPV4_OVER_VXLAN_PORT                   0                   
        SWITCH_COMPT_FEATURE_MASK                   0x0(0)              
        NON_PREFETCHABLE_PF_BAR                     False(0)            
        VF_VPD_ENABLE                               False(0)            
        STRICT_VF_MSIX_NUM                          False(0)            
        VF_NODNIC_ENABLE                            False(0)            
        NUM_PF_MSIX_VALID                           True(1)             
        NUM_OF_VFS                                  8                   
        NUM_OF_PF                                   2                   
        FPP_EN                                      True(1)             
        SRIOV_EN                                    True(1)             
        PF_LOG_BAR_SIZE                             5                   
        VF_LOG_BAR_SIZE                             1                   
        NUM_PF_MSIX                                 63                  
        NUM_VF_MSIX                                 11                  
        PCIE_CREDIT_TOKEN_TIMEOUT                   0                   
        MAX_ACC_OUT_READ                            0                   
        PCI_DOWNSTREAM_PORT_OWNER                   Array[0..15]        
        PCI_WR_ORDERING                             per_mkey(0)         
        MULTI_PORT_VHCA_EN                          False(0)            
        LOG_DCR_HASH_TABLE_SIZE                     14                  
        MAX_PACKET_LIFETIME                         0                   
        DCR_LIFO_SIZE                               16384               
        IB_PROTO_WIDTH_EN_MASK_P1                   0                   
    (same option for port 2)
        KEEP_ETH_LINK_UP_P1                         True(1)             
        KEEP_IB_LINK_UP_P1                          False(0)            
        KEEP_LINK_UP_ON_BOOT_P1                     False(0)            
        KEEP_LINK_UP_ON_STANDBY_P1                  False(0)            
        DO_NOT_CLEAR_PORT_STATS_P1                  False(0)            
        AUTO_POWER_SAVE_LINK_DOWN_P1                False(0)            
    (same options for port 2)
        LLDP_NB_DCBX_P1                             False(0)            
        LLDP_NB_RX_MODE_P1                          OFF(0)              
        LLDP_NB_TX_MODE_P1                          OFF(0)              
    (same options for port 2)
        DCBX_IEEE_P1                                True(1)             
        DCBX_CEE_P1                                 True(1)             
        DCBX_WILLING_P1                             True(1)             
    (same options for port 2)
        MEMIC_BAR_SIZE                              0                   
        MEMIC_SIZE_LIMIT                            _256KB(1)           
        DUP_MAC_ACTION_P1                           LAST_CFG(0)         
        SRIOV_IB_ROUTING_MODE_P1                    LID(1)              
        IB_ROUTING_MODE_P1                          LID(1)              
    (same options for port 2)
        NUM_OF_PLANES_P1                            0                   
        LOAD_BALANCE_MODE_P1                        DEVICE_DEFAULT(0)   
    (same options for port 2)
        DYNAMIC_VF_MSIX_TABLE                       False(0)            
        EXP_ROM_UEFI_ARM_ENABLE                     False(0)            
        EXP_ROM_UEFI_x86_ENABLE                     True(1)             
        EXP_ROM_PXE_ENABLE                          True(1)             
        FORCE_ETH_PCI_SUBCLASS                      False(0)            
        ADVANCED_PCI_SETTINGS                       True(1)             
        TX_SCHEDULER_BURST                          0                   
        PARTIAL_RESET_EN                            False(0)            
        SW_RECOVERY_ON_ERRORS                       False(0)            
        RESET_WITH_HOST_ON_ERRORS                   False(0)            
        NUM_OF_VL_P1                                _4_VLs(3)           
        NUM_OF_TC_P1                                _8_TCs(0)           
        NUM_OF_PFC_P1                               8                   
        VL15_BUFFER_SIZE_P1                         0                   
    (same options for port 2)
        ROCE_NEXT_PROTOCOL                          254                 
        ROCE_CC_PRIO_MASK_P1                        255                 
        ROCE_CC_CNP_MODERATION_P1                   DEVICE_DEFAULT(0)   
    (same options for port 2)
        CLAMP_TGT_RATE_AFTER_TIME_INC_P1            True(1)             
        CLAMP_TGT_RATE_P1                           False(0)            
        RPG_TIME_RESET_P1                           300                 
        RPG_BYTE_RESET_P1                           32767               
        RPG_THRESHOLD_P1                            1                   
        RPG_MAX_RATE_P1                             0                   
        RPG_AI_RATE_P1                              5                   
        RPG_HAI_RATE_P1                             50                  
        RPG_GD_P1                                   11                  
        RPG_MIN_DEC_FAC_P1                          50                  
        RPG_MIN_RATE_P1                             1                   
        RATE_TO_SET_ON_FIRST_CNP_P1                 0                   
        DCE_TCP_G_P1                                1019                
        DCE_TCP_RTT_P1                              1                   
        RATE_REDUCE_MONITOR_PERIOD_P1               4                   
        INITIAL_ALPHA_VALUE_P1                      1023                
        MIN_TIME_BETWEEN_CNPS_P1                    0                   
        CNP_802P_PRIO_P1                            6                   
        CNP_DSCP_MODE_P1                            DEVICE_DEFAULT(0)   
        CNP_DSCP_P1                                 48                  
    (same options for port 2)
        ROCE_RTT_RESP_DSCP_P1                       0                   
        ROCE_RTT_RESP_DSCP_MODE_P1                  DEVICE_DEFAULT(0)   
    (same options for port 2)
        BOOT_VLAN                                   1                   
        LEGACY_BOOT_PROTOCOL                        ISCSI_WO_FAIL_PXE(5)
        BOOT_INTERRUPT_DIS                          False(0)            
        BOOT_LACP_DIS                               True(1)             
        BOOT_VLAN_EN                                False(0)            
        IP_VER                                      IPv4(0)             
        BOOT_UNDI_NETWORK_WAIT                      0                   
        BOOT_DBG_LOG                                False(0)            
        BOOT_PKEY                                   0                   
        UEFI_HII_EN                                 True(1)             
        UEFI_LOGS                                   DISABLED(0)         
        CQE_COMPRESSION                             BALANCED(0)         
        IP_OVER_VXLAN_EN                            False(0)            
        MKEY_BY_NAME                                False(0)            
        UCTX_EN                                     True(1)             
        PCI_ATOMIC_MODE                             PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
        TUNNEL_ECN_COPY_DISABLE                     False(0)            
        LRO_LOG_TIMEOUT0                            6                   
        LRO_LOG_TIMEOUT1                            7                   
        LRO_LOG_TIMEOUT2                            8                   
        LRO_LOG_TIMEOUT3                            13                  
        INT_LOG_MAX_PAYLOAD_SIZE                    AUTOMATIC(0)        
        TPH_STEERING_TAG_TABLE                      DEVICE_DEFAULT(0)   
        LINK_TYPE_P1                                IB(1)               
        LINK_TYPE_P2                                ETH(2)              
        SAFE_MODE_THRESHOLD                         10                  
        SAFE_MODE_ENABLE                            True(1)             



====SR-IOV====
(For info about Infiniband SR-IOV, also read the [[nndocs::infiniband::Connect-IB]] page.)

Create virtual functions:

  [root]@[southpark][13:49:01][/sys/bus/pci/devices/0000:06:00.1]# echo 0 > sriov_drivers_autoprobe 
  [root]@[southpark][13:49:26][/sys/bus/pci/devices/0000:06:00.1]# echo 8 > sriov_numvfs 
  
  [root]@[southpark][13:49:44][/sys/bus/pci/devices/0000:06:00.1]# ip link
  (...snip...)
  8: mlx1: <BROADCAST,MULTICAST> mtu 4042 qdisc mq master green state DOWN mode DEFAULT group default qlen 1000
      link/ether b8:ce:f6:4e:bc:87 brd ff:ff:ff:ff:ff:ff
      vf 0     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 1     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 2     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 3     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 4     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 5     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 6     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
      vf 7     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off
  (...snip...)

This adapter helpfully creates new PCI devices for the VFs instead of adding more functions to device 0.

  [root]@[southpark][13:49:47][/sys/bus/pci/devices/0000:06:00.1]# lspci | grep Mel
  06:00.0 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4]
  06:00.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
  06:01.2 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:01.3 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:01.4 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:01.5 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:01.6 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:01.7 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:02.0 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
  06:02.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]

Configure a VF; this one is for the Factorio server:

  [root]@[southpark][14:26:26][~]# ip link set mlx1 vf 0 mac 00:02:c8:00:00:38 vlan 3
  [root]@[southpark][14:26:26][~]# ip link
        vf 0     link/ether 00:02:c8:00:00:38 brd ff:ff:ff:ff:ff:ff, vlan 3, spoof checking off, link-state auto, trust off, query_rss off
        vf 1     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off

(Notice vlan wasn't there before, but it's there now.)

===For a virtual machine===

The SR-IOV network pool thing in virt-manager still doesn't work. It says the PF isn't configured. I haven't dug into it at all. You have to attach the PCIe host device for the VF you want to use. Here's the working XML that virt-manager generated for factorio:

  <hostdev mode="subsystem" type="pci" managed="yes">
    <driver name="vfio"/>
    <source>
      <address domain="0x0000" bus="0x06" slot="0x01" function="0x2"/>
    </source>
    <alias name="hostdev0"/>
    <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
  </hostdev>

===For use by the host===

There's a sysfs interface for managing driver overrides, or you can use driverctl, which is not installed by default.

Warning: `driverctl list-devices` lists **all** the devices on the PCI bus: the root complex, the ports, the fabric functions, anything that looks like a bridge... on southpark, it's ~2 pages of output.

Strongly recommend grepping the output for the relevant PCI devices with a start-of-line anchor and the PCI domain:

  # driverctl list-devices | grep ^0000:06:0
  0000:06:00.0 mlx5_core
  0000:06:00.1 mlx5_core
  0000:06:01.2 vfio-pci [*]
  0000:06:01.3 mlx5_core [*]
  0000:06:01.4 (none)
  0000:06:01.5 (none)
  0000:06:01.6 (none)
  0000:06:01.7 (none)
  0000:06:02.0 (none)
  0000:06:02.1 (none)

Configure the VF as above:

  [root]@[southpark][14:26:29][~]# ip link set mlx1 vf 1 mac 00:10:18:f8:02:22 vlan 3
  [root]@[southpark][14:34:24][~]# ip link
  8: mlx1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 4042 qdisc mq master green state UP mode DEFAULT group default qlen 1000
      link/ether b8:ce:f6:4e:bc:87 brd ff:ff:ff:ff:ff:ff
      vf 0     link/ether 00:02:c8:00:00:38 brd ff:ff:ff:ff:ff:ff, vlan 3, spoof checking off, link-state auto, trust off, query_rss off
      vf 1     link/ether 00:10:18:f8:02:22 brd ff:ff:ff:ff:ff:ff, vlan 3, spoof checking off, link-state auto, trust off, query_rss off
      vf 2     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff, spoof checking off, link-state auto, trust off, query_rss off

Bind the driver to mlx5_core:

  [root]@[southpark][14:36:01][/sys/bus/pci/devices/0000:06:00.1]# driverctl set-override 0000:06:01.3 mlx5_core
  
  [root]@[southpark][14:55:00][/sys/bus/pci/devices/0000:06:00.1]## driverctl list-devices | grep ^0000:06:0
  0000:06:00.0 mlx5_core
  0000:06:00.1 mlx5_core
  0000:06:01.2 vfio-pci [*]
  0000:06:01.3 mlx5_core [*]
  (...snip...)

A new network device will appear in `ip link`. Here I have already connected it to green and turned it on:

  21: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 4042 qdisc mq master green state UP mode DEFAULT group default qlen 1000
    link/ether 00:10:18:f8:02:22 brd ff:ff:ff:ff:ff:ff

====GUIDs====
Right now I only have one ConnectX-4:
  * b8cef603004ebc86 - CX456A-FCAT installed in southpark