332 lines
9.4 KiB
Go
332 lines
9.4 KiB
Go
package netlink
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"net"
|
|
|
|
"github.com/vishvananda/netlink/nl"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// LinkAttrs represents data shared by most link types
|
|
type RdmaLinkAttrs struct {
|
|
Index uint32
|
|
Name string
|
|
FirmwareVersion string
|
|
NodeGuid string
|
|
SysImageGuid string
|
|
}
|
|
|
|
// Link represents a rdma device from netlink.
|
|
type RdmaLink struct {
|
|
Attrs RdmaLinkAttrs
|
|
}
|
|
|
|
func getProtoField(clientType int, op int) int {
|
|
return ((clientType << nl.RDMA_NL_GET_CLIENT_SHIFT) | op)
|
|
}
|
|
|
|
func uint64ToGuidString(guid uint64) string {
|
|
//Convert to byte array
|
|
sysGuidBytes := new(bytes.Buffer)
|
|
binary.Write(sysGuidBytes, binary.LittleEndian, guid)
|
|
|
|
//Convert to HardwareAddr
|
|
sysGuidNet := net.HardwareAddr(sysGuidBytes.Bytes())
|
|
|
|
//Get the String
|
|
return sysGuidNet.String()
|
|
}
|
|
|
|
func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) {
|
|
|
|
link := RdmaLink{}
|
|
|
|
reader := bytes.NewReader(data)
|
|
for reader.Len() >= 4 {
|
|
_, attrType, len, value := parseNfAttrTLV(reader)
|
|
|
|
switch attrType {
|
|
case nl.RDMA_NLDEV_ATTR_DEV_INDEX:
|
|
var Index uint32
|
|
r := bytes.NewReader(value)
|
|
binary.Read(r, nl.NativeEndian(), &Index)
|
|
link.Attrs.Index = Index
|
|
case nl.RDMA_NLDEV_ATTR_DEV_NAME:
|
|
link.Attrs.Name = string(value[0 : len-1])
|
|
case nl.RDMA_NLDEV_ATTR_FW_VERSION:
|
|
link.Attrs.FirmwareVersion = string(value[0 : len-1])
|
|
case nl.RDMA_NLDEV_ATTR_NODE_GUID:
|
|
var guid uint64
|
|
r := bytes.NewReader(value)
|
|
binary.Read(r, nl.NativeEndian(), &guid)
|
|
link.Attrs.NodeGuid = uint64ToGuidString(guid)
|
|
case nl.RDMA_NLDEV_ATTR_SYS_IMAGE_GUID:
|
|
var sysGuid uint64
|
|
r := bytes.NewReader(value)
|
|
binary.Read(r, nl.NativeEndian(), &sysGuid)
|
|
link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid)
|
|
}
|
|
if (len % 4) != 0 {
|
|
// Skip pad bytes
|
|
reader.Seek(int64(4-(len%4)), seekCurrent)
|
|
}
|
|
}
|
|
return &link, nil
|
|
}
|
|
|
|
func execRdmaSetLink(req *nl.NetlinkRequest) error {
|
|
|
|
_, err := req.Execute(unix.NETLINK_RDMA, 0)
|
|
return err
|
|
}
|
|
|
|
// RdmaLinkList gets a list of RDMA link devices.
|
|
// Equivalent to: `rdma dev show`
|
|
func RdmaLinkList() ([]*RdmaLink, error) {
|
|
return pkgHandle.RdmaLinkList()
|
|
}
|
|
|
|
// RdmaLinkList gets a list of RDMA link devices.
|
|
// Equivalent to: `rdma dev show`
|
|
func (h *Handle) RdmaLinkList() ([]*RdmaLink, error) {
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_GET)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP)
|
|
|
|
msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var res []*RdmaLink
|
|
for _, m := range msgs {
|
|
link, err := executeOneGetRdmaLink(m)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res = append(res, link)
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// RdmaLinkByName finds a link by name and returns a pointer to the object if
|
|
// found and nil error, otherwise returns error code.
|
|
func RdmaLinkByName(name string) (*RdmaLink, error) {
|
|
return pkgHandle.RdmaLinkByName(name)
|
|
}
|
|
|
|
// RdmaLinkByName finds a link by name and returns a pointer to the object if
|
|
// found and nil error, otherwise returns error code.
|
|
func (h *Handle) RdmaLinkByName(name string) (*RdmaLink, error) {
|
|
links, err := h.RdmaLinkList()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, link := range links {
|
|
if link.Attrs.Name == name {
|
|
return link, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("Rdma device %v not found", name)
|
|
}
|
|
|
|
// RdmaLinkSetName sets the name of the rdma link device. Return nil on success
|
|
// or error otherwise.
|
|
// Equivalent to: `rdma dev set $old_devname name $name`
|
|
func RdmaLinkSetName(link *RdmaLink, name string) error {
|
|
return pkgHandle.RdmaLinkSetName(link, name)
|
|
}
|
|
|
|
// RdmaLinkSetName sets the name of the rdma link device. Return nil on success
|
|
// or error otherwise.
|
|
// Equivalent to: `rdma dev set $old_devname name $name`
|
|
func (h *Handle) RdmaLinkSetName(link *RdmaLink, name string) error {
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SET)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK)
|
|
|
|
b := make([]byte, 4)
|
|
native.PutUint32(b, uint32(link.Attrs.Index))
|
|
data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b)
|
|
req.AddData(data)
|
|
|
|
b = make([]byte, len(name)+1)
|
|
copy(b, name)
|
|
data = nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_NAME, b)
|
|
req.AddData(data)
|
|
|
|
return execRdmaSetLink(req)
|
|
}
|
|
|
|
func netnsModeToString(mode uint8) string {
|
|
switch mode {
|
|
case 0:
|
|
return "exclusive"
|
|
case 1:
|
|
return "shared"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func executeOneGetRdmaNetnsMode(data []byte) (string, error) {
|
|
reader := bytes.NewReader(data)
|
|
for reader.Len() >= 4 {
|
|
_, attrType, len, value := parseNfAttrTLV(reader)
|
|
|
|
switch attrType {
|
|
case nl.RDMA_NLDEV_SYS_ATTR_NETNS_MODE:
|
|
var mode uint8
|
|
r := bytes.NewReader(value)
|
|
binary.Read(r, nl.NativeEndian(), &mode)
|
|
return netnsModeToString(mode), nil
|
|
}
|
|
if (len % 4) != 0 {
|
|
// Skip pad bytes
|
|
reader.Seek(int64(4-(len%4)), seekCurrent)
|
|
}
|
|
}
|
|
return "", fmt.Errorf("Invalid netns mode")
|
|
}
|
|
|
|
// RdmaSystemGetNetnsMode gets the net namespace mode for RDMA subsystem
|
|
// Returns mode string and error status as nil on success or returns error
|
|
// otherwise.
|
|
// Equivalent to: `rdma system show netns'
|
|
func RdmaSystemGetNetnsMode() (string, error) {
|
|
return pkgHandle.RdmaSystemGetNetnsMode()
|
|
}
|
|
|
|
// RdmaSystemGetNetnsMode gets the net namespace mode for RDMA subsystem
|
|
// Returns mode string and error status as nil on success or returns error
|
|
// otherwise.
|
|
// Equivalent to: `rdma system show netns'
|
|
func (h *Handle) RdmaSystemGetNetnsMode() (string, error) {
|
|
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SYS_GET)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK)
|
|
|
|
msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(msgs) == 0 {
|
|
return "", fmt.Errorf("No valid response from kernel")
|
|
}
|
|
return executeOneGetRdmaNetnsMode(msgs[0])
|
|
}
|
|
|
|
func netnsModeStringToUint8(mode string) (uint8, error) {
|
|
switch mode {
|
|
case "exclusive":
|
|
return 0, nil
|
|
case "shared":
|
|
return 1, nil
|
|
default:
|
|
return 0, fmt.Errorf("Invalid mode; %q", mode)
|
|
}
|
|
}
|
|
|
|
// RdmaSystemSetNetnsMode sets the net namespace mode for RDMA subsystem
|
|
// Returns nil on success or appropriate error code.
|
|
// Equivalent to: `rdma system set netns { shared | exclusive }'
|
|
func RdmaSystemSetNetnsMode(NewMode string) error {
|
|
return pkgHandle.RdmaSystemSetNetnsMode(NewMode)
|
|
}
|
|
|
|
// RdmaSystemSetNetnsMode sets the net namespace mode for RDMA subsystem
|
|
// Returns nil on success or appropriate error code.
|
|
// Equivalent to: `rdma system set netns { shared | exclusive }'
|
|
func (h *Handle) RdmaSystemSetNetnsMode(NewMode string) error {
|
|
value, err := netnsModeStringToUint8(NewMode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SYS_SET)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK)
|
|
|
|
data := nl.NewRtAttr(nl.RDMA_NLDEV_SYS_ATTR_NETNS_MODE, []byte{value})
|
|
req.AddData(data)
|
|
|
|
_, err = req.Execute(unix.NETLINK_RDMA, 0)
|
|
return err
|
|
}
|
|
|
|
// RdmaLinkSetNsFd puts the RDMA device into a new network namespace. The
|
|
// fd must be an open file descriptor to a network namespace.
|
|
// Similar to: `rdma dev set $dev netns $ns`
|
|
func RdmaLinkSetNsFd(link *RdmaLink, fd uint32) error {
|
|
return pkgHandle.RdmaLinkSetNsFd(link, fd)
|
|
}
|
|
|
|
// RdmaLinkSetNsFd puts the RDMA device into a new network namespace. The
|
|
// fd must be an open file descriptor to a network namespace.
|
|
// Similar to: `rdma dev set $dev netns $ns`
|
|
func (h *Handle) RdmaLinkSetNsFd(link *RdmaLink, fd uint32) error {
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SET)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK)
|
|
|
|
data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX,
|
|
nl.Uint32Attr(link.Attrs.Index))
|
|
req.AddData(data)
|
|
|
|
data = nl.NewRtAttr(nl.RDMA_NLDEV_NET_NS_FD, nl.Uint32Attr(fd))
|
|
req.AddData(data)
|
|
|
|
return execRdmaSetLink(req)
|
|
}
|
|
|
|
// RdmaLinkDel deletes an rdma link
|
|
//
|
|
// Similar to: rdma link delete NAME
|
|
// REF: https://man7.org/linux/man-pages/man8/rdma-link.8.html
|
|
func RdmaLinkDel(name string) error {
|
|
return pkgHandle.RdmaLinkDel(name)
|
|
}
|
|
|
|
// RdmaLinkDel deletes an rdma link.
|
|
func (h *Handle) RdmaLinkDel(name string) error {
|
|
link, err := h.RdmaLinkByName(name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_DELLINK)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK)
|
|
|
|
b := make([]byte, 4)
|
|
native.PutUint32(b, link.Attrs.Index)
|
|
req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b))
|
|
|
|
_, err = req.Execute(unix.NETLINK_RDMA, 0)
|
|
return err
|
|
}
|
|
|
|
// RdmaLinkAdd adds an rdma link for the specified type to the network device.
|
|
// Similar to: rdma link add NAME type TYPE netdev NETDEV
|
|
// NAME - specifies the new name of the rdma link to add
|
|
// TYPE - specifies which rdma type to use. Link types:
|
|
// rxe - Soft RoCE driver
|
|
// siw - Soft iWARP driver
|
|
// NETDEV - specifies the network device to which the link is bound
|
|
//
|
|
// REF: https://man7.org/linux/man-pages/man8/rdma-link.8.html
|
|
func RdmaLinkAdd(linkName, linkType, netdev string) error {
|
|
return pkgHandle.RdmaLinkAdd(linkName, linkType, netdev)
|
|
}
|
|
|
|
// RdmaLinkAdd adds an rdma link for the specified type to the network device.
|
|
func (h *Handle) RdmaLinkAdd(linkName string, linkType string, netdev string) error {
|
|
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_NEWLINK)
|
|
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK)
|
|
|
|
req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_NAME, nl.ZeroTerminated(linkName)))
|
|
req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_LINK_TYPE, nl.ZeroTerminated(linkType)))
|
|
req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_NDEV_NAME, nl.ZeroTerminated(netdev)))
|
|
_, err := req.Execute(unix.NETLINK_RDMA, 0)
|
|
return err
|
|
}
|