kata-containers/s390-tools
1
0
Fork 0
mirror of https://github.com/ibm-s390-linux/s390-tools.git synced 2026-08-05 02:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

opticsmon: Introduce opticsmon tool

Browse Source 
The optics monitoring tool opticsmon implements the user-space portion
of reporting optics data to the SE. Its basic functionality is to
collect optical module information equivalent to "ethtool --module-info"
for PCI Physical Functions and forwards this data to the SE using the
new SCLP Write Event Data Action Qualifier 3.

For the part of finding all PFs we need to look at all PCI
functions and determine which ones are PFs and what netdevs they
correspond to. This is a generally useful functionality so this part as
well as the SCLP issuing code go into a new libzpci library which also
includes a standalone example for listing PCI functions and their s390x
specific attributes. Medium term we plan to add this functionality to
lszdev.

For the opticsmon tool itself there are 2 basic operating modes:

* One-shot Mode: Without parameters opticsmon collects optical module
  data and prints a summary of the netdevice in JSON format. With
  --module-data it also includes a base64 encoded raw dump equivalent to
  ethtool --module-info <netdev> raw on.
* Monitor Mode: With the --monitor flag opticsmon runs continuously
  usually started via a systemd unit and collects new optical module
  data on a time interval (default 24h) or when the operational state
  ("/sys/class/net/<netdev/operstate") changes. The tool listens for
  changes via netlink so no polling on sysfs is necessary

Note: Both modes will *NOT* issues SCLPs without adding the
--send-report flag but will output a JSON summary for each data
collection so can be tested without firmware impact.

Reviewed-by: Halil Pasic <pasic@linux.ibm.com>
Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
This commit is contained in:
Niklas Schnelle
2024-10-31 14:23:44 +01:00
committed by Jan Höppner
parent 86b5df0d2d
commit c34adb9cab
15 changed files with 1345 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@@ -15,7 +15,7 @@ TOOL_DIRS = zipl zdump fdasd dasdfmt dasdview tunedasd \
vmcp man mon_tools dasdinfo vmur cpuplugd ipl_tools \
ziomon iucvterm hyptop cmsfs-fuse qethqoat zfcpdump zdsfs cpumf \
systemd hmcdrvfs cpacfstats zdev dump2tar zkey netboot etc zpcictl \
lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools rust
lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools rust opticsmon
else
BASELIB_DIRS =

3
include/lib/pci_sclp.h
View File

@@ -20,8 +20,10 @@
#define SCLP_ERRNOTIFY_AQ_RESET 0
#define SCLP_ERRNOTIFY_AQ_DECONF 1
#define SCLP_ERRNOTIFY_AQ_REPORT_ERR 2
#define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3
#define SCLP_ERRNOTIFY_ID_ZPCICTL 0x4713
#define SCLP_ERRNOTIFY_ID_OPTICSMON 0x4714
#define SCLP_ERRNOTIFY_DATA_SIZE 4054
@@ -31,6 +33,7 @@ struct zpci_report_error_header {
* 0: Adapter Reset Request
* 1: Deconfigure and repair action requested
* 2: Informational Report
* 3: Optics Data
*/
__u16 length; /* Length of Subsequent Data (up to 4K – SCLP header) */
} __packed;

83
opticsmon/Makefile Normal file
View File

@@ -0,0 +1,83 @@
include ../common.mak
TESTS := tests/
libs =$(rootdir)/libzpci/libzpci.a $(rootdir)/libutil/libutil.a
ifneq (${HAVE_OPENSSL},0)
check_dep_openssl:
$(call check_dep, \
"opticsmon", \
"openssl/evp.h", \
"openssl-devel", \
"HAVE_OPENSSL=0")
BUILDTARGET += check_dep_openssl
endif # HAVE_OPENSSL
ifneq (${HAVE_LIBNL3},0)
check_dep_libnl3:
$(call check_dep, \
"opticsmon", \
"netlink/socket.h", \
"libnl3-devel", \
"HAVE_LIBNL3=0")
BUILDTARGET += check_dep_libnl3
endif # HAVE_LIBNL3
ifeq (${HAVE_OPENSSL},0)
all:
$(SKIP) HAVE_OPENSSL=0
install:
$(SKIP) HAVE_OPENSSL=0
else ifeq (${HAVE_LIBNL3},0)
all:
$(SKIP) HAVE_LIBNL3=0
install:
$(SKIP) HAVE_LIBNL3=0
else
ifneq ($(shell sh -c 'command -v pkg-config'),)
LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-3.0)
LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-genl-3.0)
LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-route-3.0)
LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libcrypto)
LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-3.0)
LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-genl-3.0)
LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-route-3.0)
LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libcrypto)
else
LIB_CFLAGS += -I /usr/include/libnl3/
LIB_LFLAGS += -lnl-route-3 -lnl-genl-3 -lnl-3
LIB_CFLAGS += -I /usr/include/openssl/
LIB_LFLAGS += -lcrypto
endif
ALL_CPPFLAGS += $(LIB_CFLAGS)
LDLIBS += $(LIB_LFLAGS)
BUILDTARGET += opticsmon
all: ${BUILDTARGET}
opticsmon: opticsmon.o optics_info.o optics_sclp.o ethtool.o link_mon.o $(libs)
install: all
$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man8
$(INSTALL) -g $(GROUP) -o $(OWNER) -m 755 opticsmon $(DESTDIR)$(BINDIR)
$(INSTALL) -g $(GROUP) -o $(OWNER) -m 644 opticsmon.8 \
$(DESTDIR)$(MANDIR)/man8
endif # HAVE_OPENSSL3=0 or HAVE_LIBNL3=0
clean:
rm -f *.o *~ opticsmon core
.PHONY: all install clean

279
opticsmon/ethtool.c Normal file
View File

@@ -0,0 +1,279 @@
#include <execinfo.h>
#include <sys/wait.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <linux/netlink.h>
#include <linux/ethtool_netlink.h>
#include <netlink/socket.h>
#include <netlink/msg.h>
#include <netlink/genl/ctrl.h>
#include <netlink/genl/genl.h>
#include <netlink/handlers.h>
#include <netlink/attr.h>
#include "lib/util_libc.h"
#include "ethtool.h"
static int ethtool_nl_cb(struct nl_msg *msg, void *arg)
{
struct nlattr *attrs[ETHTOOL_A_MODULE_EEPROM_DATA + 1] = {};
struct nlmsghdr *hdr = nlmsg_hdr(msg);
struct optics **oi = arg;
int rc = 0;
size_t len;
rc = genlmsg_parse(hdr, 0, attrs, ETHTOOL_A_MODULE_EEPROM_DATA, NULL);
if (rc) {
nl_perror(rc, "genlmsg parse");
return NL_STOP;
}
len = nla_len(attrs[ETHTOOL_A_MODULE_EEPROM_DATA]);
/* Extend optics info*/
if (!(*oi)->raw)
(*oi)->raw = util_malloc(len);
else
(*oi)->raw = util_realloc((*oi)->raw, (*oi)->size + len);
memcpy((*oi)->raw + (*oi)->size, nla_data(attrs[ETHTOOL_A_MODULE_EEPROM_DATA]), len);
(*oi)->size += len;
return NL_OK;
}
int ethtool_nl_connect(struct ethtool_nl_ctx *ctx)
{
struct nl_sock *sk;
int ethtool_id;
int rc = 0;
sk = nl_socket_alloc();
if (!sk) {
nl_perror(NLE_NOMEM, "alloc");
return EXIT_FAILURE;
}
rc = genl_connect(sk);
if (rc) {
nl_perror(rc, "connect");
rc = EXIT_FAILURE;
goto err_free;
}
ethtool_id = genl_ctrl_resolve(sk, ETHTOOL_GENL_NAME);
if (ethtool_id < 0) {
if (ethtool_id == -NLE_OBJ_NOTFOUND)
fprintf(stderr, "Ethtool netlink family not found\n");
else
nl_perror(ethtool_id, "ctrl resolve");
rc = EXIT_FAILURE;
goto err_close;
}
ctx->sk = sk;
ctx->ethtool_id = ethtool_id;
return rc;
err_close:
nl_close(sk);
err_free:
nl_socket_free(sk);
return rc;
}
void ethtool_nl_close(struct ethtool_nl_ctx *ctx)
{
nl_close(ctx->sk);
nl_socket_free(ctx->sk);
}
static int ethtool_nl_put_req_hdr(struct ethtool_nl_ctx *ctx, struct nl_msg *msg, uint8_t cmd,
const char *netdev)
{
struct nlattr *opts;
void *user_hdr;
int rc = 0;
user_hdr = genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, ctx->ethtool_id, 0,
NLM_F_REQUEST | NLM_F_ACK, cmd, ETHTOOL_GENL_VERSION);
if (!user_hdr) {
fprintf(stderr, "genlmsg put failed\n");
return EXIT_FAILURE;
}
opts = nla_nest_start(msg, ETHTOOL_A_MODULE_EEPROM_HEADER);
if (!opts) {
fprintf(stderr, "nla nest for start failed\n");
return EXIT_FAILURE;
}
NLA_PUT_STRING(msg, ETHTOOL_A_HEADER_DEV_NAME, netdev);
nla_nest_end(msg, opts);
return rc;
nla_put_failure:
nla_nest_cancel(msg, opts);
return EXIT_FAILURE;
}
static int ethtool_nl_put_eeprom_get_attrs(struct nl_msg *msg, uint8_t addr, uint8_t page,
uint32_t offset)
{
NLA_PUT_U32(msg, ETHTOOL_A_MODULE_EEPROM_LENGTH, SFF8636_PAGE_SIZE);
NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_PAGE, page);
NLA_PUT_U32(msg, ETHTOOL_A_MODULE_EEPROM_OFFSET, offset);
NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_BANK, 0);
NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS, addr);
return 0;
nla_put_failure:
return EXIT_FAILURE;
}
static int ethtool_nl_get_page(struct ethtool_nl_ctx *ctx, const char *netdev, uint8_t addr,
uint8_t page, uint32_t offset)
{
struct nl_msg *msg;
int rc = 0;
msg = nlmsg_alloc();
if (!msg) {
nl_perror(NLE_NOMEM, "nlmsg alloc");
return -ENOMEM;
}
ethtool_nl_put_req_hdr(ctx, msg, ETHTOOL_MSG_MODULE_EEPROM_GET, netdev);
ethtool_nl_put_eeprom_get_attrs(msg, addr, page, offset);
rc = nl_send_auto(ctx->sk, msg);
if (rc < 0) {
nl_perror(rc, "Failed to send netlink message");
rc = -EIO;
goto free_msg;
}
rc = nl_recvmsgs_default(ctx->sk);
if (rc < 0) {
if (rc == -NLE_NODEV) {
rc = -ENODEV;
} else {
nl_perror(rc, "Failed to receive netlink message");
rc = -EIO;
}
goto free_msg;
}
/* Ethtool netlink sends ACKs need to pick them up */
rc = nl_wait_for_ack(ctx->sk);
if (rc < 0) {
nl_perror(rc, "Failed to wait for netlink ack");
rc = -EIO;
goto free_msg;
}
free_msg:
nlmsg_free(msg);
return rc;
}
static int ethtool_nl_get_sfp(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics *oi)
{
int rc = 0;
/* Page A0h upper */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, SFF8636_PAGE_SIZE);
if (rc < 0)
return rc;
/* If page A2h is not present we're done */
if (!(oi->raw[SFF8472_DIAGNOSTICS_TYPE_OFFSET] & SFF8472_DIAGNOSTICS_TYPE_MASK))
return 0;
/* Page A2h lower */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_HIGH, 0x0, 0);
if (rc < 0)
return rc;
/* Page A2h upper */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_HIGH, 0x0, SFF8636_PAGE_SIZE);
if (rc < 0)
return rc;
return 0;
}
static int ethtool_nl_get_qsfp(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics *oi)
{
int rc = 0;
/* Page 00h upper */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, SFF8636_PAGE_SIZE);
if (rc)
return rc;
/* Page 01h */
if (oi->raw[SFF8636_PAGE_OFFSET] & SFF8636_P01H) {
/* Page 01h upper only */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x1,
SFF8636_PAGE_SIZE);
if (rc < 0)
return rc;
}
/* Page 02h */
if (oi->raw[SFF8636_PAGE_OFFSET] & SFF8636_P02H) {
/* Page 02h upper only */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x2,
SFF8636_PAGE_SIZE);
if (rc < 0)
return rc;
}
/* Page 03h is present if flatmem is not set */
if (!(oi->raw[SFF8636_STATUS_2_OFFSET] & SFF8636_STATUS_FLAT_MEM)) {
/* Page 03h upper only */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x3,
SFF8636_PAGE_SIZE);
if (rc < 0)
return rc;
}
return 0;
}
int ethtool_nl_get_optics(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics **oi)
{
int rc = 0;
int type;
*oi = util_zalloc(sizeof(**oi));
nl_socket_modify_cb(ctx->sk, NL_CB_VALID, NL_CB_CUSTOM, ethtool_nl_cb, oi);
/* Page 00h lower */
rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, 0);
if (rc < 0)
goto out_err_free_oi;
type = optics_type(*oi);
switch (type) {
case OPTICS_TYPE_SFP:
rc = ethtool_nl_get_sfp(ctx, netdev, *oi);
break;
case OPTICS_TYPE_QSFP28:
rc = ethtool_nl_get_qsfp(ctx, netdev, *oi);
break;
};
if (rc < 0)
goto out_err_free_oi;
return rc;
out_err_free_oi:
free(*oi);
*oi = NULL;
return rc;
}

11
opticsmon/ethtool.h Normal file
View File

@@ -0,0 +1,11 @@
#pragma once
#include "optics_info.h"
struct ethtool_nl_ctx {
struct nl_sock *sk;
int ethtool_id;
};
int ethtool_nl_connect(struct ethtool_nl_ctx *ctx);
void ethtool_nl_close(struct ethtool_nl_ctx *ctx);
int ethtool_nl_get_optics(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics **oi);

96
opticsmon/link_mon.c Normal file
View File

@@ -0,0 +1,96 @@
/*
* Copyright IBM Corp. 2024
*
* s390-tools is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#include <errno.h>
#include <sys/epoll.h>
#include <sys/signalfd.h>
#include <signal.h>
#include <linux/if.h>
#include <netlink/route/link.h>
#include <netlink/netlink.h>
#include "link_mon.h"
#define MAX_EVENTS 32
static void nl_obj_parsed_cb(struct nl_object *obj, void *arg)
{
struct link_mon_nl_ctx *ctx = arg;
struct rtnl_link *link;
struct zpci_netdev netdev;
if (strcmp(nl_object_get_type(obj), "route/link") != 0)
return;
link = (struct rtnl_link *)obj;
netdev.name = rtnl_link_get_name(link);
netdev.operstate = rtnl_link_get_operstate(link);
ctx->cb(&netdev, ctx->arg);
}
static int nl_rtnl_lnkgrp_cb(struct nl_msg *msg, void *arg)
{
if (nl_msg_parse(msg, &nl_obj_parsed_cb, arg) < 0)
fprintf(stderr, "<<EVENT>> Unknown message type\n");
return NL_STOP;
}
void link_mon_nl_waitfd_read(struct link_mon_nl_ctx *ctx)
{
nl_recvmsgs_default(ctx->sk);
}
int link_mon_nl_waitfd_getfd(struct link_mon_nl_ctx *ctx)
{
return nl_socket_get_fd(ctx->sk);
}
int link_mon_nl_waitfd_create(struct link_mon_nl_ctx *ctx, link_mon_nl_cb cb, void *arg)
{
int ret = 0, rc = 0;
ctx->sk = nl_socket_alloc();
if (!ctx->sk)
return -ENOMEM;
ctx->cb = cb;
ctx->arg = arg;
nl_socket_disable_seq_check(ctx->sk);
nl_socket_modify_cb(ctx->sk, NL_CB_VALID, NL_CB_CUSTOM, nl_rtnl_lnkgrp_cb, ctx);
rc = nl_connect(ctx->sk, NETLINK_ROUTE);
if (rc < 0) {
ret = rc;
goto err_free;
}
rc = nl_socket_add_membership(ctx->sk, RTNLGRP_LINK);
if (rc < 0) {
ret = rc;
goto err_close;
}
rc = rtnl_link_alloc_cache(ctx->sk, AF_UNSPEC, &ctx->cache);
if (rc < 0) {
ret = rc;
goto err_close;
}
nl_cache_mngt_provide(ctx->cache);
return 0;
err_close:
nl_close(ctx->sk);
err_free:
nl_socket_free(ctx->sk);
return ret;
}
void link_mon_nl_waitfd_destroy(struct link_mon_nl_ctx *ctx)
{
nl_cache_free(ctx->cache);
nl_close(ctx->sk);
nl_socket_free(ctx->sk);
}

27
opticsmon/link_mon.h Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright IBM Corp. 2024
*
* s390-tools is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#pragma once
#include <stdint.h>
#include <linux/if.h>
#include "lib/pci_list.h"
typedef void (*link_mon_nl_cb)(struct zpci_netdev *, void *arg);
struct link_mon_nl_ctx {
/* private fields */
struct nl_sock *sk;
struct nl_cache *cache;
link_mon_nl_cb cb;
void *arg;
};
int link_mon_nl_waitfd_create(struct link_mon_nl_ctx *ctx, link_mon_nl_cb cb, void *arg);
void link_mon_nl_waitfd_read(struct link_mon_nl_ctx *ctx);
void link_mon_nl_waitfd_destroy(struct link_mon_nl_ctx *ctx);
int link_mon_nl_waitfd_getfd(struct link_mon_nl_ctx *ctx);

171
opticsmon/optics_info.c Normal file
View File

@@ -0,0 +1,171 @@
#include <stdlib.h>
#include "optics_info.h"
#define OPTICS_TYPE_OFFSET 0x0
#define OPTICS_SFP_LOS_IMPLEMENTED_OFFSET 0x41
#define OPTICS_SFP_LOS_IMPLEMENTED_MASK 0x2
#define OPTICS_SFP_A2H_OFFSET 0x100
#define OPTICS_SFP_LOS_OFFSET (OPTICS_SFP_A2H_OFFSET + 0x6e)
#define OPTICS_SFP_DATA_NOT_READY_MASK 0x1
#define OPTICS_SFP_TX_FAULT_MASK 0x4
#define OPTICS_SFP_RX_LOS_MASK 0x2
#define OPTICS_QSFP28_LOS_IMPLEMENTED_OFFSET 0xC3
#define OPTICS_QSFP28_TX_LOS_IMPLEMENTED_MASK 0x2
#define OPTICS_QSFP28_TX_FAULT_IMPLEMENTED_MASK 0x8
#define OPTICS_QSFP28_LOS_OFFSET 0x3
#define OPTICS_QSFP28_LOS_MASK 0xf
#define OPTICS_QSFP28_TX_LOS_MASK 0xf0
#define OPTICS_QSFP28_TX_LOS_SHIFT 0x4
#define OPTICS_QSFP28_TX_FAULT_OFFSET 0x4
#define OPTICS_QSFP28_TX_FAULT_MASK 0xf
const char *optics_type_str(enum optics_type type)
{
switch (type) {
case OPTICS_TYPE_UNKNOWN:
return "unknown";
case OPTICS_TYPE_SFP:
return "SFP/SFP+/SFP28";
case OPTICS_TYPE_QSFP28:
return "QSFP28";
};
return "n.a.";
}
enum optics_type optics_type(struct optics *oi)
{
if (!oi || !oi->raw || oi->size < OPTICS_TYPE_OFFSET + 1)
return OPTICS_TYPE_UNKNOWN;
switch (oi->raw[OPTICS_TYPE_OFFSET]) {
case (uint8_t)OPTICS_TYPE_SFP:
return OPTICS_TYPE_SFP;
case (uint8_t)OPTICS_TYPE_QSFP28:
return OPTICS_TYPE_QSFP28;
default:
return OPTICS_TYPE_UNKNOWN;
};
}
bool optics_los_implemented(struct optics *oi)
{
enum optics_type type = optics_type(oi);
uint8_t implemented;
if (type == OPTICS_TYPE_SFP) {
if (oi->size < OPTICS_SFP_LOS_IMPLEMENTED_OFFSET + 1)
return false;
implemented = oi->raw[OPTICS_SFP_LOS_IMPLEMENTED_OFFSET];
return !!(implemented & OPTICS_SFP_LOS_IMPLEMENTED_MASK);
} else if (type == OPTICS_TYPE_QSFP28) {
if (oi->size < OPTICS_QSFP28_LOS_OFFSET + 1)
return false;
if (oi->size < OPTICS_QSFP28_LOS_IMPLEMENTED_OFFSET)
return false;
implemented = oi->raw[OPTICS_QSFP28_LOS_IMPLEMENTED_OFFSET];
/*
* No RX LoS implemented flag take TX LOS implemented like
* ethtool
*/
return !!(implemented & OPTICS_QSFP28_TX_LOS_IMPLEMENTED_MASK);
}
return false;
}
enum optics_los optics_rx_los(struct optics *oi)
{
enum optics_los los = OPTICS_UNKNOWN_LOS;
enum optics_type type = optics_type(oi);
if (!optics_los_implemented(oi))
return los;
if (type == OPTICS_TYPE_SFP) {
los = oi->raw[OPTICS_SFP_LOS_OFFSET];
if (los & OPTICS_SFP_DATA_NOT_READY_MASK)
return OPTICS_UNKNOWN_LOS;
if (los & OPTICS_SFP_RX_LOS_MASK)
return OPTICS_LOS;
else
return OPTICS_NO_LOS;
} else if (type == OPTICS_TYPE_QSFP28) {
los = oi->raw[OPTICS_QSFP28_LOS_OFFSET];
if (los & OPTICS_QSFP28_LOS_MASK)
los = OPTICS_LOS;
else
los = OPTICS_NO_LOS;
}
return los;
}
const char *optics_los_str(enum optics_los los)
{
switch (los) {
case OPTICS_LOS:
return "yes";
case OPTICS_NO_LOS:
return "no";
case OPTICS_UNAVAILABLE_LOS:
return "unavailable";
default:
return "unknown";
}
}
enum optics_los optics_tx_fault(struct optics *oi)
{
enum optics_los los = OPTICS_UNKNOWN_LOS;
enum optics_type type = optics_type(oi);
if (!optics_los_implemented(oi))
return los;
if (type == OPTICS_TYPE_SFP) {
los = oi->raw[OPTICS_SFP_LOS_OFFSET];
if (los & OPTICS_SFP_DATA_NOT_READY_MASK)
return OPTICS_UNKNOWN_LOS;
if (los & OPTICS_SFP_TX_FAULT_MASK)
return OPTICS_LOS;
else
return OPTICS_NO_LOS;
} else if (type == OPTICS_TYPE_QSFP28) {
los = oi->raw[OPTICS_QSFP28_TX_FAULT_OFFSET];
if (los & OPTICS_QSFP28_TX_FAULT_MASK)
los = OPTICS_LOS;
else
los = OPTICS_NO_LOS;
}
return los;
}
enum optics_los optics_tx_los(struct optics *oi)
{
enum optics_los los = OPTICS_UNKNOWN_LOS;
enum optics_type type = optics_type(oi);
if (!optics_los_implemented(oi))
return los;
if (type == OPTICS_TYPE_SFP) {
return OPTICS_UNAVAILABLE_LOS;
} else if (type == OPTICS_TYPE_QSFP28) {
los = oi->raw[OPTICS_QSFP28_LOS_OFFSET];
if (los & OPTICS_QSFP28_TX_LOS_MASK)
los = OPTICS_LOS;
else
los = OPTICS_NO_LOS;
}
return los;
}
void optics_free(struct optics *oi)
{
free(oi->raw);
free(oi);
}

49
opticsmon/optics_info.h Normal file
View File

@@ -0,0 +1,49 @@
#pragma once
#include <stdbool.h>
#include <stdint.h>
#include <sys/types.h>
#define SFF8079_I2C_ADDRESS_LOW 0x50
#define SFF8079_I2C_ADDRESS_HIGH 0x51
#define SFF8472_DIAGNOSTICS_TYPE_OFFSET 0x5C
#define SFF8472_DIAGNOSTICS_TYPE_MASK (1 << 6)
#define SFF8636_PAGE_SIZE 0x80
#define SFF8636_QSFP28_LENGTH 0x100
#define SFF8636_STATUS_2_OFFSET 0x02
#define SFF8636_STATUS_FLAT_MEM (1 << 2)
#define SFF8636_PAGE_OFFSET 0xC3
#define SFF8636_P01H (1 << 6)
#define SFF8636_P02H (1 << 7)
enum optics_type {
OPTICS_TYPE_UNKNOWN = 0x0, /* Unknown or unsupported */
OPTICS_TYPE_SFP = 0x3, /* SFP/SFP+/SFP28 and later with SFF-8472 management interface */
OPTICS_TYPE_QSFP28 = 0x11 /* QSFP28 (SFF-8665 et al.)*/
};
enum optics_los {
OPTICS_NO_LOS = 0x0,
OPTICS_LOS = 0x1,
OPTICS_UNKNOWN_LOS = 0x2,
OPTICS_UNAVAILABLE_LOS = 0x3,
};
struct optics {
size_t size;
uint8_t *raw;
};
enum optics_type optics_type(struct optics *oi);
const char *optics_type_str(enum optics_type type);
const char *optics_los_str(enum optics_los los);
enum optics_los optics_rx_los(struct optics *oi);
enum optics_los optics_tx_los(struct optics *oi);
enum optics_los optics_tx_fault(struct optics *oi);
void optics_free(struct optics *oi);

50
opticsmon/optics_sclp.c Normal file
View File

@@ -0,0 +1,50 @@
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include "lib/pci_sclp.h"
#include "lib/util_libc.h"
#include "optics_sclp.h"
static struct sclp_optics_data *init_sclp_optics_data(struct optics *oi, size_t *length)
{
struct sclp_optics_data *od;
*length = sizeof(*od) + oi->size;
od = util_zalloc(*length);
od->module_present = optics_type(oi) != OPTICS_TYPE_UNKNOWN;
od->rx_los = optics_rx_los(oi) == OPTICS_LOS;
od->tx_fault = optics_tx_fault(oi) == OPTICS_LOS;
switch (optics_type(oi)) {
case OPTICS_TYPE_SFP:
od->data_identifier = 1;
break;
case OPTICS_TYPE_QSFP28:
od->data_identifier = 2;
break;
default:
od->data_identifier = 0;
}
memcpy(od->data, oi->raw, oi->size);
return od;
}
int sclp_issue_optics_report(struct zpci_dev *zdev, struct optics *oi)
{
struct sclp_optics_data *od;
size_t length;
char *pci_addr;
int rc;
if (zdev->pft != ZPCI_PFT_NETD)
return -ENOTSUP;
od = init_sclp_optics_data(oi, &length);
pci_addr = zpci_pci_addr(zdev);
rc = zpci_sclp_issue_action(pci_addr, SCLP_ERRNOTIFY_AQ_OPTICS_DATA,
(char *)od, length, SCLP_ERRNOTIFY_ID_OPTICSMON);
free(pci_addr);
free(od);
return rc;
}

21
opticsmon/optics_sclp.h Normal file
View File

@@ -0,0 +1,21 @@
#include <stdint.h>
#include "lib/pci_list.h"
#include "optics_info.h"
struct sclp_optics_data {
/* Status */
uint32_t module_present : 1;
uint32_t rx_los : 1;
uint32_t tx_fault : 1;
uint32_t reserved_status : 29;
/* Data Identifier */
uint32_t data_identifier;
/* Reserved */
uint64_t reserved[3];
/* Additional Log Data */
uint8_t data[];
} __packed;
int sclp_issue_optics_report(struct zpci_dev *zdev, struct optics *oi);

83
opticsmon/opticsmon.8 Normal file
View File

@@ -0,0 +1,83 @@
.\" Copyright IBM Corp. 2024
.\" s390-tools is free software; you can redistribute it and/or modify
.\" it under the terms of the MIT license. See LICENSE for details.
.\"
.\" Macro for inserting an option description prologue.
.\" .OD <long> [<short>] [args]
.de OD
. ds args "
. if !'\\$3'' .as args \fI\\$3\fP
. if !'\\$4'' .as args \\$4
. if !'\\$5'' .as args \fI\\$5\fP
. if !'\\$6'' .as args \\$6
. if !'\\$7'' .as args \fI\\$7\fP
. PD 0
. if !'\\$2'' .IP "\fB\-\\$2\fP \\*[args]" 4
. if !'\\$1'' .IP "\fB\-\-\\$1\fP \\*[args]" 4
. PD
..
.
.TH opticsmon 8 "Oct 2024" s390-tools zpcictl
.
.SH NAME
opticsmon - Monitor optical modules for directly attached PCI based NICs
.
.
.SH SYNOPSIS
.B "opticsmon"
.I "OPTIONS"
.
.
.SH DESCRIPTION
Use
.B opticsmon
to monitor the health of the optical modules of directly attached PCI based
NICs. When executed without the \fB--daemon\fR option it will collect optical
module data from all available PCI network interface physical functions and
print a summary in JSON format. Add the \fB--send-report\fR option to report
this data to the support element.
.
.
.SH OPTIONS
.SS Operation Options
.OD daemon "d"
Run continuously and report on link state changes and periodically
.PP
.
.OD send-report "r"
Report the optics health data to the Support Element (SE)
.PP
.
.OD quiet "q"
Be quiet and don't print optics health summary
.PP
.
.OD interval "i" "seconds"
Interval in seconds at which to collect monitoring data in the absence of link
state changes. A value larger than 24 hours (86400 seconds) is clamped down to
24 hours.
.PP
.
.OD module-info ""
Include a base64 encoded binary dump of the module's SFF-8636/8472/8024
standard data for each netdev. This matches "ethtool --module-info <netdev> raw
on".
.B Example: Extract module information for the first adapter
.CL
opticsmon --module-info | jq -r '.adapters[0].netdevs[0].optics.module_info' | base64 -d | hexdump -C
.PP
.PP
.
.SS General Options
.OD help "h" ""
Print usage information, then exit.
.PP
.
.OD version "v" ""
Print version information, then exit.
.PP

449
opticsmon/opticsmon.c Normal file
View File

@@ -0,0 +1,449 @@
/*
* opticsmon - Report optics monitoring data to firmware
*
* Copyright IBM Corp. 2024
*
* s390-tools is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <errno.h>
#include <sys/epoll.h>
#include <sys/signalfd.h>
#include <signal.h>
#include <sys/timerfd.h>
#include <time.h>
#include <linux/if.h>
#include "lib/util_list.h"
#include "lib/pci_list.h"
#include "lib/util_prg.h"
#include "lib/util_opt.h"
#include "lib/util_fmt.h"
#include "lib/util_libc.h"
#include <openssl/evp.h>
#include "optics_info.h"
#include "optics_sclp.h"
#include "ethtool.h"
#include "link_mon.h"
#define API_LEVEL 1
struct options {
bool monitor;
bool report;
bool module_info;
bool quiet;
uint32_t interval_seconds;
};
struct opticsmon_ctx {
struct options opts;
struct ethtool_nl_ctx ethtool_ctx;
struct link_mon_nl_ctx lctx;
struct util_list *zpci_list;
};
static const struct util_prg prg = {
.desc = "Use opticsmon to monitor the health of the optical modules\n"
"of directly attached PCI based NICs",
.copyright_vec = { {
.owner = "IBM Corp.",
.pub_first = 2024,
.pub_last = 2024,
},
UTIL_PRG_COPYRIGHT_END }
};
#define OPT_DUMP 128
static struct util_opt opt_vec[] = {
UTIL_OPT_SECTION("OPERATION OPTIONS"),
{
.option = { "monitor", no_argument, NULL, 'm' },
.desc = "Run continuously and report on link state changes "
"collecting optics health data when a change is detected",
},
{
.option = { "send-report", no_argument, NULL, 'r' },
.desc = "Report the optics health data to the Support Element",
},
{
.option = { "quiet", no_argument, NULL, 'q' },
.desc = "Be quiet and don't print optics health summary",
},
{
.option = { "module-info", no_argument, NULL, OPT_DUMP },
.desc = "Include a base64 encoded binary dump of the module's "
"SFF-8636/8472/8024 standard data for each netdev. "
"This matches \"ethtool --module-info <netdev> raw on\"",
.flags = UTIL_OPT_FLAG_NOSHORT,
},
UTIL_OPT_SECTION("OPTIONS WITH ARGUMENTS"),
{
.option = { "interval", required_argument, NULL, 'i' },
.argument = "seconds",
.desc = "Interval in seconds at which to collect monitoring data "
"in the absence of link state changes. A value larger than "
"24 hours (86400 seconds) is clamped down to 24 hours.",
},
UTIL_OPT_SECTION("GENERAL OPTIONS"),
UTIL_OPT_HELP,
UTIL_OPT_VERSION,
UTIL_OPT_END
};
static void parse_cmdline(int argc, char *argv[], struct options *opts)
{
uint32_t seconds;
int cmd, ret;
util_prg_init(&prg);
util_opt_init(opt_vec, NULL);
do {
cmd = util_opt_getopt_long(argc, argv);
switch (cmd) {
case 'm':
opts->monitor = true;
break;
case 'r':
opts->report = true;
break;
case 'q':
opts->quiet = true;
break;
case OPT_DUMP:
opts->module_info = true;
break;
case 'i':
ret = sscanf(optarg, "%u", &seconds);
if (ret != 1) {
fprintf(stderr,
"Failed to parse interval argument \"%s\" as seconds\n",
optarg);
exit(EXIT_FAILURE);
}
if (seconds < 86400)
opts->interval_seconds = seconds;
break;
case 'h':
util_prg_print_help();
util_opt_print_help();
exit(EXIT_SUCCESS);
case 'v':
util_prg_print_version();
exit(EXIT_SUCCESS);
case -1:
/* End of options string */
break;
}
} while (cmd != -1);
}
static int module_info_pair(struct optics *oi)
{
size_t b64_calclen, b64len;
int rc = EXIT_SUCCESS;
char *b64;
b64_calclen = (oi->size / 3) * 4;
if (oi->size % 3 > 0)
b64_calclen += 4;
b64 = util_zalloc(b64_calclen + 1); /* adds NUL byte */
b64len = EVP_EncodeBlock((unsigned char *)b64, oi->raw, oi->size);
if (b64len != b64_calclen) {
fprintf(stderr, "encoding base64 via openssl failed\n");
rc = EXIT_FAILURE;
goto out;
}
util_fmt_pair(FMT_QUOTE, "module_info", b64);
out:
free(b64);
return rc;
}
static void optics_json_print(struct opticsmon_ctx *ctx, struct zpci_netdev *nd, struct optics *oi)
{
util_fmt_obj_start(FMT_DEFAULT, "netdev");
util_fmt_pair(FMT_QUOTE, "name", nd->name);
util_fmt_pair(FMT_QUOTE, "operstate", zpci_operstate_str(nd->operstate));
util_fmt_obj_start(FMT_DEFAULT, "optics");
util_fmt_pair(FMT_QUOTE, "type", optics_type_str(optics_type(oi)));
util_fmt_pair(FMT_QUOTE, "rx_los", optics_los_str(optics_rx_los(oi)));
util_fmt_pair(FMT_QUOTE, "tx_los", optics_los_str(optics_tx_los(oi)));
util_fmt_pair(FMT_QUOTE, "tx_fault", optics_los_str(optics_rx_los(oi)));
if (ctx->opts.module_info)
module_info_pair(oi);
util_fmt_obj_end();
util_fmt_obj_end();
}
static int dump_adapter_data(struct opticsmon_ctx *ctx, struct zpci_dev *zdev)
{
struct optics **ois;
int num_ois = 0;
char *pci_addr;
int i, rc;
ois = util_zalloc(sizeof(ois[0]) * zdev->num_netdevs);
for (i = 0; i < zdev->num_netdevs; i++) {
rc = ethtool_nl_get_optics(&ctx->ethtool_ctx, zdev->netdevs[i].name, &ois[i]);
if (rc)
goto free_ois;
num_ois++;
}
if (!ctx->opts.quiet) {
util_fmt_obj_start(FMT_DEFAULT, "adapter");
util_fmt_pair(FMT_QUOTE, "pft", zpci_pft_str(zdev));
util_fmt_obj_start(FMT_DEFAULT, "ids");
util_fmt_pair(FMT_QUOTE, "fid", "0x%0x", zdev->fid);
if (zdev->uid_is_unique)
util_fmt_pair(FMT_QUOTE, "uid", "0x%0x", zdev->uid);
pci_addr = zpci_pci_addr(zdev);
util_fmt_pair(FMT_QUOTE, "pci_address", pci_addr);
free(pci_addr);
util_fmt_obj_end();
util_fmt_obj_start(FMT_LIST, "netdevs");
for (i = 0; i < zdev->num_netdevs; i++)
optics_json_print(ctx, &zdev->netdevs[i], ois[i]);
util_fmt_obj_end(); /* netdevs list */
util_fmt_obj_end(); /* adapter */
fflush(stdout);
}
if (ctx->opts.report) {
for (i = 0; i < zdev->num_netdevs; i++) {
rc = sclp_issue_optics_report(zdev, ois[i]);
if (rc == -ENOTSUP) {
fprintf(stderr, "Skipping %s which does not support reporting\n",
zdev->netdevs[i].name);
} else if (rc < 0) {
fprintf(stderr, "Error issuing SCLP for optics data failed: %s\n",
strerror(-rc));
}
}
}
free_ois:
for (i = 0; i < num_ois; i++)
optics_free(ois[i]);
free(ois);
return rc;
}
static void zpci_list_reload(struct util_list **zpci_list)
{
if (*zpci_list)
zpci_free_dev_list(*zpci_list);
*zpci_list = zpci_dev_list();
}
static void dump_all_adapter_data(struct opticsmon_ctx *ctx)
{
struct zpci_dev *zdev;
zpci_list_reload(&ctx->zpci_list);
util_list_iterate(ctx->zpci_list, zdev) {
/* Filter non-NIC devices and VFs */
if (zpci_is_vf(zdev) || !zdev->num_netdevs)
continue;
dump_adapter_data(ctx, zdev);
}
}
static int oneshot_mode(struct opticsmon_ctx *ctx)
{
util_fmt_init(stdout, FMT_JSON, FMT_DEFAULT, API_LEVEL);
if (!ctx->opts.quiet)
util_fmt_obj_start(FMT_LIST, "adapters");
dump_all_adapter_data(ctx);
if (!ctx->opts.quiet)
util_fmt_obj_end();
util_fmt_exit();
return EXIT_SUCCESS;
}
void on_link_change(struct zpci_netdev *netdev, void *arg)
{
struct opticsmon_ctx *ctx = arg;
struct zpci_dev *zdev;
int i, reloads = 1;
if (!ctx->zpci_list || util_list_is_empty(ctx->zpci_list))
zpci_list_reload(&ctx->zpci_list);
reload:
util_list_iterate(ctx->zpci_list, zdev) {
for (i = 0; i < zdev->num_netdevs; i++) {
if (!strcmp(zdev->netdevs[i].name, netdev->name)) {
reloads--;
/* Skip data collection if operational state is
* unchanged
*/
if (zdev->netdevs[i].operstate == netdev->operstate)
continue;
/* Update operation state for VFs even though
* they are skipped just for a consistent view
*/
zdev->netdevs[i].operstate = netdev->operstate;
/* Only collect optics data for PFs */
if (!zpci_is_vf(zdev))
dump_adapter_data(ctx, zdev);
}
}
}
/* Might be a new device, reload list of devices and retry */
if (reloads) {
zpci_list_reload(&ctx->zpci_list);
reloads--;
goto reload;
}
}
#define MAX_EVENTS 8
static int monitor_wait_loop(struct opticsmon_ctx *ctx, int sigfd, int timerfd)
{
struct epoll_event events[MAX_EVENTS];
struct signalfd_siginfo fdsi;
int i, nlfd, epfd, nfds;
struct epoll_event ev;
uint64_t expirations;
ssize_t sread;
epfd = epoll_create1(EPOLL_CLOEXEC);
ev.events = EPOLLIN;
ev.data.fd = sigfd;
if (epoll_ctl(epfd, EPOLL_CTL_ADD, sigfd, &ev) == -1)
return -EIO;
ev.events = EPOLLIN;
ev.data.fd = timerfd;
if (epoll_ctl(epfd, EPOLL_CTL_ADD, timerfd, &ev) == -1)
return -EIO;
nlfd = link_mon_nl_waitfd_getfd(&ctx->lctx);
ev.events = EPOLLIN;
ev.data.fd = nlfd;
if (epoll_ctl(epfd, EPOLL_CTL_ADD, nlfd, &ev) == -1)
return -EIO;
while (1) {
nfds = epoll_wait(epfd, events, MAX_EVENTS, -1);
if (nfds < 0)
return nfds;
for (i = 0; i < nfds; i++) {
/* signal fd */
if (events[i].data.fd == sigfd) {
sread = read(sigfd, &fdsi, sizeof(fdsi));
if (sread != sizeof(fdsi))
return -EIO;
switch (fdsi.ssi_signo) {
case SIGINT:
case SIGTERM:
case SIGQUIT:
return 0;
/* Unexpected signal */
default:
return -EIO;
}
/* timer fd */
} else if (events[i].data.fd == timerfd) {
sread = read(timerfd, &expirations, sizeof(uint64_t));
if (sread != sizeof(uint64_t))
return -EIO;
if (!expirations)
continue;
dump_all_adapter_data(ctx);
/* netlink fd */
} else if (events[i].data.fd == nlfd) {
link_mon_nl_waitfd_read(&ctx->lctx);
}
}
}
return 0;
}
static int monitor_mode(struct opticsmon_ctx *ctx)
{
struct itimerspec timerspec;
int sigfd, timerfd, ret;
sigset_t mask;
sigemptyset(&mask);
sigaddset(&mask, SIGINT);
sigaddset(&mask, SIGQUIT);
sigaddset(&mask, SIGTERM);
if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1)
return -EIO;
sigfd = signalfd(-1, &mask, 0);
if (sigfd == -1) {
fprintf(stderr, "Failed to create signalfd\n");
return -EIO;
}
timerfd = timerfd_create(CLOCK_MONOTONIC, 0);
if (timerfd == -1) {
fprintf(stderr, "Failed to create timerfd\n");
ret = -EIO;
goto close_signalfd;
}
/* Set initial expiration to 1 ns so we gather optics data at startup */
timerspec.it_value.tv_sec = 0;
timerspec.it_value.tv_nsec = 1;
timerspec.it_interval.tv_sec = ctx->opts.interval_seconds;
timerspec.it_interval.tv_nsec = 0;
ret = timerfd_settime(timerfd, 0, &timerspec, NULL);
if (ret == -1) {
fprintf(stderr, "Failed to arm timer\n");
goto close_timerfd;
}
util_fmt_init(stdout, FMT_JSONSEQ, FMT_DEFAULT, API_LEVEL);
ret = link_mon_nl_waitfd_create(&ctx->lctx, on_link_change, ctx);
if (ret) {
fprintf(stderr, "Failed to create link monitoring socket\n");
goto close_timerfd;
}
monitor_wait_loop(ctx, sigfd, timerfd);
link_mon_nl_waitfd_destroy(&ctx->lctx);
util_fmt_exit();
close_signalfd:
close(sigfd);
close_timerfd:
close(timerfd);
return ret;
}
int main(int argc, char **argv)
{
struct opticsmon_ctx ctx = { .opts = { .interval_seconds = 86400 } };
int ret;
parse_cmdline(argc, argv, &ctx.opts);
ethtool_nl_connect(&ctx.ethtool_ctx);
if (ctx.opts.monitor)
ret = monitor_mode(&ctx);
else
ret = oneshot_mode(&ctx);
ethtool_nl_close(&ctx.ethtool_ctx);
if (ctx.zpci_list)
zpci_free_dev_list(ctx.zpci_list);
return ret;
}

3
systemd/Makefile
View File

@@ -3,7 +3,8 @@ include ../common.mak
SYSTEM_UNITS = ttyrun-getty@.service iucvtty-login@.service \
cpacfstatsd.service cpuplugd.service \
dumpconf.service cpi.service \
mon_fsstatd.service mon_procd.service
mon_fsstatd.service mon_procd.service \
opticsmon.service
all:

20
systemd/opticsmon.service.in Normal file
View File

@@ -0,0 +1,20 @@
#
# Systemd unit for mon_fsstatd (monitor z/VM guest file systems)
#
# Copyright IBM Corp. 2024
#
# s390-tools is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
#
[Unit]
Description=Monitor health of directly attached PCI NIC optical modules
# Documentation=man:opticsmon
[Service]
Type=exec
ExecStart=@usrsbin_path@/opticsmon --send-report --monitor --interval 86400
KillMode=control-group
[Install]
WantedBy=multi-user.target