diff --git a/Makefile b/Makefile index 94efe173..e1a23058 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ TOOL_DIRS = zipl zdump fdasd dasdfmt dasdview tunedasd \ vmcp man mon_tools dasdinfo vmur cpuplugd ipl_tools \ ziomon iucvterm hyptop cmsfs-fuse qethqoat zfcpdump zdsfs cpumf \ systemd hmcdrvfs cpacfstats zdev dump2tar zkey netboot etc zpcictl \ - lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools rust + lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools rust opticsmon else BASELIB_DIRS = diff --git a/include/lib/pci_sclp.h b/include/lib/pci_sclp.h index 8713e897..688a7b76 100644 --- a/include/lib/pci_sclp.h +++ b/include/lib/pci_sclp.h @@ -20,8 +20,10 @@ #define SCLP_ERRNOTIFY_AQ_RESET 0 #define SCLP_ERRNOTIFY_AQ_DECONF 1 #define SCLP_ERRNOTIFY_AQ_REPORT_ERR 2 +#define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3 #define SCLP_ERRNOTIFY_ID_ZPCICTL 0x4713 +#define SCLP_ERRNOTIFY_ID_OPTICSMON 0x4714 #define SCLP_ERRNOTIFY_DATA_SIZE 4054 @@ -31,6 +33,7 @@ struct zpci_report_error_header { * 0: Adapter Reset Request * 1: Deconfigure and repair action requested * 2: Informational Report + * 3: Optics Data */ __u16 length; /* Length of Subsequent Data (up to 4K – SCLP header) */ } __packed; diff --git a/opticsmon/Makefile b/opticsmon/Makefile new file mode 100644 index 00000000..20ebace7 --- /dev/null +++ b/opticsmon/Makefile @@ -0,0 +1,83 @@ +include ../common.mak + +TESTS := tests/ + +libs =$(rootdir)/libzpci/libzpci.a $(rootdir)/libutil/libutil.a + +ifneq (${HAVE_OPENSSL},0) +check_dep_openssl: + $(call check_dep, \ + "opticsmon", \ + "openssl/evp.h", \ + "openssl-devel", \ + "HAVE_OPENSSL=0") +BUILDTARGET += check_dep_openssl +endif # HAVE_OPENSSL + +ifneq (${HAVE_LIBNL3},0) +check_dep_libnl3: + $(call check_dep, \ + "opticsmon", \ + "netlink/socket.h", \ + "libnl3-devel", \ + "HAVE_LIBNL3=0") +BUILDTARGET += check_dep_libnl3 +endif # HAVE_LIBNL3 + +ifeq (${HAVE_OPENSSL},0) + +all: + $(SKIP) HAVE_OPENSSL=0 + +install: + $(SKIP) HAVE_OPENSSL=0 + +else ifeq (${HAVE_LIBNL3},0) +all: + $(SKIP) HAVE_LIBNL3=0 + +install: + $(SKIP) HAVE_LIBNL3=0 + +else + +ifneq ($(shell sh -c 'command -v pkg-config'),) +LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-3.0) +LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-genl-3.0) +LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-route-3.0) + +LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libcrypto) + +LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-3.0) +LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-genl-3.0) +LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-route-3.0) + +LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libcrypto) +else +LIB_CFLAGS += -I /usr/include/libnl3/ +LIB_LFLAGS += -lnl-route-3 -lnl-genl-3 -lnl-3 + +LIB_CFLAGS += -I /usr/include/openssl/ +LIB_LFLAGS += -lcrypto +endif + +ALL_CPPFLAGS += $(LIB_CFLAGS) +LDLIBS += $(LIB_LFLAGS) + +BUILDTARGET += opticsmon + +all: ${BUILDTARGET} + +opticsmon: opticsmon.o optics_info.o optics_sclp.o ethtool.o link_mon.o $(libs) + +install: all + $(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man8 + $(INSTALL) -g $(GROUP) -o $(OWNER) -m 755 opticsmon $(DESTDIR)$(BINDIR) + $(INSTALL) -g $(GROUP) -o $(OWNER) -m 644 opticsmon.8 \ + $(DESTDIR)$(MANDIR)/man8 +endif # HAVE_OPENSSL3=0 or HAVE_LIBNL3=0 + +clean: + rm -f *.o *~ opticsmon core + +.PHONY: all install clean diff --git a/opticsmon/ethtool.c b/opticsmon/ethtool.c new file mode 100644 index 00000000..5ac11dfb --- /dev/null +++ b/opticsmon/ethtool.c @@ -0,0 +1,279 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "lib/util_libc.h" + +#include "ethtool.h" + +static int ethtool_nl_cb(struct nl_msg *msg, void *arg) +{ + struct nlattr *attrs[ETHTOOL_A_MODULE_EEPROM_DATA + 1] = {}; + struct nlmsghdr *hdr = nlmsg_hdr(msg); + struct optics **oi = arg; + int rc = 0; + size_t len; + + rc = genlmsg_parse(hdr, 0, attrs, ETHTOOL_A_MODULE_EEPROM_DATA, NULL); + if (rc) { + nl_perror(rc, "genlmsg parse"); + return NL_STOP; + } + + len = nla_len(attrs[ETHTOOL_A_MODULE_EEPROM_DATA]); + + /* Extend optics info*/ + if (!(*oi)->raw) + (*oi)->raw = util_malloc(len); + else + (*oi)->raw = util_realloc((*oi)->raw, (*oi)->size + len); + memcpy((*oi)->raw + (*oi)->size, nla_data(attrs[ETHTOOL_A_MODULE_EEPROM_DATA]), len); + (*oi)->size += len; + + return NL_OK; +} + +int ethtool_nl_connect(struct ethtool_nl_ctx *ctx) +{ + struct nl_sock *sk; + int ethtool_id; + int rc = 0; + + sk = nl_socket_alloc(); + if (!sk) { + nl_perror(NLE_NOMEM, "alloc"); + return EXIT_FAILURE; + } + + rc = genl_connect(sk); + if (rc) { + nl_perror(rc, "connect"); + rc = EXIT_FAILURE; + goto err_free; + } + + ethtool_id = genl_ctrl_resolve(sk, ETHTOOL_GENL_NAME); + if (ethtool_id < 0) { + if (ethtool_id == -NLE_OBJ_NOTFOUND) + fprintf(stderr, "Ethtool netlink family not found\n"); + else + nl_perror(ethtool_id, "ctrl resolve"); + rc = EXIT_FAILURE; + goto err_close; + } + ctx->sk = sk; + ctx->ethtool_id = ethtool_id; + return rc; + +err_close: + nl_close(sk); +err_free: + nl_socket_free(sk); + return rc; +} + +void ethtool_nl_close(struct ethtool_nl_ctx *ctx) +{ + nl_close(ctx->sk); + nl_socket_free(ctx->sk); +} + +static int ethtool_nl_put_req_hdr(struct ethtool_nl_ctx *ctx, struct nl_msg *msg, uint8_t cmd, + const char *netdev) +{ + struct nlattr *opts; + void *user_hdr; + int rc = 0; + + user_hdr = genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, ctx->ethtool_id, 0, + NLM_F_REQUEST | NLM_F_ACK, cmd, ETHTOOL_GENL_VERSION); + if (!user_hdr) { + fprintf(stderr, "genlmsg put failed\n"); + return EXIT_FAILURE; + } + + opts = nla_nest_start(msg, ETHTOOL_A_MODULE_EEPROM_HEADER); + if (!opts) { + fprintf(stderr, "nla nest for start failed\n"); + return EXIT_FAILURE; + } + + NLA_PUT_STRING(msg, ETHTOOL_A_HEADER_DEV_NAME, netdev); + nla_nest_end(msg, opts); + return rc; + +nla_put_failure: + nla_nest_cancel(msg, opts); + return EXIT_FAILURE; +} + +static int ethtool_nl_put_eeprom_get_attrs(struct nl_msg *msg, uint8_t addr, uint8_t page, + uint32_t offset) +{ + NLA_PUT_U32(msg, ETHTOOL_A_MODULE_EEPROM_LENGTH, SFF8636_PAGE_SIZE); + NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_PAGE, page); + NLA_PUT_U32(msg, ETHTOOL_A_MODULE_EEPROM_OFFSET, offset); + NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_BANK, 0); + NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS, addr); + + return 0; + +nla_put_failure: + return EXIT_FAILURE; +} + +static int ethtool_nl_get_page(struct ethtool_nl_ctx *ctx, const char *netdev, uint8_t addr, + uint8_t page, uint32_t offset) +{ + struct nl_msg *msg; + int rc = 0; + + msg = nlmsg_alloc(); + if (!msg) { + nl_perror(NLE_NOMEM, "nlmsg alloc"); + return -ENOMEM; + } + ethtool_nl_put_req_hdr(ctx, msg, ETHTOOL_MSG_MODULE_EEPROM_GET, netdev); + ethtool_nl_put_eeprom_get_attrs(msg, addr, page, offset); + rc = nl_send_auto(ctx->sk, msg); + if (rc < 0) { + nl_perror(rc, "Failed to send netlink message"); + rc = -EIO; + goto free_msg; + } + + rc = nl_recvmsgs_default(ctx->sk); + if (rc < 0) { + if (rc == -NLE_NODEV) { + rc = -ENODEV; + } else { + nl_perror(rc, "Failed to receive netlink message"); + rc = -EIO; + } + goto free_msg; + } + + /* Ethtool netlink sends ACKs need to pick them up */ + rc = nl_wait_for_ack(ctx->sk); + if (rc < 0) { + nl_perror(rc, "Failed to wait for netlink ack"); + rc = -EIO; + goto free_msg; + } +free_msg: + nlmsg_free(msg); + return rc; +} + +static int ethtool_nl_get_sfp(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics *oi) +{ + int rc = 0; + + /* Page A0h upper */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, SFF8636_PAGE_SIZE); + if (rc < 0) + return rc; + + /* If page A2h is not present we're done */ + if (!(oi->raw[SFF8472_DIAGNOSTICS_TYPE_OFFSET] & SFF8472_DIAGNOSTICS_TYPE_MASK)) + return 0; + + /* Page A2h lower */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_HIGH, 0x0, 0); + if (rc < 0) + return rc; + + /* Page A2h upper */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_HIGH, 0x0, SFF8636_PAGE_SIZE); + if (rc < 0) + return rc; + + return 0; +} + +static int ethtool_nl_get_qsfp(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics *oi) +{ + int rc = 0; + + /* Page 00h upper */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, SFF8636_PAGE_SIZE); + if (rc) + return rc; + + /* Page 01h */ + if (oi->raw[SFF8636_PAGE_OFFSET] & SFF8636_P01H) { + /* Page 01h upper only */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x1, + SFF8636_PAGE_SIZE); + if (rc < 0) + return rc; + } + + /* Page 02h */ + if (oi->raw[SFF8636_PAGE_OFFSET] & SFF8636_P02H) { + /* Page 02h upper only */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x2, + SFF8636_PAGE_SIZE); + if (rc < 0) + return rc; + } + + /* Page 03h is present if flatmem is not set */ + if (!(oi->raw[SFF8636_STATUS_2_OFFSET] & SFF8636_STATUS_FLAT_MEM)) { + /* Page 03h upper only */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x3, + SFF8636_PAGE_SIZE); + if (rc < 0) + return rc; + } + + return 0; +} + +int ethtool_nl_get_optics(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics **oi) +{ + int rc = 0; + int type; + + *oi = util_zalloc(sizeof(**oi)); + nl_socket_modify_cb(ctx->sk, NL_CB_VALID, NL_CB_CUSTOM, ethtool_nl_cb, oi); + + /* Page 00h lower */ + rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, 0); + if (rc < 0) + goto out_err_free_oi; + + type = optics_type(*oi); + switch (type) { + case OPTICS_TYPE_SFP: + rc = ethtool_nl_get_sfp(ctx, netdev, *oi); + break; + case OPTICS_TYPE_QSFP28: + rc = ethtool_nl_get_qsfp(ctx, netdev, *oi); + break; + }; + if (rc < 0) + goto out_err_free_oi; + + return rc; + +out_err_free_oi: + free(*oi); + *oi = NULL; + return rc; +} diff --git a/opticsmon/ethtool.h b/opticsmon/ethtool.h new file mode 100644 index 00000000..6cf0892c --- /dev/null +++ b/opticsmon/ethtool.h @@ -0,0 +1,11 @@ +#pragma once +#include "optics_info.h" + +struct ethtool_nl_ctx { + struct nl_sock *sk; + int ethtool_id; +}; + +int ethtool_nl_connect(struct ethtool_nl_ctx *ctx); +void ethtool_nl_close(struct ethtool_nl_ctx *ctx); +int ethtool_nl_get_optics(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics **oi); diff --git a/opticsmon/link_mon.c b/opticsmon/link_mon.c new file mode 100644 index 00000000..8c9310fa --- /dev/null +++ b/opticsmon/link_mon.c @@ -0,0 +1,96 @@ +/* + * Copyright IBM Corp. 2024 + * + * s390-tools is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "link_mon.h" + +#define MAX_EVENTS 32 + +static void nl_obj_parsed_cb(struct nl_object *obj, void *arg) +{ + struct link_mon_nl_ctx *ctx = arg; + struct rtnl_link *link; + struct zpci_netdev netdev; + + if (strcmp(nl_object_get_type(obj), "route/link") != 0) + return; + link = (struct rtnl_link *)obj; + netdev.name = rtnl_link_get_name(link); + netdev.operstate = rtnl_link_get_operstate(link); + ctx->cb(&netdev, ctx->arg); +} + +static int nl_rtnl_lnkgrp_cb(struct nl_msg *msg, void *arg) +{ + if (nl_msg_parse(msg, &nl_obj_parsed_cb, arg) < 0) + fprintf(stderr, "<> Unknown message type\n"); + return NL_STOP; +} + +void link_mon_nl_waitfd_read(struct link_mon_nl_ctx *ctx) +{ + nl_recvmsgs_default(ctx->sk); +} + +int link_mon_nl_waitfd_getfd(struct link_mon_nl_ctx *ctx) +{ + return nl_socket_get_fd(ctx->sk); +} + +int link_mon_nl_waitfd_create(struct link_mon_nl_ctx *ctx, link_mon_nl_cb cb, void *arg) +{ + int ret = 0, rc = 0; + + ctx->sk = nl_socket_alloc(); + if (!ctx->sk) + return -ENOMEM; + ctx->cb = cb; + ctx->arg = arg; + nl_socket_disable_seq_check(ctx->sk); + nl_socket_modify_cb(ctx->sk, NL_CB_VALID, NL_CB_CUSTOM, nl_rtnl_lnkgrp_cb, ctx); + + rc = nl_connect(ctx->sk, NETLINK_ROUTE); + if (rc < 0) { + ret = rc; + goto err_free; + } + + rc = nl_socket_add_membership(ctx->sk, RTNLGRP_LINK); + if (rc < 0) { + ret = rc; + goto err_close; + } + + rc = rtnl_link_alloc_cache(ctx->sk, AF_UNSPEC, &ctx->cache); + if (rc < 0) { + ret = rc; + goto err_close; + } + nl_cache_mngt_provide(ctx->cache); + return 0; + +err_close: + nl_close(ctx->sk); +err_free: + nl_socket_free(ctx->sk); + return ret; +} + +void link_mon_nl_waitfd_destroy(struct link_mon_nl_ctx *ctx) +{ + nl_cache_free(ctx->cache); + nl_close(ctx->sk); + nl_socket_free(ctx->sk); +} diff --git a/opticsmon/link_mon.h b/opticsmon/link_mon.h new file mode 100644 index 00000000..8311017b --- /dev/null +++ b/opticsmon/link_mon.h @@ -0,0 +1,27 @@ +/* + * Copyright IBM Corp. 2024 + * + * s390-tools is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ +#pragma once +#include + +#include + +#include "lib/pci_list.h" + +typedef void (*link_mon_nl_cb)(struct zpci_netdev *, void *arg); + +struct link_mon_nl_ctx { + /* private fields */ + struct nl_sock *sk; + struct nl_cache *cache; + link_mon_nl_cb cb; + void *arg; +}; + +int link_mon_nl_waitfd_create(struct link_mon_nl_ctx *ctx, link_mon_nl_cb cb, void *arg); +void link_mon_nl_waitfd_read(struct link_mon_nl_ctx *ctx); +void link_mon_nl_waitfd_destroy(struct link_mon_nl_ctx *ctx); +int link_mon_nl_waitfd_getfd(struct link_mon_nl_ctx *ctx); diff --git a/opticsmon/optics_info.c b/opticsmon/optics_info.c new file mode 100644 index 00000000..fc041da3 --- /dev/null +++ b/opticsmon/optics_info.c @@ -0,0 +1,171 @@ +#include + +#include "optics_info.h" + +#define OPTICS_TYPE_OFFSET 0x0 + +#define OPTICS_SFP_LOS_IMPLEMENTED_OFFSET 0x41 +#define OPTICS_SFP_LOS_IMPLEMENTED_MASK 0x2 + +#define OPTICS_SFP_A2H_OFFSET 0x100 +#define OPTICS_SFP_LOS_OFFSET (OPTICS_SFP_A2H_OFFSET + 0x6e) +#define OPTICS_SFP_DATA_NOT_READY_MASK 0x1 +#define OPTICS_SFP_TX_FAULT_MASK 0x4 +#define OPTICS_SFP_RX_LOS_MASK 0x2 + +#define OPTICS_QSFP28_LOS_IMPLEMENTED_OFFSET 0xC3 +#define OPTICS_QSFP28_TX_LOS_IMPLEMENTED_MASK 0x2 +#define OPTICS_QSFP28_TX_FAULT_IMPLEMENTED_MASK 0x8 + +#define OPTICS_QSFP28_LOS_OFFSET 0x3 +#define OPTICS_QSFP28_LOS_MASK 0xf +#define OPTICS_QSFP28_TX_LOS_MASK 0xf0 +#define OPTICS_QSFP28_TX_LOS_SHIFT 0x4 + +#define OPTICS_QSFP28_TX_FAULT_OFFSET 0x4 +#define OPTICS_QSFP28_TX_FAULT_MASK 0xf + +const char *optics_type_str(enum optics_type type) +{ + switch (type) { + case OPTICS_TYPE_UNKNOWN: + return "unknown"; + case OPTICS_TYPE_SFP: + return "SFP/SFP+/SFP28"; + case OPTICS_TYPE_QSFP28: + return "QSFP28"; + }; + return "n.a."; +} + +enum optics_type optics_type(struct optics *oi) +{ + if (!oi || !oi->raw || oi->size < OPTICS_TYPE_OFFSET + 1) + return OPTICS_TYPE_UNKNOWN; + + switch (oi->raw[OPTICS_TYPE_OFFSET]) { + case (uint8_t)OPTICS_TYPE_SFP: + return OPTICS_TYPE_SFP; + case (uint8_t)OPTICS_TYPE_QSFP28: + return OPTICS_TYPE_QSFP28; + default: + return OPTICS_TYPE_UNKNOWN; + }; +} + +bool optics_los_implemented(struct optics *oi) +{ + enum optics_type type = optics_type(oi); + uint8_t implemented; + + if (type == OPTICS_TYPE_SFP) { + if (oi->size < OPTICS_SFP_LOS_IMPLEMENTED_OFFSET + 1) + return false; + implemented = oi->raw[OPTICS_SFP_LOS_IMPLEMENTED_OFFSET]; + return !!(implemented & OPTICS_SFP_LOS_IMPLEMENTED_MASK); + } else if (type == OPTICS_TYPE_QSFP28) { + if (oi->size < OPTICS_QSFP28_LOS_OFFSET + 1) + return false; + if (oi->size < OPTICS_QSFP28_LOS_IMPLEMENTED_OFFSET) + return false; + implemented = oi->raw[OPTICS_QSFP28_LOS_IMPLEMENTED_OFFSET]; + /* + * No RX LoS implemented flag take TX LOS implemented like + * ethtool + */ + return !!(implemented & OPTICS_QSFP28_TX_LOS_IMPLEMENTED_MASK); + } + return false; +} + +enum optics_los optics_rx_los(struct optics *oi) +{ + enum optics_los los = OPTICS_UNKNOWN_LOS; + enum optics_type type = optics_type(oi); + + if (!optics_los_implemented(oi)) + return los; + + if (type == OPTICS_TYPE_SFP) { + los = oi->raw[OPTICS_SFP_LOS_OFFSET]; + if (los & OPTICS_SFP_DATA_NOT_READY_MASK) + return OPTICS_UNKNOWN_LOS; + if (los & OPTICS_SFP_RX_LOS_MASK) + return OPTICS_LOS; + else + return OPTICS_NO_LOS; + } else if (type == OPTICS_TYPE_QSFP28) { + los = oi->raw[OPTICS_QSFP28_LOS_OFFSET]; + if (los & OPTICS_QSFP28_LOS_MASK) + los = OPTICS_LOS; + else + los = OPTICS_NO_LOS; + } + return los; +} + +const char *optics_los_str(enum optics_los los) +{ + switch (los) { + case OPTICS_LOS: + return "yes"; + case OPTICS_NO_LOS: + return "no"; + case OPTICS_UNAVAILABLE_LOS: + return "unavailable"; + default: + return "unknown"; + } +} + +enum optics_los optics_tx_fault(struct optics *oi) +{ + enum optics_los los = OPTICS_UNKNOWN_LOS; + enum optics_type type = optics_type(oi); + + if (!optics_los_implemented(oi)) + return los; + + if (type == OPTICS_TYPE_SFP) { + los = oi->raw[OPTICS_SFP_LOS_OFFSET]; + if (los & OPTICS_SFP_DATA_NOT_READY_MASK) + return OPTICS_UNKNOWN_LOS; + if (los & OPTICS_SFP_TX_FAULT_MASK) + return OPTICS_LOS; + else + return OPTICS_NO_LOS; + } else if (type == OPTICS_TYPE_QSFP28) { + los = oi->raw[OPTICS_QSFP28_TX_FAULT_OFFSET]; + if (los & OPTICS_QSFP28_TX_FAULT_MASK) + los = OPTICS_LOS; + else + los = OPTICS_NO_LOS; + } + return los; +} + +enum optics_los optics_tx_los(struct optics *oi) +{ + enum optics_los los = OPTICS_UNKNOWN_LOS; + enum optics_type type = optics_type(oi); + + if (!optics_los_implemented(oi)) + return los; + + if (type == OPTICS_TYPE_SFP) { + return OPTICS_UNAVAILABLE_LOS; + } else if (type == OPTICS_TYPE_QSFP28) { + los = oi->raw[OPTICS_QSFP28_LOS_OFFSET]; + if (los & OPTICS_QSFP28_TX_LOS_MASK) + los = OPTICS_LOS; + else + los = OPTICS_NO_LOS; + } + return los; +} + +void optics_free(struct optics *oi) +{ + free(oi->raw); + free(oi); +} diff --git a/opticsmon/optics_info.h b/opticsmon/optics_info.h new file mode 100644 index 00000000..0b23f795 --- /dev/null +++ b/opticsmon/optics_info.h @@ -0,0 +1,49 @@ +#pragma once +#include +#include + +#include + +#define SFF8079_I2C_ADDRESS_LOW 0x50 +#define SFF8079_I2C_ADDRESS_HIGH 0x51 + +#define SFF8472_DIAGNOSTICS_TYPE_OFFSET 0x5C +#define SFF8472_DIAGNOSTICS_TYPE_MASK (1 << 6) + +#define SFF8636_PAGE_SIZE 0x80 +#define SFF8636_QSFP28_LENGTH 0x100 + +#define SFF8636_STATUS_2_OFFSET 0x02 +#define SFF8636_STATUS_FLAT_MEM (1 << 2) + +#define SFF8636_PAGE_OFFSET 0xC3 +#define SFF8636_P01H (1 << 6) +#define SFF8636_P02H (1 << 7) + +enum optics_type { + OPTICS_TYPE_UNKNOWN = 0x0, /* Unknown or unsupported */ + OPTICS_TYPE_SFP = 0x3, /* SFP/SFP+/SFP28 and later with SFF-8472 management interface */ + OPTICS_TYPE_QSFP28 = 0x11 /* QSFP28 (SFF-8665 et al.)*/ +}; + +enum optics_los { + OPTICS_NO_LOS = 0x0, + OPTICS_LOS = 0x1, + OPTICS_UNKNOWN_LOS = 0x2, + OPTICS_UNAVAILABLE_LOS = 0x3, +}; + +struct optics { + size_t size; + uint8_t *raw; +}; + +enum optics_type optics_type(struct optics *oi); +const char *optics_type_str(enum optics_type type); +const char *optics_los_str(enum optics_los los); + +enum optics_los optics_rx_los(struct optics *oi); +enum optics_los optics_tx_los(struct optics *oi); +enum optics_los optics_tx_fault(struct optics *oi); + +void optics_free(struct optics *oi); diff --git a/opticsmon/optics_sclp.c b/opticsmon/optics_sclp.c new file mode 100644 index 00000000..e6248130 --- /dev/null +++ b/opticsmon/optics_sclp.c @@ -0,0 +1,50 @@ +#include +#include +#include + +#include "lib/pci_sclp.h" +#include "lib/util_libc.h" + +#include "optics_sclp.h" + +static struct sclp_optics_data *init_sclp_optics_data(struct optics *oi, size_t *length) +{ + struct sclp_optics_data *od; + + *length = sizeof(*od) + oi->size; + od = util_zalloc(*length); + od->module_present = optics_type(oi) != OPTICS_TYPE_UNKNOWN; + od->rx_los = optics_rx_los(oi) == OPTICS_LOS; + od->tx_fault = optics_tx_fault(oi) == OPTICS_LOS; + switch (optics_type(oi)) { + case OPTICS_TYPE_SFP: + od->data_identifier = 1; + break; + case OPTICS_TYPE_QSFP28: + od->data_identifier = 2; + break; + default: + od->data_identifier = 0; + } + memcpy(od->data, oi->raw, oi->size); + + return od; +} + +int sclp_issue_optics_report(struct zpci_dev *zdev, struct optics *oi) +{ + struct sclp_optics_data *od; + size_t length; + char *pci_addr; + int rc; + + if (zdev->pft != ZPCI_PFT_NETD) + return -ENOTSUP; + od = init_sclp_optics_data(oi, &length); + pci_addr = zpci_pci_addr(zdev); + rc = zpci_sclp_issue_action(pci_addr, SCLP_ERRNOTIFY_AQ_OPTICS_DATA, + (char *)od, length, SCLP_ERRNOTIFY_ID_OPTICSMON); + free(pci_addr); + free(od); + return rc; +} diff --git a/opticsmon/optics_sclp.h b/opticsmon/optics_sclp.h new file mode 100644 index 00000000..b211e9ac --- /dev/null +++ b/opticsmon/optics_sclp.h @@ -0,0 +1,21 @@ +#include + +#include "lib/pci_list.h" + +#include "optics_info.h" + +struct sclp_optics_data { + /* Status */ + uint32_t module_present : 1; + uint32_t rx_los : 1; + uint32_t tx_fault : 1; + uint32_t reserved_status : 29; + /* Data Identifier */ + uint32_t data_identifier; + /* Reserved */ + uint64_t reserved[3]; + /* Additional Log Data */ + uint8_t data[]; +} __packed; + +int sclp_issue_optics_report(struct zpci_dev *zdev, struct optics *oi); diff --git a/opticsmon/opticsmon.8 b/opticsmon/opticsmon.8 new file mode 100644 index 00000000..1801de9d --- /dev/null +++ b/opticsmon/opticsmon.8 @@ -0,0 +1,83 @@ +.\" Copyright IBM Corp. 2024 +.\" s390-tools is free software; you can redistribute it and/or modify +.\" it under the terms of the MIT license. See LICENSE for details. +.\" +.\" Macro for inserting an option description prologue. +.\" .OD [] [args] +.de OD +. ds args " +. if !'\\$3'' .as args \fI\\$3\fP +. if !'\\$4'' .as args \\$4 +. if !'\\$5'' .as args \fI\\$5\fP +. if !'\\$6'' .as args \\$6 +. if !'\\$7'' .as args \fI\\$7\fP +. PD 0 +. if !'\\$2'' .IP "\fB\-\\$2\fP \\*[args]" 4 +. if !'\\$1'' .IP "\fB\-\-\\$1\fP \\*[args]" 4 +. PD +.. +. +.TH opticsmon 8 "Oct 2024" s390-tools zpcictl +. +.SH NAME +opticsmon - Monitor optical modules for directly attached PCI based NICs +. +. +.SH SYNOPSIS +.B "opticsmon" +.I "OPTIONS" +. +. +.SH DESCRIPTION +Use +.B opticsmon +to monitor the health of the optical modules of directly attached PCI based +NICs. When executed without the \fB--daemon\fR option it will collect optical +module data from all available PCI network interface physical functions and +print a summary in JSON format. Add the \fB--send-report\fR option to report +this data to the support element. + + +. +. +.SH OPTIONS +.SS Operation Options +.OD daemon "d" +Run continuously and report on link state changes and periodically +.PP +. +.OD send-report "r" +Report the optics health data to the Support Element (SE) +.PP +. +.OD quiet "q" +Be quiet and don't print optics health summary +.PP +. +.OD interval "i" "seconds" +Interval in seconds at which to collect monitoring data in the absence of link +state changes. A value larger than 24 hours (86400 seconds) is clamped down to +24 hours. +.PP +. +.OD module-info "" +Include a base64 encoded binary dump of the module's SFF-8636/8472/8024 +standard data for each netdev. This matches "ethtool --module-info raw +on". + +.B Example: Extract module information for the first adapter + +.CL +opticsmon --module-info | jq -r '.adapters[0].netdevs[0].optics.module_info' | base64 -d | hexdump -C +.PP +.PP +. +.SS General Options +.OD help "h" "" +Print usage information, then exit. +.PP +. +.OD version "v" "" +Print version information, then exit. +.PP + diff --git a/opticsmon/opticsmon.c b/opticsmon/opticsmon.c new file mode 100644 index 00000000..c2f355e2 --- /dev/null +++ b/opticsmon/opticsmon.c @@ -0,0 +1,449 @@ +/* + * opticsmon - Report optics monitoring data to firmware + * + * Copyright IBM Corp. 2024 + * + * s390-tools is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "lib/util_list.h" +#include "lib/pci_list.h" +#include "lib/util_prg.h" +#include "lib/util_opt.h" +#include "lib/util_fmt.h" +#include "lib/util_libc.h" + +#include + +#include "optics_info.h" +#include "optics_sclp.h" +#include "ethtool.h" +#include "link_mon.h" + +#define API_LEVEL 1 + +struct options { + bool monitor; + bool report; + bool module_info; + bool quiet; + + uint32_t interval_seconds; +}; + +struct opticsmon_ctx { + struct options opts; + struct ethtool_nl_ctx ethtool_ctx; + struct link_mon_nl_ctx lctx; + struct util_list *zpci_list; +}; + +static const struct util_prg prg = { + .desc = "Use opticsmon to monitor the health of the optical modules\n" + "of directly attached PCI based NICs", + .copyright_vec = { { + .owner = "IBM Corp.", + .pub_first = 2024, + .pub_last = 2024, + }, + UTIL_PRG_COPYRIGHT_END } +}; + +#define OPT_DUMP 128 + +static struct util_opt opt_vec[] = { + UTIL_OPT_SECTION("OPERATION OPTIONS"), + { + .option = { "monitor", no_argument, NULL, 'm' }, + .desc = "Run continuously and report on link state changes " + "collecting optics health data when a change is detected", + }, + { + .option = { "send-report", no_argument, NULL, 'r' }, + .desc = "Report the optics health data to the Support Element", + }, + { + .option = { "quiet", no_argument, NULL, 'q' }, + .desc = "Be quiet and don't print optics health summary", + }, + { + .option = { "module-info", no_argument, NULL, OPT_DUMP }, + .desc = "Include a base64 encoded binary dump of the module's " + "SFF-8636/8472/8024 standard data for each netdev. " + "This matches \"ethtool --module-info raw on\"", + .flags = UTIL_OPT_FLAG_NOSHORT, + }, + UTIL_OPT_SECTION("OPTIONS WITH ARGUMENTS"), + { + .option = { "interval", required_argument, NULL, 'i' }, + .argument = "seconds", + .desc = "Interval in seconds at which to collect monitoring data " + "in the absence of link state changes. A value larger than " + "24 hours (86400 seconds) is clamped down to 24 hours.", + }, + UTIL_OPT_SECTION("GENERAL OPTIONS"), + UTIL_OPT_HELP, + UTIL_OPT_VERSION, + UTIL_OPT_END +}; + +static void parse_cmdline(int argc, char *argv[], struct options *opts) +{ + uint32_t seconds; + int cmd, ret; + + util_prg_init(&prg); + util_opt_init(opt_vec, NULL); + + do { + cmd = util_opt_getopt_long(argc, argv); + + switch (cmd) { + case 'm': + opts->monitor = true; + break; + case 'r': + opts->report = true; + break; + case 'q': + opts->quiet = true; + break; + case OPT_DUMP: + opts->module_info = true; + break; + case 'i': + ret = sscanf(optarg, "%u", &seconds); + if (ret != 1) { + fprintf(stderr, + "Failed to parse interval argument \"%s\" as seconds\n", + optarg); + exit(EXIT_FAILURE); + } + if (seconds < 86400) + opts->interval_seconds = seconds; + break; + case 'h': + util_prg_print_help(); + util_opt_print_help(); + exit(EXIT_SUCCESS); + case 'v': + util_prg_print_version(); + exit(EXIT_SUCCESS); + case -1: + /* End of options string */ + break; + } + } while (cmd != -1); +} + +static int module_info_pair(struct optics *oi) +{ + size_t b64_calclen, b64len; + int rc = EXIT_SUCCESS; + char *b64; + + b64_calclen = (oi->size / 3) * 4; + if (oi->size % 3 > 0) + b64_calclen += 4; + + b64 = util_zalloc(b64_calclen + 1); /* adds NUL byte */ + b64len = EVP_EncodeBlock((unsigned char *)b64, oi->raw, oi->size); + if (b64len != b64_calclen) { + fprintf(stderr, "encoding base64 via openssl failed\n"); + rc = EXIT_FAILURE; + goto out; + } + util_fmt_pair(FMT_QUOTE, "module_info", b64); +out: + free(b64); + return rc; +} + +static void optics_json_print(struct opticsmon_ctx *ctx, struct zpci_netdev *nd, struct optics *oi) +{ + util_fmt_obj_start(FMT_DEFAULT, "netdev"); + util_fmt_pair(FMT_QUOTE, "name", nd->name); + util_fmt_pair(FMT_QUOTE, "operstate", zpci_operstate_str(nd->operstate)); + util_fmt_obj_start(FMT_DEFAULT, "optics"); + util_fmt_pair(FMT_QUOTE, "type", optics_type_str(optics_type(oi))); + util_fmt_pair(FMT_QUOTE, "rx_los", optics_los_str(optics_rx_los(oi))); + util_fmt_pair(FMT_QUOTE, "tx_los", optics_los_str(optics_tx_los(oi))); + util_fmt_pair(FMT_QUOTE, "tx_fault", optics_los_str(optics_rx_los(oi))); + if (ctx->opts.module_info) + module_info_pair(oi); + util_fmt_obj_end(); + util_fmt_obj_end(); +} + +static int dump_adapter_data(struct opticsmon_ctx *ctx, struct zpci_dev *zdev) +{ + struct optics **ois; + int num_ois = 0; + char *pci_addr; + int i, rc; + + ois = util_zalloc(sizeof(ois[0]) * zdev->num_netdevs); + for (i = 0; i < zdev->num_netdevs; i++) { + rc = ethtool_nl_get_optics(&ctx->ethtool_ctx, zdev->netdevs[i].name, &ois[i]); + if (rc) + goto free_ois; + num_ois++; + } + if (!ctx->opts.quiet) { + util_fmt_obj_start(FMT_DEFAULT, "adapter"); + util_fmt_pair(FMT_QUOTE, "pft", zpci_pft_str(zdev)); + util_fmt_obj_start(FMT_DEFAULT, "ids"); + util_fmt_pair(FMT_QUOTE, "fid", "0x%0x", zdev->fid); + if (zdev->uid_is_unique) + util_fmt_pair(FMT_QUOTE, "uid", "0x%0x", zdev->uid); + pci_addr = zpci_pci_addr(zdev); + util_fmt_pair(FMT_QUOTE, "pci_address", pci_addr); + free(pci_addr); + util_fmt_obj_end(); + util_fmt_obj_start(FMT_LIST, "netdevs"); + for (i = 0; i < zdev->num_netdevs; i++) + optics_json_print(ctx, &zdev->netdevs[i], ois[i]); + util_fmt_obj_end(); /* netdevs list */ + util_fmt_obj_end(); /* adapter */ + fflush(stdout); + } + if (ctx->opts.report) { + for (i = 0; i < zdev->num_netdevs; i++) { + rc = sclp_issue_optics_report(zdev, ois[i]); + if (rc == -ENOTSUP) { + fprintf(stderr, "Skipping %s which does not support reporting\n", + zdev->netdevs[i].name); + } else if (rc < 0) { + fprintf(stderr, "Error issuing SCLP for optics data failed: %s\n", + strerror(-rc)); + } + } + } +free_ois: + for (i = 0; i < num_ois; i++) + optics_free(ois[i]); + free(ois); + return rc; +} + +static void zpci_list_reload(struct util_list **zpci_list) +{ + if (*zpci_list) + zpci_free_dev_list(*zpci_list); + *zpci_list = zpci_dev_list(); +} + +static void dump_all_adapter_data(struct opticsmon_ctx *ctx) +{ + struct zpci_dev *zdev; + + zpci_list_reload(&ctx->zpci_list); + util_list_iterate(ctx->zpci_list, zdev) { + /* Filter non-NIC devices and VFs */ + if (zpci_is_vf(zdev) || !zdev->num_netdevs) + continue; + dump_adapter_data(ctx, zdev); + } +} + +static int oneshot_mode(struct opticsmon_ctx *ctx) +{ + util_fmt_init(stdout, FMT_JSON, FMT_DEFAULT, API_LEVEL); + if (!ctx->opts.quiet) + util_fmt_obj_start(FMT_LIST, "adapters"); + dump_all_adapter_data(ctx); + if (!ctx->opts.quiet) + util_fmt_obj_end(); + util_fmt_exit(); + + return EXIT_SUCCESS; +} + +void on_link_change(struct zpci_netdev *netdev, void *arg) +{ + struct opticsmon_ctx *ctx = arg; + struct zpci_dev *zdev; + int i, reloads = 1; + + if (!ctx->zpci_list || util_list_is_empty(ctx->zpci_list)) + zpci_list_reload(&ctx->zpci_list); + +reload: + util_list_iterate(ctx->zpci_list, zdev) { + for (i = 0; i < zdev->num_netdevs; i++) { + if (!strcmp(zdev->netdevs[i].name, netdev->name)) { + reloads--; + /* Skip data collection if operational state is + * unchanged + */ + if (zdev->netdevs[i].operstate == netdev->operstate) + continue; + /* Update operation state for VFs even though + * they are skipped just for a consistent view + */ + zdev->netdevs[i].operstate = netdev->operstate; + /* Only collect optics data for PFs */ + if (!zpci_is_vf(zdev)) + dump_adapter_data(ctx, zdev); + } + } + } + /* Might be a new device, reload list of devices and retry */ + if (reloads) { + zpci_list_reload(&ctx->zpci_list); + reloads--; + goto reload; + } +} + +#define MAX_EVENTS 8 + +static int monitor_wait_loop(struct opticsmon_ctx *ctx, int sigfd, int timerfd) +{ + struct epoll_event events[MAX_EVENTS]; + struct signalfd_siginfo fdsi; + int i, nlfd, epfd, nfds; + struct epoll_event ev; + uint64_t expirations; + ssize_t sread; + + epfd = epoll_create1(EPOLL_CLOEXEC); + + ev.events = EPOLLIN; + ev.data.fd = sigfd; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, sigfd, &ev) == -1) + return -EIO; + + ev.events = EPOLLIN; + ev.data.fd = timerfd; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, timerfd, &ev) == -1) + return -EIO; + + nlfd = link_mon_nl_waitfd_getfd(&ctx->lctx); + ev.events = EPOLLIN; + ev.data.fd = nlfd; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, nlfd, &ev) == -1) + return -EIO; + + while (1) { + nfds = epoll_wait(epfd, events, MAX_EVENTS, -1); + if (nfds < 0) + return nfds; + for (i = 0; i < nfds; i++) { + /* signal fd */ + if (events[i].data.fd == sigfd) { + sread = read(sigfd, &fdsi, sizeof(fdsi)); + if (sread != sizeof(fdsi)) + return -EIO; + switch (fdsi.ssi_signo) { + case SIGINT: + case SIGTERM: + case SIGQUIT: + return 0; + /* Unexpected signal */ + default: + return -EIO; + } + /* timer fd */ + } else if (events[i].data.fd == timerfd) { + sread = read(timerfd, &expirations, sizeof(uint64_t)); + if (sread != sizeof(uint64_t)) + return -EIO; + if (!expirations) + continue; + dump_all_adapter_data(ctx); + /* netlink fd */ + } else if (events[i].data.fd == nlfd) { + link_mon_nl_waitfd_read(&ctx->lctx); + } + } + } + return 0; +} + +static int monitor_mode(struct opticsmon_ctx *ctx) +{ + struct itimerspec timerspec; + int sigfd, timerfd, ret; + sigset_t mask; + + sigemptyset(&mask); + sigaddset(&mask, SIGINT); + sigaddset(&mask, SIGQUIT); + sigaddset(&mask, SIGTERM); + + if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) + return -EIO; + + sigfd = signalfd(-1, &mask, 0); + if (sigfd == -1) { + fprintf(stderr, "Failed to create signalfd\n"); + return -EIO; + } + + timerfd = timerfd_create(CLOCK_MONOTONIC, 0); + if (timerfd == -1) { + fprintf(stderr, "Failed to create timerfd\n"); + ret = -EIO; + goto close_signalfd; + } + + /* Set initial expiration to 1 ns so we gather optics data at startup */ + timerspec.it_value.tv_sec = 0; + timerspec.it_value.tv_nsec = 1; + timerspec.it_interval.tv_sec = ctx->opts.interval_seconds; + timerspec.it_interval.tv_nsec = 0; + ret = timerfd_settime(timerfd, 0, &timerspec, NULL); + if (ret == -1) { + fprintf(stderr, "Failed to arm timer\n"); + goto close_timerfd; + } + + util_fmt_init(stdout, FMT_JSONSEQ, FMT_DEFAULT, API_LEVEL); + ret = link_mon_nl_waitfd_create(&ctx->lctx, on_link_change, ctx); + if (ret) { + fprintf(stderr, "Failed to create link monitoring socket\n"); + goto close_timerfd; + } + + monitor_wait_loop(ctx, sigfd, timerfd); + + link_mon_nl_waitfd_destroy(&ctx->lctx); + util_fmt_exit(); +close_signalfd: + close(sigfd); +close_timerfd: + close(timerfd); + return ret; +} + +int main(int argc, char **argv) +{ + struct opticsmon_ctx ctx = { .opts = { .interval_seconds = 86400 } }; + int ret; + + parse_cmdline(argc, argv, &ctx.opts); + ethtool_nl_connect(&ctx.ethtool_ctx); + if (ctx.opts.monitor) + ret = monitor_mode(&ctx); + else + ret = oneshot_mode(&ctx); + ethtool_nl_close(&ctx.ethtool_ctx); + + if (ctx.zpci_list) + zpci_free_dev_list(ctx.zpci_list); + + return ret; +} diff --git a/systemd/Makefile b/systemd/Makefile index 76b1646f..9388fd9c 100644 --- a/systemd/Makefile +++ b/systemd/Makefile @@ -3,7 +3,8 @@ include ../common.mak SYSTEM_UNITS = ttyrun-getty@.service iucvtty-login@.service \ cpacfstatsd.service cpuplugd.service \ dumpconf.service cpi.service \ - mon_fsstatd.service mon_procd.service + mon_fsstatd.service mon_procd.service \ + opticsmon.service all: diff --git a/systemd/opticsmon.service.in b/systemd/opticsmon.service.in new file mode 100644 index 00000000..335d9165 --- /dev/null +++ b/systemd/opticsmon.service.in @@ -0,0 +1,20 @@ +# +# Systemd unit for mon_fsstatd (monitor z/VM guest file systems) +# +# Copyright IBM Corp. 2024 +# +# s390-tools is free software; you can redistribute it and/or modify +# it under the terms of the MIT license. See LICENSE for details. +# + +[Unit] +Description=Monitor health of directly attached PCI NIC optical modules +# Documentation=man:opticsmon + +[Service] +Type=exec +ExecStart=@usrsbin_path@/opticsmon --send-report --monitor --interval 86400 +KillMode=control-group + +[Install] +WantedBy=multi-user.target