From 147ff1bf49d1b792ca3151563963e3e0f8a8d35f Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Thu, 23 May 2024 13:58:08 +0200 Subject: [PATCH] cpuplugd: Halt cpu hotplugging on vertical polarization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On vertical polarization, kernel can be adjusting CPU capacities dynamically, and cpuplugd can interfere this with hotplug operations causing performance degradation. To prevent this, check if system has switched its polarization state, and act accordingly. If system is on vertical polarization when daemon starts, no CPU hotplug action is triggered. If system changes to vertical polarization during daemon runtime, revert cpuhotplug adjustments and stop further CPU hotplug actions. If system switches back to horizontal polarization during runtime of the daemon, start evaluating CPU hotplug rules and trigger adjustments. Reviewed-by: Steffen Eiden Reviewed-by: Gerald Schaefer Signed-off-by: Mete Durlu Signed-off-by: Jan Höppner --- cpuplugd/config.c | 92 ++++++++++++++++++++++++----------------- cpuplugd/cpu.c | 19 ++++++++- cpuplugd/cpuplugd.h | 5 +++ cpuplugd/daemon.c | 4 ++ cpuplugd/main.c | 34 +++++++++++---- cpuplugd/man/cpuplugd.8 | 3 ++ 6 files changed, 111 insertions(+), 46 deletions(-) diff --git a/cpuplugd/config.c b/cpuplugd/config.c index e48a3913..a3c8c6b7 100644 --- a/cpuplugd/config.c +++ b/cpuplugd/config.c @@ -204,6 +204,45 @@ void parse_configfile(char *file) fclose(filp); } +void apply_cpu_config(void) +{ + /* + * Check that the initial number of cpus is not below the + * minimum + */ + if (num_cpu_start < cfg.cpu_min && + get_numcpus() >= cfg.cpu_min) { + cpuplugd_debug("The number of online cpus is below "); + cpuplugd_debug("the minimum and will be increased.\n"); + while (get_num_online_cpus() < cfg.cpu_min) { + if (hotplug_one_cpu()) + break; + } + } + if (get_num_online_cpus() > cfg.cpu_max) { + cpuplugd_debug("The number of online cpus is above the maximum"); + cpuplugd_debug(" and will be decreased.\n"); + while (get_num_online_cpus() > cfg.cpu_max) { + if (hotunplug_one_cpu()) + break; + } + } + if (cfg.cpu_min > get_numcpus()) { + /* + * This check only works if nobody used the + * additional_cpus in the boot parameter section + */ + cpuplugd_debug("The minimum amount of cpus is above the "); + cpuplugd_debug("number of available cpus.\n"); + cpuplugd_exit("Detected %d available cpus\n", get_numcpus()); + } + if (get_num_online_cpus() < cfg.cpu_min) { + cpuplugd_debug("Failed to set the number of online cpus to "); + cpuplugd_debug("the minimum. "); + cpuplugd_exit("Aborting.\n"); + } +} + /* * Check if the required settings are found in the configuration file. * "Autodetect" if cpu and/or memory hotplug configuration entries @@ -261,44 +300,23 @@ void check_config() /* * Save the number of online cpus and the cmm_pagesize at startup, * so that we can enable exactly the same amount when the daemon ends + * + * Don't adjust cpus if system is on vertical polarization */ - if (cpu) { - num_cpu_start = get_num_online_cpus(); - cpuplugd_debug("Daemon started with %d active cpus.\n", - num_cpu_start); - /* - * Check that the initial number of cpus is not below the - * minimum - */ - if (num_cpu_start < cfg.cpu_min && - get_numcpus() >= cfg.cpu_min) { - cpuplugd_debug("The number of online cpus is below " - "the minimum and will be increased.\n"); - while (get_num_online_cpus() < cfg.cpu_min) { - if (hotplug_one_cpu()) - break; - } - } - if (get_num_online_cpus() > cfg.cpu_max) { - cpuplugd_debug("The number of online cpus is above the maximum" - " and will be decreased.\n"); - while (get_num_online_cpus() > cfg.cpu_max) { - if (hotunplug_one_cpu()) - break; - } - } - if (cfg.cpu_min > get_numcpus()) - /* - * This check only works if nobody used the - * additional_cpus in the boot parameter section - */ - cpuplugd_exit("The minimum amount of cpus is above " - "the number of available cpus.\n" - "Detected %d available cpus\n", - get_numcpus()); - if (get_num_online_cpus() < cfg.cpu_min) - cpuplugd_exit("Failed to set the number of online " - "cpus to the minimum. Aborting.\n"); + saved_polarization = get_polarization(); + num_cpu_start = get_num_online_cpus(); + cpuplugd_debug("Daemon started with %d active cpus.\n", + num_cpu_start); + if (saved_polarization < 0) { + cpuplugd_debug("Daemon couldn't determine system polarization\n"); + cpuplugd_debug("Starting without evaluating cpu rules\n"); + } else if (saved_polarization == PLR_VERTICAL) { + cpuplugd_debug("Daemon started with vertical polarization.\n"); + cpuplugd_debug("Cpu adjustments won't be made until system "); + cpuplugd_debug("is in horizontal polarization\n"); + } else if (saved_polarization == PLR_HORIZONTAL && + cpu == 1) { + apply_cpu_config(); } if (memory == 1) { /* diff --git a/cpuplugd/cpu.c b/cpuplugd/cpu.c index 54a6f349..97092c16 100644 --- a/cpuplugd/cpu.c +++ b/cpuplugd/cpu.c @@ -78,6 +78,23 @@ int get_num_online_cpus(void) return number; } +/* + * get_polarization() - return system polarization + */ +int get_polarization(void) +{ + int polarization; + char *path; + + path = util_path_sysfs("devices/system/cpu/dispatching"); + if (util_file_read_i(&polarization, NUM_BASE, path) < 0) { + polarization = -1; + cpuplugd_debug("failed to read system polarization\n"); + } + free(path); + return polarization; +} + /* * is_cpu_hotpluggable() - check if cpuhotplug operations are supported * for the given cpu. @@ -206,8 +223,6 @@ void reactivate_cpus(void) int cpuid, nc, count, i; char *path; - /* suppress verbose messages on exit */ - debug = 0; /* * Only enable the number of cpus which where available at * daemon startup time by checking num_cpu_start. diff --git a/cpuplugd/cpuplugd.h b/cpuplugd/cpuplugd.h index 2b06cd10..dff3fc34 100644 --- a/cpuplugd/cpuplugd.h +++ b/cpuplugd/cpuplugd.h @@ -38,6 +38,8 @@ #define MAX_VARNAME 128 #define MAX_LINESIZE 2048 #define CPUSTATS 10 +#define PLR_HORIZONTAL 0 +#define PLR_VERTICAL 1 /* * Precedence of C operators @@ -176,9 +178,11 @@ extern unsigned int history_max; extern unsigned int history_current; extern struct symbol_names sym_names[]; extern unsigned int sym_names_count; +extern int saved_polarization; int get_numcpus(); int get_num_online_cpus(); +int get_polarization(void); void get_loadavg_runnable(double *loadavg, double *runnable); void clean_up(); void reactivate_cpus(); @@ -204,6 +208,7 @@ void reload_daemon(void); int daemonize(void); int check_cmmfiles(void); void check_config(); +void apply_cpu_config(void); void set_cmm_pages(long size); int check_lpar(); void setup_history(void); diff --git a/cpuplugd/daemon.c b/cpuplugd/daemon.c index 9820eba5..0b618972 100644 --- a/cpuplugd/daemon.c +++ b/cpuplugd/daemon.c @@ -149,6 +149,8 @@ void clean_up() cpuplugd_info("terminated\n"); remove(pid_file); remove(LOCKFILE); + /* suppress verbose messages on exit */ + debug = 0; reactivate_cpus(); if (memory) cleanup_cmm(); @@ -163,6 +165,8 @@ void kill_daemon(int UNUSED(a)) cpuplugd_info("shutting down\n"); remove(pid_file); remove(LOCKFILE); + /* suppress verbose messages on exit */ + debug = 0; reactivate_cpus(); if (memory) cleanup_cmm(); diff --git a/cpuplugd/main.c b/cpuplugd/main.c index 658ca91a..de286205 100644 --- a/cpuplugd/main.c +++ b/cpuplugd/main.c @@ -54,12 +54,12 @@ struct config cfg = { .hotunplug = NULL, }; -int num_cpu_start, memory, cpu, reload_pending; -long cmm_pagesize_start; -unsigned long meminfo_size, vmstat_size, cpustat_size, varinfo_size; -char *meminfo, *vmstat, *cpustat, *varinfo; -double *timestamps; unsigned int history_max, history_current, history_prev, sym_names_count; +unsigned long meminfo_size, vmstat_size, cpustat_size, varinfo_size; +int num_cpu_start, memory, cpu, reload_pending, saved_polarization; +char *meminfo, *vmstat, *cpustat, *varinfo; +long cmm_pagesize_start; +double *timestamps; static struct symbols symbols; static jmp_buf jmpenv; @@ -77,8 +77,29 @@ static void eval_cpu_rules(void) { double diffs[CPUSTATS], diffs_total, percent_factor; char *procinfo_current, *procinfo_prev; - int nr_cpus, on_off; + int nr_cpus, on_off, polarization; + polarization = get_polarization(); + if (polarization < 0) { + cpuplugd_debug("couldn't determine system polarization\n"); + cpuplugd_debug("skipping cpu rule evaluation\n"); + return; + } + if (saved_polarization != polarization) { + saved_polarization = polarization; + if (polarization == PLR_VERTICAL) { + /* revert cpu hotplug adjustments after switching from horizontal */ + reactivate_cpus(); + } else if (polarization == PLR_HORIZONTAL) { + /* reapply cpu config after switching from vertical */ + apply_cpu_config(); + } + } + if (polarization == PLR_VERTICAL) { + cpuplugd_debug("system is running vertical polarization\n"); + cpuplugd_debug("cpuplugd won't make cpu adjustments\n"); + return; + } nr_cpus = get_numcpus(); procinfo_current = cpustat + history_current * cpustat_size; procinfo_prev = cpustat + history_prev * cpustat_size; @@ -157,7 +178,6 @@ static void eval_cpu_rules(void) printf("\n"); printf("---------------------------------------------\n"); } - on_off = 0; /* Evaluate the hotplug rule */ if (eval_term(cfg.hotplug, &symbols)) diff --git a/cpuplugd/man/cpuplugd.8 b/cpuplugd/man/cpuplugd.8 index 846db881..5b51bd90 100644 --- a/cpuplugd/man/cpuplugd.8 +++ b/cpuplugd/man/cpuplugd.8 @@ -22,6 +22,9 @@ of active CPUs are reset to the values they had before the cpuplugd was started. This program can be used to control the number of CPUs for Linux on z/VM and for Linux in LPAR mode. The memory hotplug feature (CMM page pool) applies to Linux on z/VM only. + +The cpuplugd daemon stops any CPU hot-plug operations when the system switches +to vertical polarization, thus avoiding possible performance penalties. . .SH OPTIONS .TP