cpuplugd: Halt cpu hotplugging on vertical polarization

On vertical polarization, kernel can be adjusting CPU capacities
dynamically, and cpuplugd can interfere this with hotplug operations
causing performance degradation. To prevent this, check if system has
switched its polarization state, and act accordingly.
If system is on vertical polarization when daemon starts, no CPU hotplug
action is triggered. If system changes to vertical polarization during
daemon runtime, revert cpuhotplug adjustments and stop further CPU
hotplug actions. If system switches back to horizontal polarization
during runtime of the daemon, start evaluating CPU hotplug rules and
trigger adjustments.

Reviewed-by: Steffen Eiden <seiden@linux.ibm.com>
Reviewed-by: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Signed-off-by: Mete Durlu <meted@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>

cpuplugd/config.c

@@ -204,6 +204,45 @@ void parse_configfile(char *file)
 	fclose(filp);
 }
 
+void apply_cpu_config(void)
+{
+	/*
+	 * Check that the initial number of cpus is not below the
+	 * minimum
+	 */
+	if (num_cpu_start < cfg.cpu_min &&
+	    get_numcpus() >= cfg.cpu_min) {
+		cpuplugd_debug("The number of online cpus is below ");
+		cpuplugd_debug("the minimum and will be increased.\n");
+		while (get_num_online_cpus() < cfg.cpu_min) {
+			if (hotplug_one_cpu())
+				break;
+		}
+	}
+	if (get_num_online_cpus() > cfg.cpu_max) {
+		cpuplugd_debug("The number of online cpus is above the maximum");
+		cpuplugd_debug(" and will be decreased.\n");
+		while (get_num_online_cpus() > cfg.cpu_max) {
+			if (hotunplug_one_cpu())
+				break;
+		}
+	}
+	if (cfg.cpu_min > get_numcpus()) {
+		/*
+		 * This check only works if nobody used the
+		 * additional_cpus in the boot parameter section
+		 */
+		cpuplugd_debug("The minimum amount of cpus is above the ");
+		cpuplugd_debug("number of available cpus.\n");
+		cpuplugd_exit("Detected %d available cpus\n", get_numcpus());
+	}
+	if (get_num_online_cpus() < cfg.cpu_min) {
+		cpuplugd_debug("Failed to set the number of online cpus to ");
+		cpuplugd_debug("the minimum. ");
+		cpuplugd_exit("Aborting.\n");
+	}
+}
+
 /*
  * Check if the required settings are found in the configuration file.
  * "Autodetect" if cpu and/or memory hotplug configuration entries
@@ -261,44 +300,23 @@ void check_config()
 	/*
 	* Save the number of online cpus and the cmm_pagesize at startup,
 	* so that we can enable exactly the same amount when the daemon ends
+	*
+	* Don't adjust cpus if system is on vertical polarization
 	*/
-	if (cpu) {
-		num_cpu_start = get_num_online_cpus();
-		cpuplugd_debug("Daemon started with %d active cpus.\n",
-			       num_cpu_start);
-		/*
-		 * Check that the initial number of cpus is not below the
-		 * minimum
-		 */
-		if (num_cpu_start < cfg.cpu_min &&
-		    get_numcpus() >= cfg.cpu_min) {
-			cpuplugd_debug("The number of online cpus is below "
-				       "the minimum and will be increased.\n");
-			while (get_num_online_cpus() < cfg.cpu_min) {
-				if (hotplug_one_cpu())
-					break;
-			}
-		}
-		if (get_num_online_cpus() > cfg.cpu_max) {
-			cpuplugd_debug("The number of online cpus is above the maximum"
-				       " and will be decreased.\n");
-			while (get_num_online_cpus() > cfg.cpu_max) {
-				if (hotunplug_one_cpu())
-					break;
-			}
-		}
-		if (cfg.cpu_min > get_numcpus())
-			/*
-			 * This check only works if nobody used the
-			 * additional_cpus in the boot parameter section
-			 */
-			cpuplugd_exit("The minimum amount of cpus is above "
-				      "the number of available cpus.\n"
-				      "Detected %d available cpus\n",
-				      get_numcpus());
-		if (get_num_online_cpus() < cfg.cpu_min)
-			cpuplugd_exit("Failed to set the number of online "
-				      "cpus to the minimum. Aborting.\n");
+	saved_polarization = get_polarization();
+	num_cpu_start = get_num_online_cpus();
+	cpuplugd_debug("Daemon started with %d active cpus.\n",
+		       num_cpu_start);
+	if (saved_polarization < 0) {
+		cpuplugd_debug("Daemon couldn't determine system polarization\n");
+		cpuplugd_debug("Starting without evaluating cpu rules\n");
+	} else if (saved_polarization == PLR_VERTICAL) {
+		cpuplugd_debug("Daemon started with vertical polarization.\n");
+		cpuplugd_debug("Cpu adjustments won't be made until system ");
+		cpuplugd_debug("is in horizontal polarization\n");
+	} else if (saved_polarization == PLR_HORIZONTAL &&
+		   cpu == 1) {
+		apply_cpu_config();
 	}
 	if (memory == 1) {
 		/*

cpuplugd/cpu.c

@@ -78,6 +78,23 @@ int get_num_online_cpus(void)
 	return number;
 }
 
+/*
+ * get_polarization() - return system polarization
+ */
+int get_polarization(void)
+{
+	int polarization;
+	char *path;
+
+	path = util_path_sysfs("devices/system/cpu/dispatching");
+	if (util_file_read_i(&polarization, NUM_BASE, path) < 0) {
+		polarization = -1;
+		cpuplugd_debug("failed to read system polarization\n");
+	}
+	free(path);
+	return polarization;
+}
+
 /*
  * is_cpu_hotpluggable() - check if cpuhotplug operations are supported
  * for the given cpu.
@@ -206,8 +223,6 @@ void reactivate_cpus(void)
 	int cpuid, nc, count, i;
 	char *path;
 
-	/* suppress verbose messages on exit */
-	debug = 0;
 	/*
 	 * Only enable the number of cpus which where available at
 	 * daemon startup time by checking num_cpu_start.

cpuplugd/cpuplugd.h

@@ -38,6 +38,8 @@
 #define MAX_VARNAME	128
 #define MAX_LINESIZE	2048
 #define CPUSTATS	10
+#define PLR_HORIZONTAL	0
+#define PLR_VERTICAL	1
 
 /*
  *  Precedence of C operators
@@ -176,9 +178,11 @@ extern unsigned int history_max;
 extern unsigned int history_current;
 extern struct symbol_names sym_names[];
 extern unsigned int sym_names_count;
+extern int saved_polarization;
 
 int get_numcpus();
 int get_num_online_cpus();
+int get_polarization(void);
 void get_loadavg_runnable(double *loadavg, double *runnable);
 void clean_up();
 void reactivate_cpus();
@@ -204,6 +208,7 @@ void reload_daemon(void);
 int daemonize(void);
 int check_cmmfiles(void);
 void check_config();
+void apply_cpu_config(void);
 void set_cmm_pages(long size);
 int check_lpar();
 void setup_history(void);

cpuplugd/daemon.c

@@ -149,6 +149,8 @@ void clean_up()
 	cpuplugd_info("terminated\n");
 	remove(pid_file);
 	remove(LOCKFILE);
+	/* suppress verbose messages on exit */
+	debug = 0;
 	reactivate_cpus();
 	if (memory)
 		cleanup_cmm();
@@ -163,6 +165,8 @@ void kill_daemon(int UNUSED(a))
 	cpuplugd_info("shutting down\n");
 	remove(pid_file);
 	remove(LOCKFILE);
+	/* suppress verbose messages on exit */
+	debug = 0;
 	reactivate_cpus();
 	if (memory)
 		cleanup_cmm();

cpuplugd/main.c

@@ -54,12 +54,12 @@ struct config cfg = {
 	.hotunplug = NULL,
 };
 
-int num_cpu_start, memory, cpu, reload_pending;
-long cmm_pagesize_start;
-unsigned long meminfo_size, vmstat_size, cpustat_size, varinfo_size;
-char *meminfo, *vmstat, *cpustat, *varinfo;
-double *timestamps;
 unsigned int history_max, history_current, history_prev, sym_names_count;
+unsigned long meminfo_size, vmstat_size, cpustat_size, varinfo_size;
+int num_cpu_start, memory, cpu, reload_pending, saved_polarization;
+char *meminfo, *vmstat, *cpustat, *varinfo;
+long cmm_pagesize_start;
+double *timestamps;
 
 static struct symbols symbols;
 static jmp_buf jmpenv;
@@ -77,8 +77,29 @@ static void eval_cpu_rules(void)
 {
 	double diffs[CPUSTATS], diffs_total, percent_factor;
 	char *procinfo_current, *procinfo_prev;
-	int nr_cpus, on_off;
+	int nr_cpus, on_off, polarization;
 
+	polarization = get_polarization();
+	if (polarization < 0) {
+		cpuplugd_debug("couldn't determine system polarization\n");
+		cpuplugd_debug("skipping cpu rule evaluation\n");
+		return;
+	}
+	if (saved_polarization != polarization) {
+		saved_polarization = polarization;
+		if (polarization == PLR_VERTICAL) {
+			/* revert cpu hotplug adjustments after switching from horizontal */
+			reactivate_cpus();
+		} else if (polarization == PLR_HORIZONTAL) {
+			/* reapply cpu config after switching from vertical */
+			apply_cpu_config();
+		}
+	}
+	if (polarization == PLR_VERTICAL) {
+		cpuplugd_debug("system is running vertical polarization\n");
+		cpuplugd_debug("cpuplugd won't make cpu adjustments\n");
+		return;
+	}
 	nr_cpus = get_numcpus();
 	procinfo_current = cpustat + history_current * cpustat_size;
 	procinfo_prev = cpustat + history_prev * cpustat_size;
@@ -157,7 +178,6 @@ static void eval_cpu_rules(void)
 		printf("\n");
 		printf("---------------------------------------------\n");
 	}
-
 	on_off = 0;
 	/* Evaluate the hotplug rule */
 	if (eval_term(cfg.hotplug, &symbols))

cpuplugd/man/cpuplugd.8

@@ -22,6 +22,9 @@ of active CPUs are reset to the values they had before the cpuplugd was started.
 This program can be used to control the number of CPUs for Linux on z/VM
 and for Linux in LPAR mode. The memory hotplug feature (CMM page pool) applies
 to Linux on z/VM only.
+
+The cpuplugd daemon stops any CPU hot-plug operations when the system switches
+to vertical polarization, thus avoiding possible performance penalties.
 .
 .SH OPTIONS
 .TP