zdsfs: transparent codepage conversion

Add transparent codepage conversion feature to zdsfs. Add options to specify that codepage conversion should be done using default codepages from 'CP1047' to 'UTF-8' or using user defined codepage tables. Also add a new config file allowing the user to specify codepage conversion options on a per dataset basis. When a dataset is opened and codepage conversion settings match this dataset an iconv handle is created and the conversion itself is performed by the lzds_dshandle_read() function. Signed-off-by: Stefan Haberland <sth@linux.ibm.com> Reviewed-by: Niklas Schnelle <schnelle@linux.ibm.com> Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
2026-08-05 02:14:52 +00:00 · 2021-04-28 16:23:59 +02:00
parent cdf716a7a9
commit 7244785279
1 changed files with 392 additions and 5 deletions
--- a/zdsfs/zdsfs.c
+++ b/zdsfs/zdsfs.c
@@ -25,6 +25,8 @@
 #include <time.h>
 #include <signal.h>

+#include <iconv.h>
+
 #ifdef HAVE_SETXATTR
 #include <linux/xattr.h>
 #endif
@@ -34,6 +36,7 @@
 #include "lib/zt_common.h"

 #define COMP "zdsfs: "
+
 #define METADATAFILE "metadata.txt"

 /* defaults for file and directory permissions (octal) */
@@ -42,6 +45,10 @@
 /* default timer interval 9 minutes, enq times out after 10 minutes */
 #define DEFAULT_KEEPALIVE_SEC	         540

+#define SECTION_ENTRIES 3
+static char CODEPAGE_EDF[] = "CP1047";
+static char CODEPAGE_LINUX[] = "UTF-8";
+
 struct zdsfs_info {
 	int devcount;
 	int allow_inclomplete_multi_volume;
@@ -61,6 +68,12 @@ struct zdsfs_info {
 	int active_server;
 	char *server[MAX_SERVER];
 	long keepalive;
+	int codepage_convert;
+	char *codepage_from;
+	char *codepage_to;
+	iconv_t iconv;
+	struct util_list *dsclist;
+	char *dsfile;
 };

 static struct zdsfs_info zdsfsinfo;
@@ -82,6 +95,18 @@ struct zdsfs_file_info {
 	size_t metaread; /* how many bytes have already been read */
 };

+struct dsconvert {
+	char *name;
+	char *codepage_from;
+	char *codepage_to;
+	bool keeprdw;
+};
+
+struct dsc_node {
+	struct util_list_node node;
+	struct dsconvert *dsc;
+};
+
 /* Allocate and initialize a new list of struct dsh_node. */
 static struct util_list *dshlist_alloc(void)
 {
@@ -144,6 +169,65 @@ void dshlist_remove(struct util_list *list, struct dshandle *dsh)
 	}
 }

+/* Allocate and initialize a new list of struct dsc_node. */
+static struct util_list *dsclist_alloc(void)
+{
+	struct util_list *list;
+
+	list = util_malloc(sizeof(struct util_list));
+	util_list_init(list, struct dsc_node, node);
+
+	return list;
+}
+
+/* free struct dsconvert. */
+static void dsc_free(struct dsconvert *dsc)
+{
+	free(dsc->name);
+	free(dsc->codepage_from);
+	free(dsc->codepage_to);
+	free(dsc);
+}
+
+/* free list of struct dsc_node. */
+static void dsclist_free(struct util_list *list)
+{
+	struct dsc_node *s, *n;
+
+	if (!list)
+		return;
+
+	util_list_iterate_safe(list, s, n) {
+		util_list_remove(list, s);
+		dsc_free(s->dsc);
+		free(s);
+	}
+
+	free(list);
+}
+
+/* add dsc to list */
+static void dsclist_add(struct util_list *list, struct dsconvert *dsc)
+{
+	struct dsc_node *s;
+
+	s = util_malloc(sizeof(struct dsc_node));
+	s->dsc = dsc;
+	util_list_add_tail(list, s);
+}
+
+/* Find a dsc_node by name. */
+static struct dsc_node *dsclist_find_by_name(struct util_list *list, char *name)
+{
+	struct dsc_node *s;
+
+	util_list_iterate(list, s) {
+		if (strcmp(s->dsc->name, name) == 0)
+			return s;
+	}
+
+	return NULL;
+}

 /* normalize the given path name to a dataset name
 * so that we can compare it to the names in the vtoc. This means:
@@ -176,6 +260,16 @@ static void path_to_member_name(const char *path, char *normds, size_t size)
 	}
 }

+static int setup_iconv(iconv_t *conv, const char *from, const char *to)
+{
+	*conv = iconv_open(to, from);
+	if (*conv == ((iconv_t) -1)) {
+		fprintf(stderr, "error when setting up iconv\n");
+		return -1;
+	}
+	return 0;
+}
+
 static void setup_timer(long sec)
 {
 	static struct itimerval timer;
@@ -531,6 +625,102 @@ static int zdsfs_test_restserver(void)
 	return 0;
 }

+/*
+ * check if a dsconvert entry exists that match the given DS name
+ * if the dsconvert entry ends in an asterisk only match the prefix
+ * otherwise look for an exact match
+ */
+static struct dsconvert *zdsfs_get_matching_dsc(char *name)
+{
+	struct dsconvert *dsc;
+	unsigned int length;
+	struct dsc_node *n;
+	char *match;
+
+	util_list_iterate(zdsfsinfo.dsclist, n) {
+		dsc = n->dsc;
+		match = dsc->name;
+		length = strlen(match);
+		if (strcmp(&match[length - 1], "*") == 0)
+			length--;
+		else if (strlen(name) != length)
+			continue;
+
+		if (strncmp(name, match, length) == 0)
+			return dsc;
+	}
+	return NULL;
+}
+
+/*
+ * Setup iconv conversion for a given dataset.
+ * The codepage settings can be obtained from (in descending priority)
+ *   - globally set codepage settings
+ *   - a dsconvert entry matching the DS name
+ *   - default codepage settings.
+ */
+static int zdsfs_setup_conversion(struct dshandle *dsh, struct dataset *ds)
+{
+	struct dsconvert *dsc;
+	const char *from, *to;
+	struct errorlog *log;
+	iconv_t *iconv;
+	char *dsname;
+	int rc;
+
+	from = to = NULL;
+	lzds_dataset_get_name(ds, &dsname);
+	dsc = zdsfs_get_matching_dsc(dsname);
+	/* the DS matches a dsconvert entry */
+	if (dsc) {
+		from = dsc->codepage_from;
+		to = dsc->codepage_to;
+	}
+
+	/* globally set conversion overwriting possible config file settings */
+	if (zdsfsinfo.codepage_from && zdsfsinfo.codepage_to) {
+		from = zdsfsinfo.codepage_from;
+		to = zdsfsinfo.codepage_to;
+	}
+
+	/*
+	 * globally set conversion using defaults
+	 * if not specified otherwise already
+	 */
+	if (zdsfsinfo.codepage_convert) {
+		if (!from)
+			from = CODEPAGE_EDF;
+		if  (!to)
+			to = CODEPAGE_LINUX;
+	}
+
+	/* no conversion */
+	if (!from || !to)
+		return 0;
+
+	iconv = util_malloc(sizeof(*iconv));
+	rc = setup_iconv(iconv, from, to);
+	if (rc) {
+		fprintf(stderr,	"Error when preparing iconv setting:\n");
+		lzds_dshandle_get_errorlog(dsh, &log);
+		lzds_errorlog_fprint(log, stderr);
+		rc = -rc;
+		goto out;
+	}
+	rc = lzds_dshandle_set_iconv(dsh, iconv);
+	if (rc) {
+		fprintf(stderr,	"Error when setting iconv handle:\n");
+		lzds_dshandle_get_errorlog(dsh, &log);
+		lzds_errorlog_fprint(log, stderr);
+		rc = -rc;
+		goto out;
+	}
+	return 0;
+out:
+	free(iconv);
+	return rc;
+}
+

 static int zdsfs_open(const char *path, struct fuse_file_info *fi)
 {
@@ -620,6 +810,7 @@ static int zdsfs_open(const char *path, struct fuse_file_info *fi)
 		rc = -rc;
 		goto error2;
 	}
+	zdsfs_setup_conversion(dsh, ds);

 retry:
 	if (zdsfsinfo.restapi && zdsfsinfo.active_server >= 0) {
@@ -660,7 +851,6 @@ error2:
 error1:
 	free(zfi);
 	return rc;
-
 }

 static int zdsfs_release(const char *UNUSED(path), struct fuse_file_info *fi)
@@ -1012,7 +1202,10 @@ enum {
 	KEY_TRACKS,
 	KEY_SEEKBUFFER,
 	KEY_CONFIG,
+	KEY_DSCONFIG,
 	KEY_SERVER,
+	KEY_CODE_FROM,
+	KEY_CODE_TO,
 };

 #define ZDSFS_OPT(t, p, v) { t, offsetof(struct zdsfs_info, p), v }
@@ -1026,11 +1219,15 @@ static const struct fuse_opt zdsfs_opts[] = {
 	FUSE_OPT_KEY("tracks=",         KEY_TRACKS),
 	FUSE_OPT_KEY("seekbuffer=",     KEY_SEEKBUFFER),
 	FUSE_OPT_KEY("-c %s",           KEY_CONFIG),
+	FUSE_OPT_KEY("-x %s",           KEY_DSCONFIG),
 	FUSE_OPT_KEY("restserver=",     KEY_SERVER),
+	FUSE_OPT_KEY("codepage_from=",  KEY_CODE_FROM),
+	FUSE_OPT_KEY("codepage_to=",    KEY_CODE_TO),
 	ZDSFS_OPT("rdw",                keepRDW, 1),
 	ZDSFS_OPT("ignore_incomplete",  allow_inclomplete_multi_volume, 1),
 	ZDSFS_OPT("check_host_count",   host_count, 1),
 	ZDSFS_OPT("restapi",            restapi, 1),
+	ZDSFS_OPT("codepage_convert",   codepage_convert, 1),
 	FUSE_OPT_END
 };

@@ -1052,6 +1249,8 @@ static void usage(const char *progname)
 " nodes\n"
 "    -c config_file         Text file that contains configuration options\n"
 "                           for zdsfs\n"
+"    -x ds_config_file      Text file that contains conversion options\n"
+"                           for specific datasets\n"
 "    -o rdw                 Keep record descriptor words in byte stream\n"
 "    -o ignore_incomplete   Continue processing even if parts of a multi"
 " volume\n"
@@ -1065,6 +1264,10 @@ static void usage(const char *progname)
 "                           access to datasets\n"
 "    -o restserver=URL      The URL of the z/OSMF REST server to be used for\n"
 "                           coordinated access to datasets\n"
+"    -o codepage_convert    Enable codepage conversion using default codepages\n"
+"                           from 'CP1047' to 'UTF-8'\n"
+"    -o codepage_from=from  Set codepage for source. See 'iconv -l' for a list\n"
+"    -o codepage_to=to      Set codepage for target. See 'iconv -l' for a list\n"
 		, progname);
 }

@@ -1167,7 +1370,7 @@ static void zdsfs_process_device_file(const char *devfile)
 void remove_whitespace(const char *s, char *t)
 {
 	while (*s != '\0') {
-		if (!isspace(*s)) {
+		if (!isblank(*s)) {
 			*t = *s;
 			t++;
 		}
@@ -1176,7 +1379,6 @@ void remove_whitespace(const char *s, char *t)
 	*t = '\0';
 }

-
 static void zdsfs_process_config_file(const char *config)
 {
 	char line[MAX_LINE_LENGTH];
@@ -1225,6 +1427,163 @@ static void zdsfs_process_config_file(const char *config)
 	fclose(fd);
 }

+static struct dsconvert *zdsfs_allocate_dsc(char *name, const char *config)
+{
+	struct dsconvert *dsc;
+
+	/* check for duplicate entries */
+	if (dsclist_find_by_name(zdsfsinfo.dsclist, name)) {
+		fprintf(stderr,
+			"Error in config file %s. Duplicate entry found: %s\n",
+			config, name);
+		return NULL;
+	}
+	dsc = util_zalloc(sizeof(*dsc));
+	dsc->name = util_strdup(name);
+
+	return dsc;
+}
+
+static int zdsfs_check_codepage_setting(char *from, char *to)
+{
+	iconv_t iconv;
+
+	/* no conversion is OK */
+	if (!from && !to)
+		return 0;
+
+	/* partial setup is not OK */
+	if ((from && !to) || (to && !from))
+		return 1;
+
+	/* return if the codepages are valid */
+	return setup_iconv(&iconv, from, to);
+}
+
+/*
+ * process a dataset configuration file that specifies conversion on a per dataset basis
+ *
+ * expect a section title each 3 lines
+ * the section should contain a rdw= and conv= line
+ * valid values for rdw= are 0/1
+ * valid values for conv= are 0/1 or a comma separated list
+ * of codepage_from and codepage_to arguments
+ */
+static int zdsfs_process_dataset_conf(const char *config)
+{
+	char line[MAX_LINE_LENGTH];
+	char delimiter[] = " =#\n";
+	char *tmp, *key, *value;
+	int linecount, in_section;
+	unsigned long enabled;
+	struct dsconvert *dsc;
+	int rc = 1;
+	FILE *fd;
+
+	fd = fopen(config, "r");
+	if (!fd) {
+		fprintf(stderr, "could not open file %s: %s\n",
+			config, strerror(errno));
+		return 0;
+	}
+	in_section = 0;
+	linecount = 0;
+	dsc = NULL;
+	tmp = NULL;
+	while (fgets(line, sizeof(line), fd)) {
+		linecount++;
+		/* remove all whitespaces */
+		tmp = util_malloc(strlen(line) + 1);
+		remove_whitespace(line, tmp);
+		/* skip empty lines */
+		if (*tmp == '\n' || *tmp == '#') {
+			free(tmp);
+			tmp = NULL;
+			continue;
+		}
+		if (!in_section) {
+			/* the section title should not contain a '=' */
+			if (strchr(line, '=') != NULL) {
+				fprintf(stderr,
+					"Error in config file %s line %d. Expected section title instead of %s\n",
+					config, linecount, line);
+				goto out;
+			}
+		}
+		key = strtok(tmp, delimiter);
+		if (!in_section) {
+			dsc = zdsfs_allocate_dsc(key, config);
+			if (!dsc)
+				goto out;
+			in_section = SECTION_ENTRIES;
+		} else if (strcmp(key, "rdw")  == 0) {
+			value = strtok(NULL, delimiter);
+			enabled = strtoul(value, NULL, 0);
+			if (enabled == 1)
+				dsc->keeprdw = true;
+			else
+				dsc->keeprdw = false;
+		} else if (strcmp(key, "conv") == 0) {
+			value = strtok(NULL, delimiter);
+			if (strchr(value, ',') != NULL) {
+				/* use provided codepages */
+				value = strtok(value, ",");
+				dsc->codepage_from = util_strdup(value);
+				value = strtok(NULL, delimiter);
+				dsc->codepage_to = util_strdup(value);
+			} else if (strcmp(value, "1") == 0) {
+				/* use default codepages */
+				dsc->codepage_from = CODEPAGE_EDF;
+				dsc->codepage_to = CODEPAGE_LINUX;
+			} else if  (strcmp(value, "0") == 0) {
+				/* disable conversion */
+				dsc->codepage_from = NULL;
+				dsc->codepage_to = NULL;
+			} else {
+				fprintf(stderr,
+					"Error in config file %s line %d. Invalid 'conv' statement: %s.\n",
+					config, linecount, value);
+				goto out;
+			}
+		} else {
+			fprintf(stderr,
+				"Error in config file %s line %d. Missing 'rdw' or 'conv' statement.\n",
+				config, linecount);
+			goto out;
+		}
+		in_section--;
+		/* if the section was parsed completely, add the dsc to the list */
+		if (!in_section) {
+			if (zdsfs_check_codepage_setting(dsc->codepage_from,
+							 dsc->codepage_to)) {
+				fprintf(stderr,
+					"Error in config file %s. Invalid codepage setting: %s %s.\n",
+					config, dsc->codepage_from,
+					dsc->codepage_to);
+				goto out;
+			}
+			dsclist_add(zdsfsinfo.dsclist, dsc);
+			dsc = NULL;
+		}
+		free(tmp);
+		tmp = NULL;
+	}
+	/* find incomplete last section */
+	if (in_section)
+		fprintf(stderr,
+			"Error in config file %s. Missing 'rdw' or 'conv' statement.\n",
+			config);
+	else
+		rc = 0;
+
+out:
+	fclose(fd);
+	free(tmp);
+	dsc_free(dsc);
+
+	return rc;
+}
+
 static int zdsfs_process_args(void *UNUSED(data), const char *arg, int key,
 			      struct fuse_args *outargs)
 {
@@ -1313,6 +1672,10 @@ static int zdsfs_process_args(void *UNUSED(data), const char *arg, int key,
 		/* note that arg starts with "-c" */
 		zdsfsinfo.configfile = util_strdup(arg + 2);
 		return 0;
+	case KEY_DSCONFIG:
+		/* note that arg starts with "-x" */
+		zdsfsinfo.dsfile = util_strdup(arg + 2);
+		return 0;
 	case KEY_SERVER:
 		if (zdsfsinfo.nr_server >= MAX_SERVER)
 			return 0;
@@ -1321,6 +1684,16 @@ static int zdsfs_process_args(void *UNUSED(data), const char *arg, int key,
 			util_strdup(value);
 		zdsfsinfo.nr_server++;
 		return 0;
+	case KEY_CODE_FROM:
+		value = arg + strlen("codepage_from=");
+		zdsfsinfo.codepage_from =
+			util_strdup(value);
+		return 0;
+	case KEY_CODE_TO:
+		value = arg + strlen("codepage_to=");
+		zdsfsinfo.codepage_to =
+			util_strdup(value);
+		return 0;
 	default:
 		fprintf(stderr, "Unknown argument key %x\n", key);
 		exit(1);
@@ -1340,11 +1713,14 @@ int main(int argc, char *argv[])
 	zdsfsinfo.tracks_per_frame = 128;
 	zdsfsinfo.seek_buffer_size = 1048576;
 	zdsfsinfo.configfile = "/etc/zdsfs.conf";
+	zdsfsinfo.dsfile = "/etc/zdsfs-dataset.conf";
 	zdsfsinfo.keepalive = DEFAULT_KEEPALIVE_SEC;
 	zdsfsinfo.active_server = -1;

 	rc = lzds_zdsroot_alloc(&zdsfsinfo.zdsroot);
 	open_dsh = dshlist_alloc();
+	zdsfsinfo.dsclist = dsclist_alloc();
+
 	if (rc) {
 		fprintf(stderr, "Could not allocate internal structures\n");
 		exit(1);
@@ -1354,8 +1730,19 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "Failed to parse option\n");
 		exit(1);
 	}
+	if (zdsfs_check_codepage_setting(zdsfsinfo.codepage_from,
+					 zdsfsinfo.codepage_to)) {
+		fprintf(stderr, "Ivalid codepage setting from '%s' to '%s'\n",
+			zdsfsinfo.codepage_from,
+			zdsfsinfo.codepage_to);
+		rc = -EINVAL;
+		goto cleanup;
+	}
 	zdsfs_process_config_file(zdsfsinfo.configfile);
-
+	if (zdsfs_process_dataset_conf(zdsfsinfo.dsfile)) {
+		rc = -EACCES;
+		goto cleanup;
+	}
 	if (!zdsfsinfo.devcount) {
 		fprintf(stderr, "Please specify a block device\n");
 		fprintf(stderr, "Try '%s --help' for more information\n",
@@ -1390,12 +1777,12 @@ int main(int argc, char *argv[])
 			goto cleanup;
 		}
 	}
-
 	rc = fuse_main(args.argc, args.argv, &rdf_oper, NULL);

 cleanup:
 	curl_global_cleanup();
 	dshlist_free(open_dsh);
+	dsclist_free(zdsfsinfo.dsclist);
 	lzds_zdsroot_free(zdsfsinfo.zdsroot);

 	fuse_opt_free_args(&args);