From b044aec4ffba221fb29691ee177a4e54bca0433c Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Tue, 21 Jul 2020 21:26:38 +0200 Subject: [PATCH] libzds: add curl interface to access zosmf rest api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add library functions that allow to communicate with z/OSMF REST services using libcurl. Following three functions are added: lzds_rest_get_enq() to obtain an ENQ that will mark a dataset as in use to z/OS until it is released again lzds_rest_release_enq() to release an ENQ and mark a dataset as no longer in use to z/OS lzds_rest_ping() to ping a z/OSMF REST server to check if it accessible or to refresh an ENQ and prevent it from a timeout after 10 minutes Signed-off-by: Stefan Haberland Reviewed-by: Niklas Schnelle Signed-off-by: Jan Höppner --- dasdview/Makefile | 33 ++++- include/lib/libzds.h | 7 +- libzds/Makefile | 4 + libzds/libzds.c | 281 +++++++++++++++++++++++++++++++++++++++++++ zdsfs/Makefile | 26 +++- 5 files changed, 344 insertions(+), 7 deletions(-) diff --git a/dasdview/Makefile b/dasdview/Makefile index 59fda17d..af52be56 100644 --- a/dasdview/Makefile +++ b/dasdview/Makefile @@ -1,14 +1,39 @@ include ../common.mak -ALL_CPPFLAGS += -DSYSFS - -all: dasdview - libs = $(rootdir)/libdasd/libdasd.a \ $(rootdir)/libzds/libzds.a \ $(rootdir)/libvtoc/libvtoc.a \ $(rootdir)/libutil/libutil.a +ifneq (${HAVE_CURL},0) + +check_dep: + $(call check_dep, \ + "dasdview", \ + "curl/curl.h", \ + "curl-devel or libcurl-dev", \ + "HAVE_CURL=0") + +BUILDTARGET = check_dep + +ifneq ($(shell sh -c 'command -v pkg-config'),) +CURL_CFLAGS = $(shell pkg-config --silence-errors --cflags libcurl) +CURL_LDLIBS = $(shell pkg-config --silence-errors --libs libcurl) +else +CURL_CFLAGS = -I/usr/include/s390x-linux-gnu/curl +CURL_LDLIBS = -lcurl +endif # shell + +endif # HAVE_CURL + +BUILDTARGET += dasdview + +ALL_CPPFLAGS += -DSYSFS +ALL_CFLAGS += $(CURL_CFLAGS) +LDLIBS += $(CURL_LDLIBS) + +all: $(BUILDTARGET) + dasdview: dasdview.o $(libs) install: all diff --git a/include/lib/libzds.h b/include/lib/libzds.h index 6ae767b5..5c06669c 100644 --- a/include/lib/libzds.h +++ b/include/lib/libzds.h @@ -191,6 +191,8 @@ #define ENDTOKEN 0xFFFFFFFFFFFFFFFFULL +#define MAX_LINE_LENGTH 512 + /** * @brief This structure represents the count field in an ECKD record. @@ -854,6 +856,7 @@ int lzds_zdsroot_extract_datasets_from_dasd(struct zdsroot *root, void lzds_dslist_free(struct zdsroot *root); +int lzds_ping_rest(struct dshandle *dsh, char *server); /** @} */ /* end of group libzds_functions_high */ @@ -878,6 +881,8 @@ int lzds_analyse_open_count(struct zdsroot *root, int warn); /** @} */ /* end of group libzds_functions_helper */ - +int lzds_rest_get_enq(struct dshandle *dsh, char *server); +int lzds_rest_release_enq(struct dshandle *dsh, char *server); +int lzds_rest_ping(struct dshandle *dsh, char *server); #endif /* LIB_LIBZDS_H */ diff --git a/libzds/Makefile b/libzds/Makefile index a86b4654..3618e1ec 100644 --- a/libzds/Makefile +++ b/libzds/Makefile @@ -2,6 +2,10 @@ include ../common.mak ALL_CFLAGS += -D_FILE_OFFSET_BITS=64 +ifneq (${HAVE_CURL},0) +ALL_CFLAGS += -DHAVE_CURL +endif + lib = libzds.a all: $(lib) diff --git a/libzds/libzds.c b/libzds/libzds.c index c4447690..a6c6912f 100644 --- a/libzds/libzds.c +++ b/libzds/libzds.c @@ -18,7 +18,12 @@ #include #include #include +#include +#ifdef HAVE_CURL +#include +#endif /* HAVE_CURL */ +#include "lib/util_libc.h" #include "lib/dasd_base.h" #include "lib/dasd_sys.h" #include "lib/libzds.h" @@ -300,6 +305,8 @@ struct dshandle { unsigned long long skip; /** @brief Detailed error messages in case of a problem */ struct errorlog *log; + + char *session_ref; }; /** @endcond */ @@ -2942,6 +2949,280 @@ void lzds_dshandle_close(struct dshandle *dsh) dsh->is_open = 0; } +#ifdef HAVE_CURL + +struct response_data { + char *session_ref; + unsigned long statuscode; +}; + +static size_t +parse_response_callback(void *data, size_t size, size_t member, void *target) +{ + struct response_data *response = target; + + if (strstr(data, "HTTP/1.1 500 Internal Server Error")) { + response->statuscode = 500; + } else if (strstr(data, "HTTP/1.1 200 OK")) { + response->statuscode = 200; + } else + sscanf(data, "X-IBM-Session-Ref: %m[^\n]\n", + &response->session_ref); + + return size*member; +} + +static size_t write_discard_callback(void *UNUSED(data), size_t size, size_t member, + void *UNUSED(target)) +{ + /* do nothing just pretend all data has been processed */ + return size*member; +} + +CURL *lzds_prepare_curl(char *url) +{ + CURL *curl; + + curl = curl_easy_init(); + if (!curl) + return NULL; + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + curl_easy_setopt(curl, CURLOPT_CONNECT_ONLY, 0L); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_discard_callback); + + return curl; +} + +/** + * Ping the z/OSMS REST server. + * Used to check if the server is responding and accessible and to prevent + * the ENQ from timing out. If not used it would be automatically released + * after 10 minutes. + * + * @param[in] dsh The dshandle that keeps track of the I/O operations. + * server The URL to the z/OSMF REST services + * @return 1 on success, 0 otherwise + */ +int lzds_rest_ping(struct dshandle *dsh, char *server) +{ + struct curl_slist *list = NULL; + char *release; + CURLcode res; + size_t size; + CURL *curl; + char *url; + + url = util_strcat_realloc(NULL, server); + url = util_strcat_realloc(url, "restfiles/ping"); + + curl = lzds_prepare_curl(url); + if (!curl) { + free(url); + return 0; + } + + list = curl_slist_append(list, "X-CSRF-ZOSMF-HEADER: none"); + if (dsh && dsh->session_ref) { + size = sizeof("X-IBM-Session-Ref: ") + strlen(dsh->session_ref); + release = util_zalloc(size); + snprintf(release, size, "X-IBM-Session-Ref: %s", + dsh->session_ref); + list = curl_slist_append(list, release); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + res = curl_easy_perform(curl); + curl_slist_free_all(list); + curl_easy_cleanup(curl); + + if (res == CURLE_OK) { + free(url); + return 1; + } + + fprintf(stderr, "URL: %s\n", url); + fprintf(stderr, "Error: %s\n", curl_easy_strerror(res)); + free(url); + return 0; +} + +/** + * Mark the dataset as in use for z/OS. + * Use z/OSMF REST services to read a small amount of data and get an exclusive + * ENQ that prevents z/OS applications from writing to the dataset in parallel + * until the ENQ is released. + * + * @param[in] dsh The dshandle that keeps track of the I/O operations. + * server The URL to the z/OSMF REST services + * @return 0 on success, otherwise one of the following error codes: + * - ENOTSUP Unable to setup curl and therefore no further access possible. + * - EPERM ENQ not obtained and therefore access is not allowed. + */ +int lzds_rest_get_enq(struct dshandle *dsh, char *server) +{ + struct curl_slist *list = NULL; + struct response_data response; + int first_run; + CURLcode res; + CURL *curl; + char *url; + int rc; + + url = util_strcat_realloc(NULL, server); + url = util_strcat_realloc(url, "restfiles/ds/"); + url = util_strcat_realloc(url, dsh->ds->name); + + memset(&response, 0, sizeof(response)); + /* + * in the first run provide a range statement to read only 1 record of + * the dataset to get an ENQ. + * For the unlikely case that the dataset is empty + * "500 Internal Server Error" will be returned. + * If this is the case give it a second try without a range statement + */ + first_run = 1; + list = curl_slist_append(list, "X-IBM-Record-Range: 0-1"); +retry: + rc = 1; + curl = lzds_prepare_curl(url); + if (!curl) { + free(url); + return errorlog_add_message( + &dsh->log, + NULL, ENOTSUP, + "curl handle not established for dataset %s\n", + dsh->ds->name); + } + + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_response_callback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response); + list = curl_slist_append(list, "X-CSRF-ZOSMF-HEADER: none"); + list = curl_slist_append(list, "X-IBM-Obtain-ENQ: EXCLU"); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + rc = errorlog_add_message(&dsh->log, NULL, ECONNREFUSED, + "Error: %s\n", + curl_easy_strerror(res)); + } else { + if (first_run && response.statuscode == 500) { + curl_slist_free_all(list); + curl_easy_cleanup(curl); + first_run = 0; + list = NULL; + goto retry; + } + /* expect that the callback function found a reference string, double check */ + if (response.statuscode == 200 && response.session_ref) { + dsh->session_ref = response.session_ref; + rc = 0; + } else { + rc = errorlog_add_message( + &dsh->log, + NULL, EPERM, + "no session ref obtained for dataset %s rest rc %ld\n", + dsh->ds->name, response.statuscode); + } + } + + free(url); + curl_slist_free_all(list); + curl_easy_cleanup(curl); + + return rc; +} + +/** + * Mark the dataset as no longer in use for z/OS. + * Use z/OSMF REST services to read a small amount of data and release the exclusive + * ENQ that was previously obtained. + * + * @param[in] dsh The dshandle that keeps track of the I/O operations. + * server The URL to the z/OSMF REST services + * @return 0 on success, otherwise one of the following error codes: + * - ENOTSUP Unable to release the ENQ. + */ +int lzds_rest_release_enq(struct dshandle *dsh, char *server) +{ + struct curl_slist *list = NULL; + struct response_data response; + char *release; + int first_run; + CURLcode res; + CURL *curl; + char *url; + + + if (!dsh->session_ref) { + fprintf(stderr, "No ENQ to release.\n"); + return 0; + } + + url = util_strcat_realloc(NULL, server); + url = util_strcat_realloc(url, "restfiles/ds/"); + url = util_strcat_realloc(url, dsh->ds->name); + + release = util_strcat_realloc(NULL, "X-IBM-Session-Ref: "); + release = util_strcat_realloc(release, dsh->session_ref); + + memset(&response, 0, sizeof(response)); + /* + * in the first run provide a range statement to read only 1 record of + * the dataset to release the ENQ. + * For the unlikely case that the dataset is empty + * "500 Internal Server Error" will be returned. + * If this is the case give it a second try without a range statement + */ + first_run = 1; + list = curl_slist_append(list, "X-IBM-Record-Range: 0-1"); +retry: + curl = lzds_prepare_curl(url); + if (!curl) { + free(url); + free(release); + return errorlog_add_message( + &dsh->log, + NULL, ENOTSUP, + "curl handle not established for dataset %s\n", + dsh->ds->name); + } + + list = curl_slist_append(list, "X-CSRF-ZOSMF-HEADER: none"); + list = curl_slist_append(list, "X-IBM-Release-ENQ: true"); + list = curl_slist_append(list, release); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_response_callback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response); + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + errorlog_add_message(&dsh->log, NULL, ENOTSUP, "Error: %s\n", + curl_easy_strerror(res)); + } else if (first_run && response.statuscode == 500) { + curl_slist_free_all(list); + curl_easy_cleanup(curl); + first_run = 0; + list = NULL; + goto retry; + } + + curl_slist_free_all(list); + curl_easy_cleanup(curl); + free(dsh->session_ref); + free(release); + free(url); + dsh->session_ref = NULL; + + return res; +} + +#endif /* HAVE_CURL */ + /** * This makes the data set context ready for read operations. diff --git a/zdsfs/Makefile b/zdsfs/Makefile index 07d7658e..f4aefc4f 100644 --- a/zdsfs/Makefile +++ b/zdsfs/Makefile @@ -13,6 +13,17 @@ all: install: $(SKIP) HAVE_FUSE=0 + +else + +ifeq (${HAVE_CURL},0) + +all: + $(SKIP) HAVE_CURL=0 + +install: + $(SKIP) HAVE_CURL=0 + else check_dep: @@ -21,17 +32,26 @@ check_dep: "fuse.h", \ "fuse-devel or libfuse-dev", \ "HAVE_FUSE=0") + $(call check_dep, \ + "zdsfs", \ + "curl/curl.h", \ + "curl-devel or libcurl-dev", \ + "HAVE_CURL=0") ifneq ($(shell sh -c 'command -v pkg-config'),) FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse) FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse) +CURL_CFLAGS = $(shell pkg-config --silence-errors --cflags libcurl) +CURL_LDLIBS = $(shell pkg-config --silence-errors --libs libcurl) else FUSE_CFLAGS = -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse FUSE_LDLIBS = -lfuse +CURL_CFLAGS = -I/usr/include/s390x-linux-gnu/curl +CURL_LDLIBS = -lcurl endif ALL_CPPFLAGS += -DSYSFS -ALL_CFLAGS += -DHAVE_SETXATTR -pthread $(FUSE_CFLAGS) -LDLIBS += $(FUSE_LDLIBS) -lpthread -lrt -lm +ALL_CFLAGS += -DHAVE_SETXATTR -pthread $(FUSE_CFLAGS) $(CURL_CFLAGS) +LDLIBS += $(FUSE_LDLIBS) $(CURL_LDLIBS) -lpthread -lrt -lm all: check_dep zdsfs @@ -45,6 +65,8 @@ install: all endif +endif + clean: rm -f *.o *~ zdsfs core