Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit dcf6ad384149ee0b3f91c5a8127160cc291f7157:

  Merge branch 'fio-man-page' of https://github.com/bvanassche/fio (2018-08-13 21:05:33 -0600)

are available in the git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 875e8d6fa4d443068eb1c48a29f5367e454d2a37:

  Merge branch 'wip-http-engine' of https://github.com/l-mb/fio (2018-08-14 10:09:47 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'wip-http-engine' of https://github.com/l-mb/fio

Lars Marowsky-Bree (1):
      engines/http: Add support for WebDAV and S3

 HOWTO                    |  50 +++++
 Makefile                 |   3 +
 configure                |  34 +++
 engines/http.c           | 558 +++++++++++++++++++++++++++++++++++++++++++++++
 examples/http-s3.fio     |  34 +++
 examples/http-webdav.fio |  26 +++
 fio.1                    |  40 ++++
 optgroup.h               |   2 +
 options.c                |   5 +
 9 files changed, 752 insertions(+)
 create mode 100644 engines/http.c
 create mode 100644 examples/http-s3.fio
 create mode 100644 examples/http-webdav.fio

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index 1bec806..c77dad1 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1835,6 +1835,15 @@ I/O engine
 			(RBD) via librbd without the need to use the kernel rbd driver. This
 			ioengine defines engine specific options.
 
+		**http**
+			I/O engine supporting GET/PUT requests over HTTP(S) with libcurl to
+			a WebDAV or S3 endpoint.  This ioengine defines engine specific options.
+
+			This engine only supports direct IO of iodepth=1; you need to scale this
+			via numjobs. blocksize defines the size of the objects to be created.
+
+			TRIM is translated to object deletion.
+
 		**gfapi**
 			Using GlusterFS libgfapi sync interface to direct access to
 			GlusterFS volumes without having to go through FUSE.  This ioengine
@@ -2115,6 +2124,47 @@ with the caveat that when used on the command line, they must come after the
 		transferred to the device. The writefua option is ignored with this
 		selection.
 
+.. option:: http_host=str : [http]
+
+	Hostname to connect to. For S3, this could be the bucket hostname.
+	Default is **localhost**
+
+.. option:: http_user=str : [http]
+
+	Username for HTTP authentication.
+
+.. option:: http_pass=str : [http]
+
+	Password for HTTP authentication.
+
+.. option:: https=bool : [http]
+
+	Enable HTTPS instead of http. Default is **0**
+
+.. option:: http_s3=bool : [http]
+
+	Enable S3 specific HTTP headers such as authenticating requests
+	with AWS Signature Version 4. Default is **0**
+
+.. option:: http_s3_region=str : [http]
+
+	The S3 region/zone string.
+	Default is **us-east-1**
+
+.. option:: http_s3_key=str : [http]
+
+	The S3 secret key.
+
+.. option:: http_s3_keyid=str : [http]
+
+	The S3 key/access id.
+
+.. option:: http_verbose=int : [http]
+
+	Enable verbose requests from libcurl. Useful for debugging. 1
+	turns on verbose logging from libcurl, 2 additionally enables
+	HTTP IO tracing. Default is **0**
+
 I/O depth
 ~~~~~~~~~
 
diff --git a/Makefile b/Makefile
index f9bd787..b981b45 100644
--- a/Makefile
+++ b/Makefile
@@ -101,6 +101,9 @@ endif
 ifdef CONFIG_RBD
   SOURCE += engines/rbd.c
 endif
+ifdef CONFIG_HTTP
+  SOURCE += engines/http.c
+endif
 SOURCE += oslib/asprintf.c
 ifndef CONFIG_STRSEP
   SOURCE += oslib/strsep.c
diff --git a/configure b/configure
index 9bdc7a1..103ea94 100755
--- a/configure
+++ b/configure
@@ -181,6 +181,8 @@ for opt do
   ;;
   --disable-rbd) disable_rbd="yes"
   ;;
+  --disable-http) disable_http="yes"
+  ;;
   --disable-gfapi) disable_gfapi="yes"
   ;;
   --enable-libhdfs) libhdfs="yes"
@@ -1567,6 +1569,35 @@ fi
 print_config "IPv6 helpers" "$ipv6"
 
 ##########################################
+# check for http
+if test "$http" != "yes" ; then
+  http="no"
+fi
+cat > $TMPC << EOF
+#include <curl/curl.h>
+#include <openssl/hmac.h>
+
+int main(int argc, char **argv)
+{
+  CURL *curl;
+  HMAC_CTX *ctx;
+
+  curl = curl_easy_init();
+  curl_easy_cleanup(curl);
+
+  ctx = HMAC_CTX_new();
+  HMAC_CTX_reset(ctx);
+  HMAC_CTX_free(ctx);
+  return 0;
+}
+EOF
+if test "$disable_http" != "yes"  && compile_prog "" "-lcurl -lssl -lcrypto" "curl"; then
+  LIBS="-lcurl -lssl -lcrypto $LIBS"
+  http="yes"
+fi
+print_config "http engine" "$http"
+
+##########################################
 # check for rados
 if test "$rados" != "yes" ; then
   rados="no"
@@ -2346,6 +2377,9 @@ fi
 if test "$ipv6" = "yes" ; then
   output_sym "CONFIG_IPV6"
 fi
+if test "$http" = "yes" ; then
+  output_sym "CONFIG_HTTP"
+fi
 if test "$rados" = "yes" ; then
   output_sym "CONFIG_RADOS"
 fi
diff --git a/engines/http.c b/engines/http.c
new file mode 100644
index 0000000..d3fdba8
--- /dev/null
+++ b/engines/http.c
@@ -0,0 +1,558 @@
+/*
+ * HTTP GET/PUT IO engine
+ *
+ * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
+ *
+ * Copyright (C) 2018 SUSE LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2 as published by the Free Software Foundation..
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include <pthread.h>
+#include <time.h>
+#include <curl/curl.h>
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+#include "fio.h"
+#include "../optgroup.h"
+
+
+struct http_data {
+	CURL *curl;
+};
+
+struct http_options {
+	void *pad;
+	int  https;
+	char *host;
+	char *user;
+	char *pass;
+	char *s3_key;
+	char *s3_keyid;
+	char *s3_region;
+	int verbose;
+	int s3;
+};
+
+struct http_curl_stream {
+	char *buf;
+	size_t pos;
+	size_t max;
+};
+
+static struct fio_option options[] = {
+	{
+		.name     = "https",
+		.lname    = "https",
+		.type     = FIO_OPT_BOOL,
+		.help     = "Enable https",
+		.off1     = offsetof(struct http_options, https),
+		.def      = "0",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_host",
+		.lname    = "http_host",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "Hostname (S3 bucket)",
+		.off1     = offsetof(struct http_options, host),
+		.def	  = "localhost",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_user",
+		.lname    = "http_user",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "HTTP user name",
+		.off1     = offsetof(struct http_options, user),
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_pass",
+		.lname    = "http_pass",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "HTTP password",
+		.off1     = offsetof(struct http_options, pass),
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_s3_key",
+		.lname    = "S3 secret key",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "S3 secret key",
+		.off1     = offsetof(struct http_options, s3_key),
+		.def	  = "",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_s3_keyid",
+		.lname    = "S3 key id",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "S3 key id",
+		.off1     = offsetof(struct http_options, s3_keyid),
+		.def	  = "",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_s3_region",
+		.lname    = "S3 region",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "S3 region",
+		.off1     = offsetof(struct http_options, s3_region),
+		.def	  = "us-east-1",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_s3",
+		.lname    = "S3 extensions",
+		.type     = FIO_OPT_BOOL,
+		.help     = "Whether to enable S3 specific headers",
+		.off1     = offsetof(struct http_options, s3),
+		.def	  = "0",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = "http_verbose",
+		.lname    = "CURL verbosity",
+		.type     = FIO_OPT_INT,
+		.help     = "increase http engine verbosity",
+		.off1     = offsetof(struct http_options, verbose),
+		.def	  = "0",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
+	{
+		.name     = NULL,
+	},
+};
+
+static char *_aws_uriencode(const char *uri)
+{
+	size_t bufsize = 1024;
+	char *r = malloc(bufsize);
+	char c;
+	int i, n;
+	const char *hex = "0123456789ABCDEF";
+
+	if (!r) {
+		log_err("malloc failed\n");
+		return NULL;
+	}
+
+	n = 0;
+	for (i = 0; (c = uri[i]); i++) {
+		if (n > bufsize-5) {
+			log_err("encoding the URL failed\n");
+			return NULL;
+		}
+
+		if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
+		|| (c >= '0' && c <= '9') || c == '_' || c == '-'
+		|| c == '~' || c == '.' || c == '/')
+			r[n++] = c;
+		else {
+			r[n++] = '%';
+			r[n++] = hex[(c >> 4 ) & 0xF];
+			r[n++] = hex[c & 0xF];
+		}
+	}
+	r[n++] = 0;
+	return r;
+}
+
+static char *_conv_hex(const unsigned char *p, size_t len)
+{
+	char *r;
+	int i,n;
+	const char *hex = "0123456789abcdef";
+	r = malloc(len * 2 + 1);
+	n = 0;
+	for (i = 0; i < len; i++) {
+		r[n++] = hex[(p[i] >> 4 ) & 0xF];
+		r[n++] = hex[p[i] & 0xF];
+	}
+	r[n] = 0;
+
+	return r;
+}
+
+static char *_gen_hex_sha256(const char *p, size_t len)
+{
+	unsigned char hash[SHA256_DIGEST_LENGTH];
+
+	SHA256((unsigned char*)p, len, hash);
+	return _conv_hex(hash, SHA256_DIGEST_LENGTH);
+}
+
+static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
+	HMAC_CTX *ctx;
+	unsigned int hmac_len;
+
+	ctx = HMAC_CTX_new();
+	HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
+	HMAC_Update(ctx, (unsigned char*)data, strlen(data));
+	HMAC_Final(ctx, md, &hmac_len);
+	HMAC_CTX_free(ctx);
+}
+
+static int _curl_trace(CURL *handle, curl_infotype type,
+	     char *data, size_t size,
+	     void *userp)
+{
+	const char *text;
+	(void)handle; /* prevent compiler warning */
+	(void)userp;
+
+	switch (type) {
+	case CURLINFO_TEXT:
+	fprintf(stderr, "== Info: %s", data);
+	default:
+	case CURLINFO_SSL_DATA_OUT:
+	case CURLINFO_SSL_DATA_IN:
+		return 0;
+
+	case CURLINFO_HEADER_OUT:
+		text = "=> Send header";
+		break;
+	case CURLINFO_DATA_OUT:
+		text = "=> Send data";
+		break;
+	case CURLINFO_HEADER_IN:
+		text = "<= Recv header";
+		break;
+	case CURLINFO_DATA_IN:
+		text = "<= Recv data";
+		break;
+	}
+
+	log_info("%s: %s", text, data);
+	return 0;
+}
+
+/* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
+ */
+static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
+		int op, const char *uri, char *buf, size_t len)
+{
+	char date_short[16];
+	char date_iso[32];
+	char method[8];
+	char dkey[128];
+	char creq[512];
+	char sts[256];
+	char s[512];
+	char *uri_encoded = NULL;
+	char *dsha = NULL;
+	char *csha = NULL;
+	char *signature = NULL;
+	const char *service = "s3";
+	const char *aws = "aws4_request";
+	unsigned char md[SHA256_DIGEST_LENGTH];
+
+	time_t t = time(NULL);
+	struct tm *gtm = gmtime(&t);
+
+	strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
+	strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
+	uri_encoded = _aws_uriencode(uri);
+
+	if (op == DDIR_WRITE) {
+		dsha = _gen_hex_sha256(buf, len);
+		sprintf(method, "PUT");
+	} else {
+		/* DDIR_READ && DDIR_TRIM supply an empty body */
+		if (op == DDIR_READ)
+			sprintf(method, "GET");
+		else
+			sprintf(method, "DELETE");
+		dsha = _gen_hex_sha256("", 0);
+	}
+
+	/* Create the canonical request first */
+	snprintf(creq, sizeof(creq),
+	"%s\n"
+	"%s\n"
+	"\n"
+	"host:%s\n"
+	"x-amz-content-sha256:%s\n"
+	"x-amz-date:%s\n"
+	"\n"
+	"host;x-amz-content-sha256;x-amz-date\n"
+	"%s"
+	, method
+	, uri_encoded, o->host, dsha, date_iso, dsha);
+
+	csha = _gen_hex_sha256(creq, strlen(creq));
+	snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
+		date_iso, date_short, o->s3_region, service, aws, csha);
+
+	snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
+	_hmac(md, dkey, strlen(dkey), date_short);
+	_hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
+	_hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
+	_hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
+	_hmac(md, md, SHA256_DIGEST_LENGTH, sts);
+
+	signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
+
+	/* Surpress automatic Accept: header */
+	slist = curl_slist_append(slist, "Accept:");
+
+	snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
+	slist = curl_slist_append(slist, s);
+
+	snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
+	slist = curl_slist_append(slist, s);
+
+	snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
+	"SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
+	o->s3_keyid, date_short, o->s3_region, signature);
+	slist = curl_slist_append(slist, s);
+
+	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+
+	free(uri_encoded);
+	free(csha);
+	free(dsha);
+	free(signature);
+}
+
+static void fio_http_cleanup(struct thread_data *td)
+{
+	struct http_data *http = td->io_ops_data;
+
+	if (http) {
+		curl_easy_cleanup(http->curl);
+		free(http);
+	}
+}
+
+static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+	struct http_curl_stream *state = stream;
+	size_t len = size * nmemb;
+	/* We're retrieving; nothing is supposed to be read locally */
+	if (!stream)
+		return 0;
+	if (len+state->pos > state->max)
+		len = state->max - state->pos;
+	memcpy(ptr, &state->buf[state->pos], len);
+	state->pos += len;
+	return len;
+}
+
+static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+	struct http_curl_stream *state = stream;
+	/* We're just discarding the returned body after a PUT */
+	if (!stream)
+		return nmemb;
+	if (size != 1)
+		return CURLE_WRITE_ERROR;
+	if (nmemb + state->pos > state->max)
+		return CURLE_WRITE_ERROR;
+	memcpy(&state->buf[state->pos], ptr, nmemb);
+	state->pos += nmemb;
+	return nmemb;
+}
+
+static int _http_seek(void *stream, curl_off_t offset, int origin)
+{
+	struct http_curl_stream *state = stream;
+	if (offset < state->max && origin == SEEK_SET) {
+		state->pos = offset;
+		return CURL_SEEKFUNC_OK;
+	} else
+		return CURL_SEEKFUNC_FAIL;
+}
+
+static enum fio_q_status fio_http_queue(struct thread_data *td,
+					 struct io_u *io_u)
+{
+	struct http_data *http = td->io_ops_data;
+	struct http_options *o = td->eo;
+	struct http_curl_stream _curl_stream;
+	struct curl_slist *slist = NULL;
+	char object[512];
+	char url[1024];
+	long status;
+	CURLcode res;
+	int r = -1;
+
+	fio_ro_check(td, io_u);
+	memset(&_curl_stream, 0, sizeof(_curl_stream));
+	snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, io_u->offset, io_u->xfer_buflen);
+	snprintf(url, sizeof(url), "%s://%s%s", o->https ? "https" : "http", o->host, object);
+	curl_easy_setopt(http->curl, CURLOPT_URL, url);
+	_curl_stream.buf = io_u->xfer_buf;
+	_curl_stream.max = io_u->xfer_buflen;
+	curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
+	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
+
+	if (o->s3)
+		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
+			io_u->xfer_buf, io_u->xfer_buflen);
+
+	if (io_u->ddir == DDIR_WRITE) {
+		curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
+		curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
+		curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
+		res = curl_easy_perform(http->curl);
+		if (res == CURLE_OK) {
+			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
+			if (status == 100 || (status >= 200 && status <= 204))
+				goto out;
+			log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
+			goto err;
+		}
+	} else if (io_u->ddir == DDIR_READ) {
+		curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
+		curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
+		curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
+		res = curl_easy_perform(http->curl);
+		if (res == CURLE_OK) {
+			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
+			if (status == 200)
+				goto out;
+			else if (status == 404) {
+				/* Object doesn't exist. Pretend we read
+				 * zeroes */
+				memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
+				goto out;
+			}
+			log_err("DDIR_READ failed with HTTP status code %ld\n", status);
+		}
+		goto err;
+	} else if (io_u->ddir == DDIR_TRIM) {
+		curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
+		curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+		curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, 0);
+		curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
+		curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
+		res = curl_easy_perform(http->curl);
+		if (res == CURLE_OK) {
+			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
+			if (status == 200 || status == 202 || status == 204 || status == 404)
+				goto out;
+			log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
+		}
+		goto err;
+	}
+
+	log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
+
+err:
+	io_u->error = r;
+	td_verror(td, io_u->error, "transfer");
+out:
+	curl_slist_free_all(slist);
+	return FIO_Q_COMPLETED;
+}
+
+static struct io_u *fio_http_event(struct thread_data *td, int event)
+{
+	/* sync IO engine - never any outstanding events */
+	return NULL;
+}
+
+int fio_http_getevents(struct thread_data *td, unsigned int min,
+	unsigned int max, const struct timespec *t)
+{
+	/* sync IO engine - never any outstanding events */
+	return 0;
+}
+
+static int fio_http_setup(struct thread_data *td)
+{
+	struct http_data *http = NULL;
+	struct http_options *o = td->eo;
+	int r;
+	/* allocate engine specific structure to deal with libhttp. */
+	http = calloc(1, sizeof(*http));
+	if (!http) {
+		log_err("calloc failed.\n");
+		goto cleanup;
+	}
+
+	http->curl = curl_easy_init();
+	if (o->verbose)
+		curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
+	if (o->verbose > 1)
+		curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
+	curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
+	curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
+	curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
+	curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
+	curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
+	curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, _http_seek);
+	if (o->user && o->pass) {
+		curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
+		curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
+		curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
+	}
+
+	td->io_ops_data = http;
+
+	/* Force single process mode. */
+	td->o.use_thread = 1;
+
+	return 0;
+cleanup:
+	fio_http_cleanup(td);
+	return r;
+}
+
+static int fio_http_open(struct thread_data *td, struct fio_file *f)
+{
+	return 0;
+}
+static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
+{
+	return 0;
+}
+
+static struct ioengine_ops ioengine = {
+	.name = "http",
+	.version		= FIO_IOOPS_VERSION,
+	.flags			= FIO_DISKLESSIO,
+	.setup			= fio_http_setup,
+	.queue			= fio_http_queue,
+	.getevents		= fio_http_getevents,
+	.event			= fio_http_event,
+	.cleanup		= fio_http_cleanup,
+	.open_file		= fio_http_open,
+	.invalidate		= fio_http_invalidate,
+	.options		= options,
+	.option_struct_size	= sizeof(struct http_options),
+};
+
+static void fio_init fio_http_register(void)
+{
+	register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_http_unregister(void)
+{
+	unregister_ioengine(&ioengine);
+}
diff --git a/examples/http-s3.fio b/examples/http-s3.fio
new file mode 100644
index 0000000..a9805da
--- /dev/null
+++ b/examples/http-s3.fio
@@ -0,0 +1,34 @@
+# Example test for the HTTP engine's S3 support against Amazon AWS.
+# Obviously, you have to adjust the S3 credentials; for this example,
+# they're passed in via the environment.
+#
+
+[global]
+ioengine=http
+name=test
+direct=1
+filename=/larsmb-fio-test/object
+http_verbose=0
+https=1
+http_s3=1
+http_s3_key=${S3_KEY}
+http_s3_keyid=${S3_ID}
+http_host=s3.eu-central-1.amazonaws.com
+http_s3_region=eu-central-1
+group_reporting
+
+# With verify, this both writes and reads the object
+[create]
+rw=write
+bs=4k
+size=64k
+io_size=4k
+verify=sha256
+
+[trim]
+stonewall
+rw=trim
+bs=4k
+size=64k
+io_size=4k
+
diff --git a/examples/http-webdav.fio b/examples/http-webdav.fio
new file mode 100644
index 0000000..c0624f8
--- /dev/null
+++ b/examples/http-webdav.fio
@@ -0,0 +1,26 @@
+[global]
+ioengine=http
+rw=randwrite
+name=test
+direct=1
+http_verbose=0
+http_s3=0
+https=0
+http_host=localhost
+filename_format=/dav/bucket.$jobnum
+group_reporting
+bs=64k
+size=1M
+
+[create]
+numjobs=16
+rw=randwrite
+io_size=10M
+verify=sha256
+
+# This will delete all created objects again
+[trim]
+stonewall
+numjobs=16
+rw=trim
+io_size=1M
diff --git a/fio.1 b/fio.1
index 18bf6a2..883a31b 100644
--- a/fio.1
+++ b/fio.1
@@ -1608,6 +1608,15 @@ I/O engine supporting direct access to Ceph Rados Block Devices
 (RBD) via librbd without the need to use the kernel rbd driver. This
 ioengine defines engine specific options.
 .TP
+.B http
+I/O engine supporting GET/PUT requests over HTTP(S) with libcurl to
+a WebDAV or S3 endpoint.  This ioengine defines engine specific options.
+
+This engine only supports direct IO of iodepth=1; you need to scale this
+via numjobs. blocksize defines the size of the objects to be created.
+
+TRIM is translated to object deletion.
+.TP
 .B gfapi
 Using GlusterFS libgfapi sync interface to direct access to
 GlusterFS volumes without having to go through FUSE. This ioengine
@@ -1810,6 +1819,37 @@ by default.
 Poll store instead of waiting for completion. Usually this provides better
 throughput at cost of higher(up to 100%) CPU utilization.
 .TP
+.BI (http)http_host \fR=\fPstr
+Hostname to connect to. For S3, this could be the bucket name. Default
+is \fBlocalhost\fR
+.TP
+.BI (http)http_user \fR=\fPstr
+Username for HTTP authentication.
+.TP
+.BI (http)http_pass \fR=\fPstr
+Password for HTTP authentication.
+.TP
+.BI (http)https \fR=\fPbool
+Whether to use HTTPS instead of plain HTTP. Default is \fB0\fR.
+.TP
+.BI (http)http_s3 \fR=\fPbool
+Include S3 specific HTTP headers such as authenticating requests with
+AWS Signature Version 4. Default is \fB0\fR.
+.TP
+.BI (http)http_s3_region \fR=\fPstr
+The S3 region/zone to include in the request. Default is \fBus-east-1\fR.
+.TP
+.BI (http)http_s3_key \fR=\fPstr
+The S3 secret key.
+.TP
+.BI (http)http_s3_keyid \fR=\fPstr
+The S3 key/access id.
+.TP
+.BI (http)http_verbose \fR=\fPint
+Enable verbose requests from libcurl. Useful for debugging. 1 turns on
+verbose logging from libcurl, 2 additionally enables HTTP IO tracing.
+Default is \fB0\fR
+.TP
 .BI (mtd)skip_bad \fR=\fPbool
 Skip operations against known bad blocks.
 .TP
diff --git a/optgroup.h b/optgroup.h
index d5e968d..adf4d09 100644
--- a/optgroup.h
+++ b/optgroup.h
@@ -56,6 +56,7 @@ enum opt_category_group {
 	__FIO_OPT_G_ACT,
 	__FIO_OPT_G_LATPROF,
 	__FIO_OPT_G_RBD,
+	__FIO_OPT_G_HTTP,
 	__FIO_OPT_G_GFAPI,
 	__FIO_OPT_G_MTD,
 	__FIO_OPT_G_HDFS,
@@ -91,6 +92,7 @@ enum opt_category_group {
 	FIO_OPT_G_ACT		= (1ULL << __FIO_OPT_G_ACT),
 	FIO_OPT_G_LATPROF	= (1ULL << __FIO_OPT_G_LATPROF),
 	FIO_OPT_G_RBD		= (1ULL << __FIO_OPT_G_RBD),
+	FIO_OPT_G_HTTP		= (1ULL << __FIO_OPT_G_HTTP),
 	FIO_OPT_G_GFAPI		= (1ULL << __FIO_OPT_G_GFAPI),
 	FIO_OPT_G_MTD		= (1ULL << __FIO_OPT_G_MTD),
 	FIO_OPT_G_HDFS		= (1ULL << __FIO_OPT_G_HDFS),
diff --git a/options.c b/options.c
index e53b717..9ee1ba3 100644
--- a/options.c
+++ b/options.c
@@ -1863,6 +1863,11 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 			    .help = "PMDK libpmem based IO engine",
 			  },
 #endif
+#ifdef CONFIG_HTTP
+			  { .ival = "http",
+			    .help = "HTTP (WebDAV/S3) IO engine",
+			  },
+#endif
 		},
 	},
 	{