Recent changes (master)

Jens Axboe
2018-08-15T14:52:17Z

The following changes since commit dcf6ad384149ee0b3f91c5a8127160cc291f7157:

Merge branch 'fio-man-page' of https://github.com/bvanassche/fio (2018-08-13 21:05:33 -0600)

are available in the git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 875e8d6fa4d443068eb1c48a29f5367e454d2a37:

Merge branch 'wip-http-engine' of https://github.com/l-mb/fio (2018-08-14 10:09:47 -0600)

----------------------------------------------------------------
Jens Axboe (1):
Merge branch 'wip-http-engine' of https://github.com/l-mb/fio

Lars Marowsky-Bree (1):
engines/http: Add support for WebDAV and S3

HOWTO | 50 +++++
Makefile | 3 +
configure | 34 +++
engines/http.c | 558 +++++++++++++++++++++++++++++++++++++++++++++++
examples/http-s3.fio | 34 +++
examples/http-webdav.fio | 26 +++
fio.1 | 40 ++++
optgroup.h | 2 +
options.c | 5 +
9 files changed, 752 insertions(+)
create mode 100644 engines/http.c
create mode 100644 examples/http-s3.fio
create mode 100644 examples/http-webdav.fio

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index 1bec806..c77dad1 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1835,6 +1835,15 @@ I/O engine
(RBD) via librbd without the need to use the kernel rbd driver. This
ioengine defines engine specific options.

+ **http**
+ I/O engine supporting GET/PUT requests over HTTP(S) with libcurl to
+ a WebDAV or S3 endpoint. This ioengine defines engine specific options.
+
+ This engine only supports direct IO of iodepth=1; you need to scale this
+ via numjobs. blocksize defines the size of the objects to be created.
+
+ TRIM is translated to object deletion.
+
**gfapi**
Using GlusterFS libgfapi sync interface to direct access to
GlusterFS volumes without having to go through FUSE. This ioengine
@@ -2115,6 +2124,47 @@ with the caveat that when used on the command line, they must come after the
transferred to the device. The writefua option is ignored with this
selection.

+.. option:: http_host=str : [http]
+
+ Hostname to connect to. For S3, this could be the bucket hostname.
+ Default is **localhost**
+
+.. option:: http_user=str : [http]
+
+ Username for HTTP authentication.
+
+.. option:: http_pass=str : [http]
+
+ Password for HTTP authentication.
+
+.. option:: https=bool : [http]
+
+ Enable HTTPS instead of http. Default is **0**
+
+.. option:: http_s3=bool : [http]
+
+ Enable S3 specific HTTP headers such as authenticating requests
+ with AWS Signature Version 4. Default is **0**
+
+.. option:: http_s3_region=str : [http]
+
+ The S3 region/zone string.
+ Default is **us-east-1**
+
+.. option:: http_s3_key=str : [http]
+
+ The S3 secret key.
+
+.. option:: http_s3_keyid=str : [http]
+
+ The S3 key/access id.
+
+.. option:: http_verbose=int : [http]
+
+ Enable verbose requests from libcurl. Useful for debugging. 1
+ turns on verbose logging from libcurl, 2 additionally enables
+ HTTP IO tracing. Default is **0**
+
I/O depth
~~~~~~~~~

diff --git a/Makefile b/Makefile
index f9bd787..b981b45 100644
--- a/Makefile
+++ b/Makefile
@@ -101,6 +101,9 @@ endif
ifdef CONFIG_RBD
SOURCE += engines/rbd.c
endif
+ifdef CONFIG_HTTP
+ SOURCE += engines/http.c
+endif
SOURCE += oslib/asprintf.c
ifndef CONFIG_STRSEP
SOURCE += oslib/strsep.c
diff --git a/configure b/configure
index 9bdc7a1..103ea94 100755
--- a/configure
+++ b/configure
@@ -181,6 +181,8 @@ for opt do
;;
--disable-rbd) disable_rbd="yes"
;;
+ --disable-http) disable_http="yes"
+ ;;
--disable-gfapi) disable_gfapi="yes"
;;
--enable-libhdfs) libhdfs="yes"
@@ -1567,6 +1569,35 @@ fi
print_config "IPv6 helpers" "$ipv6"

##########################################
+# check for http
+if test "$http" != "yes" ; then
+ http="no"
+fi
+cat > $TMPC << EOF
+#include <curl/curl.h>
+#include <openssl/hmac.h>
+
+int main(int argc, char **argv)
+{
+ CURL *curl;
+ HMAC_CTX *ctx;
+
+ curl = curl_easy_init();
+ curl_easy_cleanup(curl);
+
+ ctx = HMAC_CTX_new();
+ HMAC_CTX_reset(ctx);
+ HMAC_CTX_free(ctx);
+ return 0;
+}
+EOF
+if test "$disable_http" != "yes" && compile_prog "" "-lcurl -lssl -lcrypto" "curl"; then
+ LIBS="-lcurl -lssl -lcrypto $LIBS"
+ http="yes"
+fi
+print_config "http engine" "$http"
+
+##########################################
# check for rados
if test "$rados" != "yes" ; then
rados="no"
@@ -2346,6 +2377,9 @@ fi
if test "$ipv6" = "yes" ; then
output_sym "CONFIG_IPV6"
fi
+if test "$http" = "yes" ; then
+ output_sym "CONFIG_HTTP"
+fi
if test "$rados" = "yes" ; then
output_sym "CONFIG_RADOS"
fi
diff --git a/engines/http.c b/engines/http.c
new file mode 100644
index 0000000..d3fdba8
--- /dev/null
+++ b/engines/http.c
@@ -0,0 +1,558 @@
+/*
+ * HTTP GET/PUT IO engine
+ *
+ * IO engine to perform HTTP(S) GET/PUT requests via libcurl-easy.
+ *
+ * Copyright (C) 2018 SUSE LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2 as published by the Free Software Foundation..
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include <pthread.h>
+#include <time.h>
+#include <curl/curl.h>
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+#include "fio.h"
+#include "../optgroup.h"
+
+
+struct http_data {
+ CURL *curl;
+};
+
+struct http_options {
+ void *pad;
+ int https;
+ char *host;
+ char *user;
+ char *pass;
+ char *s3_key;
+ char *s3_keyid;
+ char *s3_region;
+ int verbose;
+ int s3;
+};
+
+struct http_curl_stream {
+ char *buf;
+ size_t pos;
+ size_t max;
+};
+
+static struct fio_option options[] = {
+ {
+ .name = "https",
+ .lname = "https",
+ .type = FIO_OPT_BOOL,
+ .help = "Enable https",
+ .off1 = offsetof(struct http_options, https),
+ .def = "0",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_host",
+ .lname = "http_host",
+ .type = FIO_OPT_STR_STORE,
+ .help = "Hostname (S3 bucket)",
+ .off1 = offsetof(struct http_options, host),
+ .def = "localhost",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_user",
+ .lname = "http_user",
+ .type = FIO_OPT_STR_STORE,
+ .help = "HTTP user name",
+ .off1 = offsetof(struct http_options, user),
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_pass",
+ .lname = "http_pass",
+ .type = FIO_OPT_STR_STORE,
+ .help = "HTTP password",
+ .off1 = offsetof(struct http_options, pass),
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_s3_key",
+ .lname = "S3 secret key",
+ .type = FIO_OPT_STR_STORE,
+ .help = "S3 secret key",
+ .off1 = offsetof(struct http_options, s3_key),
+ .def = "",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_s3_keyid",
+ .lname = "S3 key id",
+ .type = FIO_OPT_STR_STORE,
+ .help = "S3 key id",
+ .off1 = offsetof(struct http_options, s3_keyid),
+ .def = "",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_s3_region",
+ .lname = "S3 region",
+ .type = FIO_OPT_STR_STORE,
+ .help = "S3 region",
+ .off1 = offsetof(struct http_options, s3_region),
+ .def = "us-east-1",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_s3",
+ .lname = "S3 extensions",
+ .type = FIO_OPT_BOOL,
+ .help = "Whether to enable S3 specific headers",
+ .off1 = offsetof(struct http_options, s3),
+ .def = "0",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = "http_verbose",
+ .lname = "CURL verbosity",
+ .type = FIO_OPT_INT,
+ .help = "increase http engine verbosity",
+ .off1 = offsetof(struct http_options, verbose),
+ .def = "0",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_HTTP,
+ },
+ {
+ .name = NULL,
+ },
+};
+
+static char *_aws_uriencode(const char *uri)
+{
+ size_t bufsize = 1024;
+ char *r = malloc(bufsize);
+ char c;
+ int i, n;
+ const char *hex = "0123456789ABCDEF";
+
+ if (!r) {
+ log_err("malloc failed\n");
+ return NULL;
+ }
+
+ n = 0;
+ for (i = 0; (c = uri[i]); i++) {
+ if (n > bufsize-5) {
+ log_err("encoding the URL failed\n");
+ return NULL;
+ }
+
+ if ( (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
+ || (c >= '0' && c <= '9') || c == '_' || c == '-'
+ || c == '~' || c == '.' || c == '/')
+ r[n++] = c;
+ else {
+ r[n++] = '%';
+ r[n++] = hex[(c >> 4 ) & 0xF];
+ r[n++] = hex[c & 0xF];
+ }
+ }
+ r[n++] = 0;
+ return r;
+}
+
+static char *_conv_hex(const unsigned char *p, size_t len)
+{
+ char *r;
+ int i,n;
+ const char *hex = "0123456789abcdef";
+ r = malloc(len * 2 + 1);
+ n = 0;
+ for (i = 0; i < len; i++) {
+ r[n++] = hex[(p[i] >> 4 ) & 0xF];
+ r[n++] = hex[p[i] & 0xF];
+ }
+ r[n] = 0;
+
+ return r;
+}
+
+static char *_gen_hex_sha256(const char *p, size_t len)
+{
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+
+ SHA256((unsigned char*)p, len, hash);
+ return _conv_hex(hash, SHA256_DIGEST_LENGTH);
+}
+
+static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
+ HMAC_CTX *ctx;
+ unsigned int hmac_len;
+
+ ctx = HMAC_CTX_new();
+ HMAC_Init_ex(ctx, key, key_len, EVP_sha256(), NULL);
+ HMAC_Update(ctx, (unsigned char*)data, strlen(data));
+ HMAC_Final(ctx, md, &hmac_len);
+ HMAC_CTX_free(ctx);
+}
+
+static int _curl_trace(CURL *handle, curl_infotype type,
+ char *data, size_t size,
+ void *userp)
+{
+ const char *text;
+ (void)handle; /* prevent compiler warning */
+ (void)userp;
+
+ switch (type) {
+ case CURLINFO_TEXT:
+ fprintf(stderr, "== Info: %s", data);
+ default:
+ case CURLINFO_SSL_DATA_OUT:
+ case CURLINFO_SSL_DATA_IN:
+ return 0;
+
+ case CURLINFO_HEADER_OUT:
+ text = "=> Send header";
+ break;
+ case CURLINFO_DATA_OUT:
+ text = "=> Send data";
+ break;
+ case CURLINFO_HEADER_IN:
+ text = "<= Recv header";
+ break;
+ case CURLINFO_DATA_IN:
+ text = "<= Recv data";
+ break;
+ }
+
+ log_info("%s: %s", text, data);
+ return 0;
+}
+
+/* https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html#signing-request-intro
+ */
+static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
+ int op, const char *uri, char *buf, size_t len)
+{
+ char date_short[16];
+ char date_iso[32];
+ char method[8];
+ char dkey[128];
+ char creq[512];
+ char sts[256];
+ char s[512];
+ char *uri_encoded = NULL;
+ char *dsha = NULL;
+ char *csha = NULL;
+ char *signature = NULL;
+ const char *service = "s3";
+ const char *aws = "aws4_request";
+ unsigned char md[SHA256_DIGEST_LENGTH];
+
+ time_t t = time(NULL);
+ struct tm *gtm = gmtime(&t);
+
+ strftime (date_short, sizeof(date_short), "%Y%m%d", gtm);
+ strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
+ uri_encoded = _aws_uriencode(uri);
+
+ if (op == DDIR_WRITE) {
+ dsha = _gen_hex_sha256(buf, len);
+ sprintf(method, "PUT");
+ } else {
+ /* DDIR_READ && DDIR_TRIM supply an empty body */
+ if (op == DDIR_READ)
+ sprintf(method, "GET");
+ else
+ sprintf(method, "DELETE");
+ dsha = _gen_hex_sha256("", 0);
+ }
+
+ /* Create the canonical request first */
+ snprintf(creq, sizeof(creq),
+ "%s\n"
+ "%s\n"
+ "\n"
+ "host:%s\n"
+ "x-amz-content-sha256:%s\n"
+ "x-amz-date:%s\n"
+ "\n"
+ "host;x-amz-content-sha256;x-amz-date\n"
+ "%s"
+ , method
+ , uri_encoded, o->host, dsha, date_iso, dsha);
+
+ csha = _gen_hex_sha256(creq, strlen(creq));
+ snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
+ date_iso, date_short, o->s3_region, service, aws, csha);
+
+ snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
+ _hmac(md, dkey, strlen(dkey), date_short);
+ _hmac(md, md, SHA256_DIGEST_LENGTH, o->s3_region);
+ _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) service);
+ _hmac(md, md, SHA256_DIGEST_LENGTH, (char*) aws);
+ _hmac(md, md, SHA256_DIGEST_LENGTH, sts);
+
+ signature = _conv_hex(md, SHA256_DIGEST_LENGTH);
+
+ /* Surpress automatic Accept: header */
+ slist = curl_slist_append(slist, "Accept:");
+
+ snprintf(s, sizeof(s), "x-amz-content-sha256: %s", dsha);
+ slist = curl_slist_append(slist, s);
+
+ snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
+ slist = curl_slist_append(slist, s);
+
+ snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
+ o->s3_keyid, date_short, o->s3_region, signature);
+ slist = curl_slist_append(slist, s);
+
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+
+ free(uri_encoded);
+ free(csha);
+ free(dsha);
+ free(signature);
+}
+
+static void fio_http_cleanup(struct thread_data *td)
+{
+ struct http_data *http = td->io_ops_data;
+
+ if (http) {
+ curl_easy_cleanup(http->curl);
+ free(http);
+ }
+}
+
+static size_t _http_read(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ struct http_curl_stream *state = stream;
+ size_t len = size * nmemb;
+ /* We're retrieving; nothing is supposed to be read locally */
+ if (!stream)
+ return 0;
+ if (len+state->pos > state->max)
+ len = state->max - state->pos;
+ memcpy(ptr, &state->buf[state->pos], len);
+ state->pos += len;
+ return len;
+}
+
+static size_t _http_write(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ struct http_curl_stream *state = stream;
+ /* We're just discarding the returned body after a PUT */
+ if (!stream)
+ return nmemb;
+ if (size != 1)
+ return CURLE_WRITE_ERROR;
+ if (nmemb + state->pos > state->max)
+ return CURLE_WRITE_ERROR;
+ memcpy(&state->buf[state->pos], ptr, nmemb);
+ state->pos += nmemb;
+ return nmemb;
+}
+
+static int _http_seek(void *stream, curl_off_t offset, int origin)
+{
+ struct http_curl_stream *state = stream;
+ if (offset < state->max && origin == SEEK_SET) {
+ state->pos = offset;
+ return CURL_SEEKFUNC_OK;
+ } else
+ return CURL_SEEKFUNC_FAIL;
+}
+
+static enum fio_q_status fio_http_queue(struct thread_data *td,
+ struct io_u *io_u)
+{
+ struct http_data *http = td->io_ops_data;
+ struct http_options *o = td->eo;
+ struct http_curl_stream _curl_stream;
+ struct curl_slist *slist = NULL;
+ char object[512];
+ char url[1024];
+ long status;
+ CURLcode res;
+ int r = -1;
+
+ fio_ro_check(td, io_u);
+ memset(&_curl_stream, 0, sizeof(_curl_stream));
+ snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, io_u->offset, io_u->xfer_buflen);
+ snprintf(url, sizeof(url), "%s://%s%s", o->https ? "https" : "http", o->host, object);
+ curl_easy_setopt(http->curl, CURLOPT_URL, url);
+ _curl_stream.buf = io_u->xfer_buf;
+ _curl_stream.max = io_u->xfer_buflen;
+ curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
+ curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
+
+ if (o->s3)
+ _add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
+ io_u->xfer_buf, io_u->xfer_buflen);
+
+ if (io_u->ddir == DDIR_WRITE) {
+ curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
+ curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
+ curl_easy_setopt(http->curl, CURLOPT_UPLOAD, 1L);
+ res = curl_easy_perform(http->curl);
+ if (res == CURLE_OK) {
+ curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
+ if (status == 100 || (status >= 200 && status <= 204))
+ goto out;
+ log_err("DDIR_WRITE failed with HTTP status code %ld\n", status);
+ goto err;
+ }
+ } else if (io_u->ddir == DDIR_READ) {
+ curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
+ curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, &_curl_stream);
+ curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
+ res = curl_easy_perform(http->curl);
+ if (res == CURLE_OK) {
+ curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
+ if (status == 200)
+ goto out;
+ else if (status == 404) {
+ /* Object doesn't exist. Pretend we read
+ * zeroes */
+ memset(io_u->xfer_buf, 0, io_u->xfer_buflen);
+ goto out;
+ }
+ log_err("DDIR_READ failed with HTTP status code %ld\n", status);
+ }
+ goto err;
+ } else if (io_u->ddir == DDIR_TRIM) {
+ curl_easy_setopt(http->curl, CURLOPT_HTTPGET, 1L);
+ curl_easy_setopt(http->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+ curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, 0);
+ curl_easy_setopt(http->curl, CURLOPT_READDATA, NULL);
+ curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, NULL);
+ res = curl_easy_perform(http->curl);
+ if (res == CURLE_OK) {
+ curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
+ if (status == 200 || status == 202 || status == 204 || status == 404)
+ goto out;
+ log_err("DDIR_TRIM failed with HTTP status code %ld\n", status);
+ }
+ goto err;
+ }
+
+ log_err("WARNING: Only DDIR_READ/DDIR_WRITE/DDIR_TRIM are supported!\n");
+
+err:
+ io_u->error = r;
+ td_verror(td, io_u->error, "transfer");
+out:
+ curl_slist_free_all(slist);
+ return FIO_Q_COMPLETED;
+}
+
+static struct io_u *fio_http_event(struct thread_data *td, int event)
+{
+ /* sync IO engine - never any outstanding events */
+ return NULL;
+}
+
+int fio_http_getevents(struct thread_data *td, unsigned int min,
+ unsigned int max, const struct timespec *t)
+{
+ /* sync IO engine - never any outstanding events */
+ return 0;
+}
+
+static int fio_http_setup(struct thread_data *td)
+{
+ struct http_data *http = NULL;
+ struct http_options *o = td->eo;
+ int r;
+ /* allocate engine specific structure to deal with libhttp. */
+ http = calloc(1, sizeof(*http));
+ if (!http) {
+ log_err("calloc failed.\n");
+ goto cleanup;
+ }
+
+ http->curl = curl_easy_init();
+ if (o->verbose)
+ curl_easy_setopt(http->curl, CURLOPT_VERBOSE, 1L);
+ if (o->verbose > 1)
+ curl_easy_setopt(http->curl, CURLOPT_DEBUGFUNCTION, &_curl_trace);
+ curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
+ curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
+ curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
+ curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
+ curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, _http_seek);
+ if (o->user && o->pass) {
+ curl_easy_setopt(http->curl, CURLOPT_USERNAME, o->user);
+ curl_easy_setopt(http->curl, CURLOPT_PASSWORD, o->pass);
+ curl_easy_setopt(http->curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
+ }
+
+ td->io_ops_data = http;
+
+ /* Force single process mode. */
+ td->o.use_thread = 1;
+
+ return 0;
+cleanup:
+ fio_http_cleanup(td);
+ return r;
+}
+
+static int fio_http_open(struct thread_data *td, struct fio_file *f)
+{
+ return 0;
+}
+static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
+{
+ return 0;
+}
+
+static struct ioengine_ops ioengine = {
+ .name = "http",
+ .version = FIO_IOOPS_VERSION,
+ .flags = FIO_DISKLESSIO,
+ .setup = fio_http_setup,
+ .queue = fio_http_queue,
+ .getevents = fio_http_getevents,
+ .event = fio_http_event,
+ .cleanup = fio_http_cleanup,
+ .open_file = fio_http_open,
+ .invalidate = fio_http_invalidate,
+ .options = options,
+ .option_struct_size = sizeof(struct http_options),
+};
+
+static void fio_init fio_http_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_http_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
diff --git a/examples/http-s3.fio b/examples/http-s3.fio
new file mode 100644
index 0000000..a9805da
--- /dev/null
+++ b/examples/http-s3.fio
@@ -0,0 +1,34 @@
+# Example test for the HTTP engine's S3 support against Amazon AWS.
+# Obviously, you have to adjust the S3 credentials; for this example,
+# they're passed in via the environment.
+#
+
+[global]
+ioengine=http
+name=test
+direct=1
+filename=/larsmb-fio-test/object
+http_verbose=0
+https=1
+http_s3=1
+http_s3_key=${S3_KEY}
+http_s3_keyid=${S3_ID}
+http_host=s3.eu-central-1.amazonaws.com
+http_s3_region=eu-central-1
+group_reporting
+
+# With verify, this both writes and reads the object
+[create]
+rw=write
+bs=4k
+size=64k
+io_size=4k
+verify=sha256
+
+[trim]
+stonewall
+rw=trim
+bs=4k
+size=64k
+io_size=4k
+
diff --git a/examples/http-webdav.fio b/examples/http-webdav.fio
new file mode 100644
index 0000000..c0624f8
--- /dev/null
+++ b/examples/http-webdav.fio
@@ -0,0 +1,26 @@
+[global]
+ioengine=http
+rw=randwrite
+name=test
+direct=1
+http_verbose=0
+http_s3=0
+https=0
+http_host=localhost
+filename_format=/dav/bucket.$jobnum
+group_reporting
+bs=64k
+size=1M
+
+[create]
+numjobs=16
+rw=randwrite
+io_size=10M
+verify=sha256
+
+# This will delete all created objects again
+[trim]
+stonewall
+numjobs=16
+rw=trim
+io_size=1M
diff --git a/fio.1 b/fio.1
index 18bf6a2..883a31b 100644
--- a/fio.1
+++ b/fio.1
@@ -1608,6 +1608,15 @@ I/O engine supporting direct access to Ceph Rados Block Devices
(RBD) via librbd without the need to use the kernel rbd driver. This
ioengine defines engine specific options.
.TP
+.B http
+I/O engine supporting GET/PUT requests over HTTP(S) with libcurl to
+a WebDAV or S3 endpoint. This ioengine defines engine specific options.
+
+This engine only supports direct IO of iodepth=1; you need to scale this
+via numjobs. blocksize defines the size of the objects to be created.
+
+TRIM is translated to object deletion.
+.TP
.B gfapi
Using GlusterFS libgfapi sync interface to direct access to
GlusterFS volumes without having to go through FUSE. This ioengine
@@ -1810,6 +1819,37 @@ by default.
Poll store instead of waiting for completion. Usually this provides better
throughput at cost of higher(up to 100%) CPU utilization.
.TP
+.BI (http)http_host \fR=\fPstr
+Hostname to connect to. For S3, this could be the bucket name. Default
+is \fBlocalhost\fR
+.TP
+.BI (http)http_user \fR=\fPstr
+Username for HTTP authentication.
+.TP
+.BI (http)http_pass \fR=\fPstr
+Password for HTTP authentication.
+.TP
+.BI (http)https \fR=\fPbool
+Whether to use HTTPS instead of plain HTTP. Default is \fB0\fR.
+.TP
+.BI (http)http_s3 \fR=\fPbool
+Include S3 specific HTTP headers such as authenticating requests with
+AWS Signature Version 4. Default is \fB0\fR.
+.TP
+.BI (http)http_s3_region \fR=\fPstr
+The S3 region/zone to include in the request. Default is \fBus-east-1\fR.
+.TP
+.BI (http)http_s3_key \fR=\fPstr
+The S3 secret key.
+.TP
+.BI (http)http_s3_keyid \fR=\fPstr
+The S3 key/access id.
+.TP
+.BI (http)http_verbose \fR=\fPint
+Enable verbose requests from libcurl. Useful for debugging. 1 turns on
+verbose logging from libcurl, 2 additionally enables HTTP IO tracing.
+Default is \fB0\fR
+.TP
.BI (mtd)skip_bad \fR=\fPbool
Skip operations against known bad blocks.
.TP
diff --git a/optgroup.h b/optgroup.h
index d5e968d..adf4d09 100644
--- a/optgroup.h
+++ b/optgroup.h
@@ -56,6 +56,7 @@ enum opt_category_group {
__FIO_OPT_G_ACT,
__FIO_OPT_G_LATPROF,
__FIO_OPT_G_RBD,
+ __FIO_OPT_G_HTTP,
__FIO_OPT_G_GFAPI,
__FIO_OPT_G_MTD,
__FIO_OPT_G_HDFS,
@@ -91,6 +92,7 @@ enum opt_category_group {
FIO_OPT_G_ACT = (1ULL << __FIO_OPT_G_ACT),
FIO_OPT_G_LATPROF = (1ULL << __FIO_OPT_G_LATPROF),
FIO_OPT_G_RBD = (1ULL << __FIO_OPT_G_RBD),
+ FIO_OPT_G_HTTP = (1ULL << __FIO_OPT_G_HTTP),
FIO_OPT_G_GFAPI = (1ULL << __FIO_OPT_G_GFAPI),
FIO_OPT_G_MTD = (1ULL << __FIO_OPT_G_MTD),
FIO_OPT_G_HDFS = (1ULL << __FIO_OPT_G_HDFS),
diff --git a/options.c b/options.c
index e53b717..9ee1ba3 100644
--- a/options.c
+++ b/options.c
@@ -1863,6 +1863,11 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.help = "PMDK libpmem based IO engine",
},
#endif
+#ifdef CONFIG_HTTP
+ { .ival = "http",
+ .help = "HTTP (WebDAV/S3) IO engine",
+ },
+#endif
},
},
{