* [PATCH v8 01/20] trace-cmd library: Add support for compression algorithms
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 02/20] trace-cmd library: Internal helpers for compressing data Tzvetomir Stoyanov (VMware)
` (18 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
Added infrastructure to trace-cmd library for compression. Introduced
various new APIs to work with this new functionality:
struct tracecmd_compression
tracecmd_compress_init()
tracecmd_compress_free()
tracecmd_compress_alloc()
tracecmd_compress_destroy()
tracecmd_compress_block()
tracecmd_uncompress_block()
tracecmd_compress_reset()
tracecmd_compress_buffer_read()
tracecmd_compress_pread()
tracecmd_compress_buffer_write()
tracecmd_compress_lseek()
tracecmd_compress_proto_get_name()
tracecmd_compress_is_supported()
tracecmd_compress_protos_get()
tracecmd_compress_proto_register()
tracecmd_compress_copy_from()
tracecmd_uncompress_copy_to()
tracecmd_uncompress_chunk()
tracecmd_load_chunks_info()
The compression algorithms are not part of this patch.
Added trace-cmd library constructor and destructor routines, used to
initialize and free compression context.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/Makefile | 1 +
.../include/private/trace-cmd-private.h | 41 +
lib/trace-cmd/include/trace-cmd-local.h | 3 +
lib/trace-cmd/trace-compress.c | 977 ++++++++++++++++++
lib/trace-cmd/trace-util.c | 10 +
5 files changed, 1032 insertions(+)
create mode 100644 lib/trace-cmd/trace-compress.c
diff --git a/lib/trace-cmd/Makefile b/lib/trace-cmd/Makefile
index c9d36d1b..a7be4a42 100644
--- a/lib/trace-cmd/Makefile
+++ b/lib/trace-cmd/Makefile
@@ -25,6 +25,7 @@ ifeq ($(VSOCK_DEFINED), 1)
OBJS += trace-timesync-ptp.o
OBJS += trace-timesync-kvm.o
endif
+OBJS += trace-compress.o
# Additional util objects
OBJS += trace-blk-hack.o
diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 1efafba1..0bdb66e5 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -35,6 +35,7 @@ int *tracecmd_add_id(int *list, int id, int len);
#define FILE_VERSION_MAX 7
#define FILE_VERSION_SECTIONS 7
+#define FILE_VERSION_COMPRESSION 7
enum {
RINGBUF_TYPE_PADDING = 29,
@@ -160,6 +161,7 @@ enum {
TRACECMD_FL_IN_USECS = (1 << 2),
TRACECMD_FL_RAW_TS = (1 << 3),
TRACECMD_FL_SECTIONED = (1 << 4),
+ TRACECMD_FL_COMPRESSION = (1 << 5),
};
struct tracecmd_ftrace {
@@ -492,6 +494,45 @@ void tracecmd_tsync_free(struct tracecmd_time_sync *tsync);
int tracecmd_write_guest_time_shift(struct tracecmd_output *handle,
struct tracecmd_time_sync *tsync);
+/* --- Compression --- */
+struct tracecmd_compress_chunk {
+ unsigned int size;
+ unsigned int zsize;
+ off64_t zoffset;
+ off64_t offset;
+};
+struct tracecmd_compression;
+struct tracecmd_compression *tracecmd_compress_alloc(const char *name, const char *version,
+ int fd, struct tep_handle *tep,
+ struct tracecmd_msg_handle *msg_handle);
+void tracecmd_compress_destroy(struct tracecmd_compression *handle);
+int tracecmd_compress_block(struct tracecmd_compression *handle);
+int tracecmd_uncompress_block(struct tracecmd_compression *handle);
+void tracecmd_compress_reset(struct tracecmd_compression *handle);
+int tracecmd_compress_buffer_read(struct tracecmd_compression *handle, char *dst, int len);
+int tracecmd_compress_pread(struct tracecmd_compression *handle, char *dst, int len, off_t offset);
+int tracecmd_compress_buffer_write(struct tracecmd_compression *handle,
+ const void *data, unsigned long long size);
+off64_t tracecmd_compress_lseek(struct tracecmd_compression *handle, off64_t offset, int whence);
+int tracecmd_compress_proto_get_name(struct tracecmd_compression *compress,
+ const char **name, const char **version);
+bool tracecmd_compress_is_supported(const char *name, const char *version);
+int tracecmd_compress_protos_get(char ***names, char ***versions);
+int tracecmd_compress_proto_register(const char *name, const char *version, int weight,
+ int (*compress)(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes),
+ int (*uncompress)(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes),
+ unsigned int (*comress_size)(unsigned int bytes),
+ bool (*is_supported)(const char *name, const char *version));
+int tracecmd_compress_copy_from(struct tracecmd_compression *handle, int fd, int chunk_size,
+ unsigned long long *read_size, unsigned long long *write_size);
+int tracecmd_uncompress_copy_to(struct tracecmd_compression *handle, int fd,
+ unsigned long long *read_size, unsigned long long *write_size);
+int tracecmd_uncompress_chunk(struct tracecmd_compression *handle,
+ struct tracecmd_compress_chunk *chunk, char *data);
+int tracecmd_load_chunks_info(struct tracecmd_compression *handle,
+ struct tracecmd_compress_chunk **chunks_info);
/* --- Plugin handling --- */
extern struct tep_plugin_option trace_ftrace_options[];
diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index b4f3d8c8..d4047429 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -36,6 +36,9 @@ struct data_file_write {
unsigned long long file_data_offset;
};
+void tracecmd_compress_init(void);
+void tracecmd_compress_free(void);
+
bool check_file_state(unsigned long file_version, int current_state, int new_state);
bool check_out_state(struct tracecmd_output *handle, int new_state);
diff --git a/lib/trace-cmd/trace-compress.c b/lib/trace-cmd/trace-compress.c
new file mode 100644
index 00000000..4a4341dd
--- /dev/null
+++ b/lib/trace-cmd/trace-compress.c
@@ -0,0 +1,977 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2021, VMware, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
+ *
+ */
+#include <stdlib.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include "trace-cmd-private.h"
+#include "trace-cmd-local.h"
+
+struct compress_proto {
+ struct compress_proto *next;
+ char *proto_name;
+ char *proto_version;
+ int weight;
+
+ int (*compress_block)(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes);
+ int (*uncompress_block)(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes);
+ unsigned int (*compress_size)(unsigned int bytes);
+ bool (*is_supported)(const char *name, const char *version);
+};
+
+static struct compress_proto *proto_list;
+
+struct tracecmd_compression {
+ int fd;
+ unsigned int capacity;
+ unsigned int capacity_read;
+ unsigned long pointer;
+ char *buffer;
+ struct compress_proto *proto;
+ struct tep_handle *tep;
+ struct tracecmd_msg_handle *msg_handle;
+};
+
+static int read_fd(int fd, char *dst, int len)
+{
+ size_t size = 0;
+ int r;
+
+ do {
+ r = read(fd, dst+size, len);
+ if (r > 0) {
+ size += r;
+ len -= r;
+ } else
+ break;
+ } while (r > 0);
+
+ if (len)
+ return -1;
+ return size;
+}
+
+static long long write_fd(int fd, const void *data, size_t size)
+{
+ long long tot = 0;
+ long long w;
+
+ do {
+ w = write(fd, data + tot, size - tot);
+ tot += w;
+
+ if (!w)
+ break;
+ if (w < 0)
+ return w;
+ } while (tot != size);
+
+ return tot;
+}
+
+static long long do_write(struct tracecmd_compression *handle,
+ const void *data, unsigned long long size)
+{
+ int ret;
+
+ if (handle->msg_handle) {
+ ret = tracecmd_msg_data_send(handle->msg_handle, data, size);
+ if (ret)
+ return -1;
+ return size;
+ }
+ return write_fd(handle->fd, data, size);
+}
+
+static inline int buffer_extend(struct tracecmd_compression *handle, unsigned int size)
+{
+ int extend;
+ char *buf;
+
+ if (size <= handle->capacity)
+ return 0;
+
+ extend = (size / BUFSIZ + 1) * BUFSIZ;
+ buf = realloc(handle->buffer, extend);
+ if (!buf)
+ return -1;
+ handle->buffer = buf;
+ handle->capacity = extend;
+
+ return 0;
+}
+
+/**
+ * tracecmd_compress_lseek - Move the read/write pointer into the compression buffer
+ * @handle: compression handle
+ * @offset: number of bytes to move the pointer, can be negative or positive
+ * @whence: the starting position of the pointer movement,
+ *
+ * Returns the new file pointer on success, or -1 in case of an error.
+ */
+off64_t tracecmd_compress_lseek(struct tracecmd_compression *handle, off64_t offset, int whence)
+{
+ unsigned long p;
+
+ if (!handle || !handle->buffer)
+ return (off64_t)-1;
+
+ switch (whence) {
+ case SEEK_CUR:
+ p = handle->pointer + offset;
+ break;
+ case SEEK_END:
+ p = handle->capacity + offset;
+ break;
+ case SEEK_SET:
+ p = offset;
+ break;
+ default:
+ return (off64_t)-1;
+ }
+
+ if (buffer_extend(handle, p))
+ return (off64_t)-1;
+
+ handle->pointer = p;
+
+ return p;
+}
+
+static int compress_read(struct tracecmd_compression *handle, char *dst, int len)
+{
+
+ if (handle->pointer > handle->capacity_read)
+ return -1;
+
+ if (handle->pointer + len > handle->capacity_read)
+ len = handle->capacity_read - handle->pointer;
+
+ memcpy(dst, handle->buffer + handle->pointer, len);
+
+ return len;
+}
+
+/**
+ * tracecmd_compress_pread - pread() on compression buffer
+ * @handle: compression handle
+ * @dst: return, store the read data
+ * @len: length of data to be read
+ * @offset: offset in the buffer of data to be read
+ *
+ * Read a @len of data from the compression buffer at given @offset,
+ * without updating the buffer pointer.
+ *
+ * On success returns the number of bytes read, or -1 on failure.
+ */
+int tracecmd_compress_pread(struct tracecmd_compression *handle, char *dst, int len, off_t offset)
+{
+ int ret;
+
+ if (!handle || !handle->buffer || offset > handle->capacity_read)
+ return -1;
+
+ ret = tracecmd_compress_lseek(handle, offset, SEEK_SET);
+ if (ret < 0)
+ return ret;
+ return compress_read(handle, dst, len);
+}
+
+/**
+ * tracecmd_compress_buffer_read - read() from compression buffer
+ * @handle: compression handle
+ * @dst: return, store the read data
+ * @len: length of data to be read
+ *
+ * Read a @len of data from the compression buffer
+ *
+ * On success returns the number of bytes read, or -1 on failure.
+ */
+int tracecmd_compress_buffer_read(struct tracecmd_compression *handle, char *dst, int len)
+{
+ int ret;
+
+ if (!handle || !handle->buffer)
+ return -1;
+
+ ret = compress_read(handle, dst, len);
+ if (ret > 0)
+ handle->pointer += ret;
+
+ return ret;
+}
+
+/**
+ * tracecmd_compress_reset - Reset the compression buffer
+ * @handle: compression handle
+ *
+ * Reset the compression buffer, any data currently in the buffer
+ * will be destroyed.
+ *
+ */
+void tracecmd_compress_reset(struct tracecmd_compression *handle)
+{
+ if (!handle)
+ return;
+
+ free(handle->buffer);
+ handle->buffer = NULL;
+ handle->pointer = 0;
+ handle->capacity_read = 0;
+ handle->capacity = 0;
+}
+
+/**
+ * tracecmd_uncompress_block - uncompress a memory block
+ * @handle: compression handle
+ *
+ * Read compressed memory block from the file and uncompress it into
+ * internal buffer. The tracecmd_compress_buffer_read() can be used
+ * to read the uncompressed data from the buffer.
+ *
+ * Returns 0 on success, or -1 in case of an error.
+ */
+int tracecmd_uncompress_block(struct tracecmd_compression *handle)
+{
+ unsigned int s_uncompressed;
+ unsigned int s_compressed;
+ char *bytes = NULL;
+ char buf[4];
+ int size;
+ int ret;
+
+ if (!handle || !handle->proto || !handle->proto->uncompress_block)
+ return -1;
+
+ tracecmd_compress_reset(handle);
+
+ if (read(handle->fd, buf, 4) != 4)
+ return -1;
+
+ s_compressed = tep_read_number(handle->tep, buf, 4);
+ if (read(handle->fd, buf, 4) != 4)
+ return -1;
+
+ s_uncompressed = tep_read_number(handle->tep, buf, 4);
+ size = s_uncompressed > s_compressed ? s_uncompressed : s_compressed;
+
+ handle->buffer = malloc(size);
+ if (!handle->buffer)
+ return -1;
+
+ bytes = malloc(s_compressed);
+ if (!bytes)
+ goto error;
+
+ if (read_fd(handle->fd, bytes, s_compressed) < 0)
+ goto error;
+
+ s_uncompressed = size;
+ ret = handle->proto->uncompress_block(bytes, s_compressed,
+ handle->buffer, &s_uncompressed);
+ if (ret)
+ goto error;
+
+ free(bytes);
+ handle->pointer = 0;
+ handle->capacity_read = s_uncompressed;
+ handle->capacity = size;
+ return 0;
+error:
+ tracecmd_compress_reset(handle);
+ free(bytes);
+ return -1;
+}
+
+/**
+ * tracecmd_compress_block - compress a memory block
+ * @handle: compression handle
+ *
+ * Compress the content of the internal memory buffer and write
+ * the compressed data in the file. The tracecmd_compress_buffer_write()
+ * can be used to write data into the internal memory buffer,
+ * before calling this API.
+ *
+ * Returns 0 on success, or -1 in case of an error.
+ */
+int tracecmd_compress_block(struct tracecmd_compression *handle)
+{
+ unsigned int size;
+ char *buf;
+ int endian4;
+ int ret;
+
+ if (!handle || !handle->proto ||
+ !handle->proto->compress_size || !handle->proto->compress_block)
+ return -1;
+
+ size = handle->proto->compress_size(handle->pointer);
+
+ buf = malloc(size);
+ if (!buf)
+ return -1;
+
+ ret = handle->proto->compress_block(handle->buffer, handle->pointer, buf, &size);
+ if (ret < 0)
+ goto out;
+
+ /* Write compressed data size */
+ endian4 = tep_read_number(handle->tep, &size, 4);
+ ret = do_write(handle, &endian4, 4);
+ if (ret != 4)
+ goto out;
+
+ /* Write uncompressed data size */
+ endian4 = tep_read_number(handle->tep, &handle->pointer, 4);
+ ret = do_write(handle, &endian4, 4);
+ if (ret != 4) {
+ ret = -1;
+ goto out;
+ }
+
+ /* Write compressed data */
+ ret = do_write(handle, buf, size);
+ if (ret != size) {
+ ret = -1;
+ goto out;
+ }
+
+ ret = 0;
+ tracecmd_compress_reset(handle);
+out:
+ free(buf);
+ return ret;
+}
+
+/**
+ * tracecmd_compress_buffer_write - write() to compression buffer
+ * @handle: compression handle
+ * @data: data to be written
+ * @size: size of @data
+ *
+ * Write @data of @size in the compression buffer
+ *
+ * Returns 0 on success, or -1 on failure.
+ */
+int tracecmd_compress_buffer_write(struct tracecmd_compression *handle,
+ const void *data, unsigned long long size)
+{
+ if (!handle)
+ return -1;
+
+ if (buffer_extend(handle, handle->pointer + size))
+ return -1;
+
+ memcpy(&handle->buffer[handle->pointer], data, size);
+ handle->pointer += size;
+ if (handle->capacity_read < handle->pointer)
+ handle->capacity_read = handle->pointer;
+
+ return 0;
+}
+
+/**
+ * tracecmd_compress_init - initialize the library with available compression algorithms
+ */
+void tracecmd_compress_init(void)
+{
+ struct timeval time;
+
+ gettimeofday(&time, NULL);
+ srand((time.tv_sec * 1000) + (time.tv_usec / 1000));
+}
+
+static struct compress_proto *compress_proto_select(void)
+{
+ struct compress_proto *proto = proto_list;
+ struct compress_proto *selected = NULL;
+
+ while (proto) {
+ if (!selected || selected->weight > proto->weight)
+ selected = proto;
+ proto = proto->next;
+ }
+
+ return selected;
+}
+
+/**
+ * tracecmd_compress_alloc - Allocate a new compression context
+ * @name: name of the compression algorithm.
+ * If NULL - auto select the best available algorithm
+ * @version: version of the compression algorithm, can be NULL
+ * @fd: file descriptor for reading / writing data
+ * @tep: tep handle, used to encode the data
+ * @msg_handle: message handle, use it for reading / writing data instead of @fd
+ *
+ * Returns NULL on failure or pointer to allocated compression context.
+ * The returned context must be freed by tracecmd_compress_destroy()
+ */
+struct tracecmd_compression *tracecmd_compress_alloc(const char *name, const char *version,
+ int fd, struct tep_handle *tep,
+ struct tracecmd_msg_handle *msg_handle)
+{
+ struct tracecmd_compression *new;
+ struct compress_proto *proto;
+
+ if (name) {
+ proto = proto_list;
+ while (proto) {
+ if (proto->is_supported && proto->is_supported(name, version))
+ break;
+ proto = proto->next;
+ }
+ } else {
+ proto = compress_proto_select();
+ }
+ if (!proto)
+ return NULL;
+
+ new = calloc(1, sizeof(*new));
+ if (!new)
+ return NULL;
+
+ new->fd = fd;
+ new->tep = tep;
+ new->msg_handle = msg_handle;
+ new->proto = proto;
+ return new;
+}
+
+/**
+ * tracecmd_compress_destroy - Free a compression context
+ * @handle: handle to the compression context that will be freed
+ */
+void tracecmd_compress_destroy(struct tracecmd_compression *handle)
+{
+ tracecmd_compress_reset(handle);
+ free(handle);
+}
+
+/**
+ * tracecmd_compress_is_supported - check if compression algorithm is supported
+ * @name: name of the compression algorithm.
+ * @version: version of the compression algorithm.
+ *
+ * Checks if compression algorithm with given name and version is supported.
+ * Returns true if the algorithm is supported or false if it is not.
+ */
+bool tracecmd_compress_is_supported(const char *name, const char *version)
+{
+ struct compress_proto *proto = proto_list;
+
+ if (!name)
+ return NULL;
+
+ while (proto) {
+ if (proto->is_supported && proto->is_supported(name, version))
+ return true;
+ proto = proto->next;
+ }
+ return false;
+}
+
+/**
+ * tracecmd_compress_proto_get_name - get name and version of compression algorithm
+ * @compress: compression handle.
+ * @name: return, name of the compression algorithm.
+ * @version: return, version of the compression algorithm.
+ *
+ * Returns 0 on success, or -1 in case of an error. If 0 is returned, the name
+ * and version of the algorithm are stored in @name and @version. The returned
+ * strings must *not* be freed.
+ */
+int tracecmd_compress_proto_get_name(struct tracecmd_compression *compress,
+ const char **name, const char **version)
+{
+ if (!compress || !compress->proto)
+ return -1;
+
+ if (name)
+ *name = compress->proto->proto_name;
+ if (version)
+ *version = compress->proto->proto_version;
+
+ return 0;
+}
+
+/**
+ * tracecmd_compress_proto_register - register a new compression algorithm
+ * @name: name of the compression algorithm.
+ * @version: version of the compression algorithm.
+ * @weight: weight of the compression algorithm, lower is better.
+ * @compress: compression hook, called to compress a memory block.
+ * @uncompress: uncompression hook, called to uncompress a memory block.
+ * @compress_size: hook, called to get the required minimum size of the buffer
+ * for compression given number of bytes.
+ * @is_supported: check hook, called to check if compression with given name and
+ * version is supported by this plugin.
+ *
+ * Returns 0 on success, or -1 in case of an error. If algorithm with given name
+ * and version is already registered, -1 is returned.
+ */
+int tracecmd_compress_proto_register(const char *name, const char *version, int weight,
+ int (*compress)(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes),
+ int (*uncompress)(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes),
+ unsigned int (*compress_size)(unsigned int bytes),
+ bool (*is_supported)(const char *name, const char *version))
+{
+ struct compress_proto *new;
+
+ if (!name || !compress || !uncompress)
+ return -1;
+
+ if (tracecmd_compress_is_supported(name, version))
+ return -1;
+
+ new = calloc(1, sizeof(*new));
+ if (!new)
+ return -1;
+
+ new->proto_name = strdup(name);
+ if (!new->proto_name)
+ goto error;
+
+ new->proto_version = strdup(version);
+ if (!new->proto_version)
+ goto error;
+
+ new->compress_block = compress;
+ new->uncompress_block = uncompress;
+ new->compress_size = compress_size;
+ new->is_supported = is_supported;
+ new->weight = weight;
+ new->next = proto_list;
+ proto_list = new;
+ return 0;
+
+error:
+ free(new->proto_name);
+ free(new->proto_version);
+ free(new);
+ return -1;
+}
+
+/**
+ * tracecmd_compress_free - free the library resources, related to available compression algorithms
+ *
+ */
+void tracecmd_compress_free(void)
+{
+ struct compress_proto *proto = proto_list;
+ struct compress_proto *del;
+
+ while (proto) {
+ del = proto;
+ proto = proto->next;
+ free(del->proto_name);
+ free(del->proto_version);
+ free(del);
+ }
+ proto_list = NULL;
+}
+
+/**
+ * tracecmd_compress_protos_get - get a list of all supported compression algorithms and versions
+ * @names: return, array with names of all supported compression algorithms
+ * @versions: return, array with versions of all supported compression algorithms
+ *
+ * On success, the size of @names and @versions arrays is returned.
+ * Those arrays are allocated by the API and must be freed with free() by the
+ * caller. Both arrays are with same size, each name from @names corresponds to
+ * a version from @versions. The last element in both arrays is a NULL pointer.
+ * On error -1 is returned and @names and @versions arrays are not allocated.
+ */
+int tracecmd_compress_protos_get(char ***names, char ***versions)
+{
+ struct compress_proto *proto = proto_list;
+ char **n = NULL;
+ char **v = NULL;
+ int c, i;
+
+ for (c = 0; proto; proto = proto->next)
+ c++;
+
+ if (c < 1)
+ return c;
+
+ n = calloc(c + 1, sizeof(char *));
+ if (!n)
+ goto error;
+ v = calloc(c + 1, sizeof(char *));
+ if (!v)
+ goto error;
+
+ proto = proto_list;
+ for (i = 0; i < c && proto; i++) {
+ n[i] = proto->proto_name;
+ v[i] = proto->proto_version;
+ proto = proto->next;
+ }
+
+ n[i] = NULL;
+ v[i] = NULL;
+ *names = n;
+ *versions = v;
+ return c;
+
+error:
+ free(n);
+ free(v);
+ return -1;
+}
+
+/**
+ * tracecmd_compress_copy_from - Copy and compress data from a file
+ * @handle: compression handle
+ * @fd: file descriptor to uncompressed data to copy from
+ * @chunk_size: size of one compression chunk
+ * @read_size: Pointer to max bytes to read from. The pointer is updated
+ * with the actual size of compressed data read. If 0 is passed,
+ * read until the EOF is reached.
+ * @write_size: return, size of the compressed data written into @handle
+ *
+ * This function reads uncompressed data from given @fd, compresses the data
+ * using the @handle compression context and writes the compressed data into the
+ * fd associated with the @handle. The data is compressed on chunks with given
+ * @chunk_size size. The compressed data is written in the format:
+ * - 4 bytes, chunks count
+ * - for each chunk:
+ * - 4 bytes, size of compressed data in this chunk
+ * - 4 bytes, uncompressed size of the data in this chunk
+ * - data, bytes of <size of compressed data in this chunk>
+ *
+ * On success 0 is returned, @read_size and @write_size are updated with the size of
+ * read and written data.
+ */
+int tracecmd_compress_copy_from(struct tracecmd_compression *handle, int fd, int chunk_size,
+ unsigned long long *read_size, unsigned long long *write_size)
+{
+ unsigned int rchunk = 0;
+ unsigned int chunks = 0;
+ unsigned int wsize = 0;
+ unsigned int rsize = 0;
+ unsigned int rmax = 0;
+ unsigned int csize;
+ unsigned int size;
+ unsigned int all;
+ unsigned int r;
+ off64_t offset;
+ char *buf_from;
+ char *buf_to;
+ int endian4;
+ int ret;
+
+ if (!handle || !handle->proto ||
+ !handle->proto->compress_block || !handle->proto->compress_size)
+ return 0;
+
+ if (read_size)
+ rmax = *read_size;
+ csize = handle->proto->compress_size(chunk_size);
+ buf_from = malloc(chunk_size);
+ if (!buf_from)
+ return -1;
+
+ buf_to = malloc(csize);
+ if (!buf_to)
+ return -1;
+
+ /* save the initial offset and write 0 as initial chunk count */
+ offset = lseek64(handle->fd, 0, SEEK_CUR);
+ write_fd(handle->fd, &chunks, 4);
+
+ do {
+ all = 0;
+ if (rmax > 0 && (rmax - rsize) < chunk_size)
+ rchunk = (rmax - rsize);
+ else
+ rchunk = chunk_size;
+
+ do {
+ r = read(fd, buf_from + all, rchunk - all);
+ all += r;
+
+ if (r <= 0)
+ break;
+ } while (all != rchunk);
+
+
+ if (r < 0 || (rmax > 0 && rsize >= rmax))
+ break;
+ rsize += all;
+ size = csize;
+ if (all > 0) {
+ ret = handle->proto->compress_block(buf_from, all, buf_to, &size);
+ if (ret < 0) {
+ if (errno == EINTR)
+ continue;
+ break;
+ }
+ /* Write compressed data size */
+ endian4 = tep_read_number(handle->tep, &size, 4);
+ ret = write_fd(handle->fd, &endian4, 4);
+ if (ret != 4)
+ break;
+
+ /* Write uncompressed data size */
+ endian4 = tep_read_number(handle->tep, &all, 4);
+ ret = write_fd(handle->fd, &endian4, 4);
+ if (ret != 4)
+ break;
+
+ /* Write the compressed data */
+ ret = write_fd(handle->fd, buf_to, size);
+ if (ret != size)
+ break;
+ /* data + compress header */
+ wsize += (size + 8);
+ chunks++;
+ }
+ } while (all > 0);
+
+ free(buf_from);
+ free(buf_to);
+
+ if (all)
+ return -1;
+
+ if (lseek64(handle->fd, offset, SEEK_SET) == (off_t)-1)
+ return -1;
+
+ endian4 = tep_read_number(handle->tep, &chunks, 4);
+ /* write chunks count*/
+ write_fd(handle->fd, &chunks, 4);
+ if (lseek64(handle->fd, 0, SEEK_END) == (off_t)-1)
+ return -1;
+
+ if (read_size)
+ *read_size = rsize;
+ if (write_size)
+ *write_size = wsize;
+ return 0;
+}
+
+/**
+ * tracecmd_load_chunks_info - Read compression chunks information from the file
+ * @handle: compression handle
+ * @chunks_info: return, array with compression chunks information
+ *
+ * This function reads information of all compression chunks in the current
+ * compression block from the file and fills that information in a newly
+ * allocated array @chunks_info which is returned.
+ *
+ * On success count of compression chunks is returned. Array of that count is
+ * allocated and returned in @chunks_info. Each entry describes one compression
+ * chunk. On error -1 is returned. In case of success, @chunks_info must be
+ * freed by free().
+ */
+int tracecmd_load_chunks_info(struct tracecmd_compression *handle,
+ struct tracecmd_compress_chunk **chunks_info)
+{
+ struct tracecmd_compress_chunk *chunks = NULL;
+ unsigned long long size = 0;
+ unsigned int count = 0;
+ off64_t offset;
+ int ret = -1;
+ char buf[4];
+ int i;
+
+ if (!handle)
+ return -1;
+
+ offset = lseek64(handle->fd, 0, SEEK_CUR);
+ if (offset == (off64_t)-1)
+ return -1;
+
+ if (read(handle->fd, buf, 4) != 4)
+ return -1;
+
+ count = tep_read_number(handle->tep, buf, 4);
+ if (!count) {
+ ret = 0;
+ goto out;
+ }
+
+ chunks = calloc(count, sizeof(struct tracecmd_compress_chunk));
+ if (!chunks)
+ goto out;
+
+ for (i = 0; i < count; i++) {
+ chunks[i].zoffset = lseek64(handle->fd, 0, SEEK_CUR);
+ if (chunks[i].zoffset == (off_t)-1)
+ goto out;
+ if (read(handle->fd, buf, 4) != 4)
+ goto out;
+ chunks[i].zsize = tep_read_number(handle->tep, buf, 4);
+ chunks[i].offset = size;
+ if (read(handle->fd, buf, 4) != 4)
+ goto out;
+ chunks[i].size = tep_read_number(handle->tep, buf, 4);
+ size += chunks[i].size;
+ if (lseek64(handle->fd, chunks[i].zsize, SEEK_CUR) == (off64_t)-1)
+ goto out;
+ }
+
+ ret = count;
+out:
+ if (lseek64(handle->fd, offset, SEEK_SET) == (off64_t)-1)
+ ret = -1;
+
+ if (ret > 0 && chunks_info)
+ *chunks_info = chunks;
+ else
+ free(chunks);
+
+ return ret;
+}
+
+/**
+ * tracecmd_uncompress_chunk - Uncompress given compression chunk.
+ * @handle: compression handle
+ * @chunk: chunk, that will be uncompressed in @data
+ * @data: Preallocated memory for uncompressed data. Must have enough space
+ * to hold the uncompressed data.
+ *
+ * This function uncompresses the chunk described by @chunk and stores
+ * the uncompressed data in the preallocated memory @data.
+ *
+ * On success 0 is returned and the uncompressed data is stored in @data.
+ * On error -1 is returned.
+ */
+int tracecmd_uncompress_chunk(struct tracecmd_compression *handle,
+ struct tracecmd_compress_chunk *chunk, char *data)
+{
+ char *bytes_in = NULL;
+ unsigned int size;
+ int ret = -1;
+
+ if (!handle || !handle->proto || !handle->proto->uncompress_block || !chunk || !data)
+ return -1;
+
+ if (lseek64(handle->fd, chunk->zoffset + 8, SEEK_SET) == (off_t)-1)
+ return -1;
+
+ bytes_in = malloc(chunk->zsize);
+ if (!bytes_in)
+ return -1;
+
+ if (read_fd(handle->fd, bytes_in, chunk->zsize) < 0)
+ goto out;
+
+ size = chunk->size;
+ if (handle->proto->uncompress_block(bytes_in, chunk->zsize, data, &size))
+ goto out;
+
+ ret = 0;
+out:
+ free(bytes_in);
+ return ret;
+}
+
+/**
+ * tracecmd_uncompress_copy_to - Uncompress data and copy to a file
+ * @handle: compression handle
+ * @fd: file descriptor to uncompressed data to copy into
+ * @read_size: return, size of the compressed data read from @handle
+ * @write_size: return, size of the uncompressed data written into @fd
+ *
+ * This function reads compressed data from the fd, associated with @handle,
+ * uncompresses it using the @handle compression context and writes
+ * the uncompressed data into the fd. The compressed data must be in the format:
+ * - 4 bytes, chunks count
+ * - for each chunk:
+ * - 4 bytes, size of compressed data in this chunk
+ * - 4 bytes, uncompressed size of the data in this chunk
+ * - data, bytes of <size of compressed data in this chunk>
+ *
+ * On success 0 is returned, @read_size and @write_size are updated with
+ * the size of read and written data.
+ */
+int tracecmd_uncompress_copy_to(struct tracecmd_compression *handle, int fd,
+ unsigned long long *read_size, unsigned long long *write_size)
+{
+ unsigned int s_uncompressed;
+ unsigned int s_compressed;
+ unsigned int rsize = 0;
+ unsigned int wsize = 0;
+ char *bytes_out = NULL;
+ char *bytes_in = NULL;
+ int size_out = 0;
+ int size_in = 0;
+ int chunks;
+ char buf[4];
+ char *tmp;
+ int ret;
+
+ if (!handle || !handle->proto || !handle->proto->uncompress_block)
+ return -1;
+
+ if (read(handle->fd, buf, 4) != 4)
+ return -1;
+
+ chunks = tep_read_number(handle->tep, buf, 4);
+ rsize += 4;
+
+ while (chunks) {
+ if (read(handle->fd, buf, 4) != 4)
+ break;
+
+ s_compressed = tep_read_number(handle->tep, buf, 4);
+ rsize += 4;
+ if (read(handle->fd, buf, 4) != 4)
+ break;
+
+ s_uncompressed = tep_read_number(handle->tep, buf, 4);
+ rsize += 4;
+ if (!bytes_in || size_in < s_compressed) {
+ tmp = realloc(bytes_in, s_compressed);
+ if (!tmp)
+ break;
+
+ bytes_in = tmp;
+ size_in = s_compressed;
+ }
+
+ if (!bytes_out || size_out < s_uncompressed) {
+ tmp = realloc(bytes_out, s_uncompressed);
+ if (!tmp)
+ break;
+ bytes_out = tmp;
+ size_out = s_uncompressed;
+ }
+
+ if (read_fd(handle->fd, bytes_in, s_compressed) < 0)
+ break;
+
+ rsize += s_compressed;
+ ret = handle->proto->uncompress_block(bytes_in, s_compressed,
+ bytes_out, &s_uncompressed);
+ if (ret)
+ break;
+
+ write_fd(fd, bytes_out, s_uncompressed);
+ wsize += s_uncompressed;
+ chunks--;
+ }
+ free(bytes_in);
+ free(bytes_out);
+ if (chunks)
+ return -1;
+
+ if (read_size)
+ *read_size = rsize;
+ if (write_size)
+ *write_size = wsize;
+
+ return 0;
+}
diff --git a/lib/trace-cmd/trace-util.c b/lib/trace-cmd/trace-util.c
index 21f1b065..06071f6c 100644
--- a/lib/trace-cmd/trace-util.c
+++ b/lib/trace-cmd/trace-util.c
@@ -635,6 +635,16 @@ bool tracecmd_is_version_supported(unsigned int version)
return false;
}
+static void __attribute__ ((constructor)) tracecmd_lib_init(void)
+{
+ tracecmd_compress_init();
+}
+
+static void __attribute__((destructor)) tracecmd_lib_free(void)
+{
+ tracecmd_compress_free();
+}
+
__hidden bool check_file_state(unsigned long file_version, int current_state, int new_state)
{
switch (new_state) {
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 02/20] trace-cmd library: Internal helpers for compressing data
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 01/20] trace-cmd library: Add support for compression algorithms Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 03/20] trace-cmd library: Internal helpers for uncompressing data Tzvetomir Stoyanov (VMware)
` (17 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
New library internal helper functions are introduced, to add compression
functionality to the output trace handler.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/include/trace-cmd-local.h | 7 +++
lib/trace-cmd/trace-output.c | 65 +++++++++++++++++++++++--
2 files changed, 69 insertions(+), 3 deletions(-)
diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index d4047429..b848514e 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -42,6 +42,13 @@ void tracecmd_compress_free(void);
bool check_file_state(unsigned long file_version, int current_state, int new_state);
bool check_out_state(struct tracecmd_output *handle, int new_state);
+int out_uncompress_block(struct tracecmd_output *handle);
+int out_compression_start(struct tracecmd_output *handle, bool compress);
+int out_compression_end(struct tracecmd_output *handle, bool compress);
+void out_compression_reset(struct tracecmd_output *handle, bool compress);
+unsigned long long out_copy_fd_compress(struct tracecmd_output *handle,
+ int fd, unsigned long long max,
+ unsigned long long *write_size);
unsigned long long
out_write_section_header(struct tracecmd_output *handle, unsigned short header_id,
char *description, int flags, bool option);
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 15baef8f..87481e33 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -70,6 +70,8 @@ struct tracecmd_output {
unsigned long long options_start;
bool big_endian;
+ bool do_compress;
+ struct tracecmd_compression *compress;
struct list_head options;
struct list_head buffers;
@@ -100,6 +102,9 @@ static int save_string_section(struct tracecmd_output *handle);
static stsize_t
do_write_check(struct tracecmd_output *handle, const void *data, tsize_t size)
{
+ if (handle->do_compress)
+ return tracecmd_compress_buffer_write(handle->compress, data, size);
+
if (handle->msg_handle)
return tracecmd_msg_data_send(handle->msg_handle, data, size);
@@ -108,10 +113,21 @@ do_write_check(struct tracecmd_output *handle, const void *data, tsize_t size)
static inline off64_t do_lseek(struct tracecmd_output *handle, off_t offset, int whence)
{
+ if (handle->do_compress)
+ return tracecmd_compress_lseek(handle->compress, offset, whence);
+
if (handle->msg_handle)
return msg_lseek(handle->msg_handle, offset, whence);
- else
- return lseek64(handle->fd, offset, whence);
+
+ return lseek64(handle->fd, offset, whence);
+}
+
+static inline int do_preed(struct tracecmd_output *handle, void *dst, int len, off_t offset)
+{
+ if (handle->do_compress)
+ return tracecmd_compress_pread(handle->compress, dst, len, offset);
+
+ return pread(handle->fd, dst, len, offset);
}
static short convert_endian_2(struct tracecmd_output *handle, short val)
@@ -139,6 +155,49 @@ static unsigned long long convert_endian_8(struct tracecmd_output *handle,
return tep_read_number(handle->pevent, &val, 8);
}
+__hidden void out_compression_reset(struct tracecmd_output *handle, bool compress)
+{
+ if (!compress || !handle->compress)
+ return;
+
+ tracecmd_compress_reset(handle->compress);
+ handle->do_compress = false;
+}
+
+__hidden int out_uncompress_block(struct tracecmd_output *handle)
+{
+ int ret = 0;
+
+ if (!handle->compress)
+ return 0;
+
+ ret = tracecmd_uncompress_block(handle->compress);
+ if (!ret)
+ handle->do_compress = true;
+
+ return ret;
+}
+
+__hidden int out_compression_start(struct tracecmd_output *handle, bool compress)
+{
+ if (!compress || !handle->compress)
+ return 0;
+
+ tracecmd_compress_reset(handle->compress);
+ handle->do_compress = true;
+
+ return 0;
+}
+
+__hidden int out_compression_end(struct tracecmd_output *handle, bool compress)
+{
+ if (!compress || !handle->compress)
+ return 0;
+
+ handle->do_compress = false;
+ return tracecmd_compress_block(handle->compress);
+}
+
static long add_string(struct tracecmd_output *handle, const char *string)
{
int size = strlen(string) + 1;
@@ -1645,7 +1704,7 @@ static int append_options_v6(struct tracecmd_output *handle)
if (offset == (off_t)-1)
return -1;
- r = pread(handle->fd, &option, 2, offset);
+ r = do_preed(handle, &option, 2, offset);
if (r != 2 || option != TRACECMD_OPTION_DONE)
return -1;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 03/20] trace-cmd library: Internal helpers for uncompressing data
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 01/20] trace-cmd library: Add support for compression algorithms Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 02/20] trace-cmd library: Internal helpers for compressing data Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 04/20] trace-cmd library: Inherit compression algorithm from input file Tzvetomir Stoyanov (VMware)
` (16 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
New library internal helper functions are introduced, to add compression
functionality to the input trace handler.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/include/trace-cmd-local.h | 3 ++
lib/trace-cmd/trace-input.c | 49 ++++++++++++++++++++++---
2 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index b848514e..ca682ed9 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -49,6 +49,9 @@ void out_compression_reset(struct tracecmd_output *handle, bool compress);
unsigned long long out_copy_fd_compress(struct tracecmd_output *handle,
int fd, unsigned long long max,
unsigned long long *write_size);
+void in_uncompress_reset(struct tracecmd_input *handle);
+int in_uncompress_block(struct tracecmd_input *handle);
+
unsigned long long
out_write_section_header(struct tracecmd_output *handle, unsigned short header_id,
char *description, int flags, bool option);
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index ecb56826..4b911818 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -160,6 +160,9 @@ struct tracecmd_input {
unsigned int strings_size; /* size of the metadata strings */
char *strings; /* metadata strings */
+ bool read_compress;
+ struct tracecmd_compression *compress;
+
struct host_trace_info host;
double ts2secs;
char * cpustats;
@@ -266,13 +269,13 @@ static const char *show_records(struct page **pages, int nr_pages)
static int init_cpu(struct tracecmd_input *handle, int cpu);
-static ssize_t do_read(struct tracecmd_input *handle, void *data, size_t size)
+static ssize_t do_read_fd(int fd, void *data, size_t size)
{
ssize_t tot = 0;
ssize_t r;
do {
- r = read(handle->fd, data + tot, size - tot);
+ r = read(fd, data + tot, size - tot);
tot += r;
if (!r)
@@ -284,6 +287,22 @@ static ssize_t do_read(struct tracecmd_input *handle, void *data, size_t size)
return tot;
}
+static inline int do_lseek(struct tracecmd_input *handle, int offset, int whence)
+{
+ if (handle->read_compress)
+ return tracecmd_compress_lseek(handle->compress, offset, whence);
+ else
+ return lseek(handle->fd, offset, whence);
+}
+
+static inline ssize_t do_read(struct tracecmd_input *handle, void *data, size_t size)
+{
+ if (handle->read_compress)
+ return tracecmd_compress_buffer_read(handle->compress, data, size);
+ else
+ return do_read_fd(handle->fd, data, size);
+}
+
static ssize_t
do_read_check(struct tracecmd_input *handle, void *data, size_t size)
{
@@ -308,9 +327,7 @@ static char *read_string(struct tracecmd_input *handle)
for (;;) {
r = do_read(handle, buf, BUFSIZ);
- if (r < 0)
- goto fail;
- if (!r)
+ if (r <= 0)
goto fail;
for (i = 0; i < r; i++) {
@@ -336,7 +353,7 @@ static char *read_string(struct tracecmd_input *handle)
}
/* move the file descriptor to the end of the string */
- r = lseek(handle->fd, -(r - (i+1)), SEEK_CUR);
+ r = do_lseek(handle, -(r - (i+1)), SEEK_CUR);
if (r < 0)
goto fail;
@@ -400,6 +417,26 @@ static int read8(struct tracecmd_input *handle, unsigned long long *size)
return 0;
}
+__hidden void in_uncompress_reset(struct tracecmd_input *handle)
+{
+ if (handle->compress) {
+ handle->read_compress = false;
+ tracecmd_compress_reset(handle->compress);
+ }
+}
+
+__hidden int in_uncompress_block(struct tracecmd_input *handle)
+{
+ int ret = 0;
+
+ if (handle->compress) {
+ ret = tracecmd_uncompress_block(handle->compress);
+ if (!ret)
+ handle->read_compress = true;
+ }
+ return ret;
+}
+
static struct file_section *section_get(struct tracecmd_input *handle, int id)
{
struct file_section *sec;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 04/20] trace-cmd library: Inherit compression algorithm from input file
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (2 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 03/20] trace-cmd library: Internal helpers for uncompressing data Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 05/20] trace-cmd library: New API to configure compression on an output handler Tzvetomir Stoyanov (VMware)
` (15 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
When a new trace file output handler is allocated, based on given trace
file input handler - use the same compression algorithm.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
.../include/private/trace-cmd-private.h | 2 ++
lib/trace-cmd/trace-input.c | 18 ++++++++++++++++
lib/trace-cmd/trace-output.c | 21 +++++++++++++++++++
3 files changed, 41 insertions(+)
diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 0bdb66e5..01c4c7a7 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -261,6 +261,8 @@ tracecmd_get_cursor(struct tracecmd_input *handle, int cpu);
unsigned long tracecmd_get_in_file_version(struct tracecmd_input *handle);
size_t tracecmd_get_options_offset(struct tracecmd_input *handle);
+int tracecmd_get_file_compress_proto(struct tracecmd_input *handle,
+ const char **name, const char **version);
int tracecmd_ftrace_overrides(struct tracecmd_input *handle, struct tracecmd_ftrace *finfo);
bool tracecmd_get_use_trace_clock(struct tracecmd_input *handle);
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index 4b911818..2e75e9ac 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -4661,6 +4661,24 @@ unsigned long tracecmd_get_in_file_version(struct tracecmd_input *handle)
return handle->file_version;
}
+/**
+ * tracecmd_get_file_compress_proto - get name and version of compression algorithm
+ * @handle: input handle for the trace.dat file
+ * @name: return, name of the compression algorithm.
+ * @version: return, version of the compression algorithm.
+ *
+ * Get the name and the version of the compression algorithm, used to
+ * compress the file associated with @handle.
+ * Returns 0 on success, or -1 in case of an error. If 0 is returned,
+ * the name and version of the algorithm are stored in @name and @version.
+ * The returned strings must *not* be freed.
+ */
+int tracecmd_get_file_compress_proto(struct tracecmd_input *handle,
+ const char **name, const char **version)
+{
+ return tracecmd_compress_proto_get_name(handle->compress, name, version);
+}
+
/**
* tracecmd_get_use_trace_clock - return use_trace_clock
* @handle: input handle for the trace.dat file
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 87481e33..2b511bfc 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -1310,6 +1310,9 @@ int tracecmd_output_set_kallsyms(struct tracecmd_output *handle, const char *kal
*/
int tracecmd_output_set_from_input(struct tracecmd_output *handle, struct tracecmd_input *ihandle)
{
+ const char *cname = NULL;
+ const char *cver = NULL;
+
if (!handle || !ihandle || handle->file_state != TRACECMD_FILE_ALLOCATED)
return -1;
@@ -1321,6 +1324,16 @@ int tracecmd_output_set_from_input(struct tracecmd_output *handle, struct tracec
handle->file_version = tracecmd_get_in_file_version(ihandle);
handle->big_endian = tep_is_file_bigendian(handle->pevent);
+ if (!tracecmd_get_file_compress_proto(ihandle, &cname, &cver)) {
+ handle->compress = tracecmd_compress_alloc(cname, cver, handle->fd,
+ handle->pevent, handle->msg_handle);
+ if (!handle->compress)
+ return -1;
+
+ if (handle->file_version < FILE_VERSION_COMPRESSION)
+ handle->file_version = FILE_VERSION_COMPRESSION;
+ }
+
return 0;
}
@@ -2281,6 +2294,8 @@ struct tracecmd_output *tracecmd_get_output_handle_fd(int fd)
{
struct tracecmd_output *handle = NULL;
struct tracecmd_input *ihandle;
+ const char *cname = NULL;
+ const char *cver = NULL;
int fd2;
/* Move the file descriptor to the beginning */
@@ -2321,6 +2336,12 @@ struct tracecmd_output *tracecmd_get_output_handle_fd(int fd)
list_head_init(&handle->options);
list_head_init(&handle->buffers);
+ if (!tracecmd_get_file_compress_proto(ihandle, &cname, &cver)) {
+ handle->compress = tracecmd_compress_alloc(cname, cver, handle->fd,
+ handle->pevent, handle->msg_handle);
+ if (!handle->compress)
+ goto out_free;
+ }
tracecmd_close(ihandle);
return handle;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 05/20] trace-cmd library: New API to configure compression on an output handler
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (3 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 04/20] trace-cmd library: Inherit compression algorithm from input file Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 06/20] trace-cmd library: Write compression header in the trace file Tzvetomir Stoyanov (VMware)
` (14 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
The new API can be used to configure compression algorithm on an output
handle to a trace file.
tracecmd_output_set_compression()
The API for creation of latency trace file is extended with compression
parameter.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
.../include/private/trace-cmd-private.h | 3 +-
lib/trace-cmd/trace-output.c | 60 ++++++++++++++++++-
tracecmd/trace-record.c | 2 +-
3 files changed, 62 insertions(+), 3 deletions(-)
diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 01c4c7a7..b5dbf56d 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -302,13 +302,14 @@ int tracecmd_output_set_trace_dir(struct tracecmd_output *handle, const char *tr
int tracecmd_output_set_kallsyms(struct tracecmd_output *handle, const char *kallsyms);
int tracecmd_output_set_from_input(struct tracecmd_output *handle, struct tracecmd_input *ihandle);
int tracecmd_output_set_version(struct tracecmd_output *handle, int file_version);
+int tracecmd_output_set_compression(struct tracecmd_output *handle, const char *compression);
int tracecmd_output_write_headers(struct tracecmd_output *handle,
struct tracecmd_event_list *list);
struct tracecmd_output *tracecmd_output_create(const char *output_file);
struct tracecmd_output *tracecmd_output_create_fd(int fd);
struct tracecmd_output *tracecmd_create_file_latency(const char *output_file, int cpus,
- int file_version);
+ int file_version, const char *compression);
struct tracecmd_option *tracecmd_add_option(struct tracecmd_output *handle,
unsigned short id, int size,
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 2b511bfc..caa8cbc8 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -274,6 +274,7 @@ void tracecmd_output_free(struct tracecmd_output *handle)
free(handle->strings);
free(handle->trace_clock);
+ tracecmd_compress_destroy(handle->compress);
free(handle);
}
@@ -1353,6 +1354,55 @@ int tracecmd_output_set_version(struct tracecmd_output *handle, int file_version
if (file_version < FILE_VERSION_MIN || file_version > FILE_VERSION_MAX)
return -1;
handle->file_version = file_version;
+ if (handle->file_version < FILE_VERSION_COMPRESSION)
+ handle->compress = NULL;
+ return 0;
+}
+
+/**
+ * tracecmd_output_set_compression - Set file compression algorithm of the output handle
+ * @handle: output handle to a trace file.
+ * @compression: name of the desired compression algorithm. Can be one of:
+ * - "none" - do not use compression
+ * - "all" - use the best available compression algorithm
+ * - or specific name of the desired compression algorithm
+ *
+ * This API must be called before tracecmd_output_write_headers().
+ *
+ * Returns 0 on success, or -1 in case of an error:
+ * - the output file handle is not allocated or not in expected state.
+ * - the specified compression algorithm is not available
+ */
+int tracecmd_output_set_compression(struct tracecmd_output *handle, const char *compression)
+{
+ if (!handle || handle->file_state != TRACECMD_FILE_ALLOCATED)
+ return -1;
+
+ handle->compress = NULL;
+ if (compression && strcmp(compression, "none")) {
+ if (!strcmp(compression, "any")) {
+ handle->compress = tracecmd_compress_alloc(NULL, NULL, handle->fd,
+ handle->pevent,
+ handle->msg_handle);
+ if (!handle->compress)
+ tracecmd_warning("No compression algorithms are supported");
+ } else {
+ handle->compress = tracecmd_compress_alloc(compression, NULL, handle->fd,
+ handle->pevent,
+ handle->msg_handle);
+ if (!handle->compress) {
+ tracecmd_warning("Compression algorithm %s is not supported",
+ compression);
+ return -1;
+ }
+ }
+ }
+ if (handle->compress && handle->file_version < FILE_VERSION_COMPRESSION) {
+ handle->file_version = FILE_VERSION_COMPRESSION;
+ if (handle->msg_handle)
+ tracecmd_msg_handle_cache(handle->msg_handle);
+ }
+
return 0;
}
@@ -1963,7 +2013,7 @@ out_add_buffer_option(struct tracecmd_output *handle, const char *name,
}
struct tracecmd_output *tracecmd_create_file_latency(const char *output_file, int cpus,
- int file_version)
+ int file_version, const char *compression)
{
enum tracecmd_section_flags flags = 0;
struct tracecmd_output *handle;
@@ -1976,6 +2026,14 @@ struct tracecmd_output *tracecmd_create_file_latency(const char *output_file, in
if (file_version && tracecmd_output_set_version(handle, file_version))
goto out_free;
+
+ if (compression) {
+ if (tracecmd_output_set_compression(handle, compression))
+ goto out_free;
+ } else if (file_version >= FILE_VERSION_COMPRESSION) {
+ tracecmd_output_set_compression(handle, "any");
+ }
+
if (tracecmd_output_write_headers(handle, NULL))
goto out_free;
/*
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 4cf48c86..ece5a0c2 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -4511,7 +4511,7 @@ static void record_data(struct common_record_context *ctx)
if (latency) {
handle = tracecmd_create_file_latency(ctx->output, local_cpu_count,
- ctx->file_version);
+ ctx->file_version, NULL);
tracecmd_set_quiet(handle, quiet);
} else {
if (!local_cpu_count)
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 06/20] trace-cmd library: Write compression header in the trace file
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (4 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 05/20] trace-cmd library: New API to configure compression on an output handler Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 07/20] trace-cmd library: Compress part of " Tzvetomir Stoyanov (VMware)
` (13 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
If there is a compression configured on the output file handler and if
the file version is at least 7, write compression header in the file.
The compression header is two null terminated strings - name and version
of the compression algorithm, used to compress some parts of the file.
The header is located after the page size in the file. The new header is
mandatory for trace files version 7. If no compression is used, the
string "none" is saved as name of the compression algorithm and empty
string as compression algorithm version.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-output.c | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index caa8cbc8..76ff4419 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -1167,6 +1167,27 @@ out_free:
return ret;
}
+static int write_compression_header(struct tracecmd_output *handle)
+{
+ const char *name = NULL;
+ const char *ver = NULL;
+ int ret;
+
+ ret = tracecmd_compress_proto_get_name(handle->compress, &name, &ver);
+ if (ret < 0 || !name || !ver) {
+ name = "none";
+ ver = "";
+ }
+
+ if (do_write_check(handle, name, strlen(name) + 1))
+ return -1;
+
+ if (do_write_check(handle, ver, strlen(ver) + 1))
+ return -1;
+
+ return 0;
+}
+
/**
* tracecmd_output_create_fd - allocate new output handle to a trace file
* @fd: File descriptor for the handle to write to.
@@ -1460,6 +1481,12 @@ static int output_write_init(struct tracecmd_output *handle)
endian4 = convert_endian_4(handle, handle->page_size);
if (do_write_check(handle, &endian4, 4))
return -1;
+
+ if (handle->file_version >= FILE_VERSION_COMPRESSION) {
+ if (write_compression_header(handle))
+ return -1;
+ }
+
if (HAS_SECTIONS(handle)) {
/* Write 0 as options offset and save its location */
offset = 0;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 07/20] trace-cmd library: Compress part of the trace file
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (5 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 06/20] trace-cmd library: Write compression header in the trace file Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 08/20] trace-cmd library: Add local helper function for data compression Tzvetomir Stoyanov (VMware)
` (12 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
Compress part of the trace.dat file metadata. If there is compression
support, compress these parts of the file:
- ftrace events format
- format of recorded events
- information of the mapping of function addresses to the function names
- trace_printk() format strings
- information of the mapping a PID to a process name
- strings
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-output.c | 99 ++++++++++++++++++++++++++++++------
1 file changed, 83 insertions(+), 16 deletions(-)
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 76ff4419..3ee8d8b4 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -97,7 +97,7 @@ struct list_event_system {
#define HAS_SECTIONS(H) ((H)->file_version >= FILE_VERSION_SECTIONS)
static int write_options(struct tracecmd_output *handle);
-static int save_string_section(struct tracecmd_output *handle);
+static int save_string_section(struct tracecmd_output *handle, bool compress);
static stsize_t
do_write_check(struct tracecmd_output *handle, const void *data, tsize_t size)
@@ -288,7 +288,7 @@ void tracecmd_output_close(struct tracecmd_output *handle)
write_options(handle);
/* write strings section */
- save_string_section(handle);
+ save_string_section(handle, true);
}
if (handle->fd >= 0) {
@@ -444,6 +444,8 @@ out_write_section_header(struct tracecmd_output *handle, unsigned short header_i
return -1;
if (!HAS_SECTIONS(handle))
return 0;
+ if (!handle->compress)
+ flags &= ~TRACECMD_SEC_FL_COMPRESS;
offset = do_lseek(handle, 0, SEEK_CUR);
if (option) {
endian8 = convert_endian_8(handle, offset);
@@ -505,7 +507,7 @@ __hidden int out_update_section_header(struct tracecmd_output *handle, tsize_t o
return 0;
}
-static int save_string_section(struct tracecmd_output *handle)
+static int save_string_section(struct tracecmd_output *handle, bool compress)
{
enum tracecmd_section_flags flags = 0;
tsize_t offset;
@@ -519,13 +521,20 @@ static int save_string_section(struct tracecmd_output *handle)
return -1;
}
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_STRINGS, "strings", flags, false);
if (offset == (off64_t)-1)
return -1;
+ out_compression_start(handle, compress);
+
if (do_write_check(handle, handle->strings, handle->strings_p))
goto error;
+ if (out_compression_end(handle, compress))
+ goto error;
+
if (out_update_section_header(handle, offset))
return -1;
@@ -537,10 +546,11 @@ static int save_string_section(struct tracecmd_output *handle)
return 0;
error:
+ out_compression_reset(handle, compress);
return -1;
}
-static int read_header_files(struct tracecmd_output *handle)
+static int read_header_files(struct tracecmd_output *handle, bool compress)
{
enum tracecmd_section_flags flags = 0;
tsize_t size, check_size, endian8;
@@ -560,11 +570,14 @@ static int read_header_files(struct tracecmd_output *handle)
if (!path)
return -1;
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_HEADER_INFO,
"headers", flags, true);
if (offset == (off64_t)-1)
return -1;
+ out_compression_start(handle, compress);
ret = stat(path, &st);
if (ret < 0) {
/* old style did not show this info, just add zero */
@@ -578,6 +591,8 @@ static int read_header_files(struct tracecmd_output *handle)
goto out_close;
if (do_write_check(handle, &size, 8))
goto out_close;
+ if (out_compression_end(handle, compress))
+ goto out_close;
if (out_update_section_header(handle, offset))
goto out_close;
return 0;
@@ -630,6 +645,9 @@ static int read_header_files(struct tracecmd_output *handle)
goto out_close;
}
put_tracing_file(path);
+ if (out_compression_end(handle, compress))
+ goto out_close;
+
if (out_update_section_header(handle, offset))
goto out_close;
handle->file_state = TRACECMD_FILE_HEADERS;
@@ -637,6 +655,7 @@ static int read_header_files(struct tracecmd_output *handle)
return 0;
out_close:
+ out_compression_reset(handle, compress);
if (fd >= 0)
close(fd);
return -1;
@@ -865,7 +884,7 @@ create_event_list_item(struct tracecmd_output *handle,
tracecmd_warning("Insufficient memory");
}
-static int read_ftrace_files(struct tracecmd_output *handle)
+static int read_ftrace_files(struct tracecmd_output *handle, bool compress)
{
enum tracecmd_section_flags flags = 0;
struct list_event_system *systems = NULL;
@@ -879,14 +898,21 @@ static int read_ftrace_files(struct tracecmd_output *handle)
return -1;
}
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_FTRACE_EVENTS,
"ftrace events", flags, true);
if (offset == (off64_t)-1)
return -1;
create_event_list_item(handle, &systems, &list);
+ out_compression_start(handle, compress);
ret = copy_event_system(handle, systems);
+ if (!ret)
+ ret = out_compression_end(handle, compress);
+ else
+ out_compression_reset(handle, compress);
free_list_events(systems);
if (ret)
@@ -913,7 +939,7 @@ create_event_list(struct tracecmd_output *handle,
}
static int read_event_files(struct tracecmd_output *handle,
- struct tracecmd_event_list *event_list)
+ struct tracecmd_event_list *event_list, bool compress)
{
enum tracecmd_section_flags flags = 0;
struct list_event_system *systems;
@@ -931,6 +957,8 @@ static int read_event_files(struct tracecmd_output *handle,
return -1;
}
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_EVENT_FORMATS,
"events format", flags, true);
if (offset == (off64_t)-1)
@@ -951,7 +979,7 @@ static int read_event_files(struct tracecmd_output *handle,
for (slist = systems; slist; slist = slist->next)
count++;
-
+ out_compression_start(handle, compress);
ret = -1;
endian4 = convert_endian_4(handle, count);
if (do_write_check(handle, &endian4, 4))
@@ -966,6 +994,10 @@ static int read_event_files(struct tracecmd_output *handle,
}
ret = copy_event_system(handle, slist);
}
+ if (ret)
+ goto out_free;
+
+ ret = out_compression_end(handle, compress);
if (ret)
goto out_free;
ret = out_update_section_header(handle, offset);
@@ -973,6 +1005,8 @@ static int read_event_files(struct tracecmd_output *handle,
out_free:
if (!ret)
handle->file_state = TRACECMD_FILE_ALL_EVENTS;
+ else
+ out_compression_reset(handle, compress);
free_list_events(systems);
@@ -1019,7 +1053,7 @@ err:
tracecmd_warning("can't set kptr_restrict");
}
-static int read_proc_kallsyms(struct tracecmd_output *handle)
+static int read_proc_kallsyms(struct tracecmd_output *handle, bool compress)
{
enum tracecmd_section_flags flags = 0;
unsigned int size, check_size, endian4;
@@ -1037,11 +1071,14 @@ static int read_proc_kallsyms(struct tracecmd_output *handle)
if (handle->kallsyms)
path = handle->kallsyms;
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_KALLSYMS,
"kallsyms", flags, true);
if (offset == (off64_t)-1)
return -1;
+ out_compression_start(handle, compress);
ret = stat(path, &st);
if (ret < 0) {
/* not found */
@@ -1067,14 +1104,20 @@ static int read_proc_kallsyms(struct tracecmd_output *handle)
}
set_proc_kptr_restrict(1);
+ ret = out_compression_end(handle, compress);
+ if (ret)
+ goto out;
+
ret = out_update_section_header(handle, offset);
out:
if (!ret)
handle->file_state = TRACECMD_FILE_KALLSYMS;
+ else
+ out_compression_reset(handle, compress);
return ret;
}
-static int read_ftrace_printk(struct tracecmd_output *handle)
+static int read_ftrace_printk(struct tracecmd_output *handle, bool compress)
{
enum tracecmd_section_flags flags = 0;
unsigned int size, check_size, endian4;
@@ -1093,10 +1136,13 @@ static int read_ftrace_printk(struct tracecmd_output *handle)
if (!path)
return -1;
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_PRINTK, "printk", flags, true);
if (offset == (off64_t)-1)
return -1;
+ out_compression_start(handle, compress);
ret = stat(path, &st);
if (ret < 0) {
/* not found */
@@ -1119,12 +1165,16 @@ static int read_ftrace_printk(struct tracecmd_output *handle)
out:
put_tracing_file(path);
+ if (out_compression_end(handle, compress))
+ return -1;
+
if (out_update_section_header(handle, offset))
return -1;
handle->file_state = TRACECMD_FILE_PRINTK;
return 0;
fail:
put_tracing_file(path);
+ out_compression_reset(handle, compress);
return -1;
}
@@ -1517,21 +1567,25 @@ static int output_write_init(struct tracecmd_output *handle)
int tracecmd_output_write_headers(struct tracecmd_output *handle,
struct tracecmd_event_list *list)
{
+ bool compress = false;
+
if (!handle || handle->file_state < TRACECMD_FILE_ALLOCATED)
return -1;
/* Write init data, if not written yet */
if (handle->file_state < TRACECMD_FILE_INIT && output_write_init(handle))
return -1;
- if (read_header_files(handle))
+ if (handle->compress)
+ compress = true;
+ if (read_header_files(handle, compress))
return -1;
- if (read_ftrace_files(handle))
+ if (read_ftrace_files(handle, compress))
return -1;
- if (read_event_files(handle, list))
+ if (read_event_files(handle, list, compress))
return -1;
- if (read_proc_kallsyms(handle))
+ if (read_proc_kallsyms(handle, compress))
return -1;
- if (read_ftrace_printk(handle))
+ if (read_ftrace_printk(handle, compress))
return -1;
return 0;
}
@@ -1762,7 +1816,7 @@ int tracecmd_write_meta_strings(struct tracecmd_output *handle)
if (!HAS_SECTIONS(handle))
return 0;
- return save_string_section(handle);
+ return save_string_section(handle, true);
}
int tracecmd_write_options(struct tracecmd_output *handle)
@@ -1911,6 +1965,7 @@ static tsize_t get_buffer_file_offset(struct tracecmd_output *handle, const char
int tracecmd_write_cmdlines(struct tracecmd_output *handle)
{
enum tracecmd_section_flags flags = 0;
+ bool compress = false;
tsize_t offset;
int ret;
@@ -1920,14 +1975,26 @@ int tracecmd_write_cmdlines(struct tracecmd_output *handle)
return -1;
}
+ if (handle->compress)
+ compress = true;
+
+ if (compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_CMDLINES,
"command lines", flags, true);
if (offset == (off64_t)-1)
return -1;
+ out_compression_start(handle, compress);
+
ret = save_tracing_file_data(handle, "saved_cmdlines");
- if (ret < 0)
+ if (ret < 0) {
+ out_compression_reset(handle, compress);
return ret;
+ }
+
+ if (out_compression_end(handle, compress))
+ return -1;
if (out_update_section_header(handle, offset))
return -1;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 08/20] trace-cmd library: Add local helper function for data compression
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (6 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 07/20] trace-cmd library: Compress part of " Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 09/20] trace-cmd library: Compress the trace data Tzvetomir Stoyanov (VMware)
` (11 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
The newly added helper functions read data from a file and compress it,
before writing into the trace file. The trace data is compressed in
chunks, which are page aligned. A new local define is introduced:
PAGES_IN_CHUNK
which can be used to tune how big a compression chunk is.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-output.c | 71 +++++++++++++++++++++++++++++++++---
1 file changed, 65 insertions(+), 6 deletions(-)
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 3ee8d8b4..82f3b816 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -331,18 +331,27 @@ static unsigned long get_size(const char *file)
return size;
}
-static tsize_t copy_file_fd(struct tracecmd_output *handle, int fd)
+static tsize_t copy_file_fd(struct tracecmd_output *handle, int fd, unsigned long long max)
{
+ tsize_t rsize = BUFSIZ;
tsize_t size = 0;
char buf[BUFSIZ];
stsize_t r;
do {
- r = read(fd, buf, BUFSIZ);
+ if (max && rsize > max)
+ rsize = max;
+
+ r = read(fd, buf, rsize);
if (r > 0) {
size += r;
if (do_write_check(handle, buf, r))
return 0;
+ if (max) {
+ max -= r;
+ if (!max)
+ break;
+ }
}
} while (r > 0);
@@ -360,12 +369,62 @@ static tsize_t copy_file(struct tracecmd_output *handle,
tracecmd_warning("Can't read '%s'", file);
return 0;
}
- size = copy_file_fd(handle, fd);
+ size = copy_file_fd(handle, fd, 0);
close(fd);
return size;
}
+#define PAGES_IN_CHUNK 10
+__hidden unsigned long long out_copy_fd_compress(struct tracecmd_output *handle,
+ int fd, unsigned long long max,
+ unsigned long long *write_size)
+{
+ unsigned long long rsize = 0;
+ unsigned long long wsize = 0;
+ unsigned long long size;
+ int ret;
+
+ if (handle->compress) {
+ rsize = max;
+ ret = tracecmd_compress_copy_from(handle->compress, fd,
+ PAGES_IN_CHUNK * handle->page_size,
+ &rsize, &wsize);
+ if (ret < 0)
+ return 0;
+
+ size = rsize;
+ if (write_size)
+ *write_size = wsize;
+ } else {
+ size = copy_file_fd(handle, fd, max);
+ if (write_size)
+ *write_size = size;
+ }
+
+ return size;
+}
+
+static tsize_t copy_file_compress(struct tracecmd_output *handle,
+ const char *file, unsigned long long *write_size)
+{
+ int ret;
+ int fd;
+
+ fd = open(file, O_RDONLY);
+ if (fd < 0) {
+ tracecmd_warning("Can't read '%s'", file);
+ return 0;
+ }
+
+ ret = out_copy_fd_compress(handle, fd, 0, write_size);
+ if (!ret)
+ tracecmd_warning("Can't compress '%s'", file);
+
+ close(fd);
+ return ret;
+}
+
/*
* Finds the path to the debugfs/tracing
* Allocates the string and stores it.
@@ -612,7 +671,7 @@ static int read_header_files(struct tracecmd_output *handle, bool compress)
endian8 = convert_endian_8(handle, size);
if (do_write_check(handle, &endian8, 8))
goto out_close;
- check_size = copy_file_fd(handle, fd);
+ check_size = copy_file_fd(handle, fd, 0);
close(fd);
if (size != check_size) {
tracecmd_warning("wrong size for '%s' size=%lld read=%lld", path, size, check_size);
@@ -638,7 +697,7 @@ static int read_header_files(struct tracecmd_output *handle, bool compress)
endian8 = convert_endian_8(handle, size);
if (do_write_check(handle, &endian8, 8))
goto out_close;
- check_size = copy_file_fd(handle, fd);
+ check_size = copy_file_fd(handle, fd, 0);
close(fd);
if (size != check_size) {
tracecmd_warning("wrong size for '%s'", path);
@@ -2317,7 +2376,7 @@ __hidden int out_write_cpu_data(struct tracecmd_output *handle,
if (data[i].size) {
if (lseek64(data[i].fd, data[i].offset, SEEK_SET) == (off64_t)-1)
goto out_free;
- read_size = copy_file_fd(handle, data[i].fd);
+ read_size = copy_file_fd(handle, data[i].fd, data[i].size);
if (read_size != data_files[i].file_size) {
errno = EINVAL;
tracecmd_warning("did not match size of %lld to %lld",
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 09/20] trace-cmd library: Compress the trace data
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (7 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 08/20] trace-cmd library: Add local helper function for data compression Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 10/20] trace-cmd library: Decompress the options section, if it is compressed Tzvetomir Stoyanov (VMware)
` (10 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
If the output file handler supports compression, use it to compress the
flyrecord and latency trace data.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-output.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 82f3b816..780fb481 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -2219,11 +2219,13 @@ struct tracecmd_output *tracecmd_create_file_latency(const char *output_file, in
if (HAS_SECTIONS(handle) &&
!out_add_buffer_option(handle, "", TRACECMD_OPTION_BUFFER_TEXT, offset, 0, NULL))
goto out_free;
+ if (handle->compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
offset = out_write_section_header(handle, TRACECMD_OPTION_BUFFER_TEXT,
"buffer latency", flags, false);
- copy_file(handle, path);
+ copy_file_compress(handle, path, NULL);
if (out_update_section_header(handle, offset))
goto out_free;
@@ -2324,6 +2326,8 @@ __hidden int out_write_cpu_data(struct tracecmd_output *handle,
if (!HAS_SECTIONS(handle) && do_write_check(handle, "flyrecord", 10))
goto out_free;
+ if (handle->compress)
+ flags |= TRACECMD_SEC_FL_COMPRESS;
if (asprintf(&str, "buffer flyrecord %s", buff_name) < 1)
goto out_free;
offset = out_write_section_header(handle, TRACECMD_OPTION_BUFFER, str, flags, false);
@@ -2376,14 +2380,15 @@ __hidden int out_write_cpu_data(struct tracecmd_output *handle,
if (data[i].size) {
if (lseek64(data[i].fd, data[i].offset, SEEK_SET) == (off64_t)-1)
goto out_free;
- read_size = copy_file_fd(handle, data[i].fd, data[i].size);
+ read_size = out_copy_fd_compress(handle, data[i].fd,
+ data[i].size, &data_files[i].write_size);
+
if (read_size != data_files[i].file_size) {
errno = EINVAL;
tracecmd_warning("did not match size of %lld to %lld",
read_size, data_files[i].file_size);
goto out_free;
}
- data_files[i].write_size = read_size;
} else {
data_files[i].write_size = 0;
}
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 10/20] trace-cmd library: Decompress the options section, if it is compressed
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (8 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 09/20] trace-cmd library: Compress the trace data Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 11/20] trace-cmd library: Read compression header Tzvetomir Stoyanov (VMware)
` (9 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
In trace file version 7, options section can be compressed. Extended
the options handling decompression if needed .
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 46 ++++++++++++++++++++++++++-----------
1 file changed, 33 insertions(+), 13 deletions(-)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index 2e75e9ac..2da24a7c 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -3045,6 +3045,7 @@ static int handle_options(struct tracecmd_input *handle)
unsigned short id, flags;
char *cpustats = NULL;
struct hook_list *hook;
+ bool compress = false;
char *buf;
int cpus;
int ret;
@@ -3056,23 +3057,33 @@ static int handle_options(struct tracecmd_input *handle)
return -1;
if (id != TRACECMD_OPTION_DONE)
return -1;
+ if (flags & TRACECMD_SEC_FL_COMPRESS)
+ compress = true;
}
+ if (compress && in_uncompress_block(handle))
+ return -1;
+
for (;;) {
- if (read2(handle, &option))
- return -1;
+ ret = read2(handle, &option);
+ if (ret)
+ goto out;
if (!HAS_SECTIONS(handle) && option == TRACECMD_OPTION_DONE)
break;
/* next 4 bytes is the size of the option */
- if (read4(handle, &size))
- return -1;
+ ret = read4(handle, &size);
+ if (ret)
+ goto out;
buf = malloc(size);
- if (!buf)
- return -ENOMEM;
- if (do_read_check(handle, buf, size))
- return -1;
+ if (!buf) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ ret = do_read_check(handle, buf, size);
+ if (ret)
+ goto out;
switch (option) {
case TRACECMD_OPTION_DATE:
@@ -3126,15 +3137,17 @@ static int handle_options(struct tracecmd_input *handle)
buf + 8, 4);
ret = tsync_cpu_offsets_load(handle, buf + 12, size - 12);
if (ret < 0)
- return ret;
+ goto out;
tracecmd_enable_tsync(handle, true);
break;
case TRACECMD_OPTION_CPUSTAT:
buf[size-1] = '\n';
cpustats = realloc(handle->cpustats,
handle->cpustats_size + size + 1);
- if (!cpustats)
- return -ENOMEM;
+ if (!cpustats) {
+ ret = -ENOMEM;
+ goto out;
+ }
memcpy(cpustats + handle->cpustats_size, buf, size);
handle->cpustats_size += size;
cpustats[handle->cpustats_size] = 0;
@@ -3144,7 +3157,7 @@ static int handle_options(struct tracecmd_input *handle)
case TRACECMD_OPTION_BUFFER_TEXT:
ret = handle_buffer_option(handle, option, buf, size);
if (ret < 0)
- return ret;
+ goto out;
break;
case TRACECMD_OPTION_TRACECLOCK:
if (!handle->ts2secs)
@@ -3203,6 +3216,8 @@ static int handle_options(struct tracecmd_input *handle)
tep_read_number(handle->pevent, buf, 8), 0);
break;
case TRACECMD_OPTION_DONE:
+ if (compress)
+ in_uncompress_reset(handle);
ret = handle_option_done(handle, buf, size);
free(buf);
return ret;
@@ -3216,7 +3231,12 @@ static int handle_options(struct tracecmd_input *handle)
}
- return 0;
+ ret = 0;
+
+out:
+ if (compress)
+ in_uncompress_reset(handle);
+ return ret;
}
static int read_options_type(struct tracecmd_input *handle)
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 11/20] trace-cmd library: Read compression header
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (9 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 10/20] trace-cmd library: Decompress the options section, if it is compressed Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 12/20] trace-cmd library: Extend the input handle with trace data decompression context Tzvetomir Stoyanov (VMware)
` (8 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
Trace file version 7 introduced new mandatory compression header,
storing information about the compression algorithm used to compress the
trace file. Added code to read that header and to initialize compression
context according to it.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 37 +++++++++++++++++++++++++++++++++++--
1 file changed, 35 insertions(+), 2 deletions(-)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index 2da24a7c..20cd6c32 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -193,6 +193,7 @@ __thread struct tracecmd_input *tracecmd_curr_thread_handle;
#define CHECK_READ_STATE(H, S) ((H)->file_version < FILE_VERSION_SECTIONS && (H)->file_state >= (S))
#define HAS_SECTIONS(H) ((H)->flags & TRACECMD_FL_SECTIONED)
+#define HAS_COMPRESSION(H) ((H)->flags & TRACECMD_FL_COMPRESSION)
static int read_options_type(struct tracecmd_input *handle);
@@ -3850,7 +3851,9 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd, int flags)
char test[] = TRACECMD_MAGIC;
unsigned int page_size;
size_t offset;
- char *version;
+ char *version = NULL;
+ char *zver = NULL;
+ char *zname = NULL;
char buf[BUFSIZ];
unsigned long ver;
@@ -3888,9 +3891,12 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd, int flags)
}
handle->file_version = ver;
free(version);
+ version = NULL;
if (handle->file_version >= FILE_VERSION_SECTIONS)
handle->flags |= TRACECMD_FL_SECTIONED;
+ if (handle->file_version >= FILE_VERSION_COMPRESSION)
+ handle->flags |= TRACECMD_FL_COMPRESSION;
if (do_read_check(handle, buf, 1))
goto failed_read;
@@ -3920,6 +3926,29 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd, int flags)
handle->total_file_size = lseek64(handle->fd, 0, SEEK_END);
lseek64(handle->fd, offset, SEEK_SET);
+ if (HAS_COMPRESSION(handle)) {
+ zname = read_string(handle);
+ if (!zname)
+ goto failed_read;
+
+ zver = read_string(handle);
+ if (!zver)
+ goto failed_read;
+
+ if (strcmp(zname, "none")) {
+ handle->compress = tracecmd_compress_alloc(zname, zver,
+ handle->fd,
+ handle->pevent, NULL);
+ if (!handle->compress) {
+ tracecmd_warning("Unsupported file compression %s %s", zname, zver);
+ goto failed_read;
+ }
+ }
+
+ free(zname);
+ free(zver);
+ }
+
if (HAS_SECTIONS(handle)) {
if (read8(handle, &(handle->options_start))) {
tracecmd_warning("Filed to read the offset of the first option section");
@@ -3933,6 +3962,9 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd, int flags)
return handle;
failed_read:
+ free(version);
+ free(zname);
+ free(zver);
free(handle);
return NULL;
@@ -4132,7 +4164,8 @@ void tracecmd_close(struct tracecmd_input *handle)
if (handle->flags & TRACECMD_FL_BUFFER_INSTANCE)
tracecmd_close(handle->parent);
else {
- /* Only main handle frees plugins and pevent */
+ /* Only main handle frees plugins, pevent and compression context */
+ tracecmd_compress_destroy(handle->compress);
tep_unload_plugins(handle->plugin_list, handle->pevent);
tep_free(handle->pevent);
}
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 12/20] trace-cmd library: Extend the input handle with trace data decompression context
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (10 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 11/20] trace-cmd library: Read compression header Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:48 ` [PATCH v8 13/20] trace-cmd library: Initialize CPU data decompression logic Tzvetomir Stoyanov (VMware)
` (7 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
The CPU trace data is compressed in chunks, as chunk's size is multiple
trace pages. The input handle is extended with the necessary
structures, to control the data decompression. There are two approaches
for data decompression, both are supported and can be used in different
use cases:
- in-memory decompression, page by page.
- using a temporary file
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 59 ++++++++++++++++++++++++++++++-------
1 file changed, 48 insertions(+), 11 deletions(-)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index 20cd6c32..a6d90c01 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -54,6 +54,24 @@ struct page {
#endif
};
+struct zchunk_cache {
+ struct list_head list;
+ struct tracecmd_compress_chunk *chunk;
+ void *map;
+ int ref;
+};
+
+struct cpu_zdata {
+ /* uncompressed cpu data */
+ int fd;
+ char file[26]; /* strlen(COMPR_TEMP_FILE) */
+ unsigned int count;
+ unsigned int last_chunk;
+ struct list_head cache;
+ struct tracecmd_compress_chunk *chunks;
+};
+
+#define COMPR_TEMP_FILE "/tmp/trace_cpu_dataXXXXXX"
struct cpu_data {
/* the first two never change */
unsigned long long file_offset;
@@ -72,6 +90,7 @@ struct cpu_data {
int page_cnt;
int cpu;
int pipe_fd;
+ struct cpu_zdata compress;
};
struct cpu_file_data {
@@ -151,6 +170,8 @@ struct tracecmd_input {
bool use_trace_clock;
bool read_page;
bool use_pipe;
+ bool read_zpage; /* uncompress pages in memory, do not use tmp files */
+ bool cpu_compressed;
int file_version;
unsigned int cpustats_size;
struct cpu_data *cpu_data;
@@ -3316,6 +3337,7 @@ static int init_cpu_data(struct tracecmd_input *handle)
endian = KBUFFER_ENDIAN_LITTLE;
for (cpu = 0; cpu < handle->cpus; cpu++) {
+ handle->cpu_data[cpu].compress.fd = -1;
handle->cpu_data[cpu].kbuf = kbuffer_alloc(long_size, endian);
if (!handle->cpu_data[cpu].kbuf)
goto out_free;
@@ -4100,6 +4122,7 @@ static inline void free_buffer(struct input_buffer_instance *buf)
*/
void tracecmd_close(struct tracecmd_input *handle)
{
+ struct zchunk_cache *cache;
struct file_section *del_sec;
int cpu;
int i;
@@ -4119,17 +4142,31 @@ void tracecmd_close(struct tracecmd_input *handle)
/* The tracecmd_peek_data may have cached a record */
free_next(handle, cpu);
free_page(handle, cpu);
- if (handle->cpu_data && handle->cpu_data[cpu].kbuf) {
- kbuffer_free(handle->cpu_data[cpu].kbuf);
- if (handle->cpu_data[cpu].page_map)
- free_page_map(handle->cpu_data[cpu].page_map);
-
- if (handle->cpu_data[cpu].page_cnt)
- tracecmd_warning("%d pages still allocated on cpu %d%s",
- handle->cpu_data[cpu].page_cnt, cpu,
- show_records(handle->cpu_data[cpu].pages,
- handle->cpu_data[cpu].nr_pages));
- free(handle->cpu_data[cpu].pages);
+ if (handle->cpu_data) {
+ if (handle->cpu_data[cpu].kbuf) {
+ kbuffer_free(handle->cpu_data[cpu].kbuf);
+ if (handle->cpu_data[cpu].page_map)
+ free_page_map(handle->cpu_data[cpu].page_map);
+
+ if (handle->cpu_data[cpu].page_cnt)
+ tracecmd_warning("%d pages still allocated on cpu %d%s",
+ handle->cpu_data[cpu].page_cnt, cpu,
+ show_records(handle->cpu_data[cpu].pages,
+ handle->cpu_data[cpu].nr_pages));
+ free(handle->cpu_data[cpu].pages);
+ }
+ if (handle->cpu_data[cpu].compress.fd >= 0) {
+ close(handle->cpu_data[cpu].compress.fd);
+ unlink(handle->cpu_data[cpu].compress.file);
+ }
+ while (!list_empty(&handle->cpu_data[cpu].compress.cache)) {
+ cache = container_of(handle->cpu_data[cpu].compress.cache.next,
+ struct zchunk_cache, list);
+ list_del(&cache->list);
+ free(cache->map);
+ free(cache);
+ }
+ free(handle->cpu_data[cpu].compress.chunks);
}
}
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 13/20] trace-cmd library: Initialize CPU data decompression logic
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (11 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 12/20] trace-cmd library: Extend the input handle with trace data decompression context Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:48 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 14/20] trace-cmd library: Add logic for in-memory decompression Tzvetomir Stoyanov (VMware)
` (6 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:48 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
On CPU data initialization stage, initialize decompression context for
both in-memory and temporary file decompression logics.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 77 +++++++++++++++++++++++++++++++++++--
1 file changed, 74 insertions(+), 3 deletions(-)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index a6d90c01..192f6596 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -1266,6 +1266,7 @@ static void *allocate_page_map(struct tracecmd_input *handle,
off64_t map_offset;
void *map;
int ret;
+ int fd;
if (handle->read_page) {
map = malloc(handle->page_size);
@@ -1305,12 +1306,15 @@ static void *allocate_page_map(struct tracecmd_input *handle,
map_size -= map_offset + map_size -
(cpu_data->file_offset + cpu_data->file_size);
+ if (cpu_data->compress.fd >= 0)
+ fd = cpu_data->compress.fd;
+ else
+ fd = handle->fd;
again:
page_map->size = map_size;
page_map->offset = map_offset;
- page_map->map = mmap(NULL, map_size, PROT_READ, MAP_PRIVATE,
- handle->fd, map_offset);
+ page_map->map = mmap(NULL, map_size, PROT_READ, MAP_PRIVATE, fd, map_offset);
if (page_map->map == MAP_FAILED) {
/* Try a smaller map */
@@ -2498,16 +2502,81 @@ tracecmd_read_prev(struct tracecmd_input *handle, struct tep_record *record)
/* Not reached */
}
-static int init_cpu(struct tracecmd_input *handle, int cpu)
+static int init_cpu_zfile(struct tracecmd_input *handle, int cpu)
+{
+ struct cpu_data *cpu_data;
+ unsigned long long size;
+ off64_t offset;
+
+ cpu_data = &handle->cpu_data[cpu];
+ offset = lseek64(handle->fd, 0, SEEK_CUR);
+ if (lseek64(handle->fd, cpu_data->file_offset, SEEK_SET) == (off_t)-1)
+ return -1;
+
+ strcpy(cpu_data->compress.file, COMPR_TEMP_FILE);
+ cpu_data->compress.fd = mkstemp(cpu_data->compress.file);
+ if (cpu_data->compress.fd < 0)
+ return -1;
+
+ if (tracecmd_uncompress_copy_to(handle->compress, cpu_data->compress.fd, NULL, &size))
+ return -1;
+
+ if (lseek64(handle->fd, offset, SEEK_SET) == (off_t)-1)
+ return -1;
+
+ cpu_data->offset = 0;
+ cpu_data->file_offset = 0;
+ cpu_data->file_size = size;
+ cpu_data->size = size;
+ return 0;
+}
+
+static int init_cpu_zpage(struct tracecmd_input *handle, int cpu)
{
struct cpu_data *cpu_data = &handle->cpu_data[cpu];
+ int count;
int i;
+ if (lseek64(handle->fd, cpu_data->file_offset, SEEK_SET) == (off_t)-1)
+ return -1;
+
+ count = tracecmd_load_chunks_info(handle->compress, &cpu_data->compress.chunks);
+ if (count < 0)
+ return -1;
+
+ cpu_data->compress.count = count;
+ cpu_data->compress.last_chunk = 0;
+
+ cpu_data->file_offset = 0;
+ cpu_data->file_size = 0;
+ for (i = 0; i < count; i++)
+ cpu_data->file_size += cpu_data->compress.chunks[i].size;
cpu_data->offset = cpu_data->file_offset;
cpu_data->size = cpu_data->file_size;
+ return 0;
+}
+
+static int init_cpu(struct tracecmd_input *handle, int cpu)
+{
+ struct cpu_data *cpu_data = &handle->cpu_data[cpu];
+ int ret;
+ int i;
+
+ if (handle->cpu_compressed && cpu_data->file_size > 0) {
+ if (handle->read_zpage)
+ ret = init_cpu_zpage(handle, cpu);
+ else
+ ret = init_cpu_zfile(handle, cpu);
+ if (ret)
+ return ret;
+ } else {
+ cpu_data->offset = cpu_data->file_offset;
+ cpu_data->size = cpu_data->file_size;
+ }
cpu_data->timestamp = 0;
list_head_init(&cpu_data->page_maps);
+ list_head_init(&cpu_data->compress.cache);
if (!cpu_data->size) {
printf("CPU %d is empty\n", cpu);
@@ -3394,6 +3463,8 @@ static int init_buffer_cpu_data(struct tracecmd_input *handle, struct input_buff
return -1;
if (read_section_header(handle, &id, &flags, NULL, NULL))
return -1;
+ if (flags & TRACECMD_SEC_FL_COMPRESS)
+ handle->cpu_compressed = true;
if (buffer->latency) {
handle->file_state = TRACECMD_FILE_CPU_LATENCY;
return init_latency_data(handle) == 0 ? 1 : -1;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 14/20] trace-cmd library: Add logic for in-memory decompression
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (12 preceding siblings ...)
2022-01-26 9:48 ` [PATCH v8 13/20] trace-cmd library: Initialize CPU data decompression logic Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 15/20] trace-cmd library: Read compressed latency data Tzvetomir Stoyanov (VMware)
` (5 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
There are two approaches to read compressed trace data:
- use a temporary file to decompress entire trace data before reading
- use in-memory decompression of requested trace data chunk only
In-memory decompression seems to be more efficient, but selecting which
approach to use depends in the use case.
A compression chunk consists of multiple trace pages, that's why a small
cache with uncompressed chunks is implemented. The chunk stays in the
cache until there are pages which have reference to it.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 124 ++++++++++++++++++++++++++++++++++++
1 file changed, 124 insertions(+)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index 192f6596..c52aa951 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -29,6 +29,9 @@
#define COMMIT_MASK ((1 << 27) - 1)
+/* force uncompressing in memory */
+#define INMEMORY_DECOMPRESS
+
/* for debugging read instead of mmap */
static int force_read = 0;
@@ -1257,6 +1260,119 @@ static void free_page_map(struct page_map *page_map)
free(page_map);
}
+#define CHUNK_CHECK_OFFSET(C, O) ((O) >= (C)->offset && (O) < ((C)->offset + (C)->size))
+
+static int chunk_cmp(const void *A, const void *B)
+{
+ const struct tracecmd_compress_chunk *a = A;
+ const struct tracecmd_compress_chunk *b = B;
+
+ if (CHUNK_CHECK_OFFSET(b, a->offset))
+ return 0;
+
+ if (b->offset < a->offset)
+ return -1;
+
+ return 1;
+}
+
+static struct tracecmd_compress_chunk *get_zchunk(struct cpu_data *cpu, off64_t offset)
+{
+ struct cpu_zdata *cpuz = &cpu->compress;
+ struct tracecmd_compress_chunk *chunk;
+ struct tracecmd_compress_chunk key;
+
+ if (!cpuz->chunks)
+ return NULL;
+
+ if (offset > (cpuz->chunks[cpuz->count - 1].offset + cpuz->chunks[cpuz->count - 1].size))
+ return NULL;
+
+ /* check if the requested offset is in the last requested chunk or in the next chunk */
+ if (CHUNK_CHECK_OFFSET(cpuz->chunks + cpuz->last_chunk, offset))
+ return cpuz->chunks + cpuz->last_chunk;
+
+ cpuz->last_chunk++;
+ if (cpuz->last_chunk < cpuz->count &&
+ CHUNK_CHECK_OFFSET(cpuz->chunks + cpuz->last_chunk, offset))
+ return cpuz->chunks + cpuz->last_chunk;
+
+ key.offset = offset;
+ chunk = bsearch(&key, cpuz->chunks, cpuz->count, sizeof(*chunk), chunk_cmp);
+
+ if (!chunk) /* should never happen */
+ return NULL;
+
+ cpuz->last_chunk = chunk - cpuz->chunks;
+ return chunk;
+}
+
+static void free_zpage(struct cpu_data *cpu_data, void *map)
+{
+ struct zchunk_cache *cache;
+
+ list_for_each_entry(cache, &cpu_data->compress.cache, list) {
+ if (map <= cache->map && map > (cache->map + cache->chunk->size))
+ goto found;
+ }
+ return;
+
+found:
+ cache->ref--;
+ if (cache->ref)
+ return;
+ list_del(&cache->list);
+ free(cache->map);
+ free(cache);
+}
+
+static void *read_zpage(struct tracecmd_input *handle, int cpu, off64_t offset)
+{
+ struct cpu_data *cpu_data = &handle->cpu_data[cpu];
+ struct tracecmd_compress_chunk *chunk;
+ struct zchunk_cache *cache;
+ void *map = NULL;
+ int pindex;
+ int size;
+
+ /* Look in the cache of already loaded chunks */
+ list_for_each_entry(cache, &cpu_data->compress.cache, list) {
+ if (CHUNK_CHECK_OFFSET(cache->chunk, offset)) {
+ cache->ref++;
+ goto out;
+ }
+ }
+
+ chunk = get_zchunk(cpu_data, offset);
+ if (!chunk)
+ return NULL;
+
+ size = handle->page_size > chunk->size ? handle->page_size : chunk->size;
+ map = malloc(size);
+ if (!map)
+ return NULL;
+
+ if (tracecmd_uncompress_chunk(handle->compress, chunk, map) < 0)
+ goto error;
+
+ cache = calloc(1, sizeof(struct zchunk_cache));
+ if (!cache)
+ goto error;
+
+ cache->ref = 1;
+ cache->chunk = chunk;
+ cache->map = map;
+ list_add(&cache->list, &cpu_data->compress.cache);
+
+ /* a chunk can hold multiple pages, get the requested one */
+out:
+ pindex = (offset - cache->chunk->offset) / handle->page_size;
+ return cache->map + (pindex * handle->page_size);
+error:
+ free(map);
+ return NULL;
+}
+
static void *allocate_page_map(struct tracecmd_input *handle,
struct page *page, int cpu, off64_t offset)
{
@@ -1268,6 +1384,9 @@ static void *allocate_page_map(struct tracecmd_input *handle,
int ret;
int fd;
+ if (handle->cpu_compressed && handle->read_zpage)
+ return read_zpage(handle, cpu, offset);
+
if (handle->read_page) {
map = malloc(handle->page_size);
if (!map)
@@ -1410,6 +1529,8 @@ static void __free_page(struct tracecmd_input *handle, struct page *page)
if (handle->read_page)
free(page->map);
+ else if (handle->read_zpage)
+ free_zpage(cpu_data, page->map);
else
free_page_map(page->page_map);
@@ -3960,6 +4081,9 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd, int flags)
/* By default, use usecs, unless told otherwise */
handle->flags |= TRACECMD_FL_IN_USECS;
+#ifdef INMEMORY_DECOMPRESS
+ handle->read_zpage = 1;
+#endif
if (do_read_check(handle, buf, 3))
goto failed_read;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 15/20] trace-cmd library: Read compressed latency data
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (13 preceding siblings ...)
2022-01-26 9:49 ` [PATCH v8 14/20] trace-cmd library: Add logic for in-memory decompression Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 16/20] trace-cmd library: Decompress file sections on reading Tzvetomir Stoyanov (VMware)
` (4 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
Extended the latency read logic for reading comperssed latency trace
data. Both decompressing approaches are supported: in-memory and
using temporary file.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 77 +++++++++++++++++++++++++++++++++----
1 file changed, 70 insertions(+), 7 deletions(-)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index c52aa951..7d8f0fe4 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -177,6 +177,7 @@ struct tracecmd_input {
bool cpu_compressed;
int file_version;
unsigned int cpustats_size;
+ struct cpu_zdata latz;
struct cpu_data *cpu_data;
long long ts_offset;
struct tsc2nsec tsc_calc;
@@ -3485,20 +3486,55 @@ static int read_options_type(struct tracecmd_input *handle)
int tracecmd_latency_data_read(struct tracecmd_input *handle, char **buf, size_t *size)
{
+ struct cpu_zdata *zdata = &handle->latz;
+ void *data;
+ int rsize;
+ int fd = -1;
+ int id;
+
if (!handle || !buf || !size)
return -1;
if (handle->file_state != TRACECMD_FILE_CPU_LATENCY)
return -1;
+ if (!handle->cpu_compressed) {
+ fd = handle->fd;
+ } else if (!handle->read_zpage) {
+ if (zdata->fd < 0)
+ return -1;
+ fd = zdata->fd;
+ }
+
/* Read data from a file */
- if (!(*buf)) {
- *size = BUFSIZ;
- *buf = malloc(*size);
- if (!(*buf))
+ if (fd >= 0) {
+ if (!(*buf)) {
+ *size = BUFSIZ;
+ *buf = malloc(*size);
+ if (!(*buf))
+ return -1;
+ }
+ return do_read_fd(fd, *buf, *size);
+ }
+
+ /* Uncompress data in memory */
+ if (zdata->last_chunk >= zdata->count)
+ return 0;
+
+ id = zdata->last_chunk;
+ if (!*buf || *size < zdata->chunks[id].size) {
+ data = realloc(*buf, zdata->chunks[id].size);
+ if (!data)
return -1;
+ *buf = data;
+ *size = zdata->chunks[id].size;
}
- return do_read(handle, *buf, *size);
+ if (tracecmd_uncompress_chunk(handle->compress, &zdata->chunks[id], *buf))
+ return -1;
+
+ rsize = zdata->chunks[id].size;
+ zdata->last_chunk++;
+ return rsize;
}
static int init_cpu_data(struct tracecmd_input *handle)
@@ -3566,7 +3602,29 @@ static int init_cpu_data(struct tracecmd_input *handle)
int init_latency_data(struct tracecmd_input *handle)
{
- /* To do */
+ unsigned long long wsize;
+ int ret;
+
+ if (!handle->cpu_compressed)
+ return 0;
+
+ if (handle->read_zpage) {
+ handle->latz.count = tracecmd_load_chunks_info(handle->compress, &handle->latz.chunks);
+ if (handle->latz.count < 0)
+ return -1;
+ } else {
+ strcpy(handle->latz.file, COMPR_TEMP_FILE);
+ handle->latz.fd = mkstemp(handle->latz.file);
+ if (handle->latz.fd < 0)
+ return -1;
+
+ ret = tracecmd_uncompress_copy_to(handle->compress, handle->latz.fd, NULL, &wsize);
+ if (ret)
+ return -1;
+
+ lseek64(handle->latz.fd, 0, SEEK_SET);
+ }
+
return 0;
}
@@ -4078,6 +4136,7 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd, int flags)
handle->fd = fd;
handle->ref = 1;
+ handle->latz.fd = -1;
/* By default, use usecs, unless told otherwise */
handle->flags |= TRACECMD_FL_IN_USECS;
@@ -4372,7 +4431,11 @@ void tracecmd_close(struct tracecmd_input *handle)
free(handle->strings);
free(handle->version);
close(handle->fd);
-
+ free(handle->latz.chunks);
+ if (handle->latz.fd >= 0) {
+ close(handle->latz.fd);
+ unlink(handle->latz.file);
+ }
while (handle->sections) {
del_sec = handle->sections;
handle->sections = handle->sections->next;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 16/20] trace-cmd library: Decompress file sections on reading
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (14 preceding siblings ...)
2022-01-26 9:49 ` [PATCH v8 15/20] trace-cmd library: Read compressed latency data Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 17/20] trace-cmd library: Add zlib compression algorithm Tzvetomir Stoyanov (VMware)
` (3 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
When reading sections from trace file v7 - check section flags to find
out if the section is compressed and decompress it.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
lib/trace-cmd/trace-input.c | 26 ++++++++++++++++++++++++--
1 file changed, 24 insertions(+), 2 deletions(-)
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index 7d8f0fe4..3f7971c4 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -484,12 +484,17 @@ static struct file_section *section_open(struct tracecmd_input *handle, int id)
if (lseek64(handle->fd, sec->data_offset, SEEK_SET) == (off64_t)-1)
return NULL;
+
+ if ((sec->flags & TRACECMD_SEC_FL_COMPRESS) && in_uncompress_block(handle))
+ return NULL;
+
return sec;
}
static void section_close(struct tracecmd_input *handle, struct file_section *sec)
{
- /* To Do */
+ if (sec->flags & TRACECMD_SEC_FL_COMPRESS)
+ in_uncompress_reset(handle);
}
static int section_add_or_update(struct tracecmd_input *handle, int id, int flags,
@@ -1116,6 +1121,8 @@ static int handle_section(struct tracecmd_input *handle, struct file_section *se
return -1;
section->data_offset = lseek64(handle->fd, 0, SEEK_CUR);
+ if ((section->flags & TRACECMD_SEC_FL_COMPRESS) && in_uncompress_block(handle))
+ return -1;
switch (section->id) {
case TRACECMD_OPTION_HEADER_INFO:
@@ -1141,6 +1148,9 @@ static int handle_section(struct tracecmd_input *handle, struct file_section *se
break;
}
+ if (section->flags & TRACECMD_SEC_FL_COMPRESS)
+ in_uncompress_reset(handle);
+
return ret;
}
@@ -4080,6 +4090,7 @@ static int read_metadata_strings(struct tracecmd_input *handle)
unsigned short flags;
int found = 0;
unsigned short id;
+ unsigned int csize, rsize;
unsigned long long size;
off64_t offset;
@@ -4089,7 +4100,18 @@ static int read_metadata_strings(struct tracecmd_input *handle)
break;
if (id == TRACECMD_OPTION_STRINGS) {
found++;
- init_metadata_strings(handle, size);
+ if ((flags & TRACECMD_SEC_FL_COMPRESS)) {
+ read4(handle, &csize);
+ read4(handle, &rsize);
+ do_lseek(handle, -8, SEEK_CUR);
+ if (in_uncompress_block(handle))
+ break;
+ } else {
+ rsize = size;
+ }
+ init_metadata_strings(handle, rsize);
+ if (flags & TRACECMD_SEC_FL_COMPRESS)
+ in_uncompress_reset(handle);
} else {
if (lseek64(handle->fd, size, SEEK_CUR) == (off_t)-1)
break;
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 17/20] trace-cmd library: Add zlib compression algorithm
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (15 preceding siblings ...)
2022-01-26 9:49 ` [PATCH v8 16/20] trace-cmd library: Decompress file sections on reading Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 18/20] trace-cmd list: Show supported compression algorithms Tzvetomir Stoyanov (VMware)
` (2 subsequent siblings)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
Added implementation for zlib compression algorithm, using libz
library, if available.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
Makefile | 7 ++
lib/trace-cmd/Makefile | 7 ++
lib/trace-cmd/include/trace-cmd-local.h | 4 +
lib/trace-cmd/trace-compress-zlib.c | 109 ++++++++++++++++++++++++
lib/trace-cmd/trace-compress.c | 4 +
tracecmd/Makefile | 4 +
6 files changed, 135 insertions(+)
create mode 100644 lib/trace-cmd/trace-compress-zlib.c
diff --git a/Makefile b/Makefile
index 91f62859..c9faf8db 100644
--- a/Makefile
+++ b/Makefile
@@ -300,6 +300,13 @@ ifeq ($(PERF_DEFINED), 1)
CFLAGS += -DPERF
endif
+ZLIB_INSTALLED := $(shell if (printf "$(pound)include <zlib.h>\n void main(){deflateInit(NULL, Z_BEST_COMPRESSION);}" | $(CC) -o /dev/null -x c - -lz >/dev/null 2>&1) ; then echo 1; else echo 0 ; fi)
+ifeq ($(ZLIB_INSTALLED), 1)
+export ZLIB_INSTALLED
+CFLAGS += -DHAVE_ZLIB
+$(info Have zlib compression support)
+endif
+
CUNIT_INSTALLED := $(shell if (printf "$(pound)include <CUnit/Basic.h>\n void main(){CU_initialize_registry();}" | $(CC) -o /dev/null -x c - -lcunit >/dev/null 2>&1) ; then echo 1; else echo 0 ; fi)
export CUNIT_INSTALLED
diff --git a/lib/trace-cmd/Makefile b/lib/trace-cmd/Makefile
index a7be4a42..1820c67b 100644
--- a/lib/trace-cmd/Makefile
+++ b/lib/trace-cmd/Makefile
@@ -26,6 +26,9 @@ OBJS += trace-timesync-ptp.o
OBJS += trace-timesync-kvm.o
endif
OBJS += trace-compress.o
+ifeq ($(ZLIB_INSTALLED), 1)
+OBJS += trace-compress-zlib.o
+endif
# Additional util objects
OBJS += trace-blk-hack.o
@@ -47,6 +50,10 @@ $(LIBTRACECMD_STATIC): $(OBJS)
LIBS = $(LIBTRACEEVENT_LDLAGS) $(LIBTRACEFS_LDLAGS) -lpthread
+ifeq ($(ZLIB_INSTALLED), 1)
+LIBS += -lz
+endif
+
$(LIBTRACECMD_SHARED_VERSION): $(LIBTRACECMD_SHARED)
@ln -sf $(<F) $@
diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index ca682ed9..3d75bd63 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -26,6 +26,10 @@ void tracecmd_info(const char *fmt, ...);
#endif
#endif
+#ifdef HAVE_ZLIB
+int tracecmd_zlib_init(void);
+#endif
+
struct data_file_write {
unsigned long long file_size;
unsigned long long write_size;
diff --git a/lib/trace-cmd/trace-compress-zlib.c b/lib/trace-cmd/trace-compress-zlib.c
new file mode 100644
index 00000000..a697cc61
--- /dev/null
+++ b/lib/trace-cmd/trace-compress-zlib.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2021, VMware, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
+ *
+ */
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <zlib.h>
+#include <errno.h>
+
+#include "trace-cmd-private.h"
+
+#define __ZLIB_NAME "zlib"
+#define __ZLIB_WEIGTH 10
+
+static int zlib_compress(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes)
+{
+ unsigned long out_size = *out_bytes;
+ int ret;
+
+ ret = compress2((unsigned char *)out, &out_size,
+ (unsigned char *)in, (unsigned long)in_bytes, Z_BEST_COMPRESSION);
+ *out_bytes = out_size;
+ errno = 0;
+ switch (ret) {
+ case Z_OK:
+ return 0;
+ case Z_BUF_ERROR:
+ errno = -ENOBUFS;
+ break;
+ case Z_MEM_ERROR:
+ errno = -ENOMEM;
+ break;
+ case Z_STREAM_ERROR:
+ errno = -EINVAL;
+ break;
+ default:
+ errno = -EFAULT;
+ break;
+ }
+
+ return -1;
+}
+
+static int zlib_decompress(const char *in, unsigned int in_bytes,
+ char *out, unsigned int *out_bytes)
+{
+ unsigned long out_size = *out_bytes;
+ int ret;
+
+ ret = uncompress((unsigned char *)out, &out_size,
+ (unsigned char *)in, (unsigned long)in_bytes);
+ *out_bytes = out_size;
+ errno = 0;
+ switch (ret) {
+ case Z_OK:
+ return 0;
+ case Z_BUF_ERROR:
+ errno = -ENOBUFS;
+ break;
+ case Z_MEM_ERROR:
+ errno = -ENOMEM;
+ break;
+ case Z_DATA_ERROR:
+ errno = -EINVAL;
+ break;
+ default:
+ errno = -EFAULT;
+ break;
+ }
+
+ return -1;
+}
+
+static unsigned int zlib_compress_bound(unsigned int in_bytes)
+{
+ return compressBound(in_bytes);
+}
+
+static bool zlib_is_supported(const char *name, const char *version)
+{
+ const char *zver;
+
+ if (!name)
+ return false;
+ if (strlen(name) != strlen(__ZLIB_NAME) || strcmp(name, __ZLIB_NAME))
+ return false;
+
+ if (!version)
+ return true;
+
+ zver = zlibVersion();
+ if (!zver)
+ return false;
+
+ /* Compare the major version number */
+ if (atoi(version) <= atoi(zver))
+ return true;
+
+ return false;
+}
+
+int tracecmd_zlib_init(void)
+{
+ return tracecmd_compress_proto_register(__ZLIB_NAME, zlibVersion(), __ZLIB_WEIGTH,
+ zlib_compress, zlib_decompress,
+ zlib_compress_bound, zlib_is_supported);
+}
diff --git a/lib/trace-cmd/trace-compress.c b/lib/trace-cmd/trace-compress.c
index 4a4341dd..210d58b6 100644
--- a/lib/trace-cmd/trace-compress.c
+++ b/lib/trace-cmd/trace-compress.c
@@ -386,6 +386,10 @@ void tracecmd_compress_init(void)
gettimeofday(&time, NULL);
srand((time.tv_sec * 1000) + (time.tv_usec / 1000));
+
+#ifdef HAVE_ZLIB
+ tracecmd_zlib_init();
+#endif
}
static struct compress_proto *compress_proto_select(void)
diff --git a/tracecmd/Makefile b/tracecmd/Makefile
index 80c69bbb..b7a23dc4 100644
--- a/tracecmd/Makefile
+++ b/tracecmd/Makefile
@@ -51,6 +51,10 @@ CONFIG_INCLUDES =
CONFIG_LIBS = -lrt -lpthread $(TRACE_LIBS)
CONFIG_FLAGS =
+ifeq ($(ZLIB_INSTALLED), 1)
+CONFIG_LIBS += -lz
+endif
+
all: $(TARGETS)
$(bdir):
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 18/20] trace-cmd list: Show supported compression algorithms
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (16 preceding siblings ...)
2022-01-26 9:49 ` [PATCH v8 17/20] trace-cmd library: Add zlib compression algorithm Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 19/20] trace-cmd record: Add compression to the trace context Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 20/20] trace-cmd report: Add new parameter for trace file compression Tzvetomir Stoyanov (VMware)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
Add new parameter "trace-cmd list -c" to show supported compression
algorithms.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
Documentation/trace-cmd/trace-cmd-list.1.txt | 3 +++
tracecmd/trace-list.c | 26 ++++++++++++++++++++
tracecmd/trace-usage.c | 1 +
3 files changed, 30 insertions(+)
diff --git a/Documentation/trace-cmd/trace-cmd-list.1.txt b/Documentation/trace-cmd/trace-cmd-list.1.txt
index a5c6b16c..b77e3460 100644
--- a/Documentation/trace-cmd/trace-cmd-list.1.txt
+++ b/Documentation/trace-cmd/trace-cmd-list.1.txt
@@ -71,6 +71,9 @@ OPTIONS
List defined clocks that can be used with trace-cmd record -C.
The one in brackets ([]) is the active clock.
+*-c*::
+ List the available trace file compression algorithms.
+
SEE ALSO
--------
trace-cmd(1), trace-cmd-record(1), trace-cmd-report(1), trace-cmd-start(1),
diff --git a/tracecmd/trace-list.c b/tracecmd/trace-list.c
index d060c810..900da73b 100644
--- a/tracecmd/trace-list.c
+++ b/tracecmd/trace-list.c
@@ -549,6 +549,24 @@ static void show_plugins(void)
tep_free(pevent);
}
+static void show_compression(void)
+{
+ char **versions, **names;
+ int c, i;
+
+ c = tracecmd_compress_protos_get(&names, &versions);
+ if (c <= 0) {
+ printf("No compression algorithms are supported\n");
+ return;
+ }
+ printf("Supported compression algorithms:\n");
+ for (i = 0; i < c; i++)
+ printf("\t%s, %s\n", names[i], versions[i]);
+
+ free(names);
+ free(versions);
+}
+
void trace_list(int argc, char **argv)
{
int events = 0;
@@ -562,6 +580,7 @@ void trace_list(int argc, char **argv)
int flags = 0;
int systems = 0;
int show_all = 1;
+ int compression = 0;
int i;
const char *arg;
const char *funcre = NULL;
@@ -626,6 +645,10 @@ void trace_list(int argc, char **argv)
systems = 1;
show_all = 0;
break;
+ case 'c':
+ compression = 1;
+ show_all = 0;
+ break;
case '-':
if (strcmp(argv[i], "--debug") == 0) {
tracecmd_set_debug(true);
@@ -670,6 +693,8 @@ void trace_list(int argc, char **argv)
show_clocks();
if (systems)
show_systems();
+ if (compression)
+ show_compression();
if (show_all) {
printf("event systems:\n");
show_systems();
@@ -679,6 +704,7 @@ void trace_list(int argc, char **argv)
show_tracers();
printf("\noptions:\n");
show_options();
+ show_compression();
}
return;
diff --git a/tracecmd/trace-usage.c b/tracecmd/trace-usage.c
index ac12b066..34c6cc35 100644
--- a/tracecmd/trace-usage.c
+++ b/tracecmd/trace-usage.c
@@ -348,6 +348,7 @@ static struct usage_help usage_help[] = {
" -O list plugin options\n"
" -B list defined buffer instances\n"
" -C list the defined clocks (and active one)\n"
+ " -c list the supported trace file compression algorithms\n"
},
{
"restore",
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 19/20] trace-cmd record: Add compression to the trace context
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (17 preceding siblings ...)
2022-01-26 9:49 ` [PATCH v8 18/20] trace-cmd list: Show supported compression algorithms Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
2022-01-26 9:49 ` [PATCH v8 20/20] trace-cmd report: Add new parameter for trace file compression Tzvetomir Stoyanov (VMware)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
As the trace-cmd library supports trace file compression, trace-cmd
record command should have a way to configure this functionality. Trace
context is extended to hold the compression algorithm, used to
compress the file.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
tracecmd/trace-record.c | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index ece5a0c2..4df97d7d 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -199,6 +199,7 @@ struct common_record_context {
char *date2ts;
char *user;
const char *clock;
+ const char *compression;
struct tsc_nsec tsc2nsec;
int data_flags;
int tsync_loop_interval;
@@ -3702,6 +3703,14 @@ static struct tracecmd_output *create_net_output(struct common_record_context *c
goto error;
if (tracecmd_output_set_msg(out, msg_handle))
goto error;
+
+ if (ctx->compression) {
+ if (tracecmd_output_set_compression(out, ctx->compression))
+ goto error;
+ } else if (ctx->file_version >= FILE_VERSION_COMPRESSION) {
+ tracecmd_output_set_compression(out, "any");
+ }
+
if (tracecmd_output_write_headers(out, listed_events))
goto error;
@@ -3748,6 +3757,14 @@ setup_connection(struct buffer_instance *instance, struct common_record_context
goto error;
if (tracecmd_output_set_version(network_handle, ctx->file_version))
goto error;
+
+ if (ctx->compression) {
+ if (tracecmd_output_set_compression(network_handle, ctx->compression))
+ goto error;
+ } else if (ctx->file_version >= FILE_VERSION_COMPRESSION) {
+ tracecmd_output_set_compression(network_handle, "any");
+ }
+
if (tracecmd_output_write_headers(network_handle, listed_events))
goto error;
tracecmd_set_quiet(network_handle, quiet);
@@ -4477,6 +4494,14 @@ static struct tracecmd_output *create_output(struct common_record_context *ctx)
goto error;
if (ctx->file_version && tracecmd_output_set_version(out, ctx->file_version))
goto error;
+
+ if (ctx->compression) {
+ if (tracecmd_output_set_compression(out, ctx->compression))
+ goto error;
+ } else if (ctx->file_version >= FILE_VERSION_COMPRESSION) {
+ tracecmd_output_set_compression(out, "any");
+ }
+
if (tracecmd_output_write_headers(out, listed_events))
goto error;
@@ -4511,7 +4536,7 @@ static void record_data(struct common_record_context *ctx)
if (latency) {
handle = tracecmd_create_file_latency(ctx->output, local_cpu_count,
- ctx->file_version, NULL);
+ ctx->file_version, ctx->compression);
tracecmd_set_quiet(handle, quiet);
} else {
if (!local_cpu_count)
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v8 20/20] trace-cmd report: Add new parameter for trace file compression
2022-01-26 9:48 [PATCH v8 00/20] Trace file version 7 - compression Tzvetomir Stoyanov (VMware)
` (18 preceding siblings ...)
2022-01-26 9:49 ` [PATCH v8 19/20] trace-cmd record: Add compression to the trace context Tzvetomir Stoyanov (VMware)
@ 2022-01-26 9:49 ` Tzvetomir Stoyanov (VMware)
19 siblings, 0 replies; 21+ messages in thread
From: Tzvetomir Stoyanov (VMware) @ 2022-01-26 9:49 UTC (permalink / raw)
To: rostedt; +Cc: linux-trace-devel
A new parameter is added, which can be used to set desired compression
the output trace file.
"trace-cmd report --compression <compression>"
Where the <compression> string can be compression algorithm name, "any"
for the best available algorithm or "none" for no compression.
Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
tracecmd/trace-record.c | 13 +++++++++++++
tracecmd/trace-usage.c | 5 +++++
2 files changed, 18 insertions(+)
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 4df97d7d..5dd769d1 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -5810,6 +5810,7 @@ void init_top_instance(void)
}
enum {
+ OPT_compression = 237,
OPT_file_ver = 238,
OPT_verbose = 239,
OPT_tsc2nsec = 240,
@@ -6250,6 +6251,7 @@ static void parse_record_options(int argc,
{"tsc2nsec", no_argument, NULL, OPT_tsc2nsec},
{"poll", no_argument, NULL, OPT_poll},
{"verbose", optional_argument, NULL, OPT_verbose},
+ {"compression", required_argument, NULL, OPT_compression},
{"file-version", required_argument, NULL, OPT_file_ver},
{NULL, 0, NULL, 0}
};
@@ -6676,6 +6678,17 @@ static void parse_record_options(int argc,
cmd_check_die(ctx, CMD_set, *(argv+1), "--poll");
recorder_flags |= TRACECMD_RECORD_POLL;
break;
+ case OPT_compression:
+ cmd_check_die(ctx, CMD_start, *(argv+1), "--compression");
+ cmd_check_die(ctx, CMD_set, *(argv+1), "--compression");
+ cmd_check_die(ctx, CMD_extract, *(argv+1), "--compression");
+ cmd_check_die(ctx, CMD_stream, *(argv+1), "--compression");
+ cmd_check_die(ctx, CMD_profile, *(argv+1), "--compression");
+ if (strcmp(optarg, "any") && strcmp(optarg, "none") &&
+ !tracecmd_compress_is_supported(optarg, NULL))
+ die("Compression algorithm %s is not supported", optarg);
+ ctx->compression = strdup(optarg);
+ break;
case OPT_file_ver:
if (ctx->curr_cmd != CMD_record && ctx->curr_cmd != CMD_record_agent)
die("--file_version has no effect with the command %s\n",
diff --git a/tracecmd/trace-usage.c b/tracecmd/trace-usage.c
index 34c6cc35..77898c1c 100644
--- a/tracecmd/trace-usage.c
+++ b/tracecmd/trace-usage.c
@@ -70,6 +70,11 @@ static struct usage_help usage_help[] = {
" at the beginnig and at the end of the trace\n"
" --poll don't block while reading from the trace buffer\n"
" --file-version set the desired trace file version\n"
+ " --compression compress the trace output file, one of these strings can be passed:\n"
+ " any - auto select the best available compression algorithm\n"
+ " none - do not compress the trace file\n"
+ " name - the name of the desired compression algorithms\n"
+ " available algorithms can be listed with trace-cmd list -c\n"
},
{
"set",
--
2.34.1
^ permalink raw reply related [flat|nested] 21+ messages in thread