[PATCH 1/1] add sparse support for f2fs

[Yang Jin via Linux-f2fs-devel <linux-f2fs-devel@lists.sourceforge.net>]

Description:
modify I/O operations to support writing in sparse format
build host executable for making sparse image

Tests:
Using host executable:
./make_f2fs_host -S2000000000 userdata.img
fastboot flash userdata userdata.img

Using target executable:
adb push $OUT/system/bin/make_f2fs /system/bin
fastboot erase userdata

Analyzing the output image:
./make_f2fs_host -S2000000000 userdata.img
./make_f2fs_host test.img
simg2img userdata.img output.img
hexdump -C output.img >> output.txt
hexdump -C test.img >> test.txt

Bug: 62493641

Signed-off-by: Yang Jin <yajin@google.com>
---
 Android.mk              | 35 ++++++++++++++++++------
 include/f2fs_fs.h       |  2 ++
 lib/libf2fs.c           | 14 ++++++++--
 lib/libf2fs_io.c        | 73 +++++++++++++++++++++++++++++++++++++++++++++++--
 mkfs/f2fs_format_main.c | 27 +++++++++++++++++-
 5 files changed, 137 insertions(+), 14 deletions(-)

diff --git a/Android.mk b/Android.mk
index afd719b..2a32022 100644
--- a/Android.mk
+++ b/Android.mk
@@ -11,7 +11,7 @@ common_CFLAGS := -DWITH_ANDROID $(version_CFLAGS)
 target_CFLAGS := $(common_CFLAGS) -include sys/sysmacros.h
 
 # external/e2fsprogs/lib is needed for uuid/uuid.h
-common_C_INCLUDES := $(LOCAL_PATH)/include external/e2fsprogs/lib/
+common_C_INCLUDES := $(LOCAL_PATH)/include external/e2fsprogs/lib/ system/core/libsparse/include
 
 #----------------------------------------------------------
 include $(CLEAR_VARS)
@@ -52,10 +52,10 @@ LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(common_CFLAGS)
 LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/include $(LOCAL_PATH)/mkfs
 LOCAL_STATIC_LIBRARIES := \
-     libf2fs_ioutils_host \
-     libext2_uuid \
-     libsparse \
-     libz
+	libf2fs_ioutils_host \
+	libext2_uuid \
+	libsparse \
+	libz
 # LOCAL_LDLIBS := -ldl
 include $(BUILD_HOST_SHARED_LIBRARY)
 
@@ -74,7 +74,11 @@ LOCAL_SRC_FILES := \
 	mkfs/f2fs_format_main.c
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(target_CFLAGS)
-LOCAL_STATIC_LIBRARIES := libc libf2fs_fmt libext2_uuid
+LOCAL_STATIC_LIBRARIES := \
+	libf2fs_fmt \
+	libext2_uuid \
+	libsparse \
+	libz
 LOCAL_MODULE_TAGS := optional
 include $(BUILD_EXECUTABLE)
 
@@ -88,11 +92,25 @@ LOCAL_SRC_FILES := \
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(target_CFLAGS)
 LOCAL_STATIC_LIBRARIES := libf2fs_fmt
-LOCAL_SHARED_LIBRARIES := libext2_uuid
+LOCAL_SHARED_LIBRARIES := libext2_uuid libsparse
 LOCAL_SYSTEM_SHARED_LIBRARIES := libc
 LOCAL_MODULE_TAGS := optional
 include $(BUILD_EXECUTABLE)
 
+#----------------------------------------------------------
+include $(CLEAR_VARS)
+LOCAL_MODULE := make_f2fs_host
+
+LOCAL_SRC_FILES := \
+	mkfs/f2fs_format_main.c \
+	lib/libf2fs_io.c \
+
+LOCAL_C_INCLUDES := $(common_C_INCLUDES)
+LOCAL_CFLAGS := $(common_CFLAGS)
+LOCAL_STATIC_LIBRARIES := libf2fs_fmt_host
+LOCAL_SHARED_LIBRARIES := libext2_uuid libsparse
+include $(BUILD_HOST_EXECUTABLE)
+
 #----------------------------------------------------------
 include $(CLEAR_VARS)
 # The LOCAL_MODULE name is referenced by the code. Don't change it.
@@ -107,7 +125,7 @@ LOCAL_SRC_FILES := \
 
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(target_CFLAGS)
-LOCAL_SHARED_LIBRARIES := libext2_uuid
+LOCAL_SHARED_LIBRARIES := libext2_uuid libsparse
 LOCAL_SYSTEM_SHARED_LIBRARIES := libc
 LOCAL_MODULE_TAGS := optional
 include $(BUILD_EXECUTABLE)
@@ -125,6 +143,7 @@ LOCAL_SRC_FILES := \
 
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(common_CFLAGS)
+LOCAL_SHARED_LIBRARIES := libsparse
 LOCAL_HOST_SHARED_LIBRARIES :=  libext2_uuid
 include $(BUILD_HOST_EXECUTABLE)
 
diff --git a/include/f2fs_fs.h b/include/f2fs_fs.h
index 2316869..dd2635b 100644
--- a/include/f2fs_fs.h
+++ b/include/f2fs_fs.h
@@ -256,6 +256,7 @@ struct device_info {
 struct f2fs_configuration {
 	u_int32_t reserved_segments;
 	u_int32_t new_reserved_segments;
+	int sparse_mode;
 	int zoned_mode;
 	int zoned_model;
 	size_t zone_blocks;
@@ -268,6 +269,7 @@ struct f2fs_configuration {
 	u_int32_t start_sector;
 	u_int32_t total_segments;
 	u_int32_t sector_size;
+	u_int64_t device_size;
 	u_int64_t total_sectors;
 	u_int64_t wanted_total_sectors;
 	u_int64_t target_sectors;
diff --git a/lib/libf2fs.c b/lib/libf2fs.c
index c96506a..31836db 100644
--- a/lib/libf2fs.c
+++ b/lib/libf2fs.c
@@ -32,6 +32,10 @@
 #define MODELINQUIRY	0x12,0x00,0x00,0x00,0x4A,0x00
 #endif
 
+#ifndef _WIN32 /* O_BINARY is windows-specific flag */
+#define O_BINARY 0
+#endif
+
 /*
  * UTF conversion codes are Copied from exfat tools.
  */
@@ -689,7 +693,11 @@ int get_device_info(int i)
 #endif
 	struct device_info *dev = c.devices + i;
 
-	fd = open((char *)dev->path, O_RDWR);
+	if (c.sparse_mode) {
+		fd = open((char *)dev->path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
+	} else {
+		fd = open((char *)dev->path, O_RDWR);
+	}
 	if (fd < 0) {
 		MSG(0, "\tError: Failed to open the device!\n");
 		return -1;
@@ -710,7 +718,9 @@ int get_device_info(int i)
 		return -1;
 	}
 
-	if (S_ISREG(stat_buf.st_mode)) {
+	if (c.sparse_mode) {
+		dev->total_sectors = c.device_size / dev->sector_size;
+	} else if (S_ISREG(stat_buf.st_mode)) {
 		dev->total_sectors = stat_buf.st_size / dev->sector_size;
 	} else if (S_ISBLK(stat_buf.st_mode)) {
 		if (ioctl(fd, BLKSSZGET, &sector_size) < 0)
diff --git a/lib/libf2fs_io.c b/lib/libf2fs_io.c
index c09db36..aa99068 100644
--- a/lib/libf2fs_io.c
+++ b/lib/libf2fs_io.c
@@ -25,6 +25,19 @@
 
 struct f2fs_configuration c;
 
+#ifdef WITH_ANDROID
+#include <sparse/sparse.h>
+struct sparse_file *f2fs_sparse_file;
+
+struct buf_item {
+	void *buf;
+	size_t len;
+	struct buf_item *next;
+};
+
+struct buf_item *buf_list;
+#endif
+
 static int __get_device_fd(__u64 *offset)
 {
 	__u64 blk_addr = *offset >> F2FS_BLKSIZE_BITS;
@@ -46,6 +59,8 @@ static int __get_device_fd(__u64 *offset)
  */
 int dev_read_version(void *buf, __u64 offset, size_t len)
 {
+	if (c.sparse_mode)
+		return 0;
 	if (lseek64(c.kd, (off64_t)offset, SEEK_SET) < 0)
 		return -1;
 	if (read(c.kd, buf, len) < 0)
@@ -55,8 +70,12 @@ int dev_read_version(void *buf, __u64 offset, size_t len)
 
 int dev_read(void *buf, __u64 offset, size_t len)
 {
-	int fd = __get_device_fd(&offset);
+	int fd;
+
+	if (c.sparse_mode)
+		return 0;
 
+	fd = __get_device_fd(&offset);
 	if (fd < 0)
 		return fd;
 
@@ -80,10 +99,40 @@ int dev_readahead(__u64 offset, size_t len)
 #endif
 }
 
+#ifdef WITH_ANDROID
+static int dev_write_sparse(void *buf, __u64 byte_offset, size_t byte_len)
+{
+	struct buf_item *bi = calloc(1, sizeof(struct buf_item));
+
+	if (bi == NULL) {
+		return -1;
+	}
+	bi->buf = malloc(byte_len);
+	if (bi->buf == NULL) {
+		free(bi);
+		return -1;
+	}
+
+	bi->len = byte_len;
+	memcpy(bi->buf, buf, byte_len);
+	bi->next = buf_list;
+	buf_list = bi;
+
+	sparse_file_add_data(f2fs_sparse_file, bi->buf, byte_len, byte_offset/F2FS_BLKSIZE);
+	return 0;
+}
+#else
+static int dev_write_sparse(void *buf, __u64 byte_offset, size_t byte_len) { return 0; }
+#endif
+
 int dev_write(void *buf, __u64 offset, size_t len)
 {
-	int fd = __get_device_fd(&offset);
+	int fd;
+
+	if (c.sparse_mode)
+		return dev_write_sparse(buf, offset, len);
 
+	fd = __get_device_fd(&offset);
 	if (fd < 0)
 		return fd;
 
@@ -110,8 +159,12 @@ int dev_write_dump(void *buf, __u64 offset, size_t len)
 
 int dev_fill(void *buf, __u64 offset, size_t len)
 {
-	int fd = __get_device_fd(&offset);
+	int fd;
+
+	if (c.sparse_mode)
+		return 0;
 
+	fd = __get_device_fd(&offset);
 	if (fd < 0)
 		return fd;
 
@@ -144,6 +197,20 @@ void f2fs_finalize_device(void)
 {
 	int i;
 
+#ifdef WITH_ANDROID
+	if (c.sparse_mode) {
+		sparse_file_write(f2fs_sparse_file, c.devices[0].fd, /*gzip*/0, /*sparse*/1, /*crc*/0);
+		sparse_file_destroy(f2fs_sparse_file);
+		while (buf_list) {
+			struct buf_item *bi = buf_list;
+			buf_list = buf_list->next;
+			free(bi->buf);
+			free(bi);
+		}
+		f2fs_sparse_file = NULL;
+	}
+#endif
+
 	/*
 	 * We should call fsync() to flush out all the dirty pages
 	 * in the block device page cache.
diff --git a/mkfs/f2fs_format_main.c b/mkfs/f2fs_format_main.c
index a3652a9..5525d1c 100644
--- a/mkfs/f2fs_format_main.c
+++ b/mkfs/f2fs_format_main.c
@@ -27,6 +27,11 @@
 #include "f2fs_fs.h"
 #include "f2fs_format_utils.h"
 
+#ifdef WITH_ANDROID
+#include <sparse/sparse.h>
+extern struct sparse_file *f2fs_sparse_file;
+#endif
+
 extern struct f2fs_configuration c;
 static int force_overwrite = 0;
 
@@ -45,6 +50,7 @@ static void mkfs_usage()
 	MSG(0, "  -O [feature list] e.g. \"encrypt\"\n");
 	MSG(0, "  -q quiet mode\n");
 	MSG(0, "  -s # of segments per section [default:1]\n");
+	MSG(0, "  -S sparse mode\n");
 	MSG(0, "  -t 0: nodiscard, 1: discard [default:1]\n");
 	MSG(0, "  -z # of sections per zone [default:1]\n");
 	MSG(0, "sectors: number of sectors. [default: determined by device size]\n");
@@ -82,7 +88,7 @@ static void parse_feature(const char *features)
 
 static void f2fs_parse_options(int argc, char *argv[])
 {
-	static const char *option_string = "qa:c:d:e:l:mo:O:s:z:t:f";
+	static const char *option_string = "qa:c:d:e:l:mo:O:s:S:z:t:f";
 	int32_t option=0;
 
 	while ((option = getopt(argc,argv,option_string)) != EOF) {
@@ -132,6 +138,11 @@ static void f2fs_parse_options(int argc, char *argv[])
 		case 's':
 			c.segs_per_sec = atoi(optarg);
 			break;
+		case 'S':
+			c.device_size = atoll(optarg);
+			c.device_size &= (~((u_int64_t)(F2FS_BLKSIZE - 1)));
+			c.sparse_mode = 1;
+			break;
 		case 'z':
 			c.secs_per_zone = atoi(optarg);
 			break;
@@ -164,6 +175,9 @@ static void f2fs_parse_options(int argc, char *argv[])
 		c.wanted_total_sectors = atoll(argv[optind+1]);
 	}
 
+	if (c.sparse_mode)
+		c.trim = 0;
+
 	if (c.zoned_mode)
 		c.feature |= cpu_to_le32(F2FS_FEATURE_BLKZONED);
 }
@@ -277,6 +291,17 @@ int main(int argc, char *argv[])
 		return -1;
 	}
 
+	if (c.sparse_mode) {
+#ifndef WITH_ANDROID
+		MSG(0, "\tError: Sparse mode is only supported for android\n");
+		return -1;
+#else
+		if (f2fs_sparse_file)
+			sparse_file_destroy(f2fs_sparse_file);
+		f2fs_sparse_file = sparse_file_new(F2FS_BLKSIZE, c.device_size);
+#endif
+	}
+
 	if (f2fs_format_device() < 0)
 		return -1;
 
-- 
2.13.1.518.g3df882009-goog


------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot