[PATCH v5 5/5] btrfs: introduce new read_policy device

[Anand Jain <anand.jain@oracle.com>]

A new read policy 'device' is introduced with this patch, which when set
can pick the read_preferred device for reading. This tunable is for
advance users who knows what they are doing so that they have the
flexibility of making sure that reads are read from the device they
prefer.

For example:

$ cat /sys/fs/btrfs/UUID/read_policy
[pid] device

$echo 1 > /sys/fs/btrfs/UUID/devinfo/1/read_preferred

$echo device > /sys/fs/btrfs/UUID/read_policy

$cat /sys/fs/btrfs/UUID/read_policy
pid [device]

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
v5: born

 fs/btrfs/sysfs.c   | 26 +++++++++++++++++++++++++-
 fs/btrfs/volumes.c | 36 ++++++++++++++++++++++++++++++++++--
 fs/btrfs/volumes.h |  1 +
 3 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 638ce3f4abe3..e142a31c62e7 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -832,7 +832,8 @@ static int btrfs_strmatch(const char *given, const char *golden)
 	return -EINVAL;
 }
 
-static const char* const btrfs_read_policy_name[] = { "pid" };
+/* Must follow the order as in enum btrfs_read_policy */
+static const char* const btrfs_read_policy_name[] = { "pid", "device" };
 
 static ssize_t btrfs_read_policy_show(struct kobject *kobj,
 				      struct kobj_attribute *a, char *buf)
@@ -857,6 +858,25 @@ static ssize_t btrfs_read_policy_show(struct kobject *kobj,
 	return ret;
 }
 
+static bool btrfs_has_read_pref_device(struct btrfs_fs_devices *fs_devices)
+{
+	struct btrfs_device *device;
+
+	/*
+	 * rcu is good enough. If read preferred device is deleted by another
+	 * thread, the find_live_mirror will reset the policy to default.
+	 */
+	rcu_read_lock();
+	list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
+		if (test_bit(BTRFS_DEV_STATE_READ_PREFERRED,
+			     &device->dev_state))
+			return true;
+	}
+	rcu_read_unlock();
+
+	return false;
+}
+
 static ssize_t btrfs_read_policy_store(struct kobject *kobj,
 				       struct kobj_attribute *a,
 				       const char *buf, size_t len)
@@ -866,6 +886,10 @@ static ssize_t btrfs_read_policy_store(struct kobject *kobj,
 
 	for (i = 0; i < BTRFS_NR_READ_POLICY; i++) {
 		if (btrfs_strmatch(buf, btrfs_read_policy_name[i]) == 0) {
+			if (i == BTRFS_READ_POLICY_DEVICE &&
+			    ! btrfs_has_read_pref_device(fs_devices))
+					return -EINVAL;
+
 			if (i != fs_devices->read_policy) {
 				fs_devices->read_policy = i;
 				btrfs_info(fs_devices->fs_info,
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index b6efb87bb0ae..53020699c6cc 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -5341,6 +5341,25 @@ int btrfs_is_parity_mirror(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
 	return ret;
 }
 
+static int btrfs_find_read_preferred(struct map_lookup *map, int num_stripe)
+{
+	int i;
+
+	/*
+	 * If there are more than one read preferred devices, then just pick the
+	 * first found read preferred device as of now. Once we have the Qdepth
+	 * based device selection, we could pick the least busy device among the
+	 * read preferred devices.
+	 */
+	for (i = 0; i < num_stripe; i++) {
+		if (test_bit(BTRFS_DEV_STATE_READ_PREFERRED,
+			     &map->stripes[i].dev->dev_state))
+			return i;
+        }
+
+	return -EINVAL;
+}
+
 static int find_live_mirror(struct btrfs_fs_info *fs_info,
 			    struct map_lookup *map, int first,
 			    int dev_replace_is_ongoing)
@@ -5360,15 +5379,28 @@ static int find_live_mirror(struct btrfs_fs_info *fs_info,
 		num_stripes = map->num_stripes;
 
 	switch (fs_info->fs_devices->read_policy) {
+	case BTRFS_READ_POLICY_DEVICE:
+		preferred_mirror = btrfs_find_read_preferred(map, num_stripes);
+		if (preferred_mirror != -EINVAL) {
+			preferred_mirror = first + preferred_mirror;
+			break;
+		}
+		/* Reset the read_policy to the default */
+		fs_info->fs_devices->read_policy = BTRFS_READ_POLICY_PID;
+		btrfs_warn_rl(fs_info,
+		      "No read preferred device found, fallback to default");
+		/* Fall through if there isn't any read preferred device */
+	case BTRFS_READ_POLICY_PID:
+		preferred_mirror = first + current->pid % num_stripes;
+		break;
 	default:
 		/*
 		 * Shouldn't happen, just warn and use pid instead of failing.
 		 */
+		preferred_mirror = first + current->pid % num_stripes;
 		btrfs_warn_rl(fs_info,
 			      "unknown read_policy type %u, fallback to pid",
 			      fs_info->fs_devices->read_policy);
-	case BTRFS_READ_POLICY_PID:
-		preferred_mirror = first + current->pid % num_stripes;
 	}
 
 	if (dev_replace_is_ongoing &&
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index 07962a0ce898..9c3c6ba7aad5 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -216,6 +216,7 @@ struct btrfs_device {
  */
 enum btrfs_read_policy {
 	BTRFS_READ_POLICY_PID,
+	BTRFS_READ_POLICY_DEVICE,
 	BTRFS_NR_READ_POLICY,
 };
 
-- 
1.8.3.1