From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-eopbgr680112.outbound.protection.outlook.com ([40.107.68.112]:63837 "EHLO NAM04-BN3-obe.outbound.protection.outlook.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1729247AbeJAHOY (ORCPT ); Mon, 1 Oct 2018 03:14:24 -0400 From: Sasha Levin To: "stable@vger.kernel.org" , "linux-kernel@vger.kernel.org" CC: Joe Thornber , Mike Snitzer , Sasha Levin Subject: [PATCH AUTOSEL 4.18 48/65] dm thin metadata: try to avoid ever aborting transactions Date: Mon, 1 Oct 2018 00:38:32 +0000 Message-ID: <20181001003754.146961-48-alexander.levin@microsoft.com> References: <20181001003754.146961-1-alexander.levin@microsoft.com> In-Reply-To: <20181001003754.146961-1-alexander.levin@microsoft.com> Content-Language: en-US Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Sender: stable-owner@vger.kernel.org List-ID: From: Joe Thornber [ Upstream commit 3ab91828166895600efd9cdc3a0eb32001f7204a ] Committing a transaction can consume some metadata of it's own, we now reserve a small amount of metadata to cover this. Free metadata reported by the kernel will not include this reserve. If any of the reserve has been used after a commit we enter a new internal state PM_OUT_OF_METADATA_SPACE. This is reported as PM_READ_ONLY, so no userland changes are needed. If the metadata device is resized the pool will move back to PM_WRITE. These changes mean we never need to abort and rollback a transaction due to running out of metadata space. This is particularly important because there have been a handful of reports of data corruption against DM thin-provisioning that can all be attributed to the thin-pool having ran out of metadata space. Signed-off-by: Joe Thornber Signed-off-by: Mike Snitzer Signed-off-by: Sasha Levin --- drivers/md/dm-thin-metadata.c | 36 ++++++++++++++++- drivers/md/dm-thin.c | 73 +++++++++++++++++++++++++++++++---- 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c index 72142021b5c9..74f6770c70b1 100644 --- a/drivers/md/dm-thin-metadata.c +++ b/drivers/md/dm-thin-metadata.c @@ -188,6 +188,12 @@ struct dm_pool_metadata { unsigned long flags; sector_t data_block_size; =20 + /* + * We reserve a section of the metadata for commit overhead. + * All reported space does *not* include this. + */ + dm_block_t metadata_reserve; + /* * Set if a transaction has to be aborted but the attempt to roll back * to the previous (good) transaction failed. The only pool metadata @@ -816,6 +822,22 @@ static int __commit_transaction(struct dm_pool_metadat= a *pmd) return dm_tm_commit(pmd->tm, sblock); } =20 +static void __set_metadata_reserve(struct dm_pool_metadata *pmd) +{ + int r; + dm_block_t total; + dm_block_t max_blocks =3D 4096; /* 16M */ + + r =3D dm_sm_get_nr_blocks(pmd->metadata_sm, &total); + if (r) { + DMERR("could not get size of metadata device"); + pmd->metadata_reserve =3D max_blocks; + } else { + sector_div(total, 10); + pmd->metadata_reserve =3D min(max_blocks, total); + } +} + struct dm_pool_metadata *dm_pool_metadata_open(struct block_device *bdev, sector_t data_block_size, bool format_device) @@ -849,6 +871,8 @@ struct dm_pool_metadata *dm_pool_metadata_open(struct b= lock_device *bdev, return ERR_PTR(r); } =20 + __set_metadata_reserve(pmd); + return pmd; } =20 @@ -1820,6 +1844,13 @@ int dm_pool_get_free_metadata_block_count(struct dm_= pool_metadata *pmd, down_read(&pmd->root_lock); if (!pmd->fail_io) r =3D dm_sm_get_nr_free(pmd->metadata_sm, result); + + if (!r) { + if (*result < pmd->metadata_reserve) + *result =3D 0; + else + *result -=3D pmd->metadata_reserve; + } up_read(&pmd->root_lock); =20 return r; @@ -1932,8 +1963,11 @@ int dm_pool_resize_metadata_dev(struct dm_pool_metad= ata *pmd, dm_block_t new_cou int r =3D -EINVAL; =20 down_write(&pmd->root_lock); - if (!pmd->fail_io) + if (!pmd->fail_io) { r =3D __resize_space_map(pmd->metadata_sm, new_count); + if (!r) + __set_metadata_reserve(pmd); + } up_write(&pmd->root_lock); =20 return r; diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 1087f6a1ac79..b512efd4050c 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -200,7 +200,13 @@ struct dm_thin_new_mapping; enum pool_mode { PM_WRITE, /* metadata may be changed */ PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be = allocated */ + + /* + * Like READ_ONLY, except may switch back to WRITE on metadata resize. Re= ported as READ_ONLY. + */ + PM_OUT_OF_METADATA_SPACE, PM_READ_ONLY, /* metadata may not be changed */ + PM_FAIL, /* all I/O fails */ }; =20 @@ -1388,7 +1394,35 @@ static void set_pool_mode(struct pool *pool, enum po= ol_mode new_mode); =20 static void requeue_bios(struct pool *pool); =20 -static void check_for_space(struct pool *pool) +static bool is_read_only_pool_mode(enum pool_mode mode) +{ + return (mode =3D=3D PM_OUT_OF_METADATA_SPACE || mode =3D=3D PM_READ_ONLY)= ; +} + +static bool is_read_only(struct pool *pool) +{ + return is_read_only_pool_mode(get_pool_mode(pool)); +} + +static void check_for_metadata_space(struct pool *pool) +{ + int r; + const char *ooms_reason =3D NULL; + dm_block_t nr_free; + + r =3D dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free); + if (r) + ooms_reason =3D "Could not get free metadata blocks"; + else if (!nr_free) + ooms_reason =3D "No free metadata blocks"; + + if (ooms_reason && !is_read_only(pool)) { + DMERR("%s", ooms_reason); + set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE); + } +} + +static void check_for_data_space(struct pool *pool) { int r; dm_block_t nr_free; @@ -1414,14 +1448,16 @@ static int commit(struct pool *pool) { int r; =20 - if (get_pool_mode(pool) >=3D PM_READ_ONLY) + if (get_pool_mode(pool) >=3D PM_OUT_OF_METADATA_SPACE) return -EINVAL; =20 r =3D dm_pool_commit_metadata(pool->pmd); if (r) metadata_operation_failed(pool, "dm_pool_commit_metadata", r); - else - check_for_space(pool); + else { + check_for_metadata_space(pool); + check_for_data_space(pool); + } =20 return r; } @@ -1487,6 +1523,19 @@ static int alloc_data_block(struct thin_c *tc, dm_bl= ock_t *result) return r; } =20 + r =3D dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks); + if (r) { + metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count",= r); + return r; + } + + if (!free_blocks) { + /* Let's commit before we use up the metadata reserve. */ + r =3D commit(pool); + if (r) + return r; + } + return 0; } =20 @@ -1518,6 +1567,7 @@ static blk_status_t should_error_unserviceable_bio(st= ruct pool *pool) case PM_OUT_OF_DATA_SPACE: return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0; =20 + case PM_OUT_OF_METADATA_SPACE: case PM_READ_ONLY: case PM_FAIL: return BLK_STS_IOERR; @@ -2481,8 +2531,9 @@ static void set_pool_mode(struct pool *pool, enum poo= l_mode new_mode) error_retry_list(pool); break; =20 + case PM_OUT_OF_METADATA_SPACE: case PM_READ_ONLY: - if (old_mode !=3D new_mode) + if (!is_read_only_pool_mode(old_mode)) notify_of_pool_mode_change(pool, "read-only"); dm_pool_metadata_read_only(pool->pmd); pool->process_bio =3D process_bio_read_only; @@ -3420,6 +3471,10 @@ static int maybe_resize_metadata_dev(struct dm_targe= t *ti, bool *need_commit) DMINFO("%s: growing the metadata device from %llu to %llu blocks", dm_device_name(pool->pool_md), sb_metadata_dev_size, metadata_dev_size); + + if (get_pool_mode(pool) =3D=3D PM_OUT_OF_METADATA_SPACE) + set_pool_mode(pool, PM_WRITE); + r =3D dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size); if (r) { metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r); @@ -3724,7 +3779,7 @@ static int pool_message(struct dm_target *ti, unsigne= d argc, char **argv, struct pool_c *pt =3D ti->private; struct pool *pool =3D pt->pool; =20 - if (get_pool_mode(pool) >=3D PM_READ_ONLY) { + if (get_pool_mode(pool) >=3D PM_OUT_OF_METADATA_SPACE) { DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL m= ode", dm_device_name(pool->pool_md)); return -EOPNOTSUPP; @@ -3798,6 +3853,7 @@ static void pool_status(struct dm_target *ti, status_= type_t type, dm_block_t nr_blocks_data; dm_block_t nr_blocks_metadata; dm_block_t held_root; + enum pool_mode mode; char buf[BDEVNAME_SIZE]; char buf2[BDEVNAME_SIZE]; struct pool_c *pt =3D ti->private; @@ -3868,9 +3924,10 @@ static void pool_status(struct dm_target *ti, status= _type_t type, else DMEMIT("- "); =20 - if (pool->pf.mode =3D=3D PM_OUT_OF_DATA_SPACE) + mode =3D get_pool_mode(pool); + if (mode =3D=3D PM_OUT_OF_DATA_SPACE) DMEMIT("out_of_data_space "); - else if (pool->pf.mode =3D=3D PM_READ_ONLY) + else if (is_read_only_pool_mode(mode)) DMEMIT("ro "); else DMEMIT("rw "); --=20 2.17.1