[PATCH 0/2] xfs: help mkfs shed its AG initialization code

[PATCH 0/2] xfs: help mkfs shed its AG initialization code

[Darrick J. Wong]

Hi all,

In this series, we start by adapting the libxfs AG construction code to
be aware that an internal log can be placed with an AG that is being
initialized.  This is necessary to refactor mkfs to use the AG
construction set instead of its own open-coded initialization work.

In userspace, the next thing we have to do is to fix the uncached buffer
code so that libxfs_putbuf won't try to suck them into the buffer cache;
and then fix delwri_{queue,submit} so that IO errors are returned by the
submit function.

The final patch in the xfsprogs series replaces all of mkfs' AG
initialization functions with a single call to the functions in libxfs.

If you're going to start using this mess, you probably ought to just
pull from my git trees, which are linked below.

This is an extraordinary way to destroy everything.  Enjoy!
Comments and questions are, as always, welcome.

--D

kernel git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfs-linux.git/log/?h=mkfs-refactor

xfsprogs git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfsprogs-dev.git/log/?h=mkfs-refactor

[PATCH 1/2] xfs: refactor free space btree record initialization

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

Refactor the code that populates the free space btrees of a new AG so
that we can avoid code duplication once things start getting
complicated.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 fs/xfs/libxfs/xfs_ag.c |   27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)


diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c
index 5efb82744664..80a3df7ccab3 100644
--- a/fs/xfs/libxfs/xfs_ag.c
+++ b/fs/xfs/libxfs/xfs_ag.c
@@ -56,37 +56,42 @@ xfs_btroot_init(
 	xfs_btree_init_block(mp, bp, id->type, 0, 0, id->agno);
 }
 
-/*
- * Alloc btree root block init functions
- */
+/* Finish initializing a free space btree. */
 static void
-xfs_bnoroot_init(
+xfs_freesp_init_recs(
 	struct xfs_mount	*mp,
 	struct xfs_buf		*bp,
 	struct aghdr_init_data	*id)
 {
 	struct xfs_alloc_rec	*arec;
 
-	xfs_btree_init_block(mp, bp, XFS_BTNUM_BNO, 0, 1, id->agno);
 	arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1);
 	arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks);
 	arec->ar_blockcount = cpu_to_be32(id->agsize -
 					  be32_to_cpu(arec->ar_startblock));
 }
 
+/*
+ * Alloc btree root block init functions
+ */
 static void
-xfs_cntroot_init(
+xfs_bnoroot_init(
 	struct xfs_mount	*mp,
 	struct xfs_buf		*bp,
 	struct aghdr_init_data	*id)
 {
-	struct xfs_alloc_rec	*arec;
+	xfs_btree_init_block(mp, bp, XFS_BTNUM_BNO, 0, 1, id->agno);
+	xfs_freesp_init_recs(mp, bp, id);
+}
 
+static void
+xfs_cntroot_init(
+	struct xfs_mount	*mp,
+	struct xfs_buf		*bp,
+	struct aghdr_init_data	*id)
+{
 	xfs_btree_init_block(mp, bp, XFS_BTNUM_CNT, 0, 1, id->agno);
-	arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1);
-	arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks);
-	arec->ar_blockcount = cpu_to_be32(id->agsize -
-					  be32_to_cpu(arec->ar_startblock));
+	xfs_freesp_init_recs(mp, bp, id);
 }
 
 /*

[PATCH 2/2] xfs: account for log space when formatting new AGs

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

When we're writing out a fresh new AG, make sure that we don't list an
internal log as free and that we create the rmap for the region.  growfs
never does this, but we will need it when we hook up mkfs.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 fs/xfs/libxfs/xfs_ag.c |   66 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)


diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c
index 80a3df7ccab3..57aa85d8e3aa 100644
--- a/fs/xfs/libxfs/xfs_ag.c
+++ b/fs/xfs/libxfs/xfs_ag.c
@@ -10,6 +10,7 @@
 #include "xfs_shared.h"
 #include "xfs_format.h"
 #include "xfs_trans_resv.h"
+#include "xfs_bit.h"
 #include "xfs_sb.h"
 #include "xfs_mount.h"
 #include "xfs_btree.h"
@@ -44,6 +45,12 @@ xfs_get_aghdr_buf(
 	return bp;
 }
 
+static inline bool is_log_ag(struct xfs_mount *mp, struct aghdr_init_data *id)
+{
+	return mp->m_sb.sb_logstart > 0 &&
+	       id->agno == XFS_FSB_TO_AGNO(mp, mp->m_sb.sb_logstart);
+}
+
 /*
  * Generic btree root block init function
  */
@@ -64,11 +71,50 @@ xfs_freesp_init_recs(
 	struct aghdr_init_data	*id)
 {
 	struct xfs_alloc_rec	*arec;
+	struct xfs_btree_block	*block = XFS_BUF_TO_BLOCK(bp);
 
 	arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1);
 	arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks);
+
+	if (is_log_ag(mp, id)) {
+		struct xfs_alloc_rec	*nrec;
+		xfs_agblock_t		start = XFS_FSB_TO_AGBNO(mp,
+							mp->m_sb.sb_logstart);
+
+		ASSERT(start >= mp->m_ag_prealloc_blocks);
+		if (start != mp->m_ag_prealloc_blocks) {
+			/*
+			 * Modify first record to pad stripe align of log
+			 */
+			arec->ar_blockcount = cpu_to_be32(start -
+						mp->m_ag_prealloc_blocks);
+			nrec = arec + 1;
+			/*
+			 * Insert second record at start of internal log
+			 * which then gets trimmed.
+			 */
+			nrec->ar_startblock = cpu_to_be32(
+					be32_to_cpu(arec->ar_startblock) +
+					be32_to_cpu(arec->ar_blockcount));
+			arec = nrec;
+			be16_add_cpu(&block->bb_numrecs, 1);
+		}
+		/*
+		 * Change record start to after the internal log
+		 */
+		be32_add_cpu(&arec->ar_startblock, mp->m_sb.sb_logblocks);
+	}
+
+	/*
+	 * Calculate the record block count and check for the case where
+	 * the log might have consumed all available space in the AG. If
+	 * so, reset the record count to 0 to avoid exposure of an invalid
+	 * record start block.
+	 */
 	arec->ar_blockcount = cpu_to_be32(id->agsize -
 					  be32_to_cpu(arec->ar_startblock));
+	if (!arec->ar_blockcount)
+		block->bb_numrecs = 0;
 }
 
 /*
@@ -154,6 +200,18 @@ xfs_rmaproot_init(
 		rrec->rm_offset = 0;
 		be16_add_cpu(&block->bb_numrecs, 1);
 	}
+
+	/* account for the log space */
+	if (is_log_ag(mp, id)) {
+		rrec = XFS_RMAP_REC_ADDR(block,
+				be16_to_cpu(block->bb_numrecs) + 1);
+		rrec->rm_startblock = cpu_to_be32(
+				XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart));
+		rrec->rm_blockcount = cpu_to_be32(mp->m_sb.sb_logblocks);
+		rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_LOG);
+		rrec->rm_offset = 0;
+		be16_add_cpu(&block->bb_numrecs, 1);
+	}
 }
 
 /*
@@ -214,6 +272,14 @@ xfs_agfblock_init(
 		agf->agf_refcount_level = cpu_to_be32(1);
 		agf->agf_refcount_blocks = cpu_to_be32(1);
 	}
+
+	if (is_log_ag(mp, id)) {
+		int64_t	logblocks = mp->m_sb.sb_logblocks;
+
+		be32_add_cpu(&agf->agf_freeblks, -logblocks);
+		agf->agf_longest = cpu_to_be32(id->agsize -
+			XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart) - logblocks);
+	}
 }
 
 static void

Re: [PATCH 1/2] xfs: refactor free space btree record initialization

[Allison Collins]

On 6/20/19 11:42 AM, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Refactor the code that populates the free space btrees of a new AG so
> that we can avoid code duplication once things start getting
> complicated.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Looks fine
Reviewed-by: Allison Collins <allison.henderson@oracle.com>

> ---
>   fs/xfs/libxfs/xfs_ag.c |   27 ++++++++++++++++-----------
>   1 file changed, 16 insertions(+), 11 deletions(-)
> 
> 
> diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c
> index 5efb82744664..80a3df7ccab3 100644
> --- a/fs/xfs/libxfs/xfs_ag.c
> +++ b/fs/xfs/libxfs/xfs_ag.c
> @@ -56,37 +56,42 @@ xfs_btroot_init(
>   	xfs_btree_init_block(mp, bp, id->type, 0, 0, id->agno);
>   }
>   
> -/*
> - * Alloc btree root block init functions
> - */
> +/* Finish initializing a free space btree. */
>   static void
> -xfs_bnoroot_init(
> +xfs_freesp_init_recs(
>   	struct xfs_mount	*mp,
>   	struct xfs_buf		*bp,
>   	struct aghdr_init_data	*id)
>   {
>   	struct xfs_alloc_rec	*arec;
>   
> -	xfs_btree_init_block(mp, bp, XFS_BTNUM_BNO, 0, 1, id->agno);
>   	arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1);
>   	arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks);
>   	arec->ar_blockcount = cpu_to_be32(id->agsize -
>   					  be32_to_cpu(arec->ar_startblock));
>   }
>   
> +/*
> + * Alloc btree root block init functions
> + */
>   static void
> -xfs_cntroot_init(
> +xfs_bnoroot_init(
>   	struct xfs_mount	*mp,
>   	struct xfs_buf		*bp,
>   	struct aghdr_init_data	*id)
>   {
> -	struct xfs_alloc_rec	*arec;
> +	xfs_btree_init_block(mp, bp, XFS_BTNUM_BNO, 0, 1, id->agno);
> +	xfs_freesp_init_recs(mp, bp, id);
> +}
>   
> +static void
> +xfs_cntroot_init(
> +	struct xfs_mount	*mp,
> +	struct xfs_buf		*bp,
> +	struct aghdr_init_data	*id)
> +{
>   	xfs_btree_init_block(mp, bp, XFS_BTNUM_CNT, 0, 1, id->agno);
> -	arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1);
> -	arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks);
> -	arec->ar_blockcount = cpu_to_be32(id->agsize -
> -					  be32_to_cpu(arec->ar_startblock));
> +	xfs_freesp_init_recs(mp, bp, id);
>   }
>   
>   /*
> 

Re: [PATCH 2/2] xfs: account for log space when formatting new AGs

[Allison Collins]

On 6/20/19 11:43 AM, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> When we're writing out a fresh new AG, make sure that we don't list an
> internal log as free and that we create the rmap for the region.  growfs
> never does this, but we will need it when we hook up mkfs.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Looks ok to me.  Thanks for the comments they help!
Reviewed-by: Allison Collins <allison.henderson@oracle.com>

> ---
>   fs/xfs/libxfs/xfs_ag.c |   66 ++++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 66 insertions(+)
> 
> 
> diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c
> index 80a3df7ccab3..57aa85d8e3aa 100644
> --- a/fs/xfs/libxfs/xfs_ag.c
> +++ b/fs/xfs/libxfs/xfs_ag.c
> @@ -10,6 +10,7 @@
>   #include "xfs_shared.h"
>   #include "xfs_format.h"
>   #include "xfs_trans_resv.h"
> +#include "xfs_bit.h"
>   #include "xfs_sb.h"
>   #include "xfs_mount.h"
>   #include "xfs_btree.h"
> @@ -44,6 +45,12 @@ xfs_get_aghdr_buf(
>   	return bp;
>   }
>   
> +static inline bool is_log_ag(struct xfs_mount *mp, struct aghdr_init_data *id)
> +{
> +	return mp->m_sb.sb_logstart > 0 &&
> +	       id->agno == XFS_FSB_TO_AGNO(mp, mp->m_sb.sb_logstart);
> +}
> +
>   /*
>    * Generic btree root block init function
>    */
> @@ -64,11 +71,50 @@ xfs_freesp_init_recs(
>   	struct aghdr_init_data	*id)
>   {
>   	struct xfs_alloc_rec	*arec;
> +	struct xfs_btree_block	*block = XFS_BUF_TO_BLOCK(bp);
>   
>   	arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1);
>   	arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks);
> +
> +	if (is_log_ag(mp, id)) {
> +		struct xfs_alloc_rec	*nrec;
> +		xfs_agblock_t		start = XFS_FSB_TO_AGBNO(mp,
> +							mp->m_sb.sb_logstart);
> +
> +		ASSERT(start >= mp->m_ag_prealloc_blocks);
> +		if (start != mp->m_ag_prealloc_blocks) {
> +			/*
> +			 * Modify first record to pad stripe align of log
> +			 */
> +			arec->ar_blockcount = cpu_to_be32(start -
> +						mp->m_ag_prealloc_blocks);
> +			nrec = arec + 1;
> +			/*
> +			 * Insert second record at start of internal log
> +			 * which then gets trimmed.
> +			 */
> +			nrec->ar_startblock = cpu_to_be32(
> +					be32_to_cpu(arec->ar_startblock) +
> +					be32_to_cpu(arec->ar_blockcount));
> +			arec = nrec;
> +			be16_add_cpu(&block->bb_numrecs, 1);
> +		}
> +		/*
> +		 * Change record start to after the internal log
> +		 */
> +		be32_add_cpu(&arec->ar_startblock, mp->m_sb.sb_logblocks);
> +	}
> +
> +	/*
> +	 * Calculate the record block count and check for the case where
> +	 * the log might have consumed all available space in the AG. If
> +	 * so, reset the record count to 0 to avoid exposure of an invalid
> +	 * record start block.
> +	 */
>   	arec->ar_blockcount = cpu_to_be32(id->agsize -
>   					  be32_to_cpu(arec->ar_startblock));
> +	if (!arec->ar_blockcount)
> +		block->bb_numrecs = 0;
>   }
>   
>   /*
> @@ -154,6 +200,18 @@ xfs_rmaproot_init(
>   		rrec->rm_offset = 0;
>   		be16_add_cpu(&block->bb_numrecs, 1);
>   	}
> +
> +	/* account for the log space */
> +	if (is_log_ag(mp, id)) {
> +		rrec = XFS_RMAP_REC_ADDR(block,
> +				be16_to_cpu(block->bb_numrecs) + 1);
> +		rrec->rm_startblock = cpu_to_be32(
> +				XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart));
> +		rrec->rm_blockcount = cpu_to_be32(mp->m_sb.sb_logblocks);
> +		rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_LOG);
> +		rrec->rm_offset = 0;
> +		be16_add_cpu(&block->bb_numrecs, 1);
> +	}
>   }
>   
>   /*
> @@ -214,6 +272,14 @@ xfs_agfblock_init(
>   		agf->agf_refcount_level = cpu_to_be32(1);
>   		agf->agf_refcount_blocks = cpu_to_be32(1);
>   	}
> +
> +	if (is_log_ag(mp, id)) {
> +		int64_t	logblocks = mp->m_sb.sb_logblocks;
> +
> +		be32_add_cpu(&agf->agf_freeblks, -logblocks);
> +		agf->agf_longest = cpu_to_be32(id->agsize -
> +			XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart) - logblocks);
> +	}
>   }
>   
>   static void
>