[PATCH 01/10] string: introduce memweight

[PATCH 01/10] string: introduce memweight

[Akinobu Mita]

memweight() is the function that counts the total number of bits set
in memory area.  The memory area doesn't need to be aligned to
long-word boundary unlike bitmap_weight().

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Anders Larsen <al@alarsen.net>
Cc: Alasdair Kergon <agk@redhat.com>
Cc: dm-devel@redhat.com
Cc: linux-fsdevel@vger.kernel.org
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: linux-media@vger.kernel.org
Cc: Mark Fasheh <mfasheh@suse.com>
Cc: Joel Becker <jlbec@evilplan.org>
Cc: ocfs2-devel@oss.oracle.com
Cc: Jan Kara <jack@suse.cz>
Cc: linux-ext4@vger.kernel.org
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andreas Dilger <adilger.kernel@dilger.ca>
Cc: "Theodore Ts'o" <tytso@mit.edu>
---
 include/linux/string.h |    3 +++
 lib/string.c           |   37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 0 deletions(-)

diff --git a/include/linux/string.h b/include/linux/string.h
index e033564..ffe0442 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -145,4 +145,7 @@ static inline bool strstarts(const char *str, const char *prefix)
 	return strncmp(str, prefix, strlen(prefix)) == 0;
 }
 #endif
+
+extern size_t memweight(const void *ptr, size_t bytes);
+
 #endif /* _LINUX_STRING_H_ */
diff --git a/lib/string.c b/lib/string.c
index e5878de..c8b92a0 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -26,6 +26,7 @@
 #include <linux/export.h>
 #include <linux/bug.h>
 #include <linux/errno.h>
+#include <linux/bitmap.h>
 
 #ifndef __HAVE_ARCH_STRNICMP
 /**
@@ -824,3 +825,39 @@ void *memchr_inv(const void *start, int c, size_t bytes)
 	return check_bytes8(start, value, bytes % 8);
 }
 EXPORT_SYMBOL(memchr_inv);
+
+/**
+ * memweight - count the total number of bits set in memory area
+ * @ptr: pointer to the start of the area
+ * @bytes: the size of the area
+ */
+size_t memweight(const void *ptr, size_t bytes)
+{
+	size_t w = 0;
+	size_t longs;
+	union {
+		const void *ptr;
+		const unsigned char *b;
+		unsigned long address;
+	} bitmap;
+
+	for (bitmap.ptr = ptr; bytes > 0 && bitmap.address % sizeof(long);
+			bytes--, bitmap.address++)
+		w += hweight8(*bitmap.b);
+
+	for (longs = bytes / sizeof(long); longs > 0; ) {
+		size_t bits = min_t(size_t, INT_MAX & ~(BITS_PER_LONG - 1),
+					longs * BITS_PER_LONG);
+
+		w += bitmap_weight(bitmap.ptr, bits);
+		bytes -= bits / BITS_PER_BYTE;
+		bitmap.address += bits / BITS_PER_BYTE;
+		longs -= bits / BITS_PER_LONG;
+	}
+
+	for (; bytes > 0; bytes--, bitmap.address++)
+		w += hweight8(*bitmap.b);
+
+	return w;
+}
+EXPORT_SYMBOL(memweight);
-- 
1.7.7.6

[PATCH 05/10] affs: use memweight()

[Akinobu Mita]

Use memweight() to count the total number of bits set in memory area.

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: linux-fsdevel@vger.kernel.org
---
 fs/affs/bitmap.c |   28 ++--------------------------
 1 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/fs/affs/bitmap.c b/fs/affs/bitmap.c
index 3e26271..3dfdcd2 100644
--- a/fs/affs/bitmap.c
+++ b/fs/affs/bitmap.c
@@ -10,30 +10,6 @@
 #include <linux/slab.h>
 #include "affs.h"
 
-/* This is, of course, shamelessly stolen from fs/minix */
-
-static const int nibblemap[] = { 0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4 };
-
-static u32
-affs_count_free_bits(u32 blocksize, const void *data)
-{
-	const u32 *map;
-	u32 free;
-	u32 tmp;
-
-	map = data;
-	free = 0;
-	for (blocksize /= 4; blocksize > 0; blocksize--) {
-		tmp = *map++;
-		while (tmp) {
-			free += nibblemap[tmp & 0xf];
-			tmp >>= 4;
-		}
-	}
-
-	return free;
-}
-
 u32
 affs_count_free_blocks(struct super_block *sb)
 {
@@ -317,7 +293,7 @@ int affs_init_bitmap(struct super_block *sb, int *flags)
 			goto out;
 		}
 		pr_debug("AFFS: read bitmap block %d: %d\n", blk, bm->bm_key);
-		bm->bm_free = affs_count_free_bits(sb->s_blocksize - 4, bh->b_data + 4);
+		bm->bm_free = memweight(bh->b_data + 4, sb->s_blocksize - 4);
 
 		/* Don't try read the extension if this is the last block,
 		 * but we also need the right bm pointer below
@@ -367,7 +343,7 @@ int affs_init_bitmap(struct super_block *sb, int *flags)
 
 	/* recalculate bitmap count for last block */
 	bm--;
-	bm->bm_free = affs_count_free_bits(sb->s_blocksize - 4, bh->b_data + 4);
+	bm->bm_free = memweight(bh->b_data + 4, sb->s_blocksize - 4);
 
 out:
 	affs_brelse(bh);
-- 
1.7.7.6

Re: [PATCH 01/10] string: introduce memweight

[Jan Kara]

On Sun 20-05-12 22:23:14, Akinobu Mita wrote:
> memweight() is the function that counts the total number of bits set
> in memory area.  The memory area doesn't need to be aligned to
> long-word boundary unlike bitmap_weight().
  Thanks for the patch. I have some comments below.

> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
> Cc: Anders Larsen <al@alarsen.net>
> Cc: Alasdair Kergon <agk@redhat.com>
> Cc: dm-devel@redhat.com
> Cc: linux-fsdevel@vger.kernel.org
> Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
> Cc: linux-media@vger.kernel.org
> Cc: Mark Fasheh <mfasheh@suse.com>
> Cc: Joel Becker <jlbec@evilplan.org>
> Cc: ocfs2-devel@oss.oracle.com
> Cc: Jan Kara <jack@suse.cz>
> Cc: linux-ext4@vger.kernel.org
> Cc: Andrew Morton <akpm@linux-foundation.org>
> Cc: Andreas Dilger <adilger.kernel@dilger.ca>
> Cc: "Theodore Ts'o" <tytso@mit.edu>
> ---
>  include/linux/string.h |    3 +++
>  lib/string.c           |   37 +++++++++++++++++++++++++++++++++++++
>  2 files changed, 40 insertions(+), 0 deletions(-)
> 
> diff --git a/include/linux/string.h b/include/linux/string.h
> index e033564..ffe0442 100644
> --- a/include/linux/string.h
> +++ b/include/linux/string.h
> @@ -145,4 +145,7 @@ static inline bool strstarts(const char *str, const char *prefix)
>  	return strncmp(str, prefix, strlen(prefix)) == 0;
>  }
>  #endif
> +
> +extern size_t memweight(const void *ptr, size_t bytes);
> +
>  #endif /* _LINUX_STRING_H_ */
> diff --git a/lib/string.c b/lib/string.c
> index e5878de..c8b92a0 100644
> --- a/lib/string.c
> +++ b/lib/string.c
> @@ -26,6 +26,7 @@
>  #include <linux/export.h>
>  #include <linux/bug.h>
>  #include <linux/errno.h>
> +#include <linux/bitmap.h>
>  
>  #ifndef __HAVE_ARCH_STRNICMP
>  /**
> @@ -824,3 +825,39 @@ void *memchr_inv(const void *start, int c, size_t bytes)
>  	return check_bytes8(start, value, bytes % 8);
>  }
>  EXPORT_SYMBOL(memchr_inv);
> +
> +/**
> + * memweight - count the total number of bits set in memory area
> + * @ptr: pointer to the start of the area
> + * @bytes: the size of the area
> + */
> +size_t memweight(const void *ptr, size_t bytes)
> +{
> +	size_t w = 0;
> +	size_t longs;
> +	union {
> +		const void *ptr;
> +		const unsigned char *b;
> +		unsigned long address;
> +	} bitmap;
  Ugh, this is ugly and mostly unnecessary. Just use "const unsigned char
*bitmap".

> +
> +	for (bitmap.ptr = ptr; bytes > 0 && bitmap.address % sizeof(long);
> +			bytes--, bitmap.address++)
> +		w += hweight8(*bitmap.b);
  This can be:
	count = ((unsigned long)bitmap) % sizeof(long);
	while (count--) {
		w += hweight(*bitmap);
		bitmap++;
		bytes--;
	}
> +
> +	for (longs = bytes / sizeof(long); longs > 0; ) {
> +		size_t bits = min_t(size_t, INT_MAX & ~(BITS_PER_LONG - 1),
> +					longs * BITS_PER_LONG);
  I find it highly unlikely that someone would have such a large bitmap
(256 MB or more on 32-bit). Also the condition as you wrote it can just
overflow so it won't have the desired effect. Just do
	BUG_ON(longs >= ULONG_MAX / BITS_PER_LONG);
and remove the loop completely. If someone comes with such a huge bitmap,
the code can be modified easily (after really closely inspecting whether
such a huge bitmap is really well justified).

> +
> +		w += bitmap_weight(bitmap.ptr, bits);
> +		bytes -= bits / BITS_PER_BYTE;
> +		bitmap.address += bits / BITS_PER_BYTE;
> +		longs -= bits / BITS_PER_LONG;
> +	}
> +
> +	for (; bytes > 0; bytes--, bitmap.address++)
> +		w += hweight8(*bitmap.b);
> +
> +	return w;
> +}
> +EXPORT_SYMBOL(memweight);

									Honza
-- 
Jan Kara <jack@suse.cz>
SUSE Labs, CR

Re: [PATCH 01/10] string: introduce memweight

[Akinobu Mita]

2012/5/23 Jan Kara <jack@suse.cz>:
> On Sun 20-05-12 22:23:14, Akinobu Mita wrote:
>> memweight() is the function that counts the total number of bits set
>> in memory area.  The memory area doesn't need to be aligned to
>> long-word boundary unlike bitmap_weight().
>  Thanks for the patch. I have some comments below.

Thanks for the review.

>> @@ -824,3 +825,39 @@ void *memchr_inv(const void *start, int c, size_t bytes)
>>       return check_bytes8(start, value, bytes % 8);
>>  }
>>  EXPORT_SYMBOL(memchr_inv);
>> +
>> +/**
>> + * memweight - count the total number of bits set in memory area
>> + * @ptr: pointer to the start of the area
>> + * @bytes: the size of the area
>> + */
>> +size_t memweight(const void *ptr, size_t bytes)
>> +{
>> +     size_t w = 0;
>> +     size_t longs;
>> +     union {
>> +             const void *ptr;
>> +             const unsigned char *b;
>> +             unsigned long address;
>> +     } bitmap;
>  Ugh, this is ugly and mostly unnecessary. Just use "const unsigned char
> *bitmap".
>
>> +
>> +     for (bitmap.ptr = ptr; bytes > 0 && bitmap.address % sizeof(long);
>> +                     bytes--, bitmap.address++)
>> +             w += hweight8(*bitmap.b);
>  This can be:
>        count = ((unsigned long)bitmap) % sizeof(long);

The count should be the size of unaligned area and it can be greater than
bytes. So

        count = min(bytes,
                    sizeof(long) - ((unsigned long)bitmap) % sizeof(long));

>        while (count--) {
>                w += hweight(*bitmap);
>                bitmap++;
>                bytes--;
>        }
>> +
>> +     for (longs = bytes / sizeof(long); longs > 0; ) {
>> +             size_t bits = min_t(size_t, INT_MAX & ~(BITS_PER_LONG - 1),
>> +                                     longs * BITS_PER_LONG);
>  I find it highly unlikely that someone would have such a large bitmap
> (256 MB or more on 32-bit). Also the condition as you wrote it can just
> overflow so it won't have the desired effect. Just do
>        BUG_ON(longs >= ULONG_MAX / BITS_PER_LONG);

The bits argument of bitmap_weight() is int type. So this should be

        BUG_ON(longs >= INT_MAX / BITS_PER_LONG);

> and remove the loop completely. If someone comes with such a huge bitmap,
> the code can be modified easily (after really closely inspecting whether
> such a huge bitmap is really well justified).

size_t memweight(const void *ptr, size_t bytes)
{
	size_t w = 0;
	size_t longs;
	const unsigned char *bitmap = ptr;

	for (; bytes > 0 && ((unsigned long)bitmap) % sizeof(long);
			bytes--, bitmap++)
		w += hweight8(*bitmap);

	longs = bytes / sizeof(long);
	BUG_ON(longs >= INT_MAX / BITS_PER_LONG);
	w += bitmap_weight((unsigned long *)bitmap, longs * BITS_PER_LONG);
	bytes -= longs * sizeof(long);
	bitmap += longs * sizeof(long);

	for (; bytes > 0; bytes--, bitmap++)
		w += hweight8(*bitmap);

	return w;
}
--
To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 01/10] string: introduce memweight

[Jan Kara]

On Wed 23-05-12 21:12:18, Akinobu Mita wrote:
> 2012/5/23 Jan Kara <jack@suse.cz>:
> > On Sun 20-05-12 22:23:14, Akinobu Mita wrote:
> >> memweight() is the function that counts the total number of bits set
> >> in memory area.  The memory area doesn't need to be aligned to
> >> long-word boundary unlike bitmap_weight().
> >  Thanks for the patch. I have some comments below.
> 
> Thanks for the review.
> 
> >> @@ -824,3 +825,39 @@ void *memchr_inv(const void *start, int c, size_t bytes)
> >>       return check_bytes8(start, value, bytes % 8);
> >>  }
> >>  EXPORT_SYMBOL(memchr_inv);
> >> +
> >> +/**
> >> + * memweight - count the total number of bits set in memory area
> >> + * @ptr: pointer to the start of the area
> >> + * @bytes: the size of the area
> >> + */
> >> +size_t memweight(const void *ptr, size_t bytes)
> >> +{
> >> +     size_t w = 0;
> >> +     size_t longs;
> >> +     union {
> >> +             const void *ptr;
> >> +             const unsigned char *b;
> >> +             unsigned long address;
> >> +     } bitmap;
> >  Ugh, this is ugly and mostly unnecessary. Just use "const unsigned char
> > *bitmap".
> >
> >> +
> >> +     for (bitmap.ptr = ptr; bytes > 0 && bitmap.address % sizeof(long);
> >> +                     bytes--, bitmap.address++)
> >> +             w += hweight8(*bitmap.b);
> >  This can be:
> >        count = ((unsigned long)bitmap) % sizeof(long);
> 
> The count should be the size of unaligned area and it can be greater than
> bytes. So
> 
>         count = min(bytes,
>                     sizeof(long) - ((unsigned long)bitmap) % sizeof(long));
  You are right, I didn't quite think this through.
 
> >        while (count--) {
> >                w += hweight(*bitmap);
> >                bitmap++;
> >                bytes--;
> >        }
> >> +
> >> +     for (longs = bytes / sizeof(long); longs > 0; ) {
> >> +             size_t bits = min_t(size_t, INT_MAX & ~(BITS_PER_LONG - 1),
> >> +                                     longs * BITS_PER_LONG);
> >  I find it highly unlikely that someone would have such a large bitmap
> > (256 MB or more on 32-bit). Also the condition as you wrote it can just
> > overflow so it won't have the desired effect. Just do
> >        BUG_ON(longs >= ULONG_MAX / BITS_PER_LONG);
> 
> The bits argument of bitmap_weight() is int type. So this should be
> 
>         BUG_ON(longs >= INT_MAX / BITS_PER_LONG);
  OK, I didn't check and thought it's size_t.

> > and remove the loop completely. If someone comes with such a huge bitmap,
> > the code can be modified easily (after really closely inspecting whether
> > such a huge bitmap is really well justified).
> 
> size_t memweight(const void *ptr, size_t bytes)
> {
> 	size_t w = 0;
> 	size_t longs;
> 	const unsigned char *bitmap = ptr;
> 
> 	for (; bytes > 0 && ((unsigned long)bitmap) % sizeof(long);
> 			bytes--, bitmap++)
> 		w += hweight8(*bitmap);
> 
> 	longs = bytes / sizeof(long);
> 	BUG_ON(longs >= INT_MAX / BITS_PER_LONG);
> 	w += bitmap_weight((unsigned long *)bitmap, longs * BITS_PER_LONG);
> 	bytes -= longs * sizeof(long);
> 	bitmap += longs * sizeof(long);
> 
> 	for (; bytes > 0; bytes--, bitmap++)
> 		w += hweight8(*bitmap);
> 
> 	return w;
> }
  Yup, this looks much more readable. Thanks!

								Honza
  
-- 
Jan Kara <jack@suse.cz>
SUSE Labs, CR
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 01/10] string: introduce memweight

[Matthew Wilcox]

On Wed, May 23, 2012 at 09:12:18PM +0900, Akinobu Mita wrote:
> size_t memweight(const void *ptr, size_t bytes)

Why should this return size_t instead of unsigned long?

> {
> 	size_t w = 0;
> 	size_t longs;
> 	const unsigned char *bitmap = ptr;
> 
> 	for (; bytes > 0 && ((unsigned long)bitmap) % sizeof(long);
> 			bytes--, bitmap++)
> 		w += hweight8(*bitmap);
> 
> 	longs = bytes / sizeof(long);
> 	BUG_ON(longs >= INT_MAX / BITS_PER_LONG);
> 	w += bitmap_weight((unsigned long *)bitmap, longs * BITS_PER_LONG);
> 	bytes -= longs * sizeof(long);
> 	bitmap += longs * sizeof(long);
> 
> 	for (; bytes > 0; bytes--, bitmap++)
> 		w += hweight8(*bitmap);
> 
> 	return w;
> }

bitmap_weight copes with a bitmask that isn't a multiple of BITS_PER_LONG
in size already.  So I think this can be done as:

unsigned long memweight(const void *s, size_t n)
{
	const unsigned char *ptr = s;
	unsigned long r = 0;

	while (n > 0 && (unsigned long)ptr % sizeof(long)) {
		r += hweight8(*ptr);
		n--;
		ptr++;
	}

	BUG_ON(n >= INT_MAX / 8)

	return r + bitmap_weight((unsigned long *)ptr, n * 8);
}

-- 
Matthew Wilcox				Intel Open Source Technology Centre
"Bill, look, we understand that you're interested in selling us this
operating system, but compare it to ours.  We can't possibly take such
a retrograde step."

Re: [PATCH 01/10] string: introduce memweight

[Akinobu Mita]

2012/5/23 Matthew Wilcox <matthew@wil.cx>:
> On Wed, May 23, 2012 at 09:12:18PM +0900, Akinobu Mita wrote:
>> size_t memweight(const void *ptr, size_t bytes)
>
> Why should this return size_t instead of unsigned long?

I just use the same type as the bytes argument without mature
consideration.  If unsigned long is better than size_t, I'll
change the return type.

>> {
>>       size_t w = 0;
>>       size_t longs;
>>       const unsigned char *bitmap = ptr;
>>
>>       for (; bytes > 0 && ((unsigned long)bitmap) % sizeof(long);
>>                       bytes--, bitmap++)
>>               w += hweight8(*bitmap);
>>
>>       longs = bytes / sizeof(long);
>>       BUG_ON(longs >= INT_MAX / BITS_PER_LONG);
>>       w += bitmap_weight((unsigned long *)bitmap, longs * BITS_PER_LONG);
>>       bytes -= longs * sizeof(long);
>>       bitmap += longs * sizeof(long);
>>
>>       for (; bytes > 0; bytes--, bitmap++)
>>               w += hweight8(*bitmap);
>>
>>       return w;
>> }
>
> bitmap_weight copes with a bitmask that isn't a multiple of BITS_PER_LONG
> in size already.  So I think this can be done as:
>
> unsigned long memweight(const void *s, size_t n)
> {
>        const unsigned char *ptr = s;
>        unsigned long r = 0;
>
>        while (n > 0 && (unsigned long)ptr % sizeof(long)) {
>                r += hweight8(*ptr);
>                n--;
>                ptr++;
>        }
>
>        BUG_ON(n >= INT_MAX / 8)
>
>        return r + bitmap_weight((unsigned long *)ptr, n * 8);
> }

This works perfectly on little-endian machines.  But it doesn't work
on big-endian machines, if the bottom edge of memory area is not
aligned on long word boundary.
--
To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html