* [RFC PATCH 6/8] [NET]: uninline skb_trim, de-bloats
From: Ilpo Järvinen @ 2008-02-20 13:47 UTC (permalink / raw)
To: netdev, linux-kernel
Cc: David Miller, Arnaldo Carvalho de Melo, Ilpo J�rvinen
In-Reply-To: <1203515238-22848-6-git-send-email-ilpo.jarvinen@helsinki.fi>
-10976 209 funcs, 123 +, 11099 -, diff: -10976 --- skb_trim
Signed-off-by: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
---
include/linux/skbuff.h | 16 +---------------
net/core/skbuff.c | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index c11f248..75d8a66 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1156,21 +1156,7 @@ static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
skb_set_tail_pointer(skb, len);
}
-/**
- * skb_trim - remove end from a buffer
- * @skb: buffer to alter
- * @len: new length
- *
- * Cut the length of a buffer down by removing data from the tail. If
- * the buffer is already under the length specified it is not modified.
- * The skb must be linear.
- */
-static inline void skb_trim(struct sk_buff *skb, unsigned int len)
-{
- if (skb->len > len)
- __skb_trim(skb, len);
-}
-
+extern void skb_trim(struct sk_buff *skb, unsigned int len);
static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
{
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 05d43fd..b57cadb 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -931,6 +931,22 @@ unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
}
EXPORT_SYMBOL(skb_pull);
+/**
+ * skb_trim - remove end from a buffer
+ * @skb: buffer to alter
+ * @len: new length
+ *
+ * Cut the length of a buffer down by removing data from the tail. If
+ * the buffer is already under the length specified it is not modified.
+ * The skb must be linear.
+ */
+void skb_trim(struct sk_buff *skb, unsigned int len)
+{
+ if (skb->len > len)
+ __skb_trim(skb, len);
+}
+EXPORT_SYMBOL(skb_trim);
+
/* Trims skb to length len. It can change skb pointers.
*/
--
1.5.2.2
^ permalink raw reply related
* [RFC PATCH 2/8] [NET]: uninline skb_pull, de-bloats a lot
From: Ilpo Järvinen @ 2008-02-20 13:47 UTC (permalink / raw)
To: netdev, linux-kernel
Cc: David Miller, Arnaldo Carvalho de Melo, Ilpo J�rvinen
In-Reply-To: <1203515238-22848-2-git-send-email-ilpo.jarvinen@helsinki.fi>
-28162 354 funcs, 3005 +, 31167 -, diff: -28162 --- skb_pull
Signed-off-by: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
---
include/linux/skbuff.h | 15 +--------------
net/core/skbuff.c | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 5925435..a9f8f15 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -930,20 +930,7 @@ static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
return skb->data += len;
}
-/**
- * skb_pull - remove data from the start of a buffer
- * @skb: buffer to use
- * @len: amount of data to remove
- *
- * This function removes data from the start of a buffer, returning
- * the memory to the headroom. A pointer to the next data in the buffer
- * is returned. Once the data has been pulled future pushes will overwrite
- * the old data.
- */
-static inline unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
-{
- return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
-}
+extern unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 661d439..14f462b 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -878,6 +878,22 @@ unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
}
EXPORT_SYMBOL(skb_put);
+/**
+ * skb_pull - remove data from the start of a buffer
+ * @skb: buffer to use
+ * @len: amount of data to remove
+ *
+ * This function removes data from the start of a buffer, returning
+ * the memory to the headroom. A pointer to the next data in the buffer
+ * is returned. Once the data has been pulled future pushes will overwrite
+ * the old data.
+ */
+unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
+{
+ return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
+}
+EXPORT_SYMBOL(skb_pull);
+
/* Trims skb to length len. It can change skb pointers.
*/
--
1.5.2.2
^ permalink raw reply related
* [RFC PATCH 7/8] [SCTP]: uninline sctp_add_cmd_sf
From: Ilpo Järvinen @ 2008-02-20 13:47 UTC (permalink / raw)
To: netdev, linux-kernel
Cc: David Miller, Arnaldo Carvalho de Melo, Ilpo J�rvinen,
Vlad Yasevich
In-Reply-To: <1203515238-22848-7-git-send-email-ilpo.jarvinen@helsinki.fi>
I added inline to sctp_add_cmd and appropriate comment there to
avoid adding another call into the call chain. This works at least
with "gcc (GCC) 4.1.2 20070626 (Red Hat 4.1.2-13)". Alternatively,
__sctp_add_cmd could be introduced to .h.
net/sctp/sm_statefuns.c:
sctp_sf_cookie_wait_prm_abort | -125
sctp_sf_cookie_wait_prm_shutdown | -75
sctp_sf_do_9_1_prm_abort | -75
sctp_sf_shutdown_sent_prm_abort | -50
sctp_sf_pdiscard | -25
sctp_stop_t1_and_abort | -100
sctp_sf_do_9_2_start_shutdown | -154
__sctp_sf_do_9_1_abort | -50
sctp_send_stale_cookie_err | -29
sctp_sf_abort_violation | -181
sctp_sf_do_9_2_shutdown_ack | -154
sctp_sf_do_9_2_reshutack | -86
sctp_sf_tabort_8_4_8 | -28
sctp_sf_heartbeat | -52
sctp_sf_shut_8_4_5 | -27
sctp_eat_data | -246
sctp_sf_shutdown_sent_abort | -58
sctp_sf_check_restart_addrs | -50
sctp_sf_do_unexpected_init | -110
sctp_sf_sendbeat_8_3 | -107
sctp_sf_unk_chunk | -65
sctp_sf_do_prm_asoc | -129
sctp_sf_do_prm_send | -25
sctp_sf_do_9_2_prm_shutdown | -50
sctp_sf_error_closed | -25
sctp_sf_error_shutdown | -25
sctp_sf_shutdown_pending_prm_abort | -25
sctp_sf_do_prm_requestheartbeat | -28
sctp_sf_do_prm_asconf | -75
sctp_sf_do_6_3_3_rtx | -104
sctp_sf_do_6_2_sack | -25
sctp_sf_t1_init_timer_expire | -133
sctp_sf_t1_cookie_timer_expire | -104
sctp_sf_t2_timer_expire | -161
sctp_sf_t4_timer_expire | -175
sctp_sf_t5_timer_expire | -75
sctp_sf_autoclose_timer_expire | -50
sctp_sf_do_5_2_4_dupcook | -579
sctp_sf_do_4_C | -125
sctp_sf_shutdown_pending_abort | -32
sctp_sf_do_5_1E_ca | -186
sctp_sf_backbeat_8_3 | -27
sctp_sf_cookie_echoed_err | -300
sctp_sf_eat_data_6_2 | -146
sctp_sf_eat_data_fast_4_4 | -125
sctp_sf_eat_sack_6_2 | -29
sctp_sf_operr_notify | -25
sctp_sf_do_9_2_final | -152
sctp_sf_do_asconf | -64
sctp_sf_do_asconf_ack | -284
sctp_sf_eat_fwd_tsn_fast | -160
sctp_sf_eat_auth | -86
sctp_sf_do_5_1B_init | -110
sctp_sf_do_5_1C_ack | -204
sctp_sf_do_9_2_shutdown | -78
sctp_sf_do_ecn_cwr | -24
sctp_sf_do_ecne | -32
sctp_sf_eat_fwd_tsn | -135
sctp_sf_do_5_1D_ce | -197
sctp_sf_beat_8_3 | -28
60 functions changed, 6184 bytes removed, diff: -6184
net/sctp/sm_sideeffect.c:
sctp_side_effects | -3873
sctp_do_sm | +3429
2 functions changed, 3429 bytes added, 3873 bytes removed, diff: -444
kernel/uninlined.c:
sctp_add_cmd_sf | +35
1 function changed, 35 bytes added, diff: +35
vmlinux.o:
63 functions changed, 3464 bytes added, 10057 bytes removed, diff: -6593
Signed-off-by: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
Cc: Vlad Yasevich <vladislav.yasevich@hp.com>
---
include/net/sctp/sm.h | 8 ++------
net/sctp/command.c | 12 +++++++++++-
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/include/net/sctp/sm.h b/include/net/sctp/sm.h
index ef9e7ed..6740b11 100644
--- a/include/net/sctp/sm.h
+++ b/include/net/sctp/sm.h
@@ -385,13 +385,9 @@ static inline int ADDIP_SERIAL_gte(__u16 s, __u16 t)
return (((s) == (t)) || (((t) - (s)) & ADDIP_SERIAL_SIGN_BIT));
}
-
/* Run sctp_add_cmd() generating a BUG() if there is a failure. */
-static inline void sctp_add_cmd_sf(sctp_cmd_seq_t *seq, sctp_verb_t verb, sctp_arg_t obj)
-{
- if (unlikely(!sctp_add_cmd(seq, verb, obj)))
- BUG();
-}
+extern void sctp_add_cmd_sf(sctp_cmd_seq_t *seq, sctp_verb_t verb,
+ sctp_arg_t obj);
/* Check VTAG of the packet matches the sender's own tag. */
static inline int
diff --git a/net/sctp/command.c b/net/sctp/command.c
index bb97733..187da2d 100644
--- a/net/sctp/command.c
+++ b/net/sctp/command.c
@@ -51,8 +51,11 @@ int sctp_init_cmd_seq(sctp_cmd_seq_t *seq)
/* Add a command to a sctp_cmd_seq_t.
* Return 0 if the command sequence is full.
+ *
+ * Inline here is not a mistake, this way sctp_add_cmd_sf doesn't need extra
+ * calls, size penalty is of insignificant magnitude here
*/
-int sctp_add_cmd(sctp_cmd_seq_t *seq, sctp_verb_t verb, sctp_arg_t obj)
+inline int sctp_add_cmd(sctp_cmd_seq_t *seq, sctp_verb_t verb, sctp_arg_t obj)
{
if (seq->next_free_slot >= SCTP_MAX_NUM_COMMANDS)
goto fail;
@@ -66,6 +69,13 @@ fail:
return 0;
}
+/* Run sctp_add_cmd() generating a BUG() if there is a failure. */
+void sctp_add_cmd_sf(sctp_cmd_seq_t *seq, sctp_verb_t verb, sctp_arg_t obj)
+{
+ if (unlikely(!sctp_add_cmd(seq, verb, obj)))
+ BUG();
+}
+
/* Return the next command structure in a sctp_cmd_seq.
* Returns NULL at the end of the sequence.
*/
--
1.5.2.2
^ permalink raw reply related
* [RFC PATCH 8/8] Jhash in too big for inlining, move under lib/
From: Ilpo Järvinen @ 2008-02-20 13:47 UTC (permalink / raw)
To: netdev, linux-kernel
Cc: David Miller, Arnaldo Carvalho de Melo, Ilpo J�rvinen
In-Reply-To: <1203515238-22848-8-git-send-email-ilpo.jarvinen@helsinki.fi>
vmlinux.o:
62 functions changed, 66 bytes added, 10935 bytes removed, diff: -10869
...+ these to lib/jhash.o:
jhash_3words: 112
jhash2: 276
jhash: 475
select for networking code might need a more fine-grained approach.
Signed-off-by: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
---
drivers/infiniband/Kconfig | 1 +
drivers/net/Kconfig | 1 +
fs/Kconfig | 1 +
fs/dlm/Kconfig | 1 +
fs/gfs2/Kconfig | 1 +
include/linux/jhash.h | 99 +------------------------------------
init/Kconfig | 2 +
lib/Kconfig | 6 ++
lib/Makefile | 1 +
lib/jhash.c | 116 ++++++++++++++++++++++++++++++++++++++++++++
net/Kconfig | 1 +
11 files changed, 134 insertions(+), 96 deletions(-)
create mode 100644 lib/jhash.c
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index a5dc78a..421ab71 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -2,6 +2,7 @@ menuconfig INFINIBAND
tristate "InfiniBand support"
depends on PCI || BROKEN
depends on HAS_IOMEM
+ select JHASH
---help---
Core support for InfiniBand (IB). Make sure to also select
any protocols you wish to use as well as drivers for your
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index f337800..8257648 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -2496,6 +2496,7 @@ config CHELSIO_T3
tristate "Chelsio Communications T3 10Gb Ethernet support"
depends on PCI
select FW_LOADER
+ select JHASH
help
This driver supports Chelsio T3-based gigabit and 10Gb Ethernet
adapters.
diff --git a/fs/Kconfig b/fs/Kconfig
index d731282..693fe71 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -1667,6 +1667,7 @@ config NFSD
select LOCKD
select SUNRPC
select EXPORTFS
+ select JHASH
select NFSD_V2_ACL if NFSD_V3_ACL
select NFS_ACL_SUPPORT if NFSD_V2_ACL
select NFSD_TCP if NFSD_V4
diff --git a/fs/dlm/Kconfig b/fs/dlm/Kconfig
index 2dbb422..f27a99a 100644
--- a/fs/dlm/Kconfig
+++ b/fs/dlm/Kconfig
@@ -4,6 +4,7 @@ menuconfig DLM
depends on SYSFS && (IPV6 || IPV6=n)
select CONFIGFS_FS
select IP_SCTP
+ select JHASH
help
A general purpose distributed lock manager for kernel or userspace
applications.
diff --git a/fs/gfs2/Kconfig b/fs/gfs2/Kconfig
index de8e64c..b9dcabf 100644
--- a/fs/gfs2/Kconfig
+++ b/fs/gfs2/Kconfig
@@ -3,6 +3,7 @@ config GFS2_FS
depends on EXPERIMENTAL
select FS_POSIX_ACL
select CRC32
+ select JHASH
help
A cluster filesystem.
diff --git a/include/linux/jhash.h b/include/linux/jhash.h
index 2a2f99f..14200c6 100644
--- a/include/linux/jhash.h
+++ b/include/linux/jhash.h
@@ -1,25 +1,6 @@
#ifndef _LINUX_JHASH_H
#define _LINUX_JHASH_H
-/* jhash.h: Jenkins hash support.
- *
- * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
- *
- * http://burtleburtle.net/bob/hash/
- *
- * These are the credits from Bob's sources:
- *
- * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
- * hash(), hash2(), hash3, and mix() are externally useful functions.
- * Routines to test the hash are included if SELF_TEST is defined.
- * You can use this free for any purpose. It has no warranty.
- *
- * Copyright (C) 2003 David S. Miller (davem@redhat.com)
- *
- * I've modified Bob's hash to be useful in the Linux kernel, and
- * any bugs present are surely my fault. -DaveM
- */
-
/* NOTE: Arguments are modified. */
#define __jhash_mix(a, b, c) \
{ \
@@ -41,77 +22,12 @@
* of bytes. No alignment or length assumptions are made about
* the input key.
*/
-static inline u32 jhash(const void *key, u32 length, u32 initval)
-{
- u32 a, b, c, len;
- const u8 *k = key;
-
- len = length;
- a = b = JHASH_GOLDEN_RATIO;
- c = initval;
-
- while (len >= 12) {
- a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24));
- b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24));
- c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24));
-
- __jhash_mix(a,b,c);
-
- k += 12;
- len -= 12;
- }
-
- c += length;
- switch (len) {
- case 11: c += ((u32)k[10]<<24);
- case 10: c += ((u32)k[9]<<16);
- case 9 : c += ((u32)k[8]<<8);
- case 8 : b += ((u32)k[7]<<24);
- case 7 : b += ((u32)k[6]<<16);
- case 6 : b += ((u32)k[5]<<8);
- case 5 : b += k[4];
- case 4 : a += ((u32)k[3]<<24);
- case 3 : a += ((u32)k[2]<<16);
- case 2 : a += ((u32)k[1]<<8);
- case 1 : a += k[0];
- };
-
- __jhash_mix(a,b,c);
-
- return c;
-}
+extern u32 jhash(const void *key, u32 length, u32 initval);
/* A special optimized version that handles 1 or more of u32s.
* The length parameter here is the number of u32s in the key.
*/
-static inline u32 jhash2(const u32 *k, u32 length, u32 initval)
-{
- u32 a, b, c, len;
-
- a = b = JHASH_GOLDEN_RATIO;
- c = initval;
- len = length;
-
- while (len >= 3) {
- a += k[0];
- b += k[1];
- c += k[2];
- __jhash_mix(a, b, c);
- k += 3; len -= 3;
- }
-
- c += length * 4;
-
- switch (len) {
- case 2 : b += k[1];
- case 1 : a += k[0];
- };
-
- __jhash_mix(a,b,c);
-
- return c;
-}
-
+extern u32 jhash2(const u32 *k, u32 length, u32 initval);
/* A special ultra-optimized versions that knows they are hashing exactly
* 3, 2 or 1 word(s).
@@ -119,16 +35,7 @@ static inline u32 jhash2(const u32 *k, u32 length, u32 initval)
* NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
* done at the end is not done here.
*/
-static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
-{
- a += JHASH_GOLDEN_RATIO;
- b += JHASH_GOLDEN_RATIO;
- c += initval;
-
- __jhash_mix(a, b, c);
-
- return c;
-}
+extern u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval);
static inline u32 jhash_2words(u32 a, u32 b, u32 initval)
{
diff --git a/init/Kconfig b/init/Kconfig
index 824d48c..3210626 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -601,6 +601,7 @@ config FUTEX
bool "Enable futex support" if EMBEDDED
default y
select RT_MUTEXES
+ select JHASH
help
Disabling this option will cause the kernel to be built without
support for "fast userspace mutexes". The resulting kernel may not
@@ -718,6 +719,7 @@ config PROFILING
config MARKERS
bool "Activate markers"
+ select JHASH
help
Place an empty function call at each marker site. Can be
dynamically changed for a probe function.
diff --git a/lib/Kconfig b/lib/Kconfig
index ba3d104..23d5507 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -85,6 +85,12 @@ config GENERIC_ALLOCATOR
boolean
#
+# Jenkins has support is selected if needed
+#
+config JHASH
+ boolean
+
+#
# reed solomon support is select'ed if needed
#
config REED_SOLOMON
diff --git a/lib/Makefile b/lib/Makefile
index 23de261..a100a49 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -49,6 +49,7 @@ obj-$(CONFIG_CRC32) += crc32.o
obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
+obj-$(CONFIG_JHASH) += jhash.o
obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/
obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/
diff --git a/lib/jhash.c b/lib/jhash.c
new file mode 100644
index 0000000..ee4083f
--- /dev/null
+++ b/lib/jhash.c
@@ -0,0 +1,116 @@
+/* jhash.c: Jenkins hash support.
+ *
+ * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
+ *
+ * http://burtleburtle.net/bob/hash/
+ *
+ * These are the credits from Bob's sources:
+ *
+ * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
+ * hash(), hash2(), hash3, and mix() are externally useful functions.
+ * Routines to test the hash are included if SELF_TEST is defined.
+ * You can use this free for any purpose. It has no warranty.
+ *
+ * Copyright (C) 2003 David S. Miller (davem@redhat.com)
+ *
+ * I've modified Bob's hash to be useful in the Linux kernel, and
+ * any bugs present are surely my fault. -DaveM
+ */
+#include <linux/kernel.h>
+#include <linux/jhash.h>
+#include <linux/module.h>
+
+/* The most generic version, hashes an arbitrary sequence
+ * of bytes. No alignment or length assumptions are made about
+ * the input key.
+ */
+u32 jhash(const void *key, u32 length, u32 initval)
+{
+ u32 a, b, c, len;
+ const u8 *k = key;
+
+ len = length;
+ a = b = JHASH_GOLDEN_RATIO;
+ c = initval;
+
+ while (len >= 12) {
+ a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24));
+ b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24));
+ c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24));
+
+ __jhash_mix(a,b,c);
+
+ k += 12;
+ len -= 12;
+ }
+
+ c += length;
+ switch (len) {
+ case 11: c += ((u32)k[10]<<24);
+ case 10: c += ((u32)k[9]<<16);
+ case 9 : c += ((u32)k[8]<<8);
+ case 8 : b += ((u32)k[7]<<24);
+ case 7 : b += ((u32)k[6]<<16);
+ case 6 : b += ((u32)k[5]<<8);
+ case 5 : b += k[4];
+ case 4 : a += ((u32)k[3]<<24);
+ case 3 : a += ((u32)k[2]<<16);
+ case 2 : a += ((u32)k[1]<<8);
+ case 1 : a += k[0];
+ };
+
+ __jhash_mix(a,b,c);
+
+ return c;
+}
+EXPORT_SYMBOL(jhash);
+
+/* A special optimized version that handles 1 or more of u32s.
+ * The length parameter here is the number of u32s in the key.
+ */
+u32 jhash2(const u32 *k, u32 length, u32 initval)
+{
+ u32 a, b, c, len;
+
+ a = b = JHASH_GOLDEN_RATIO;
+ c = initval;
+ len = length;
+
+ while (len >= 3) {
+ a += k[0];
+ b += k[1];
+ c += k[2];
+ __jhash_mix(a, b, c);
+ k += 3; len -= 3;
+ }
+
+ c += length * 4;
+
+ switch (len) {
+ case 2 : b += k[1];
+ case 1 : a += k[0];
+ };
+
+ __jhash_mix(a,b,c);
+
+ return c;
+}
+EXPORT_SYMBOL(jhash2);
+
+/* A special ultra-optimized versions that knows they are hashing exactly
+ * 3, 2 or 1 word(s).
+ *
+ * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
+ * done at the end is not done here.
+ */
+u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
+{
+ a += JHASH_GOLDEN_RATIO;
+ b += JHASH_GOLDEN_RATIO;
+ c += initval;
+
+ __jhash_mix(a, b, c);
+
+ return c;
+}
+EXPORT_SYMBOL(jhash_3words);
diff --git a/net/Kconfig b/net/Kconfig
index 6627c6a..0749381 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -6,6 +6,7 @@ menu "Networking"
config NET
bool "Networking support"
+ select JHASH
---help---
Unless you really know what you are doing, you should say Y here.
The reason is that some programs need kernel networking support even
--
1.5.2.2
^ permalink raw reply related
* [RFC PATCH 5/8] [NET]: uninline dst_release
From: Ilpo Järvinen @ 2008-02-20 13:47 UTC (permalink / raw)
To: netdev, linux-kernel
Cc: David Miller, Arnaldo Carvalho de Melo, Ilpo J�rvinen
In-Reply-To: <1203515238-22848-5-git-send-email-ilpo.jarvinen@helsinki.fi>
Codiff stats:
-16420 187 funcs, 103 +, 16523 -, diff: -16420 --- dst_release
Signed-off-by: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
---
include/net/dst.h | 10 +---------
net/core/dst.c | 10 ++++++++++
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/include/net/dst.h b/include/net/dst.h
index e3ac7d0..bf33471 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -158,15 +158,7 @@ struct dst_entry * dst_clone(struct dst_entry * dst)
return dst;
}
-static inline
-void dst_release(struct dst_entry * dst)
-{
- if (dst) {
- WARN_ON(atomic_read(&dst->__refcnt) < 1);
- smp_mb__before_atomic_dec();
- atomic_dec(&dst->__refcnt);
- }
-}
+extern void dst_release(struct dst_entry *dst);
/* Children define the path of the packet through the
* Linux networking. Thus, destinations are stackable.
diff --git a/net/core/dst.c b/net/core/dst.c
index 7deef48..cc2e724 100644
--- a/net/core/dst.c
+++ b/net/core/dst.c
@@ -259,6 +259,16 @@ again:
return NULL;
}
+void dst_release(struct dst_entry *dst)
+{
+ if (dst) {
+ WARN_ON(atomic_read(&dst->__refcnt) < 1);
+ smp_mb__before_atomic_dec();
+ atomic_dec(&dst->__refcnt);
+ }
+}
+EXPORT_SYMBOL(dst_release);
+
/* Dirty hack. We did it in 2.2 (in __dst_free),
* we have _very_ good reasons not to repeat
* this mistake in 2.3, but we have no choice
--
1.5.2.2
^ permalink raw reply related
* Re: [RFC PATCH 1/8] [NET]: uninline skb_put, de-bloats a lot
From: Patrick McHardy @ 2008-02-20 13:54 UTC (permalink / raw)
To: Ilpo Järvinen
Cc: netdev, linux-kernel, David Miller, Arnaldo Carvalho de Melo
In-Reply-To: <1203515238-22848-2-git-send-email-ilpo.jarvinen@helsinki.fi>
Ilpo Järvinen wrote:
> ~500 files changed
> ...
> kernel/uninlined.c:
> skb_put | +104
> 1 function changed, 104 bytes added, diff: +104
>
> vmlinux.o:
> 869 functions changed, 198 bytes added, 111003 bytes removed, diff: -110805
>
> This change is INCOMPLETE, I think that the call to current_text_addr()
> should be rethought but I don't have a clue how to do that.
I guess __builtin_return_address(0) would work.
^ permalink raw reply
* PowerPC toolchain for x86 [Was: pci_device_id cleanups]
From: Sam Ravnborg @ 2008-02-20 13:55 UTC (permalink / raw)
To: Jonas Bonn; +Cc: linux-kernel, netdev, greg, Andrew Morton
In-Reply-To: <47BC2AB7.4040402@southpole.se>
On Wed, Feb 20, 2008 at 02:27:19PM +0100, Jonas Bonn wrote:
> Sam Ravnborg wrote:
> >On Wed, Feb 20, 2008 at 01:53:36PM +0100, Jonas Bonn wrote:
> >>The PCI_DEVICE_TABLE patch I sent earlier doesn't necessarily make
> >>much sense by itself... here is a set of patches that apply
> >>this macro, in turn moving a lot of this data into __devinitconst
> >>which is discardable in certain situations.
> >>Hopefully the benefit of this approach is a bit clearer now.
> >[shorter lines please..]
>
> Sorry...
>
> >
> >Can you please confirm that this does not break powerpc (64 bit)
> >as they have troubles with the constification..
>
> I do not have access to any PowerPC machine... Olof Johansson built the
> tree I posted earlier on PowerPC; there's nothing really new here except
> the wrapping of the definition in a macro.
And you added const and a specific section.
Exactly what could break on PowerPC.
To do the build break check is easy.
Google for "crosstool" and build your own powerpc toolchain.
Andrew has something precompiled somewhere but I lost the link.
Sam
^ permalink raw reply
* Re: [RFC PATCH 1/8] [NET]: uninline skb_put, de-bloats a lot
From: Eric Dumazet @ 2008-02-20 13:57 UTC (permalink / raw)
To: "Ilpo Järvinen"
Cc: netdev, linux-kernel, David Miller, Arnaldo Carvalho de Melo
In-Reply-To: <1203515238-22848-2-git-send-email-ilpo.jarvinen@helsinki.fi>
On Wed, 20 Feb 2008 15:47:11 +0200
"Ilpo Järvinen" <ilpo.jarvinen@helsinki.fi> wrote:
> ~500 files changed
> ...
> kernel/uninlined.c:
> skb_put | +104
> 1 function changed, 104 bytes added, diff: +104
>
> vmlinux.o:
> 869 functions changed, 198 bytes added, 111003 bytes removed, diff: -110805
>
> This change is INCOMPLETE, I think that the call to current_text_addr()
> should be rethought but I don't have a clue how to do that.
You want to use __builtin_return_address(0)
>
> Signed-off-by: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
> ---
> include/linux/skbuff.h | 20 +-------------------
> net/core/skbuff.c | 21 +++++++++++++++++++++
> 2 files changed, 22 insertions(+), 19 deletions(-)
>
> diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
> index 412672a..5925435 100644
> --- a/include/linux/skbuff.h
> +++ b/include/linux/skbuff.h
> @@ -896,25 +896,7 @@ static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
> return tmp;
> }
>
> -/**
> - * skb_put - add data to a buffer
> - * @skb: buffer to use
> - * @len: amount of data to add
> - *
> - * This function extends the used data area of the buffer. If this would
> - * exceed the total buffer size the kernel will panic. A pointer to the
> - * first byte of the extra data is returned.
> - */
> -static inline unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
> -{
> - unsigned char *tmp = skb_tail_pointer(skb);
> - SKB_LINEAR_ASSERT(skb);
> - skb->tail += len;
> - skb->len += len;
> - if (unlikely(skb->tail > skb->end))
> - skb_over_panic(skb, len, current_text_addr());
> - return tmp;
> -}
> +extern unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
>
> static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
> {
> diff --git a/net/core/skbuff.c b/net/core/skbuff.c
> index 4e35422..661d439 100644
> --- a/net/core/skbuff.c
> +++ b/net/core/skbuff.c
> @@ -857,6 +857,27 @@ free_skb:
> return err;
> }
>
> +/**
> + * skb_put - add data to a buffer
> + * @skb: buffer to use
> + * @len: amount of data to add
> + *
> + * This function extends the used data area of the buffer. If this would
> + * exceed the total buffer size the kernel will panic. A pointer to the
> + * first byte of the extra data is returned.
> + */
> +unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
> +{
> + unsigned char *tmp = skb_tail_pointer(skb);
> + SKB_LINEAR_ASSERT(skb);
> + skb->tail += len;
> + skb->len += len;
> + if (unlikely(skb->tail > skb->end))
> + skb_over_panic(skb, len, current_text_addr());
> + return tmp;
> +}
> +EXPORT_SYMBOL(skb_put);
> +
> /* Trims skb to length len. It can change skb pointers.
> */
>
> --
> 1.5.2.2
>
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
^ permalink raw reply
* [PATCH] SUNRPC: Mark buffer used for debug printks with __maybe_unused
From: Pavel Emelyanov @ 2008-02-20 14:02 UTC (permalink / raw)
To: J. Bruce Fields; +Cc: Linux Netdev List
There are tree places, which declare the char buf[...] on the stack
to push it later into dprintk(). Since the dprintk sometimes (if the
CONFIG_SYSCTL=n) becomes an empty do { } while (0) stub, these buffers
cause gcc to produce appropriate warnings.
Mark them as __maybe_unused.
Signed-off-by: Pavel Emelyanov <xemul@openvz.org>
---
diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index 1d3e5fc..303f105 100644
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -175,7 +175,7 @@ static int svc_sendto(struct svc_rqst *rqstp, struct xdr_buf *xdr)
size_t base = xdr->page_base;
unsigned int pglen = xdr->page_len;
unsigned int flags = MSG_MORE;
- char buf[RPC_MAX_ADDRBUFLEN];
+ char buf[RPC_MAX_ADDRBUFLEN] __maybe_unused;
slen = xdr->len;
@@ -716,7 +716,7 @@ static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt)
struct socket *newsock;
struct svc_sock *newsvsk;
int err, slen;
- char buf[RPC_MAX_ADDRBUFLEN];
+ char buf[RPC_MAX_ADDRBUFLEN] __maybe_unused;
dprintk("svc: tcp_accept %p sock %p\n", svsk, sock);
if (!sock)
@@ -1206,7 +1206,7 @@ static struct svc_xprt *svc_create_socket(struct svc_serv *serv,
struct socket *sock;
int error;
int type;
- char buf[RPC_MAX_ADDRBUFLEN];
+ char buf[RPC_MAX_ADDRBUFLEN] __maybe_unused;
struct sockaddr_storage addr;
struct sockaddr *newsin = (struct sockaddr *)&addr;
int newlen;
^ permalink raw reply related
* Re: PowerPC toolchain for x86 [Was: pci_device_id cleanups]
From: Jonas Bonn @ 2008-02-20 14:03 UTC (permalink / raw)
To: Sam Ravnborg; +Cc: linux-kernel, netdev, greg, Andrew Morton
In-Reply-To: <20080220135533.GA19024@uranus.ravnborg.org>
Sam Ravnborg wrote:
> On Wed, Feb 20, 2008 at 02:27:19PM +0100, Jonas Bonn wrote:
>> Sam Ravnborg wrote:
>>> On Wed, Feb 20, 2008 at 01:53:36PM +0100, Jonas Bonn wrote:
>>>> The PCI_DEVICE_TABLE patch I sent earlier doesn't necessarily make
>>>> much sense by itself... here is a set of patches that apply
>>>> this macro, in turn moving a lot of this data into __devinitconst
>>>> which is discardable in certain situations.
>>>> Hopefully the benefit of this approach is a bit clearer now.
>>> [shorter lines please..]
>> Sorry...
>>
>>> Can you please confirm that this does not break powerpc (64 bit)
>>> as they have troubles with the constification..
>> I do not have access to any PowerPC machine... Olof Johansson built the
>> tree I posted earlier on PowerPC; there's nothing really new here except
>> the wrapping of the definition in a macro.
> And you added const and a specific section.
No... once the macro is expanded the code is exactly the same as that
which built cleanly on powerpc previously (which Olof, built, I mean)...
nothing new here.
> Exactly what could break on PowerPC.
>
> To do the build break check is easy.
> Google for "crosstool" and build your own powerpc toolchain.
>
Thanks... I'll throw together a cross compiler and see what I can do.
/Jonas
> Andrew has something precompiled somewhere but I lost the link.
>
>
> Sam
>
^ permalink raw reply
* [PATCH 03/28] mm: slb: add knowledge of reserve pages
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: reserve-slub.patch --]
[-- Type: text/plain, Size: 11004 bytes --]
Restrict objects from reserve slabs (ALLOC_NO_WATERMARKS) to allocation
contexts that are entitled to it. This is done to ensure reserve pages don't
leak out and get consumed.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/slub_def.h | 1
mm/slab.c | 60 +++++++++++++++++++++++++++++++++++++++--------
mm/slub.c | 42 +++++++++++++++++++++-----------
3 files changed, 80 insertions(+), 23 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c
+++ linux-2.6/mm/slub.c
@@ -21,11 +21,12 @@
#include <linux/ctype.h>
#include <linux/kallsyms.h>
#include <linux/memory.h>
+#include "internal.h"
/*
* Lock order:
* 1. slab_lock(page)
- * 2. slab->list_lock
+ * 2. node->list_lock
*
* The slab_lock protects operations on the object of a particular
* slab and its metadata in the page struct. If the slab lock
@@ -1098,15 +1099,15 @@ static struct page *allocate_slab(struct
return page;
}
-static void setup_object(struct kmem_cache *s, struct page *page,
- void *object)
+static void setup_object(struct kmem_cache *s, struct page *page, void *object)
{
setup_object_debug(s, page, object);
if (unlikely(s->ctor))
s->ctor(s, object);
}
-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
+static
+struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node, int *reserve)
{
struct page *page;
struct kmem_cache_node *n;
@@ -1121,6 +1122,7 @@ static struct page *new_slab(struct kmem
if (!page)
goto out;
+ *reserve = page->reserve;
n = get_node(s, page_to_nid(page));
if (n)
atomic_long_inc(&n->nr_slabs);
@@ -1228,8 +1230,7 @@ static __always_inline int slab_trylock(
/*
* Management of partially allocated slabs
*/
-static void add_partial(struct kmem_cache_node *n,
- struct page *page, int tail)
+static void add_partial(struct kmem_cache_node *n, struct page *page, int tail)
{
spin_lock(&n->list_lock);
n->nr_partial++;
@@ -1240,8 +1241,7 @@ static void add_partial(struct kmem_cach
spin_unlock(&n->list_lock);
}
-static void remove_partial(struct kmem_cache *s,
- struct page *page)
+static void remove_partial(struct kmem_cache *s, struct page *page)
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
@@ -1256,7 +1256,8 @@ static void remove_partial(struct kmem_c
*
* Must hold list_lock.
*/
-static inline int lock_and_freeze_slab(struct kmem_cache_node *n, struct page *page)
+static inline
+int lock_and_freeze_slab(struct kmem_cache_node *n, struct page *page)
{
if (slab_trylock(page)) {
list_del(&page->lru);
@@ -1514,11 +1515,21 @@ static void *__slab_alloc(struct kmem_ca
{
void **object;
struct page *new;
+ int reserve;
#ifdef SLUB_FASTPATH
unsigned long flags;
local_irq_save(flags);
#endif
+ if (unlikely(c->reserve)) {
+ /*
+ * If the current slab is a reserve slab and the current
+ * allocation context does not allow access to the reserves we
+ * must force an allocation to test the current levels.
+ */
+ if (!(gfp_to_alloc_flags(gfpflags) & ALLOC_NO_WATERMARKS))
+ goto grow_slab;
+ }
if (!c->page)
goto new_slab;
@@ -1530,7 +1541,7 @@ load_freelist:
object = c->page->freelist;
if (unlikely(object == c->page->end))
goto another_slab;
- if (unlikely(SlabDebug(c->page)))
+ if (unlikely(SlabDebug(c->page) || c->reserve))
goto debug;
object = c->page->freelist;
@@ -1557,16 +1568,18 @@ new_slab:
goto load_freelist;
}
+grow_slab:
if (gfpflags & __GFP_WAIT)
local_irq_enable();
- new = new_slab(s, gfpflags, node);
+ new = new_slab(s, gfpflags, node, &reserve);
if (gfpflags & __GFP_WAIT)
local_irq_disable();
if (new) {
c = get_cpu_slab(s, smp_processor_id());
+ c->reserve = reserve;
stat(c, ALLOC_SLAB);
if (c->page)
flush_slab(s, c);
@@ -1594,8 +1607,8 @@ new_slab:
return NULL;
debug:
- object = c->page->freelist;
- if (!alloc_debug_processing(s, c->page, object, addr))
+ if (SlabDebug(c->page) &&
+ !alloc_debug_processing(s, c->page, object, addr))
goto another_slab;
c->page->inuse++;
@@ -2153,10 +2166,11 @@ static struct kmem_cache_node *early_kme
struct page *page;
struct kmem_cache_node *n;
unsigned long flags;
+ int reserve;
BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node));
- page = new_slab(kmalloc_caches, gfpflags, node);
+ page = new_slab(kmalloc_caches, gfpflags, node, &reserve);
BUG_ON(!page);
if (page_to_nid(page) != node) {
Index: linux-2.6/include/linux/slub_def.h
===================================================================
--- linux-2.6.orig/include/linux/slub_def.h
+++ linux-2.6/include/linux/slub_def.h
@@ -37,6 +37,7 @@ struct kmem_cache_cpu {
int node; /* The node of the page (or -1 for debug) */
unsigned int offset; /* Freepointer offset (in word units) */
unsigned int objsize; /* Size of an object (from kmem_cache) */
+ int reserve; /* Did the current page come from the reserve */
#ifdef CONFIG_SLUB_STATS
unsigned stat[NR_SLUB_STAT_ITEMS];
#endif
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c
+++ linux-2.6/mm/slab.c
@@ -115,6 +115,8 @@
#include <asm/tlbflush.h>
#include <asm/page.h>
+#include "internal.h"
+
/*
* DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
* 0 for faster, smaller code (especially in the critical paths).
@@ -265,7 +267,8 @@ struct array_cache {
unsigned int avail;
unsigned int limit;
unsigned int batchcount;
- unsigned int touched;
+ unsigned int touched:1,
+ reserve:1;
spinlock_t lock;
void *entry[]; /*
* Must have this definition in here for the proper
@@ -761,6 +764,27 @@ static inline struct array_cache *cpu_ca
return cachep->array[smp_processor_id()];
}
+/*
+ * If the last page came from the reserves, and the current allocation context
+ * does not have access to them, force an allocation to test the watermarks.
+ */
+static inline int slab_force_alloc(struct kmem_cache *cachep, gfp_t flags)
+{
+ if (unlikely(cpu_cache_get(cachep)->reserve) &&
+ !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS))
+ return 1;
+
+ return 0;
+}
+
+static inline void slab_set_reserve(struct kmem_cache *cachep, int reserve)
+{
+ struct array_cache *ac = cpu_cache_get(cachep);
+
+ if (unlikely(ac->reserve != reserve))
+ ac->reserve = reserve;
+}
+
static inline struct kmem_cache *__find_general_cachep(size_t size,
gfp_t gfpflags)
{
@@ -960,6 +984,7 @@ static struct array_cache *alloc_arrayca
nc->limit = entries;
nc->batchcount = batchcount;
nc->touched = 0;
+ nc->reserve = 0;
spin_lock_init(&nc->lock);
}
return nc;
@@ -1663,7 +1688,8 @@ __initcall(cpucache_init);
* did not request dmaable memory, we might get it, but that
* would be relatively rare and ignorable.
*/
-static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
+static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid,
+ int *reserve)
{
struct page *page;
int nr_pages;
@@ -1685,6 +1711,7 @@ static void *kmem_getpages(struct kmem_c
if (!page)
return NULL;
+ *reserve = page->reserve;
nr_pages = (1 << cachep->gfporder);
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
add_zone_page_state(page_zone(page),
@@ -2113,6 +2140,7 @@ static int __init_refok setup_cpu_cache(
cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
cpu_cache_get(cachep)->batchcount = 1;
cpu_cache_get(cachep)->touched = 0;
+ cpu_cache_get(cachep)->reserve = 0;
cachep->batchcount = 1;
cachep->limit = BOOT_CPUCACHE_ENTRIES;
return 0;
@@ -2768,6 +2796,7 @@ static int cache_grow(struct kmem_cache
size_t offset;
gfp_t local_flags;
struct kmem_list3 *l3;
+ int reserve;
/*
* Be lazy and only check for valid flags here, keeping it out of the
@@ -2806,7 +2835,7 @@ static int cache_grow(struct kmem_cache
* 'nodeid'.
*/
if (!objp)
- objp = kmem_getpages(cachep, local_flags, nodeid);
+ objp = kmem_getpages(cachep, local_flags, nodeid, &reserve);
if (!objp)
goto failed;
@@ -2823,6 +2852,7 @@ static int cache_grow(struct kmem_cache
if (local_flags & __GFP_WAIT)
local_irq_disable();
check_irq_off();
+ slab_set_reserve(cachep, reserve);
spin_lock(&l3->list_lock);
/* Make slab active. */
@@ -2957,7 +2987,8 @@ bad:
#define check_slabp(x,y) do { } while(0)
#endif
-static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
+static void *cache_alloc_refill(struct kmem_cache *cachep,
+ gfp_t flags, int must_refill)
{
int batchcount;
struct kmem_list3 *l3;
@@ -2967,6 +2998,8 @@ static void *cache_alloc_refill(struct k
node = numa_node_id();
check_irq_off();
+ if (unlikely(must_refill))
+ goto force_grow;
ac = cpu_cache_get(cachep);
retry:
batchcount = ac->batchcount;
@@ -3035,11 +3068,14 @@ alloc_done:
if (unlikely(!ac->avail)) {
int x;
+force_grow:
x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);
/* cache_grow can reenable interrupts, then ac could change. */
ac = cpu_cache_get(cachep);
- if (!x && ac->avail == 0) /* no objects in sight? abort */
+
+ /* no objects in sight? abort */
+ if (!x && (ac->avail == 0 || must_refill))
return NULL;
if (!ac->avail) /* objects refilled by interrupt? */
@@ -3194,17 +3230,18 @@ static inline void *____cache_alloc(stru
{
void *objp;
struct array_cache *ac;
+ int must_refill = slab_force_alloc(cachep, flags);
check_irq_off();
ac = cpu_cache_get(cachep);
- if (likely(ac->avail)) {
+ if (likely(ac->avail && !must_refill)) {
STATS_INC_ALLOCHIT(cachep);
ac->touched = 1;
objp = ac->entry[--ac->avail];
} else {
STATS_INC_ALLOCMISS(cachep);
- objp = cache_alloc_refill(cachep, flags);
+ objp = cache_alloc_refill(cachep, flags, must_refill);
}
return objp;
}
@@ -3246,7 +3283,7 @@ static void *fallback_alloc(struct kmem_
gfp_t local_flags;
struct zone **z;
void *obj = NULL;
- int nid;
+ int nid, reserve;
if (flags & __GFP_THISNODE)
return NULL;
@@ -3280,10 +3317,11 @@ retry:
if (local_flags & __GFP_WAIT)
local_irq_enable();
kmem_flagcheck(cache, flags);
- obj = kmem_getpages(cache, flags, -1);
+ obj = kmem_getpages(cache, flags, -1, &reserve);
if (local_flags & __GFP_WAIT)
local_irq_disable();
if (obj) {
+ slab_set_reserve(cache, reserve);
/*
* Insert into the appropriate per node queues
*/
@@ -3322,6 +3360,9 @@ static void *____cache_alloc_node(struct
l3 = cachep->nodelists[nodeid];
BUG_ON(!l3);
+ if (unlikely(slab_force_alloc(cachep, flags)))
+ goto force_grow;
+
retry:
check_irq_off();
spin_lock(&l3->list_lock);
@@ -3359,6 +3400,7 @@ retry:
must_grow:
spin_unlock(&l3->list_lock);
+force_grow:
x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL);
if (x)
goto retry;
--
^ permalink raw reply
* Re: TG3 network data corruption regression 2.6.24/2.6.23.4
From: Tony Battersby @ 2008-02-20 15:01 UTC (permalink / raw)
To: Michael Chan; +Cc: David Miller, herbert, netdev, gregkh, linux-kernel
In-Reply-To: <1203465163.13495.102.camel@dell>
Michael Chan wrote:
> On Tue, 2008-02-19 at 17:14 -0500, Tony Battersby wrote:
>
>
>> Update: when I revert Herbert's patch in addition to applying your
>> patch, the iSCSI performance goes back up to 115 MB/s again in both
>> directions. So it looks like turning off SG for TX didn't itself cause
>> the performance drop, but rather that the performance drop is just
>> another manifestation of whatever bug is causing the data corruption.
>>
>> I do not regularly use wireshark or look at network packet dumps, so I
>> am not really sure what to look for. Given the above information, do
>> you still believe that there is value in examining the packet dump?
>>
>>
>
> Can you confirm whether you're getting TCP checksum errors on the other
> side that is receiving packets from the 5701? You can just check
> statistics using netstat -s. I suspect that after we turn off SG,
> checksum is no longer offloaded and we are getting lots of TCP checksum
> errors instead that are slowing the performance.
>
>
>
Confirmed. With a 100 MB read/write test, netstat -s shows 75 bad
segments received, and performance in the one direction is about 5
MB/s. When I switch to the SysKonnect NIC, netstat -s shows 0 bad
segments received, and performance is 115 MB/s. So that solves that
mystery - there is still data corruption, but the software-computed TCP
checksum causes the bad packets to be retransmitted rather than being
passed on to the application.
Tony
^ permalink raw reply
* [PATCH 02/28] mm: tag reseve pages
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: page_alloc-reserve.patch --]
[-- Type: text/plain, Size: 1265 bytes --]
Tag pages allocated from the reserves with a non-zero page->reserve.
This allows us to distinguish and account reserve pages.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/mm_types.h | 1 +
mm/page_alloc.c | 4 +++-
2 files changed, 4 insertions(+), 1 deletion(-)
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h
+++ linux-2.6/include/linux/mm_types.h
@@ -73,6 +73,7 @@ struct page {
union {
pgoff_t index; /* Our offset within mapping. */
void *freelist; /* SLUB: freelist req. slab lock */
+ int reserve; /* page_alloc: page is a reserve page */
};
struct list_head lru; /* Pageout list, eg. active_list
* protected by zone->lru_lock !
Index: linux-2.6/mm/page_alloc.c
===================================================================
--- linux-2.6.orig/mm/page_alloc.c
+++ linux-2.6/mm/page_alloc.c
@@ -1418,8 +1418,10 @@ zonelist_scan:
}
page = buffered_rmqueue(zonelist, zone, order, gfp_mask);
- if (page)
+ if (page) {
+ page->reserve = !!(alloc_flags & ALLOC_NO_WATERMARKS);
break;
+ }
this_zone_full:
if (NUMA_BUILD)
zlc_mark_zone_full(zonelist, z);
--
^ permalink raw reply
* [PATCH 04/28] mm: kmem_estimate_pages()
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: mm-kmem_estimate_pages.patch --]
[-- Type: text/plain, Size: 6384 bytes --]
Provide a method to get the upper bound on the pages needed to allocate
a given number of objects from a given kmem_cache.
This lays the foundation for a generic reserve framework as presented in
a later patch in this series. This framework needs to convert object demand
(kmalloc() bytes, kmem_cache_alloc() objects) to pages.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/slab.h | 4 ++
mm/slab.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++
mm/slub.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 161 insertions(+)
Index: linux-2.6/include/linux/slab.h
===================================================================
--- linux-2.6.orig/include/linux/slab.h
+++ linux-2.6/include/linux/slab.h
@@ -60,6 +60,8 @@ void kmem_cache_free(struct kmem_cache *
unsigned int kmem_cache_size(struct kmem_cache *);
const char *kmem_cache_name(struct kmem_cache *);
int kmem_ptr_validate(struct kmem_cache *cachep, const void *ptr);
+unsigned kmem_estimate_pages(struct kmem_cache *cachep,
+ gfp_t flags, int objects);
/*
* Please use this macro to create slab caches. Simply specify the
@@ -94,6 +96,8 @@ int kmem_ptr_validate(struct kmem_cache
void * __must_check krealloc(const void *, size_t, gfp_t);
void kfree(const void *);
size_t ksize(const void *);
+unsigned kestimate_single(size_t, gfp_t, int);
+unsigned kestimate(gfp_t, size_t);
/*
* Allocator specific definitions. These are mainly used to establish optimized
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c
+++ linux-2.6/mm/slub.c
@@ -2465,6 +2465,37 @@ const char *kmem_cache_name(struct kmem_
EXPORT_SYMBOL(kmem_cache_name);
/*
+ * return the max number of pages required to allocated count
+ * objects from the given cache
+ */
+unsigned kmem_estimate_pages(struct kmem_cache *s, gfp_t flags, int objects)
+{
+ unsigned long slabs;
+
+ if (WARN_ON(!s) || WARN_ON(!s->objects))
+ return 0;
+
+ slabs = DIV_ROUND_UP(objects, s->objects);
+
+ /*
+ * Account the possible additional overhead if the slab holds more that
+ * one object.
+ */
+ if (s->objects > 1) {
+ /*
+ * Account the possible additional overhead if per cpu slabs
+ * are currently empty and have to be allocated. This is very
+ * unlikely but a possible scenario immediately after
+ * kmem_cache_shrink.
+ */
+ slabs += num_online_cpus();
+ }
+
+ return slabs << s->order;
+}
+EXPORT_SYMBOL_GPL(kmem_estimate_pages);
+
+/*
* Attempt to free all slabs on a node. Return the number of slabs we
* were unable to free.
*/
@@ -2818,6 +2849,57 @@ static unsigned long count_partial(struc
}
/*
+ * return the max number of pages required to allocate @count objects
+ * of @size bytes from kmalloc given @flags.
+ */
+unsigned kestimate_single(size_t size, gfp_t flags, int count)
+{
+ struct kmem_cache *s = get_slab(size, flags);
+ if (!s)
+ return 0;
+
+ return kmem_estimate_pages(s, flags, count);
+
+}
+EXPORT_SYMBOL_GPL(kestimate_single);
+
+/*
+ * return the max number of pages required to allocate @bytes from kmalloc
+ * in an unspecified number of allocation of heterogeneous size.
+ */
+unsigned kestimate(gfp_t flags, size_t bytes)
+{
+ int i;
+ unsigned long pages;
+
+ /*
+ * multiply by two, in order to account the worst case slack space
+ * due to the power-of-two allocation sizes.
+ */
+ pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE);
+
+ /*
+ * add the kmem_cache overhead of each possible kmalloc cache
+ */
+ for (i = 1; i < PAGE_SHIFT; i++) {
+ struct kmem_cache *s;
+
+#ifdef CONFIG_ZONE_DMA
+ if (unlikely(flags & SLUB_DMA))
+ s = dma_kmalloc_cache(i, flags);
+ else
+#endif
+ s = &kmalloc_caches[i];
+
+ if (s)
+ pages += kmem_estimate_pages(s, flags, 0);
+ }
+
+ return pages;
+}
+EXPORT_SYMBOL_GPL(kestimate);
+
+/*
* kmem_cache_shrink removes empty slabs from the partial lists and sorts
* the remaining slabs by the number of items in use. The slabs with the
* most items in use come first. New allocations will then fill those up
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c
+++ linux-2.6/mm/slab.c
@@ -3851,6 +3851,81 @@ const char *kmem_cache_name(struct kmem_
EXPORT_SYMBOL_GPL(kmem_cache_name);
/*
+ * return the max number of pages required to allocated count
+ * objects from the given cache
+ */
+unsigned kmem_estimate_pages(struct kmem_cache *cachep,
+ gfp_t flags, int objects)
+{
+ /*
+ * (1) memory for objects,
+ */
+ unsigned nr_slabs = DIV_ROUND_UP(objects, cachep->num);
+ unsigned nr_pages = nr_slabs << cachep->gfporder;
+
+ /*
+ * (2) memory for each per-cpu queue (nr_cpu_ids),
+ * (3) memory for each per-node alien queues (nr_cpu_ids), and
+ * (4) some amount of memory for the slab management structures
+ *
+ * XXX: truely account these
+ */
+ nr_pages += 1 + ilog2(nr_pages);
+
+ return nr_pages;
+}
+
+/*
+ * return the max number of pages required to allocate @count objects
+ * of @size bytes from kmalloc given @flags.
+ */
+unsigned kestimate_single(size_t size, gfp_t flags, int count)
+{
+ struct kmem_cache *s = kmem_find_general_cachep(size, flags);
+ if (!s)
+ return 0;
+
+ return kmem_estimate_pages(s, flags, count);
+}
+EXPORT_SYMBOL_GPL(kestimate_single);
+
+/*
+ * return the max number of pages required to allocate @bytes from kmalloc
+ * in an unspecified number of allocation of heterogeneous size.
+ */
+unsigned kestimate(gfp_t flags, size_t bytes)
+{
+ unsigned long pages;
+ struct cache_sizes *csizep = malloc_sizes;
+
+ /*
+ * multiply by two, in order to account the worst case slack space
+ * due to the power-of-two allocation sizes.
+ */
+ pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE);
+
+ /*
+ * add the kmem_cache overhead of each possible kmalloc cache
+ */
+ for (csizep = malloc_sizes; csizep->cs_cachep; csizep++) {
+ struct kmem_cache *s;
+
+#ifdef CONFIG_ZONE_DMA
+ if (unlikely(flags & __GFP_DMA))
+ s = csizep->cs_dmacachep;
+ else
+#endif
+ s = csizep->cs_cachep;
+
+ if (s)
+ pages += kmem_estimate_pages(s, flags, 0);
+ }
+
+ return pages;
+}
+EXPORT_SYMBOL_GPL(kestimate);
+
+/*
* This initializes kmem_list3 or resizes various caches for all nodes.
*/
static int alloc_kmemlist(struct kmem_cache *cachep)
--
^ permalink raw reply
* [PATCH 12/28] net: wrap sk->sk_backlog_rcv()
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: net-backlog.patch --]
[-- Type: text/plain, Size: 2909 bytes --]
Wrap calling sk->sk_backlog_rcv() in a function. This will allow extending the
generic sk_backlog_rcv behaviour.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/net/sock.h | 5 +++++
include/net/tcp.h | 2 +-
net/core/sock.c | 4 ++--
net/ipv4/tcp.c | 2 +-
net/ipv4/tcp_timer.c | 2 +-
5 files changed, 10 insertions(+), 5 deletions(-)
Index: linux-2.6/include/net/sock.h
===================================================================
--- linux-2.6.orig/include/net/sock.h
+++ linux-2.6/include/net/sock.h
@@ -474,6 +474,11 @@ static inline void sk_add_backlog(struct
skb->next = NULL;
}
+static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+ return sk->sk_backlog_rcv(sk, skb);
+}
+
#define sk_wait_event(__sk, __timeo, __condition) \
({ int __rc; \
release_sock(__sk); \
Index: linux-2.6/net/core/sock.c
===================================================================
--- linux-2.6.orig/net/core/sock.c
+++ linux-2.6/net/core/sock.c
@@ -325,7 +325,7 @@ int sk_receive_skb(struct sock *sk, stru
*/
mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
- rc = sk->sk_backlog_rcv(sk, skb);
+ rc = sk_backlog_rcv(sk, skb);
mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
} else
@@ -1360,7 +1360,7 @@ static void __release_sock(struct sock *
struct sk_buff *next = skb->next;
skb->next = NULL;
- sk->sk_backlog_rcv(sk, skb);
+ sk_backlog_rcv(sk, skb);
/*
* We are in process context here with softirqs
Index: linux-2.6/net/ipv4/tcp.c
===================================================================
--- linux-2.6.orig/net/ipv4/tcp.c
+++ linux-2.6/net/ipv4/tcp.c
@@ -1158,7 +1158,7 @@ static void tcp_prequeue_process(struct
* necessary */
local_bh_disable();
while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
- sk->sk_backlog_rcv(sk, skb);
+ sk_backlog_rcv(sk, skb);
local_bh_enable();
/* Clear memory counter. */
Index: linux-2.6/net/ipv4/tcp_timer.c
===================================================================
--- linux-2.6.orig/net/ipv4/tcp_timer.c
+++ linux-2.6/net/ipv4/tcp_timer.c
@@ -203,7 +203,7 @@ static void tcp_delack_timer(unsigned lo
NET_INC_STATS_BH(LINUX_MIB_TCPSCHEDULERFAILED);
while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
- sk->sk_backlog_rcv(sk, skb);
+ sk_backlog_rcv(sk, skb);
tp->ucopy.memory = 0;
}
Index: linux-2.6/include/net/tcp.h
===================================================================
--- linux-2.6.orig/include/net/tcp.h
+++ linux-2.6/include/net/tcp.h
@@ -879,7 +879,7 @@ static inline int tcp_prequeue(struct so
BUG_ON(sock_owned_by_user(sk));
while ((skb1 = __skb_dequeue(&tp->ucopy.prequeue)) != NULL) {
- sk->sk_backlog_rcv(sk, skb1);
+ sk_backlog_rcv(sk, skb1);
NET_INC_STATS_BH(LINUX_MIB_TCPPREQUEUEDROPPED);
}
--
^ permalink raw reply
* [PATCH 10/28] mm: memory reserve management
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: mm-reserve.patch --]
[-- Type: text/plain, Size: 14372 bytes --]
Generic reserve management code.
It provides methods to reserve and charge. Upon this, generic alloc/free style
reserve pools could be build, which could fully replace mempool_t
functionality.
It should also allow for a Banker's algorithm replacement of __GFP_NOFAIL.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/reserve.h | 54 ++++++
mm/Makefile | 2
mm/reserve.c | 429 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 484 insertions(+), 1 deletion(-)
Index: linux-2.6/include/linux/reserve.h
===================================================================
--- /dev/null
+++ linux-2.6/include/linux/reserve.h
@@ -0,0 +1,54 @@
+/*
+ * Memory reserve management.
+ *
+ * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
+ *
+ * This file contains the public data structure and API definitions.
+ */
+
+#ifndef _LINUX_RESERVE_H
+#define _LINUX_RESERVE_H
+
+#include <linux/list.h>
+#include <linux/spinlock.h>
+
+struct mem_reserve {
+ struct mem_reserve *parent;
+ struct list_head children;
+ struct list_head siblings;
+
+ const char *name;
+
+ long pages;
+ long limit;
+ long usage;
+ spinlock_t lock; /* protects limit and usage */
+};
+
+extern struct mem_reserve mem_reserve_root;
+
+void mem_reserve_init(struct mem_reserve *res, const char *name,
+ struct mem_reserve *parent);
+int mem_reserve_connect(struct mem_reserve *new_child,
+ struct mem_reserve *node);
+int mem_reserve_disconnect(struct mem_reserve *node);
+
+int mem_reserve_pages_set(struct mem_reserve *res, long pages);
+int mem_reserve_pages_add(struct mem_reserve *res, long pages);
+int mem_reserve_pages_charge(struct mem_reserve *res, long pages,
+ int overcommit);
+
+int mem_reserve_kmalloc_set(struct mem_reserve *res, long bytes);
+int mem_reserve_kmalloc_charge(struct mem_reserve *res, long bytes,
+ int overcommit);
+
+struct kmem_cache;
+
+int mem_reserve_kmem_cache_set(struct mem_reserve *res,
+ struct kmem_cache *s,
+ int objects);
+int mem_reserve_kmem_cache_charge(struct mem_reserve *res,
+ long objs,
+ int overcommit);
+
+#endif /* _LINUX_RESERVE_H */
Index: linux-2.6/mm/Makefile
===================================================================
--- linux-2.6.orig/mm/Makefile
+++ linux-2.6/mm/Makefile
@@ -11,7 +11,7 @@ obj-y := bootmem.o filemap.o mempool.o
page_alloc.o page-writeback.o pdflush.o \
readahead.o swap.o truncate.o vmscan.o \
prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \
- page_isolation.o $(mmu-y)
+ page_isolation.o reserve.o $(mmu-y)
obj-$(CONFIG_PROC_PAGE_MONITOR) += pagewalk.o
obj-$(CONFIG_BOUNCE) += bounce.o
Index: linux-2.6/mm/reserve.c
===================================================================
--- /dev/null
+++ linux-2.6/mm/reserve.c
@@ -0,0 +1,429 @@
+/*
+ * Memory reserve management.
+ *
+ * Copyright (C) 2007, Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
+ *
+ * Description:
+ *
+ * Manage a set of memory reserves.
+ *
+ * A memory reserve is a reserve for a specified number of object of specified
+ * size. Since memory is managed in pages, this reserve demand is then
+ * translated into a page unit.
+ *
+ * So each reserve has a specified object limit, an object usage count and a
+ * number of pages required to back these objects.
+ *
+ * Usage is charged against a reserve, if the charge fails, the resource must
+ * not be allocated/used.
+ *
+ * The reserves are managed in a tree, and the resource demands (pages and
+ * limit) are propagated up the tree. Obviously the object limit will be
+ * meaningless as soon as the unit starts mixing, but the required page reserve
+ * (being of one unit) is still valid at the root.
+ *
+ * It is the page demand of the root node that is used to set the global
+ * reserve (adjust_memalloc_reserve() which sets zone->pages_emerg).
+ *
+ * As long as a subtree has the same usage unit, an aggregate node can be used
+ * to charge against, instead of the leaf nodes. However, do be consistent with
+ * who is charged, resource usage is not propagated up the tree (for
+ * performance reasons).
+ */
+
+#include <linux/reserve.h>
+#include <linux/mutex.h>
+#include <linux/mmzone.h>
+#include <linux/log2.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+static DEFINE_MUTEX(mem_reserve_mutex);
+
+/**
+ * @mem_reserve_root - the global reserve root
+ *
+ * The global reserve is empty, and has no limit unit, it merely
+ * acts as an aggregation point for reserves and an interface to
+ * adjust_memalloc_reserve().
+ */
+struct mem_reserve mem_reserve_root = {
+ .children = LIST_HEAD_INIT(mem_reserve_root.children),
+ .siblings = LIST_HEAD_INIT(mem_reserve_root.siblings),
+ .name = "total reserve",
+ .lock = __SPIN_LOCK_UNLOCKED(mem_reserve_root.lock),
+};
+EXPORT_SYMBOL_GPL(mem_reserve_root);
+
+/**
+ * mem_reserve_init - initialize a memory reserve object
+ * @res - the new reserve object
+ * @name - a name for this reserve
+ */
+void mem_reserve_init(struct mem_reserve *res, const char *name,
+ struct mem_reserve *parent)
+{
+ memset(res, 0, sizeof(*res));
+ INIT_LIST_HEAD(&res->children);
+ INIT_LIST_HEAD(&res->siblings);
+ res->name = name;
+ spin_lock_init(&res->lock);
+
+ if (parent)
+ mem_reserve_connect(res, parent);
+}
+EXPORT_SYMBOL_GPL(mem_reserve_init);
+
+/*
+ * propagate the pages and limit changes up the tree.
+ */
+static void __calc_reserve(struct mem_reserve *res, long pages, long limit)
+{
+ unsigned long flags;
+
+ for ( ; res; res = res->parent) {
+ res->pages += pages;
+
+ if (limit) {
+ spin_lock_irqsave(&res->lock, flags);
+ res->limit += limit;
+ spin_unlock_irqrestore(&res->lock, flags);
+ }
+ }
+}
+
+/**
+ * __mem_reserve_add - primitive to change the size of a reserve
+ * @res - reserve to change
+ * @pages - page delta
+ * @limit - usage limit delta
+ *
+ * Returns -ENOMEM when a size increase is not possible atm.
+ */
+static int __mem_reserve_add(struct mem_reserve *res, long pages, long limit)
+{
+ int ret = 0;
+ long reserve;
+
+ reserve = mem_reserve_root.pages;
+ __calc_reserve(res, pages, 0);
+ reserve = mem_reserve_root.pages - reserve;
+
+ if (reserve) {
+ ret = adjust_memalloc_reserve(reserve);
+ if (ret)
+ __calc_reserve(res, -pages, 0);
+ }
+
+ if (!ret)
+ __calc_reserve(res, 0, limit);
+
+ return ret;
+}
+
+/**
+ * __mem_reserve_charge - primitive to charge object usage to a reserve
+ * @res - reserve to charge
+ * @charge - size of the charge
+ * @overcommit - allow despite of limit (use with caution!)
+ *
+ * Returns non-zero on success, zero on failure.
+ */
+static
+int __mem_reserve_charge(struct mem_reserve *res, long charge, int overcommit)
+{
+ unsigned long flags;
+ int ret = 0;
+
+ spin_lock_irqsave(&res->lock, flags);
+ if (charge < 0 || res->usage + charge < res->limit || overcommit) {
+ res->usage += charge;
+ if (unlikely(res->usage < 0))
+ res->usage = 0;
+ ret = 1;
+ }
+ spin_unlock_irqrestore(&res->lock, flags);
+
+ return ret;
+}
+
+/**
+ * mem_reserve_connect - connect a reserve to another in a child-parent relation
+ * @new_child - the reserve node to connect (child)
+ * @node - the reserve node to connect to (parent)
+ *
+ * Returns -ENOMEM when the new connection would increase the reserve (parent
+ * is connected to mem_reserve_root) and there is no memory to do so.
+ *
+ * The child is _NOT_ connected on error.
+ */
+int mem_reserve_connect(struct mem_reserve *new_child, struct mem_reserve *node)
+{
+ int ret;
+
+ WARN_ON(!new_child->name);
+
+ mutex_lock(&mem_reserve_mutex);
+ new_child->parent = node;
+ list_add(&new_child->siblings, &node->children);
+ ret = __mem_reserve_add(node, new_child->pages, new_child->limit);
+ if (ret) {
+ new_child->parent = NULL;
+ list_del_init(&new_child->siblings);
+ }
+ mutex_unlock(&mem_reserve_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(mem_reserve_connect);
+
+/**
+ * mem_reserve_disconnect - sever a nodes connection to the reserve tree
+ * @node - the node to disconnect
+ *
+ * Could, in theory, return -ENOMEM, but since disconnecting a node _should_
+ * only decrease the reserves that _should_ not happen.
+ */
+int mem_reserve_disconnect(struct mem_reserve *node)
+{
+ int ret;
+
+ BUG_ON(!node->parent);
+
+ mutex_lock(&mem_reserve_mutex);
+ ret = __mem_reserve_add(node->parent, -node->pages, -node->limit);
+ if (!ret) {
+ node->parent = NULL;
+ list_del_init(&node->siblings);
+ }
+ mutex_unlock(&mem_reserve_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(mem_reserve_disconnect);
+
+#ifdef CONFIG_PROC_FS
+
+/*
+ * Simple output of the reserve tree in: /proc/reserve_info
+ * Example:
+ *
+ * localhost ~ # cat /proc/reserve_info
+ * total reserve 8156K (0/544817)
+ * total network reserve 8156K (0/544817)
+ * network TX reserve 196K (0/49)
+ * protocol TX pages 196K (0/49)
+ * network RX reserve 7960K (0/544768)
+ * IPv6 route cache 1372K (0/4096)
+ * IPv4 route cache 5468K (0/16384)
+ * SKB data reserve 1120K (0/524288)
+ * IPv6 fragment cache 560K (0/262144)
+ * IPv4 fragment cache 560K (0/262144)
+ */
+
+static void mem_reserve_show_item(struct seq_file *m, struct mem_reserve *res,
+ int nesting)
+{
+ int i;
+ struct mem_reserve *child;
+
+ for (i = 0; i < nesting; i++)
+ seq_puts(m, " ");
+
+ seq_printf(m, "%-30s %ldK (%ld/%ld)\n",
+ res->name, res->pages << (PAGE_SHIFT - 10),
+ res->usage, res->limit);
+
+ list_for_each_entry(child, &res->children, siblings)
+ mem_reserve_show_item(m, child, nesting+1);
+}
+
+static int mem_reserve_show(struct seq_file *m, void *v)
+{
+ mutex_lock(&mem_reserve_mutex);
+ mem_reserve_show_item(m, &mem_reserve_root, 0);
+ mutex_unlock(&mem_reserve_mutex);
+
+ return 0;
+}
+
+static int mem_reserve_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, mem_reserve_show, NULL);
+}
+
+static const struct file_operations mem_reserve_opterations = {
+ .open = mem_reserve_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static __init int mem_reserve_proc_init(void)
+{
+ struct proc_dir_entry *entry;
+
+ entry = create_proc_entry("reserve_info", S_IRUSR, NULL);
+ if (entry)
+ entry->proc_fops = &mem_reserve_opterations;
+
+ return 0;
+}
+
+__initcall(mem_reserve_proc_init);
+
+#endif
+
+/*
+ * alloc_page helpers
+ */
+
+/**
+ * mem_reserve_pages_set - set reserves size in pages
+ * @res - reserve to set
+ * @pages - size in pages to set it to
+ *
+ * Returns -ENOMEM when it fails to set the reserve. On failure the old size
+ * is preserved.
+ */
+int mem_reserve_pages_set(struct mem_reserve *res, long pages)
+{
+ int ret;
+
+ mutex_lock(&mem_reserve_mutex);
+ pages -= res->pages;
+ ret = __mem_reserve_add(res, pages, pages);
+ mutex_unlock(&mem_reserve_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(mem_reserve_pages_set);
+
+/**
+ * mem_reserve_pages_add - change the size in a relative way
+ * @res - reserve to change
+ * @pages - number of pages to add (or subtract when negative)
+ *
+ * Similar to mem_reserve_pages_set, except that the argument is relative
+ * instead of absolute.
+ *
+ * Returns -ENOMEM when it fails to increase.
+ */
+int mem_reserve_pages_add(struct mem_reserve *res, long pages)
+{
+ int ret;
+
+ mutex_lock(&mem_reserve_mutex);
+ ret = __mem_reserve_add(res, pages, pages);
+ mutex_unlock(&mem_reserve_mutex);
+
+ return ret;
+}
+
+/**
+ * mem_reserve_pages_charge - charge page usage to a reserve
+ * @res - reserve to charge
+ * @pages - size to charge
+ * @overcommit - disregard the usage limit (use with caution!)
+ *
+ * Returns non-zero on success.
+ */
+int mem_reserve_pages_charge(struct mem_reserve *res,
+ long pages, int overcommit)
+{
+ return __mem_reserve_charge(res, pages, overcommit);
+}
+EXPORT_SYMBOL_GPL(mem_reserve_pages_charge);
+
+/*
+ * kmalloc helpers
+ */
+
+/**
+ * mem_reserve_kmalloc_set - set this reserve to bytes worth of kmalloc
+ * @res - reserve to change
+ * @bytes - size in bytes to reserve
+ *
+ * Returns -ENOMEM on failure.
+ */
+int mem_reserve_kmalloc_set(struct mem_reserve *res, long bytes)
+{
+ int ret;
+ long pages;
+
+ mutex_lock(&mem_reserve_mutex);
+ pages = kestimate(GFP_ATOMIC, bytes);
+ pages -= res->pages;
+ bytes -= res->limit;
+ ret = __mem_reserve_add(res, pages, bytes);
+ mutex_unlock(&mem_reserve_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(mem_reserve_kmalloc_set);
+
+/**
+ * mem_reserve_kmalloc_charge - charge bytes to a reserve
+ * @res - reserve to charge
+ * @bytes - bytes to charge
+ * @overcommit - disregard the usage limit (use with caution!)
+ *
+ * Returns non-zero on success.
+ */
+int mem_reserve_kmalloc_charge(struct mem_reserve *res, long bytes,
+ int overcommit)
+{
+ if (bytes < 0)
+ bytes = -roundup_pow_of_two(-bytes);
+ else
+ bytes = roundup_pow_of_two(bytes);
+
+ return __mem_reserve_charge(res, bytes, overcommit);
+}
+EXPORT_SYMBOL_GPL(mem_reserve_kmalloc_charge);
+
+/*
+ * kmem_cache helpers
+ */
+
+/**
+ * mem_reserve_kmem_cache_set - set reserve to @objects worth of kmem_cache_alloc of @s
+ * @res - reserve to set
+ * @s - kmem_cache to reserve from
+ * @objects - number of objects to reserve
+ *
+ * Returns -ENOMEM on failure.
+ */
+int mem_reserve_kmem_cache_set(struct mem_reserve *res, struct kmem_cache *s,
+ int objects)
+{
+ int ret;
+ long pages;
+
+ mutex_lock(&mem_reserve_mutex);
+ pages = kmem_estimate_pages(s, GFP_ATOMIC, objects);
+ pages -= res->pages;
+ objects -= res->limit;
+ ret = __mem_reserve_add(res, pages, objects);
+ mutex_unlock(&mem_reserve_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(mem_reserve_kmem_cache_set);
+
+/**
+ * mem_reserve_kmem_cache_charge - charge (or uncharge) usage of objs
+ * @res - reserve to charge
+ * @objs - objects to charge for
+ * @overcommit - disregard the usage limit (use with caution!)
+ *
+ * Returns non-zero on success.
+ */
+int mem_reserve_kmem_cache_charge(struct mem_reserve *res, long objs,
+ int overcommit)
+{
+ return __mem_reserve_charge(res, objs, overcommit);
+}
+EXPORT_SYMBOL_GPL(mem_reserve_kmem_cache_charge);
--
^ permalink raw reply
* [PATCH 07/28] mm: emergency pool
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: mm-page_alloc-emerg.patch --]
[-- Type: text/plain, Size: 6847 bytes --]
Provide means to reserve a specific amount of pages.
The emergency pool is separated from the min watermark because ALLOC_HARDER
and ALLOC_HIGH modify the watermark in a relative way and thus do not ensure
a strict minimum.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/mmzone.h | 3 +
mm/page_alloc.c | 84 +++++++++++++++++++++++++++++++++++++++++++------
mm/vmstat.c | 6 +--
3 files changed, 79 insertions(+), 14 deletions(-)
Index: linux-2.6/include/linux/mmzone.h
===================================================================
--- linux-2.6.orig/include/linux/mmzone.h
+++ linux-2.6/include/linux/mmzone.h
@@ -213,7 +213,7 @@ enum zone_type {
struct zone {
/* Fields commonly accessed by the page allocator */
- unsigned long pages_min, pages_low, pages_high;
+ unsigned long pages_emerg, pages_min, pages_low, pages_high;
/*
* We don't know if the memory that we're going to allocate will be freeable
* or/and it will be released eventually, so to avoid totally wasting several
@@ -683,6 +683,7 @@ int sysctl_min_unmapped_ratio_sysctl_han
struct file *, void __user *, size_t *, loff_t *);
int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
struct file *, void __user *, size_t *, loff_t *);
+int adjust_memalloc_reserve(int pages);
extern int numa_zonelist_order_handler(struct ctl_table *, int,
struct file *, void __user *, size_t *, loff_t *);
Index: linux-2.6/mm/page_alloc.c
===================================================================
--- linux-2.6.orig/mm/page_alloc.c
+++ linux-2.6/mm/page_alloc.c
@@ -118,6 +118,8 @@ static char * const zone_names[MAX_NR_ZO
static DEFINE_SPINLOCK(min_free_lock);
int min_free_kbytes = 1024;
+static DEFINE_MUTEX(var_free_mutex);
+int var_free_kbytes;
unsigned long __meminitdata nr_kernel_pages;
unsigned long __meminitdata nr_all_pages;
@@ -1240,7 +1242,7 @@ int zone_watermark_ok(struct zone *z, in
if (alloc_flags & ALLOC_HARDER)
min -= min / 4;
- if (free_pages <= min + z->lowmem_reserve[classzone_idx])
+ if (free_pages <= min+z->lowmem_reserve[classzone_idx]+z->pages_emerg)
return 0;
for (o = 0; o < order; o++) {
/* At the next order, this order's pages become unavailable */
@@ -1569,7 +1571,7 @@ __alloc_pages(gfp_t gfp_mask, unsigned i
struct reclaim_state reclaim_state;
struct task_struct *p = current;
int do_retry;
- int alloc_flags;
+ int alloc_flags = 0;
int did_some_progress;
might_sleep_if(wait);
@@ -1721,8 +1723,8 @@ nofail_alloc:
nopage:
if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) {
printk(KERN_WARNING "%s: page allocation failure."
- " order:%d, mode:0x%x\n",
- p->comm, order, gfp_mask);
+ " order:%d, mode:0x%x, alloc_flags:0x%x, pflags:0x%x\n",
+ p->comm, order, gfp_mask, alloc_flags, p->flags);
dump_stack();
show_mem();
}
@@ -1937,9 +1939,9 @@ void show_free_areas(void)
"\n",
zone->name,
K(zone_page_state(zone, NR_FREE_PAGES)),
- K(zone->pages_min),
- K(zone->pages_low),
- K(zone->pages_high),
+ K(zone->pages_emerg + zone->pages_min),
+ K(zone->pages_emerg + zone->pages_low),
+ K(zone->pages_emerg + zone->pages_high),
K(zone_page_state(zone, NR_ACTIVE)),
K(zone_page_state(zone, NR_INACTIVE)),
K(zone->present_pages),
@@ -4125,7 +4127,7 @@ static void calculate_totalreserve_pages
}
/* we treat pages_high as reserved pages. */
- max += zone->pages_high;
+ max += zone->pages_high + zone->pages_emerg;
if (max > zone->present_pages)
max = zone->present_pages;
@@ -4182,7 +4184,8 @@ static void setup_per_zone_lowmem_reserv
*/
static void __setup_per_zone_pages_min(void)
{
- unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
+ unsigned pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
+ unsigned pages_emerg = var_free_kbytes >> (PAGE_SHIFT - 10);
unsigned long lowmem_pages = 0;
struct zone *zone;
unsigned long flags;
@@ -4194,11 +4197,13 @@ static void __setup_per_zone_pages_min(v
}
for_each_zone(zone) {
- u64 tmp;
+ u64 tmp, tmp_emerg;
spin_lock_irqsave(&zone->lru_lock, flags);
tmp = (u64)pages_min * zone->present_pages;
do_div(tmp, lowmem_pages);
+ tmp_emerg = (u64)pages_emerg * zone->present_pages;
+ do_div(tmp_emerg, lowmem_pages);
if (is_highmem(zone)) {
/*
* __GFP_HIGH and PF_MEMALLOC allocations usually don't
@@ -4217,12 +4222,14 @@ static void __setup_per_zone_pages_min(v
if (min_pages > 128)
min_pages = 128;
zone->pages_min = min_pages;
+ zone->pages_emerg = 0;
} else {
/*
* If it's a lowmem zone, reserve a number of pages
* proportionate to the zone's size.
*/
zone->pages_min = tmp;
+ zone->pages_emerg = tmp_emerg;
}
zone->pages_low = zone->pages_min + (tmp >> 2);
@@ -4244,6 +4251,63 @@ void setup_per_zone_pages_min(void)
spin_unlock_irqrestore(&min_free_lock, flags);
}
+static void __adjust_memalloc_reserve(int pages)
+{
+ var_free_kbytes += pages << (PAGE_SHIFT - 10);
+ BUG_ON(var_free_kbytes < 0);
+ setup_per_zone_pages_min();
+}
+
+static int test_reserve_limits(void)
+{
+ struct zone *zone;
+ int node;
+
+ for_each_zone(zone)
+ wakeup_kswapd(zone, 0);
+
+ for_each_online_node(node) {
+ struct page *page = alloc_pages_node(node, GFP_KERNEL, 0);
+ if (!page)
+ return -ENOMEM;
+
+ __free_page(page);
+ }
+
+ return 0;
+}
+
+/**
+ * adjust_memalloc_reserve - adjust the memalloc reserve
+ * @pages: number of pages to add
+ *
+ * It adds a number of pages to the memalloc reserve; if
+ * the number was positive it kicks reclaim into action to
+ * satisfy the higher watermarks.
+ *
+ * returns -ENOMEM when it failed to satisfy the watermarks.
+ */
+int adjust_memalloc_reserve(int pages)
+{
+ int err = 0;
+
+ mutex_lock(&var_free_mutex);
+ __adjust_memalloc_reserve(pages);
+ if (pages > 0) {
+ err = test_reserve_limits();
+ if (err) {
+ __adjust_memalloc_reserve(-pages);
+ goto unlock;
+ }
+ }
+ printk(KERN_DEBUG "Emergency reserve: %d\n", var_free_kbytes);
+
+unlock:
+ mutex_unlock(&var_free_mutex);
+ return err;
+}
+EXPORT_SYMBOL_GPL(adjust_memalloc_reserve);
+
/*
* Initialise min_free_kbytes.
*
Index: linux-2.6/mm/vmstat.c
===================================================================
--- linux-2.6.orig/mm/vmstat.c
+++ linux-2.6/mm/vmstat.c
@@ -754,9 +754,9 @@ static void zoneinfo_show_print(struct s
"\n spanned %lu"
"\n present %lu",
zone_page_state(zone, NR_FREE_PAGES),
- zone->pages_min,
- zone->pages_low,
- zone->pages_high,
+ zone->pages_emerg + zone->pages_min,
+ zone->pages_emerg + zone->pages_low,
+ zone->pages_emerg + zone->pages_high,
zone->pages_scanned,
zone->nr_scan_active, zone->nr_scan_inactive,
zone->spanned_pages,
--
^ permalink raw reply
* [PATCH 13/28] net: packet split receive api
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: net-ps_rx.patch --]
[-- Type: text/plain, Size: 9721 bytes --]
Add some packet-split receive hooks.
For one this allows to do NUMA node affine page allocs. Later on these hooks
will be extended to do emergency reserve allocations for fragments.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
drivers/net/bnx2.c | 8 +++-----
drivers/net/e1000/e1000_main.c | 8 ++------
drivers/net/e1000e/netdev.c | 7 ++-----
drivers/net/igb/igb_main.c | 8 ++------
drivers/net/ixgbe/ixgbe_main.c | 10 +++-------
drivers/net/sky2.c | 16 ++++++----------
include/linux/skbuff.h | 23 +++++++++++++++++++++++
net/core/skbuff.c | 20 ++++++++++++++++++++
8 files changed, 61 insertions(+), 39 deletions(-)
Index: linux-2.6/drivers/net/e1000/e1000_main.c
===================================================================
--- linux-2.6.orig/drivers/net/e1000/e1000_main.c
+++ linux-2.6/drivers/net/e1000/e1000_main.c
@@ -4478,12 +4478,8 @@ e1000_clean_rx_irq_ps(struct e1000_adapt
pci_unmap_page(pdev, ps_page_dma->ps_page_dma[j],
PAGE_SIZE, PCI_DMA_FROMDEVICE);
ps_page_dma->ps_page_dma[j] = 0;
- skb_fill_page_desc(skb, j, ps_page->ps_page[j], 0,
- length);
+ skb_add_rx_frag(skb, j, ps_page->ps_page[j], 0, length);
ps_page->ps_page[j] = NULL;
- skb->len += length;
- skb->data_len += length;
- skb->truesize += length;
}
/* strip the ethernet crc, problem is we're using pages now so
@@ -4691,7 +4687,7 @@ e1000_alloc_rx_buffers_ps(struct e1000_a
if (j < adapter->rx_ps_pages) {
if (likely(!ps_page->ps_page[j])) {
ps_page->ps_page[j] =
- alloc_page(GFP_ATOMIC);
+ netdev_alloc_page(netdev);
if (unlikely(!ps_page->ps_page[j])) {
adapter->alloc_rx_buff_failed++;
goto no_buffers;
Index: linux-2.6/include/linux/skbuff.h
===================================================================
--- linux-2.6.orig/include/linux/skbuff.h
+++ linux-2.6/include/linux/skbuff.h
@@ -846,6 +846,9 @@ static inline void skb_fill_page_desc(st
skb_shinfo(skb)->nr_frags = i + 1;
}
+extern void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page,
+ int off, int size);
+
#define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags)
#define SKB_FRAG_ASSERT(skb) BUG_ON(skb_shinfo(skb)->frag_list)
#define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
@@ -1339,6 +1342,26 @@ static inline struct sk_buff *netdev_all
return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
}
+extern struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask);
+
+/**
+ * netdev_alloc_page - allocate a page for ps-rx on a specific device
+ * @dev: network device to receive on
+ *
+ * Allocate a new page node local to the specified device.
+ *
+ * %NULL is returned if there is no free memory.
+ */
+static inline struct page *netdev_alloc_page(struct net_device *dev)
+{
+ return __netdev_alloc_page(dev, GFP_ATOMIC);
+}
+
+static inline void netdev_free_page(struct net_device *dev, struct page *page)
+{
+ __free_page(page);
+}
+
/**
* skb_clone_writable - is the header of a clone writable
* @skb: buffer to check
Index: linux-2.6/net/core/skbuff.c
===================================================================
--- linux-2.6.orig/net/core/skbuff.c
+++ linux-2.6/net/core/skbuff.c
@@ -263,6 +263,26 @@ struct sk_buff *__netdev_alloc_skb(struc
return skb;
}
+struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)
+{
+ int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
+ struct page *page;
+
+ page = alloc_pages_node(node, gfp_mask, 0);
+ return page;
+}
+EXPORT_SYMBOL(__netdev_alloc_page);
+
+void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
+ int size)
+{
+ skb_fill_page_desc(skb, i, page, off, size);
+ skb->len += size;
+ skb->data_len += size;
+ skb->truesize += size;
+}
+EXPORT_SYMBOL(skb_add_rx_frag);
+
static void skb_drop_list(struct sk_buff **listp)
{
struct sk_buff *list = *listp;
Index: linux-2.6/drivers/net/sky2.c
===================================================================
--- linux-2.6.orig/drivers/net/sky2.c
+++ linux-2.6/drivers/net/sky2.c
@@ -1216,7 +1216,7 @@ static struct sk_buff *sky2_rx_alloc(str
}
for (i = 0; i < sky2->rx_nfrags; i++) {
- struct page *page = alloc_page(GFP_ATOMIC);
+ struct page *page = netdev_alloc_page(sky2->netdev);
if (!page)
goto free_partial;
@@ -2088,8 +2088,8 @@ static struct sk_buff *receive_copy(stru
}
/* Adjust length of skb with fragments to match received data */
-static void skb_put_frags(struct sk_buff *skb, unsigned int hdr_space,
- unsigned int length)
+static void skb_put_frags(struct sky2_port *sky2, struct sk_buff *skb,
+ unsigned int hdr_space, unsigned int length)
{
int i, num_frags;
unsigned int size;
@@ -2106,15 +2106,11 @@ static void skb_put_frags(struct sk_buff
if (length == 0) {
/* don't need this page */
- __free_page(frag->page);
+ netdev_free_page(sky2->netdev, frag->page);
--skb_shinfo(skb)->nr_frags;
} else {
size = min(length, (unsigned) PAGE_SIZE);
-
- frag->size = size;
- skb->data_len += size;
- skb->truesize += size;
- skb->len += size;
+ skb_add_rx_frag(skb, i, frag->page, 0, size);
length -= size;
}
}
@@ -2141,7 +2137,7 @@ static struct sk_buff *receive_new(struc
sky2_rx_map_skb(sky2->hw->pdev, re, hdr_space);
if (skb_shinfo(skb)->nr_frags)
- skb_put_frags(skb, hdr_space, length);
+ skb_put_frags(sky2, skb, hdr_space, length);
else
skb_put(skb, length);
return skb;
Index: linux-2.6/drivers/net/bnx2.c
===================================================================
--- linux-2.6.orig/drivers/net/bnx2.c
+++ linux-2.6/drivers/net/bnx2.c
@@ -2356,7 +2356,7 @@ bnx2_alloc_rx_page(struct bnx2 *bp, u16
struct sw_pg *rx_pg = &bp->rx_pg_ring[index];
struct rx_bd *rxbd =
&bp->rx_pg_desc_ring[RX_RING(index)][RX_IDX(index)];
- struct page *page = alloc_page(GFP_ATOMIC);
+ struct page *page = netdev_alloc_page(bp->dev);
if (!page)
return -ENOMEM;
@@ -2381,7 +2381,7 @@ bnx2_free_rx_page(struct bnx2 *bp, u16 i
pci_unmap_page(bp->pdev, pci_unmap_addr(rx_pg, mapping), PAGE_SIZE,
PCI_DMA_FROMDEVICE);
- __free_page(page);
+ netdev_free_page(bp->dev, page);
rx_pg->page = NULL;
}
@@ -2705,9 +2705,7 @@ bnx2_rx_skb(struct bnx2 *bp, struct bnx2
}
frag_size -= frag_len;
- skb->data_len += frag_len;
- skb->truesize += frag_len;
- skb->len += frag_len;
+ skb_add_rx_frag(skb, i, rx_pg->page, 0, frag_len);
pg_prod = NEXT_RX_BD(pg_prod);
pg_cons = RX_PG_RING_IDX(NEXT_RX_BD(pg_cons));
Index: linux-2.6/drivers/net/e1000e/netdev.c
===================================================================
--- linux-2.6.orig/drivers/net/e1000e/netdev.c
+++ linux-2.6/drivers/net/e1000e/netdev.c
@@ -252,7 +252,7 @@ static void e1000_alloc_rx_buffers_ps(st
continue;
}
if (!ps_page->page) {
- ps_page->page = alloc_page(GFP_ATOMIC);
+ ps_page->page = netdev_alloc_page(netdev);
if (!ps_page->page) {
adapter->alloc_rx_buff_failed++;
goto no_buffers;
@@ -714,11 +714,8 @@ static bool e1000_clean_rx_irq_ps(struct
pci_unmap_page(pdev, ps_page->dma, PAGE_SIZE,
PCI_DMA_FROMDEVICE);
ps_page->dma = 0;
- skb_fill_page_desc(skb, j, ps_page->page, 0, length);
+ skb_add_rx_frag(skb, j, ps_page->page, 0, length);
ps_page->page = NULL;
- skb->len += length;
- skb->data_len += length;
- skb->truesize += length;
}
copydone:
Index: linux-2.6/drivers/net/igb/igb_main.c
===================================================================
--- linux-2.6.orig/drivers/net/igb/igb_main.c
+++ linux-2.6/drivers/net/igb/igb_main.c
@@ -3506,13 +3506,9 @@ static bool igb_clean_rx_irq_adv(struct
pci_unmap_page(pdev, buffer_info->page_dma,
PAGE_SIZE, PCI_DMA_FROMDEVICE);
buffer_info->page_dma = 0;
- skb_fill_page_desc(skb, j, buffer_info->page,
- 0, length);
+ skb_add_rx_frag(skb, j, buffer_info->page, 0, length);
buffer_info->page = NULL;
- skb->len += length;
- skb->data_len += length;
- skb->truesize += length;
rx_desc->wb.upper.status_error = 0;
if (staterr & E1000_RXD_STAT_EOP)
break;
@@ -3614,7 +3610,7 @@ static void igb_alloc_rx_buffers_adv(str
rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
if (adapter->rx_ps_hdr_size && !buffer_info->page) {
- buffer_info->page = alloc_page(GFP_ATOMIC);
+ buffer_info->page = netdev_alloc_page(netdev);
if (!buffer_info->page) {
adapter->alloc_rx_buff_failed++;
goto no_buffers;
Index: linux-2.6/drivers/net/ixgbe/ixgbe_main.c
===================================================================
--- linux-2.6.orig/drivers/net/ixgbe/ixgbe_main.c
+++ linux-2.6/drivers/net/ixgbe/ixgbe_main.c
@@ -367,7 +367,7 @@ static void ixgbe_alloc_rx_buffers(struc
if (!rx_buffer_info->page &&
(adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) {
- rx_buffer_info->page = alloc_page(GFP_ATOMIC);
+ rx_buffer_info->page = netdev_alloc_page(netdev);
if (!rx_buffer_info->page) {
adapter->alloc_rx_page_failed++;
goto no_buffers;
@@ -490,13 +490,9 @@ static bool ixgbe_clean_rx_irq(struct ix
pci_unmap_page(pdev, rx_buffer_info->page_dma,
PAGE_SIZE, PCI_DMA_FROMDEVICE);
rx_buffer_info->page_dma = 0;
- skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
- rx_buffer_info->page, 0, upper_len);
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+ rx_buffer_info->page, 0, upper_len);
rx_buffer_info->page = NULL;
-
- skb->len += upper_len;
- skb->data_len += upper_len;
- skb->truesize += upper_len;
}
i++;
--
^ permalink raw reply
* [PATCH 15/28] netvm: network reserve infrastructure
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: netvm-reserve.patch --]
[-- Type: text/plain, Size: 7304 bytes --]
Provide the basic infrastructure to reserve and charge/account network memory.
We provide the following reserve tree:
1) total network reserve
2) network TX reserve
3) protocol TX pages
4) network RX reserve
5) SKB data reserve
[1] is used to make all the network reserves a single subtree, for easy
manipulation.
[2] and [4] are merely for eastetic reasons.
The TX pages reserve [3] is assumed bounded by it being the upper bound of
memory that can be used for sending pages (not quite true, but good enough)
The SKB reserve [5] is an aggregate reserve, which is used to charge SKB data
against in the fallback path.
The consumers for these reserves are sockets marked with:
SOCK_MEMALLOC
Such sockets are to be used to service the VM (iow. to swap over). They
must be handled kernel side, exposing such a socket to user-space is a BUG.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/net/sock.h | 35 +++++++++++++++-
net/Kconfig | 3 +
net/core/sock.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 150 insertions(+), 1 deletion(-)
Index: linux-2.6/include/net/sock.h
===================================================================
--- linux-2.6.orig/include/net/sock.h
+++ linux-2.6/include/net/sock.h
@@ -51,6 +51,7 @@
#include <linux/skbuff.h> /* struct sk_buff */
#include <linux/mm.h>
#include <linux/security.h>
+#include <linux/reserve.h>
#include <linux/filter.h>
@@ -405,6 +406,7 @@ enum sock_flags {
SOCK_RCVTSTAMPNS, /* %SO_TIMESTAMPNS setting */
SOCK_LOCALROUTE, /* route locally only, %SO_DONTROUTE setting */
SOCK_QUEUE_SHRUNK, /* write queue has been shrunk recently */
+ SOCK_MEMALLOC, /* the VM depends on us - make sure we're serviced */
};
static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
@@ -427,9 +429,40 @@ static inline int sock_flag(struct sock
return test_bit(flag, &sk->sk_flags);
}
+static inline int sk_has_memalloc(struct sock *sk)
+{
+ return sock_flag(sk, SOCK_MEMALLOC);
+}
+
+/*
+ * Guestimate the per request queue TX upper bound.
+ *
+ * Max packet size is 64k, and we need to reserve that much since the data
+ * might need to bounce it. Double it to be on the safe side.
+ */
+#define TX_RESERVE_PAGES DIV_ROUND_UP(2*65536, PAGE_SIZE)
+
+extern atomic_t memalloc_socks;
+
+extern struct mem_reserve net_rx_reserve;
+extern struct mem_reserve net_skb_reserve;
+
+static inline int sk_memalloc_socks(void)
+{
+ return atomic_read(&memalloc_socks);
+}
+
+extern int rx_emergency_get(int bytes);
+extern int rx_emergency_get_overcommit(int bytes);
+extern void rx_emergency_put(int bytes);
+
+extern int sk_adjust_memalloc(int socks, long tx_reserve_pages);
+extern int sk_set_memalloc(struct sock *sk);
+extern int sk_clear_memalloc(struct sock *sk);
+
static inline gfp_t sk_allocation(struct sock *sk, gfp_t gfp_mask)
{
- return gfp_mask;
+ return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
}
static inline void sk_acceptq_removed(struct sock *sk)
Index: linux-2.6/net/core/sock.c
===================================================================
--- linux-2.6.orig/net/core/sock.c
+++ linux-2.6/net/core/sock.c
@@ -112,6 +112,7 @@
#include <linux/tcp.h>
#include <linux/init.h>
#include <linux/highmem.h>
+#include <linux/reserve.h>
#include <asm/uaccess.h>
#include <asm/system.h>
@@ -213,6 +214,111 @@ __u32 sysctl_rmem_default __read_mostly
/* Maximal space eaten by iovec or ancilliary data plus some space */
int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
+atomic_t memalloc_socks;
+
+static struct mem_reserve net_reserve;
+struct mem_reserve net_rx_reserve;
+EXPORT_SYMBOL_GPL(net_rx_reserve); /* modular ipv6 only */
+struct mem_reserve net_skb_reserve;
+EXPORT_SYMBOL_GPL(net_skb_reserve); /* modular ipv6 only */
+static struct mem_reserve net_tx_reserve;
+static struct mem_reserve net_tx_pages;
+
+
+/*
+ * is there room for another emergency packet?
+ */
+static int __rx_emergency_get(int bytes, bool overcommit)
+{
+ return mem_reserve_kmalloc_charge(&net_skb_reserve, bytes, overcommit);
+}
+
+int rx_emergency_get(int bytes)
+{
+ return __rx_emergency_get(bytes, false);
+}
+
+int rx_emergency_get_overcommit(int bytes)
+{
+ return __rx_emergency_get(bytes, true);
+}
+
+void rx_emergency_put(int bytes)
+{
+ mem_reserve_kmalloc_charge(&net_skb_reserve, -bytes, 0);
+}
+
+/**
+ * sk_adjust_memalloc - adjust the global memalloc reserve for critical RX
+ * @socks: number of new %SOCK_MEMALLOC sockets
+ * @tx_resserve_pages: number of pages to (un)reserve for TX
+ *
+ * This function adjusts the memalloc reserve based on system demand.
+ * The RX reserve is a limit, and only added once, not for each socket.
+ *
+ * NOTE:
+ * @tx_reserve_pages is an upper-bound of memory used for TX hence
+ * we need not account the pages like we do for RX pages.
+ */
+int sk_adjust_memalloc(int socks, long tx_reserve_pages)
+{
+ int nr_socks;
+ int err;
+
+ err = mem_reserve_pages_add(&net_tx_pages, tx_reserve_pages);
+ if (err)
+ return err;
+
+ nr_socks = atomic_read(&memalloc_socks);
+ if (!nr_socks && socks > 0)
+ err = mem_reserve_connect(&net_reserve, &mem_reserve_root);
+ nr_socks = atomic_add_return(socks, &memalloc_socks);
+ if (!nr_socks && socks)
+ err = mem_reserve_disconnect(&net_reserve);
+
+ if (err)
+ mem_reserve_pages_add(&net_tx_pages, -tx_reserve_pages);
+
+ return err;
+}
+
+/**
+ * sk_set_memalloc - sets %SOCK_MEMALLOC
+ * @sk: socket to set it on
+ *
+ * Set %SOCK_MEMALLOC on a socket and increase the memalloc reserve
+ * accordingly.
+ */
+int sk_set_memalloc(struct sock *sk)
+{
+ int set = sock_flag(sk, SOCK_MEMALLOC);
+#ifndef CONFIG_NETVM
+ BUG();
+#endif
+ if (!set) {
+ int err = sk_adjust_memalloc(1, 0);
+ if (err)
+ return err;
+
+ sock_set_flag(sk, SOCK_MEMALLOC);
+ sk->sk_allocation |= __GFP_MEMALLOC;
+ }
+ return !set;
+}
+EXPORT_SYMBOL_GPL(sk_set_memalloc);
+
+int sk_clear_memalloc(struct sock *sk)
+{
+ int set = sock_flag(sk, SOCK_MEMALLOC);
+ if (set) {
+ sk_adjust_memalloc(-1, 0);
+ sock_reset_flag(sk, SOCK_MEMALLOC);
+ sk->sk_allocation &= ~__GFP_MEMALLOC;
+ }
+ return set;
+}
+EXPORT_SYMBOL_GPL(sk_clear_memalloc);
+
static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
{
struct timeval tv;
@@ -968,6 +1074,7 @@ void sk_free(struct sock *sk)
{
struct sk_filter *filter;
+ sk_clear_memalloc(sk);
if (sk->sk_destruct)
sk->sk_destruct(sk);
@@ -1095,6 +1202,12 @@ void __init sk_init(void)
sysctl_wmem_max = 131071;
sysctl_rmem_max = 131071;
}
+
+ mem_reserve_init(&net_reserve, "total network reserve", NULL);
+ mem_reserve_init(&net_rx_reserve, "network RX reserve", &net_reserve);
+ mem_reserve_init(&net_skb_reserve, "SKB data reserve", &net_rx_reserve);
+ mem_reserve_init(&net_tx_reserve, "network TX reserve", &net_reserve);
+ mem_reserve_init(&net_tx_pages, "protocol TX pages", &net_tx_reserve);
}
/*
Index: linux-2.6/net/Kconfig
===================================================================
--- linux-2.6.orig/net/Kconfig
+++ linux-2.6/net/Kconfig
@@ -250,6 +250,9 @@ endmenu
source "net/rfkill/Kconfig"
source "net/9p/Kconfig"
+config NETVM
+ def_bool n
+
endif # if NET
endmenu # Networking
--
^ permalink raw reply
* [PATCH 27/28] nfs: enable swap on NFS
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: nfs-swap_ops.patch --]
[-- Type: text/plain, Size: 10795 bytes --]
Implement all the new swapfile a_ops for NFS. This will set the NFS socket to
SOCK_MEMALLOC and run socket reconnect under PF_MEMALLOC as well as reset
SOCK_MEMALLOC before engaging the protocol ->connect() method.
PF_MEMALLOC should allow the allocation of struct socket and related objects
and the early (re)setting of SOCK_MEMALLOC should allow us to receive the
packets required for the TCP connection buildup.
(swapping continues over a server reset during heavy network traffic)
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
fs/Kconfig | 17 +++++++++++
fs/nfs/file.c | 12 ++++++++
fs/nfs/write.c | 19 +++++++++++++
include/linux/nfs_fs.h | 2 +
include/linux/sunrpc/xprt.h | 5 ++-
net/sunrpc/sched.c | 9 ++++--
net/sunrpc/xprtsock.c | 63 ++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 124 insertions(+), 3 deletions(-)
Index: linux-2.6/fs/nfs/file.c
===================================================================
--- linux-2.6.orig/fs/nfs/file.c
+++ linux-2.6/fs/nfs/file.c
@@ -373,6 +373,13 @@ static int nfs_launder_page(struct page
return nfs_wb_page(page_file_mapping(page)->host, page);
}
+#ifdef CONFIG_NFS_SWAP
+static int nfs_swapfile(struct address_space *mapping, int enable)
+{
+ return xs_swapper(NFS_CLIENT(mapping->host)->cl_xprt, enable);
+}
+#endif
+
const struct address_space_operations nfs_file_aops = {
.readpage = nfs_readpage,
.readpages = nfs_readpages,
@@ -387,6 +394,11 @@ const struct address_space_operations nf
.direct_IO = nfs_direct_IO,
#endif
.launder_page = nfs_launder_page,
+#ifdef CONFIG_NFS_SWAP
+ .swapfile = nfs_swapfile,
+ .swap_out = nfs_swap_out,
+ .swap_in = nfs_readpage,
+#endif
};
static int nfs_vm_page_mkwrite(struct vm_area_struct *vma, struct page *page)
Index: linux-2.6/fs/nfs/write.c
===================================================================
--- linux-2.6.orig/fs/nfs/write.c
+++ linux-2.6/fs/nfs/write.c
@@ -362,6 +362,25 @@ int nfs_writepage(struct page *page, str
return ret;
}
+int nfs_swap_out(struct file *file, struct page *page,
+ struct writeback_control *wbc)
+{
+ struct nfs_open_context *ctx = nfs_file_open_context(file);
+ int status;
+
+ status = nfs_writepage_setup(ctx, page, 0, nfs_page_length(page));
+ if (status < 0) {
+ nfs_set_pageerror(page);
+ goto out;
+ }
+
+ status = nfs_writepage_locked(page, wbc);
+
+out:
+ unlock_page(page);
+ return status;
+}
+
static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
int ret;
Index: linux-2.6/include/linux/nfs_fs.h
===================================================================
--- linux-2.6.orig/include/linux/nfs_fs.h
+++ linux-2.6/include/linux/nfs_fs.h
@@ -453,6 +453,8 @@ extern int nfs_flush_incompatible(struc
extern int nfs_updatepage(struct file *, struct page *, unsigned int, unsigned int);
extern int nfs_writeback_done(struct rpc_task *, struct nfs_write_data *);
extern void nfs_writedata_release(void *);
+extern int nfs_swap_out(struct file *file, struct page *page,
+ struct writeback_control *wbc);
/*
* Try to write back everything synchronously (but check the
Index: linux-2.6/fs/Kconfig
===================================================================
--- linux-2.6.orig/fs/Kconfig
+++ linux-2.6/fs/Kconfig
@@ -1661,6 +1661,18 @@ config NFS_DIRECTIO
causes open() to return EINVAL if a file residing in NFS is
opened with the O_DIRECT flag.
+config NFS_SWAP
+ bool "Provide swap over NFS support"
+ default n
+ depends on NFS_FS
+ select SUNRPC_SWAP
+ help
+ This option enables swapon to work on files located on NFS mounts.
+
+ For more details, see Documentation/vm_deadlock.txt
+
+ If unsure, say N.
+
config NFSD
tristate "NFS server support"
depends on INET
@@ -1794,6 +1806,11 @@ config SUNRPC_BIND34
If unsure, say N to get traditional behavior (version 2 rpcbind
requests only).
+config SUNRPC_SWAP
+ def_bool n
+ depends on SUNRPC
+ select NETVM
+
config RPCSEC_GSS_KRB5
tristate "Secure RPC: Kerberos V mechanism (EXPERIMENTAL)"
depends on SUNRPC && EXPERIMENTAL
Index: linux-2.6/include/linux/sunrpc/xprt.h
===================================================================
--- linux-2.6.orig/include/linux/sunrpc/xprt.h
+++ linux-2.6/include/linux/sunrpc/xprt.h
@@ -143,7 +143,9 @@ struct rpc_xprt {
unsigned int max_reqs; /* total slots */
unsigned long state; /* transport state */
unsigned char shutdown : 1, /* being shut down */
- resvport : 1; /* use a reserved port */
+ resvport : 1, /* use a reserved port */
+ swapper : 1; /* we're swapping over this
+ transport */
unsigned int bind_index; /* bind function index */
/*
@@ -241,6 +243,7 @@ void xprt_complete_rqst(struct rpc_tas
void xprt_release_rqst_cong(struct rpc_task *task);
void xprt_disconnect_done(struct rpc_xprt *xprt);
void xprt_force_disconnect(struct rpc_xprt *xprt);
+int xs_swapper(struct rpc_xprt *xprt, int enable);
/*
* Reserved bit positions in xprt->state
Index: linux-2.6/net/sunrpc/sched.c
===================================================================
--- linux-2.6.orig/net/sunrpc/sched.c
+++ linux-2.6/net/sunrpc/sched.c
@@ -766,7 +766,10 @@ struct rpc_buffer {
void *rpc_malloc(struct rpc_task *task, size_t size)
{
struct rpc_buffer *buf;
- gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
+ gfp_t gfp = GFP_NOWAIT;
+
+ if (RPC_IS_SWAPPER(task))
+ gfp |= __GFP_MEMALLOC;
size += sizeof(struct rpc_buffer);
if (size <= RPC_BUFFER_MAXSIZE)
@@ -839,6 +842,8 @@ static void rpc_init_task(struct rpc_tas
kref_get(&task->tk_client->cl_kref);
if (task->tk_client->cl_softrtry)
task->tk_flags |= RPC_TASK_SOFT;
+ if (task->tk_client->cl_xprt->swapper)
+ task->tk_flags |= RPC_TASK_SWAPPER;
}
if (task->tk_ops->rpc_call_prepare != NULL)
@@ -865,7 +870,7 @@ static void rpc_init_task(struct rpc_tas
static struct rpc_task *
rpc_alloc_task(void)
{
- return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
+ return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOIO);
}
static void rpc_free_task(struct rcu_head *rcu)
Index: linux-2.6/net/sunrpc/xprtsock.c
===================================================================
--- linux-2.6.orig/net/sunrpc/xprtsock.c
+++ linux-2.6/net/sunrpc/xprtsock.c
@@ -1451,6 +1451,9 @@ static void xs_udp_finish_connecting(str
transport->sock = sock;
transport->inet = sk;
+ if (xprt->swapper)
+ sk_set_memalloc(sk);
+
write_unlock_bh(&sk->sk_callback_lock);
}
xs_udp_do_set_buffer_size(xprt);
@@ -1468,11 +1471,15 @@ static void xs_udp_connect_worker4(struc
container_of(work, struct sock_xprt, connect_worker.work);
struct rpc_xprt *xprt = &transport->xprt;
struct socket *sock = transport->sock;
+ unsigned long pflags = current->flags;
int err, status = -EIO;
if (xprt->shutdown || !xprt_bound(xprt))
goto out;
+ if (xprt->swapper)
+ current->flags |= PF_MEMALLOC;
+
/* Start by resetting any existing state */
xs_close(xprt);
@@ -1495,6 +1502,7 @@ static void xs_udp_connect_worker4(struc
out:
xprt_wake_pending_tasks(xprt, status);
xprt_clear_connecting(xprt);
+ tsk_restore_flags(current, pflags, PF_MEMALLOC);
}
/**
@@ -1509,11 +1517,15 @@ static void xs_udp_connect_worker6(struc
container_of(work, struct sock_xprt, connect_worker.work);
struct rpc_xprt *xprt = &transport->xprt;
struct socket *sock = transport->sock;
+ unsigned long pflags = current->flags;
int err, status = -EIO;
if (xprt->shutdown || !xprt_bound(xprt))
goto out;
+ if (xprt->swapper)
+ current->flags |= PF_MEMALLOC;
+
/* Start by resetting any existing state */
xs_close(xprt);
@@ -1536,6 +1548,7 @@ static void xs_udp_connect_worker6(struc
out:
xprt_wake_pending_tasks(xprt, status);
xprt_clear_connecting(xprt);
+ tsk_restore_flags(current, pflags, PF_MEMALLOC);
}
/*
@@ -1595,6 +1608,9 @@ static int xs_tcp_finish_connecting(stru
write_unlock_bh(&sk->sk_callback_lock);
}
+ if (xprt->swapper)
+ sk_set_memalloc(transport->inet);
+
/* Tell the socket layer to start connecting... */
xprt->stat.connect_count++;
xprt->stat.connect_start = jiffies;
@@ -1613,11 +1629,15 @@ static void xs_tcp_connect_worker4(struc
container_of(work, struct sock_xprt, connect_worker.work);
struct rpc_xprt *xprt = &transport->xprt;
struct socket *sock = transport->sock;
+ unsigned long pflags = current->flags;
int err, status = -EIO;
if (xprt->shutdown || !xprt_bound(xprt))
goto out;
+ if (xprt->swapper)
+ current->flags |= PF_MEMALLOC;
+
if (!sock) {
/* start from scratch */
if ((err = sock_create_kern(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock)) < 0) {
@@ -1659,6 +1679,7 @@ out:
xprt_wake_pending_tasks(xprt, status);
out_clear:
xprt_clear_connecting(xprt);
+ tsk_restore_flags(current, pflags, PF_MEMALLOC);
}
/**
@@ -1673,11 +1694,15 @@ static void xs_tcp_connect_worker6(struc
container_of(work, struct sock_xprt, connect_worker.work);
struct rpc_xprt *xprt = &transport->xprt;
struct socket *sock = transport->sock;
+ unsigned long pflags = current->flags;
int err, status = -EIO;
if (xprt->shutdown || !xprt_bound(xprt))
goto out;
+ if (xprt->swapper)
+ current->flags |= PF_MEMALLOC;
+
if (!sock) {
/* start from scratch */
if ((err = sock_create_kern(PF_INET6, SOCK_STREAM, IPPROTO_TCP, &sock)) < 0) {
@@ -1718,6 +1743,7 @@ out:
xprt_wake_pending_tasks(xprt, status);
out_clear:
xprt_clear_connecting(xprt);
+ tsk_restore_flags(current, pflags, PF_MEMALLOC);
}
/**
@@ -2055,6 +2081,43 @@ int init_socket_xprt(void)
return 0;
}
+#ifdef CONFIG_SUNRPC_SWAP
+#define RPC_BUF_RESERVE_PAGES \
+ kestimate_single(sizeof(struct rpc_rqst), GFP_KERNEL, RPC_MAX_SLOT_TABLE)
+#define RPC_RESERVE_PAGES (RPC_BUF_RESERVE_PAGES + TX_RESERVE_PAGES)
+
+/**
+ * xs_swapper - Tag this transport as being used for swap.
+ * @xprt: transport to tag
+ * @enable: enable/disable
+ *
+ */
+int xs_swapper(struct rpc_xprt *xprt, int enable)
+{
+ struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
+ int err = 0;
+
+ if (enable) {
+ /*
+ * keep one extra sock reference so the reserve won't dip
+ * when the socket gets reconnected.
+ */
+ err = sk_adjust_memalloc(1, RPC_RESERVE_PAGES);
+ if (!err) {
+ sk_set_memalloc(transport->inet);
+ xprt->swapper = 1;
+ }
+ } else if (xprt->swapper) {
+ xprt->swapper = 0;
+ sk_clear_memalloc(transport->inet);
+ sk_adjust_memalloc(-1, -RPC_RESERVE_PAGES);
+ }
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(xs_swapper);
+#endif
+
/**
* cleanup_socket_xprt - remove xprtsock's sysctls, unregister
*
--
^ permalink raw reply
* [PATCH 08/28] mm: system wide ALLOC_NO_WATERMARK
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: global-ALLOC_NO_WATERMARKS.patch --]
[-- Type: text/plain, Size: 853 bytes --]
Change ALLOC_NO_WATERMARK page allocation such that the reserves are system
wide - which they are per setup_per_zone_pages_min(), when we scrape the
barrel, do it properly.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
mm/page_alloc.c | 6 ++++++
1 file changed, 6 insertions(+)
Index: linux-2.6/mm/page_alloc.c
===================================================================
--- linux-2.6.orig/mm/page_alloc.c
+++ linux-2.6/mm/page_alloc.c
@@ -1552,6 +1552,12 @@ restart:
rebalance:
if (alloc_flags & ALLOC_NO_WATERMARKS) {
nofail_alloc:
+ /*
+ * break out of mempolicy boundaries
+ */
+ zonelist = NODE_DATA(numa_node_id())->node_zonelists +
+ gfp_zone(gfp_mask);
+
/* go through the zonelist yet again, ignoring mins */
page = get_page_from_freelist(gfp_mask, order, zonelist,
ALLOC_NO_WATERMARKS);
--
^ permalink raw reply
* [PATCH 17/28] netvm: hook skb allocation to reserves
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: netvm-skbuff-reserve.patch --]
[-- Type: text/plain, Size: 14777 bytes --]
Change the skb allocation api to indicate RX usage and use this to fall back to
the reserve when needed. SKBs allocated from the reserve are tagged in
skb->emergency.
Teach all other skb ops about emergency skbs and the reserve accounting.
Use the (new) packet split API to allocate and track fragment pages from the
emergency reserve. Do this using an atomic counter in page->index. This is
needed because the fragments have a different sharing semantic than that
indicated by skb_shinfo()->dataref.
Note that the decision to distinguish between regular and emergency SKBs allows
the accounting overhead to be limited to the later kind.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/mm_types.h | 1
include/linux/skbuff.h | 26 +++++-
net/core/skbuff.c | 177 +++++++++++++++++++++++++++++++++++++++++------
3 files changed, 177 insertions(+), 27 deletions(-)
Index: linux-2.6/include/linux/skbuff.h
===================================================================
--- linux-2.6.orig/include/linux/skbuff.h
+++ linux-2.6/include/linux/skbuff.h
@@ -308,7 +308,9 @@ struct sk_buff {
__u16 tc_verd; /* traffic control verdict */
#endif
#endif
- /* 2 byte hole */
+ __u8 emergency:1;
+ /* 7 bit hole */
+ /* 1 byte hole */
#ifdef CONFIG_NET_DMA
dma_cookie_t dma_cookie;
@@ -339,10 +341,22 @@ struct sk_buff {
#include <asm/system.h>
+#define SKB_ALLOC_FCLONE 0x01
+#define SKB_ALLOC_RX 0x02
+
+static inline bool skb_emergency(const struct sk_buff *skb)
+{
+#ifdef CONFIG_NETVM
+ return unlikely(skb->emergency);
+#else
+ return false;
+#endif
+}
+
extern void kfree_skb(struct sk_buff *skb);
extern void __kfree_skb(struct sk_buff *skb);
extern struct sk_buff *__alloc_skb(unsigned int size,
- gfp_t priority, int fclone, int node);
+ gfp_t priority, int flags, int node);
static inline struct sk_buff *alloc_skb(unsigned int size,
gfp_t priority)
{
@@ -352,7 +366,7 @@ static inline struct sk_buff *alloc_skb(
static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
gfp_t priority)
{
- return __alloc_skb(size, priority, 1, -1);
+ return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, -1);
}
extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
@@ -1297,7 +1311,8 @@ static inline void __skb_queue_purge(str
static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
gfp_t gfp_mask)
{
- struct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask);
+ struct sk_buff *skb =
+ __alloc_skb(length + NET_SKB_PAD, gfp_mask, SKB_ALLOC_RX, -1);
if (likely(skb))
skb_reserve(skb, NET_SKB_PAD);
return skb;
@@ -1343,6 +1358,7 @@ static inline struct sk_buff *netdev_all
}
extern struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask);
+extern void __netdev_free_page(struct net_device *dev, struct page *page);
/**
* netdev_alloc_page - allocate a page for ps-rx on a specific device
@@ -1359,7 +1375,7 @@ static inline struct page *netdev_alloc_
static inline void netdev_free_page(struct net_device *dev, struct page *page)
{
- __free_page(page);
+ __netdev_free_page(dev, page);
}
/**
Index: linux-2.6/net/core/skbuff.c
===================================================================
--- linux-2.6.orig/net/core/skbuff.c
+++ linux-2.6/net/core/skbuff.c
@@ -179,21 +179,28 @@ EXPORT_SYMBOL(skb_truesize_bug);
* %GFP_ATOMIC.
*/
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
- int fclone, int node)
+ int flags, int node)
{
struct kmem_cache *cache;
struct skb_shared_info *shinfo;
struct sk_buff *skb;
u8 *data;
+ int emergency = 0, memalloc = sk_memalloc_socks();
- cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
+ size = SKB_DATA_ALIGN(size);
+ cache = (flags & SKB_ALLOC_FCLONE)
+ ? skbuff_fclone_cache : skbuff_head_cache;
+#ifdef CONFIG_NETVM
+ if (memalloc && (flags & SKB_ALLOC_RX))
+ gfp_mask |= __GFP_NOMEMALLOC|__GFP_NOWARN;
+retry_alloc:
+#endif
/* Get the HEAD */
skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
if (!skb)
- goto out;
+ goto noskb;
- size = SKB_DATA_ALIGN(size);
data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
gfp_mask, node);
if (!data)
@@ -203,6 +210,7 @@ struct sk_buff *__alloc_skb(unsigned int
* See comment in sk_buff definition, just before the 'tail' member
*/
memset(skb, 0, offsetof(struct sk_buff, tail));
+ skb->emergency = emergency;
skb->truesize = size + sizeof(struct sk_buff);
atomic_set(&skb->users, 1);
skb->head = data;
@@ -219,7 +227,7 @@ struct sk_buff *__alloc_skb(unsigned int
shinfo->ip6_frag_id = 0;
shinfo->frag_list = NULL;
- if (fclone) {
+ if (flags & SKB_ALLOC_FCLONE) {
struct sk_buff *child = skb + 1;
atomic_t *fclone_ref = (atomic_t *) (child + 1);
@@ -227,12 +235,31 @@ struct sk_buff *__alloc_skb(unsigned int
atomic_set(fclone_ref, 1);
child->fclone = SKB_FCLONE_UNAVAILABLE;
+ child->emergency = skb->emergency;
}
out:
return skb;
+
nodata:
kmem_cache_free(cache, skb);
skb = NULL;
+noskb:
+#ifdef CONFIG_NETVM
+ /* Attempt emergency allocation when RX skb. */
+ if (likely(!(flags & SKB_ALLOC_RX) || !memalloc))
+ goto out;
+
+ if (!emergency) {
+ if (rx_emergency_get(size)) {
+ gfp_mask &= ~(__GFP_NOMEMALLOC|__GFP_NOWARN);
+ gfp_mask |= __GFP_MEMALLOC;
+ emergency = 1;
+ goto retry_alloc;
+ }
+ } else
+ rx_emergency_put(size);
+#endif
+
goto out;
}
@@ -255,7 +282,7 @@ struct sk_buff *__netdev_alloc_skb(struc
int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
struct sk_buff *skb;
- skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
+ skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, SKB_ALLOC_RX, node);
if (likely(skb)) {
skb_reserve(skb, NET_SKB_PAD);
skb->dev = dev;
@@ -268,11 +295,36 @@ struct page *__netdev_alloc_page(struct
int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
struct page *page;
+#ifdef CONFIG_NETVM
+ gfp_mask |= __GFP_NOMEMALLOC | __GFP_NOWARN;
+#endif
+
page = alloc_pages_node(node, gfp_mask, 0);
+
+#ifdef CONFIG_NETVM
+ if (!page && rx_emergency_get(PAGE_SIZE)) {
+ gfp_mask &= ~(__GFP_NOMEMALLOC | __GFP_NOWARN);
+ gfp_mask |= __GFP_MEMALLOC;
+ page = alloc_pages_node(node, gfp_mask, 0);
+ if (!page)
+ rx_emergency_put(PAGE_SIZE);
+ }
+#endif
+
return page;
}
EXPORT_SYMBOL(__netdev_alloc_page);
+void __netdev_free_page(struct net_device *dev, struct page *page)
+{
+#ifdef CONFIG_NETVM
+ if (unlikely(page->reserve))
+ rx_emergency_put(PAGE_SIZE);
+#endif
+ __free_page(page);
+}
+EXPORT_SYMBOL(__netdev_free_page);
+
void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
int size)
{
@@ -280,6 +332,34 @@ void skb_add_rx_frag(struct sk_buff *skb
skb->len += size;
skb->data_len += size;
skb->truesize += size;
+
+#ifdef CONFIG_NETVM
+ /*
+ * Fix-up the emergency accounting; make sure all pages match
+ * skb->emergency.
+ *
+ * This relies on page->reserve to be preserved between
+ * the call to __netdev_alloc_page() and this call.
+ */
+ if (skb_emergency(skb)) {
+ /*
+ * If the page was not an emergency alloc (ALLOC_NO_WATERMARK)
+ * we can use overcommit accounting, since we already have the
+ * memory.
+ */
+ if (!page->reserve)
+ rx_emergency_get_overcommit(PAGE_SIZE);
+ atomic_set(&page->frag_count, 1);
+ } else if (unlikely(page->reserve)) {
+ /*
+ * Rare case; the skb wasn't allocated under pressure but
+ * the page was. We need to return the page. This can offset
+ * the accounting a little, but its a constant shift, it does
+ * not accumulate.
+ */
+ rx_emergency_put(PAGE_SIZE);
+ }
+#endif
}
EXPORT_SYMBOL(skb_add_rx_frag);
@@ -309,21 +389,47 @@ static void skb_clone_fraglist(struct sk
skb_get(list);
}
+static inline void skb_get_page(struct sk_buff *skb, struct page *page)
+{
+ get_page(page);
+ if (skb_emergency(skb))
+ atomic_inc(&page->frag_count);
+}
+
+static inline void skb_put_page(struct sk_buff *skb, struct page *page)
+{
+ if (skb_emergency(skb) && atomic_dec_and_test(&page->frag_count))
+ rx_emergency_put(PAGE_SIZE);
+ put_page(page);
+}
+
static void skb_release_data(struct sk_buff *skb)
{
if (!skb->cloned ||
!atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
&skb_shinfo(skb)->dataref)) {
+ int size;
+
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+ size = skb->end;
+#else
+ size = skb->end - skb->head;
+#endif
+
if (skb_shinfo(skb)->nr_frags) {
int i;
- for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
- put_page(skb_shinfo(skb)->frags[i].page);
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ skb_put_page(skb,
+ skb_shinfo(skb)->frags[i].page);
+ }
}
if (skb_shinfo(skb)->frag_list)
skb_drop_fraglist(skb);
kfree(skb->head);
+ if (skb_emergency(skb))
+ rx_emergency_put(size);
}
}
@@ -444,6 +550,7 @@ static void __copy_skb_header(struct sk_
#if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
new->ipvs_property = old->ipvs_property;
#endif
+ new->emergency = old->emergency;
new->protocol = old->protocol;
new->mark = old->mark;
__nf_copy(new, old);
@@ -532,6 +639,9 @@ struct sk_buff *skb_clone(struct sk_buff
n->fclone = SKB_FCLONE_CLONE;
atomic_inc(fclone_ref);
} else {
+ if (skb_emergency(skb))
+ gfp_mask |= __GFP_MEMALLOC;
+
n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
if (!n)
return NULL;
@@ -563,6 +673,14 @@ static void copy_skb_header(struct sk_bu
skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
}
+static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
+{
+ if (skb_emergency(skb))
+ return SKB_ALLOC_RX;
+
+ return 0;
+}
+
/**
* skb_copy - create private copy of an sk_buff
* @skb: buffer to copy
@@ -583,15 +701,17 @@ static void copy_skb_header(struct sk_bu
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
{
int headerlen = skb->data - skb->head;
+ int size;
/*
* Allocate the copy buffer
*/
struct sk_buff *n;
#ifdef NET_SKBUFF_DATA_USES_OFFSET
- n = alloc_skb(skb->end + skb->data_len, gfp_mask);
+ size = skb->end + skb->data_len;
#else
- n = alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);
+ size = skb->end - skb->head + skb->data_len;
#endif
+ n = __alloc_skb(size, gfp_mask, skb_alloc_rx_flag(skb), -1);
if (!n)
return NULL;
@@ -626,12 +746,14 @@ struct sk_buff *pskb_copy(struct sk_buff
/*
* Allocate the copy buffer
*/
+ int size;
struct sk_buff *n;
#ifdef NET_SKBUFF_DATA_USES_OFFSET
- n = alloc_skb(skb->end, gfp_mask);
+ size = skb->end;
#else
- n = alloc_skb(skb->end - skb->head, gfp_mask);
+ size = skb->end - skb->head;
#endif
+ n = __alloc_skb(size, gfp_mask, skb_alloc_rx_flag(skb), -1);
if (!n)
goto out;
@@ -650,8 +772,9 @@ struct sk_buff *pskb_copy(struct sk_buff
int i;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
- skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
- get_page(skb_shinfo(n)->frags[i].page);
+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+ skb_shinfo(n)->frags[i] = *frag;
+ skb_get_page(n, frag->page);
}
skb_shinfo(n)->nr_frags = i;
}
@@ -699,6 +822,14 @@ int pskb_expand_head(struct sk_buff *skb
size = SKB_DATA_ALIGN(size);
+ if (skb_emergency(skb)) {
+ if (rx_emergency_get(size))
+ gfp_mask |= __GFP_MEMALLOC;
+ else
+ goto nodata;
+ } else
+ gfp_mask |= __GFP_NOMEMALLOC;
+
data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
if (!data)
goto nodata;
@@ -714,7 +845,7 @@ int pskb_expand_head(struct sk_buff *skb
sizeof(struct skb_shared_info));
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
- get_page(skb_shinfo(skb)->frags[i].page);
+ skb_get_page(skb, skb_shinfo(skb)->frags[i].page);
if (skb_shinfo(skb)->frag_list)
skb_clone_fraglist(skb);
@@ -793,8 +924,8 @@ struct sk_buff *skb_copy_expand(const st
/*
* Allocate the copy buffer
*/
- struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
- gfp_mask);
+ struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
+ gfp_mask, skb_alloc_rx_flag(skb), -1);
int oldheadroom = skb_headroom(skb);
int head_copy_len, head_copy_off;
int off;
@@ -911,7 +1042,7 @@ drop_pages:
skb_shinfo(skb)->nr_frags = i;
for (; i < nfrags; i++)
- put_page(skb_shinfo(skb)->frags[i].page);
+ skb_put_page(skb, skb_shinfo(skb)->frags[i].page);
if (skb_shinfo(skb)->frag_list)
skb_drop_fraglist(skb);
@@ -1080,7 +1211,7 @@ pull_pages:
k = 0;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
if (skb_shinfo(skb)->frags[i].size <= eat) {
- put_page(skb_shinfo(skb)->frags[i].page);
+ skb_put_page(skb, skb_shinfo(skb)->frags[i].page);
eat -= skb_shinfo(skb)->frags[i].size;
} else {
skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
@@ -1852,6 +1983,7 @@ static inline void skb_split_no_header(s
skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
if (pos < len) {
+ struct page *page = skb_shinfo(skb)->frags[i].page;
/* Split frag.
* We have two variants in this case:
* 1. Move all the frag to the second
@@ -1860,7 +1992,7 @@ static inline void skb_split_no_header(s
* where splitting is expensive.
* 2. Split is accurately. We make this.
*/
- get_page(skb_shinfo(skb)->frags[i].page);
+ skb_get_page(skb1, page);
skb_shinfo(skb1)->frags[0].page_offset += len - pos;
skb_shinfo(skb1)->frags[0].size -= len - pos;
skb_shinfo(skb)->frags[i].size = len - pos;
@@ -2190,7 +2322,8 @@ struct sk_buff *skb_segment(struct sk_bu
if (hsize > len || !sg)
hsize = len;
- nskb = alloc_skb(hsize + doffset + headroom, GFP_ATOMIC);
+ nskb = __alloc_skb(hsize + doffset + headroom, GFP_ATOMIC,
+ skb_alloc_rx_flag(skb), -1);
if (unlikely(!nskb))
goto err;
@@ -2235,7 +2368,7 @@ struct sk_buff *skb_segment(struct sk_bu
BUG_ON(i >= nfrags);
*frag = skb_shinfo(skb)->frags[i];
- get_page(frag->page);
+ skb_get_page(nskb, frag->page);
size = frag->size;
if (pos < offset) {
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h
+++ linux-2.6/include/linux/mm_types.h
@@ -74,6 +74,7 @@ struct page {
pgoff_t index; /* Our offset within mapping. */
void *freelist; /* SLUB: freelist req. slab lock */
int reserve; /* page_alloc: page is a reserve page */
+ atomic_t frag_count; /* skb fragment use count */
};
struct list_head lru; /* Pageout list, eg. active_list
* protected by zone->lru_lock !
--
^ permalink raw reply
* [PATCH 23/28] mm: methods for teaching filesystems about PG_swapcache pages
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: mm-page_file_methods.patch --]
[-- Type: text/plain, Size: 3457 bytes --]
In order to teach filesystems to handle swap cache pages, two new page
functions are introduced:
pgoff_t page_file_index(struct page *);
struct address_space *page_file_mapping(struct page *);
page_file_index - gives the offset of this page in the file in PAGE_CACHE_SIZE
blocks. Like page->index is for mapped pages, this function also gives the
correct index for PG_swapcache pages.
page_file_mapping - gives the mapping backing the actual page; that is for
swap cache pages it will give swap_file->f_mapping.
page_offset() is modified to use page_file_index(), so that it will give the
expected result, even for PG_swapcache pages.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/mm.h | 25 +++++++++++++++++++++++++
include/linux/pagemap.h | 2 +-
mm/swapfile.c | 19 +++++++++++++++++++
3 files changed, 45 insertions(+), 1 deletion(-)
Index: linux-2.6/include/linux/mm.h
===================================================================
--- linux-2.6.orig/include/linux/mm.h
+++ linux-2.6/include/linux/mm.h
@@ -600,6 +600,17 @@ static inline struct address_space *page
return mapping;
}
+extern struct address_space *__page_file_mapping(struct page *);
+
+static inline
+struct address_space *page_file_mapping(struct page *page)
+{
+ if (unlikely(PageSwapCache(page)))
+ return __page_file_mapping(page);
+
+ return page->mapping;
+}
+
static inline int PageAnon(struct page *page)
{
return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
@@ -616,6 +627,20 @@ static inline pgoff_t page_index(struct
return page->index;
}
+extern pgoff_t __page_file_index(struct page *page);
+
+/*
+ * Return the file index of the page. Regular pagecache pages use ->index
+ * whereas swapcache pages use swp_offset(->private)
+ */
+static inline pgoff_t page_file_index(struct page *page)
+{
+ if (unlikely(PageSwapCache(page)))
+ return __page_file_index(page);
+
+ return page->index;
+}
+
/*
* The atomic page->_mapcount, like _count, starts from -1:
* so that transitions both from it and to it can be tracked,
Index: linux-2.6/include/linux/pagemap.h
===================================================================
--- linux-2.6.orig/include/linux/pagemap.h
+++ linux-2.6/include/linux/pagemap.h
@@ -145,7 +145,7 @@ extern void __remove_from_page_cache(str
*/
static inline loff_t page_offset(struct page *page)
{
- return ((loff_t)page->index) << PAGE_CACHE_SHIFT;
+ return ((loff_t)page_file_index(page)) << PAGE_CACHE_SHIFT;
}
static inline pgoff_t linear_page_index(struct vm_area_struct *vma,
Index: linux-2.6/mm/swapfile.c
===================================================================
--- linux-2.6.orig/mm/swapfile.c
+++ linux-2.6/mm/swapfile.c
@@ -1818,6 +1818,25 @@ struct swap_info_struct *page_swap_info(
}
/*
+ * out-of-line __page_file_ methods to avoid include hell.
+ */
+
+struct address_space *__page_file_mapping(struct page *page)
+{
+ VM_BUG_ON(!PageSwapCache(page));
+ return page_swap_info(page)->swap_file->f_mapping;
+}
+EXPORT_SYMBOL_GPL(__page_file_mapping);
+
+pgoff_t __page_file_index(struct page *page)
+{
+ swp_entry_t swap = { .val = page_private(page) };
+ VM_BUG_ON(!PageSwapCache(page));
+ return swp_offset(swap);
+}
+EXPORT_SYMBOL_GPL(__page_file_index);
+
+/*
* swap_lock prevents swap_map being freed. Don't grab an extra
* reference on the swaphandle, it doesn't matter if it becomes unused.
*/
--
^ permalink raw reply
* [PATCH 25/28] nfs: teach the NFS client how to treat PG_swapcache pages
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: nfs-swapcache.patch --]
[-- Type: text/plain, Size: 12476 bytes --]
Replace all relevant occurences of page->index and page->mapping in the NFS
client with the new page_file_index() and page_file_mapping() functions.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
fs/nfs/file.c | 6 +++---
fs/nfs/internal.h | 7 ++++---
fs/nfs/pagelist.c | 6 +++---
fs/nfs/read.c | 6 +++---
fs/nfs/write.c | 51 ++++++++++++++++++++++++++-------------------------
5 files changed, 39 insertions(+), 37 deletions(-)
Index: linux-2.6/fs/nfs/file.c
===================================================================
--- linux-2.6.orig/fs/nfs/file.c
+++ linux-2.6/fs/nfs/file.c
@@ -359,7 +359,7 @@ static void nfs_invalidate_page(struct p
if (offset != 0)
return;
/* Cancel any unstarted writes on this page */
- nfs_wb_page_cancel(page->mapping->host, page);
+ nfs_wb_page_cancel(page_file_mapping(page)->host, page);
}
static int nfs_release_page(struct page *page, gfp_t gfp)
@@ -370,7 +370,7 @@ static int nfs_release_page(struct page
static int nfs_launder_page(struct page *page)
{
- return nfs_wb_page(page->mapping->host, page);
+ return nfs_wb_page(page_file_mapping(page)->host, page);
}
const struct address_space_operations nfs_file_aops = {
@@ -397,7 +397,7 @@ static int nfs_vm_page_mkwrite(struct vm
struct address_space *mapping;
lock_page(page);
- mapping = page->mapping;
+ mapping = page_file_mapping(page);
if (mapping != vma->vm_file->f_path.dentry->d_inode->i_mapping)
goto out_unlock;
Index: linux-2.6/fs/nfs/pagelist.c
===================================================================
--- linux-2.6.orig/fs/nfs/pagelist.c
+++ linux-2.6/fs/nfs/pagelist.c
@@ -76,11 +76,11 @@ nfs_create_request(struct nfs_open_conte
* update_nfs_request below if the region is not locked. */
req->wb_page = page;
atomic_set(&req->wb_complete, 0);
- req->wb_index = page->index;
+ req->wb_index = page_file_index(page);
page_cache_get(page);
BUG_ON(PagePrivate(page));
BUG_ON(!PageLocked(page));
- BUG_ON(page->mapping->host != inode);
+ BUG_ON(page_file_mapping(page)->host != inode);
req->wb_offset = offset;
req->wb_pgbase = offset;
req->wb_bytes = count;
@@ -376,7 +376,7 @@ void nfs_pageio_cond_complete(struct nfs
* nfs_scan_list - Scan a list for matching requests
* @nfsi: NFS inode
* @dst: Destination list
- * @idx_start: lower bound of page->index to scan
+ * @idx_start: lower bound of page_file_index(page) to scan
* @npages: idx_start + npages sets the upper bound to scan.
* @tag: tag to scan for
*
Index: linux-2.6/fs/nfs/read.c
===================================================================
--- linux-2.6.orig/fs/nfs/read.c
+++ linux-2.6/fs/nfs/read.c
@@ -458,11 +458,11 @@ static const struct rpc_call_ops nfs_rea
int nfs_readpage(struct file *file, struct page *page)
{
struct nfs_open_context *ctx;
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
int error;
dprintk("NFS: nfs_readpage (%p %ld@%lu)\n",
- page, PAGE_CACHE_SIZE, page->index);
+ page, PAGE_CACHE_SIZE, page_file_index(page));
nfs_inc_stats(inode, NFSIOS_VFSREADPAGE);
nfs_add_stats(inode, NFSIOS_READPAGES, 1);
@@ -509,7 +509,7 @@ static int
readpage_async_filler(void *data, struct page *page)
{
struct nfs_readdesc *desc = (struct nfs_readdesc *)data;
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
struct nfs_page *new;
unsigned int len;
int error;
Index: linux-2.6/fs/nfs/write.c
===================================================================
--- linux-2.6.orig/fs/nfs/write.c
+++ linux-2.6/fs/nfs/write.c
@@ -126,7 +126,7 @@ static struct nfs_page *nfs_page_find_re
static struct nfs_page *nfs_page_find_request(struct page *page)
{
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
struct nfs_page *req = NULL;
spin_lock(&inode->i_lock);
@@ -138,13 +138,13 @@ static struct nfs_page *nfs_page_find_re
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
loff_t end, i_size = i_size_read(inode);
pgoff_t end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
- if (i_size > 0 && page->index < end_index)
+ if (i_size > 0 && page_file_index(page) < end_index)
return;
- end = ((loff_t)page->index << PAGE_CACHE_SHIFT) + ((loff_t)offset+count);
+ end = page_offset(page) + ((loff_t)offset+count);
if (i_size >= end)
return;
nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
@@ -155,7 +155,7 @@ static void nfs_grow_file(struct page *p
static void nfs_set_pageerror(struct page *page)
{
SetPageError(page);
- nfs_zap_mapping(page->mapping->host, page->mapping);
+ nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
}
/* We can set the PG_uptodate flag if we see that a write request
@@ -185,7 +185,7 @@ static int nfs_writepage_setup(struct nf
ret = PTR_ERR(req);
if (ret != -EBUSY)
return ret;
- ret = nfs_wb_page(page->mapping->host, page);
+ ret = nfs_wb_page(page_file_mapping(page)->host, page);
if (ret != 0)
return ret;
}
@@ -219,7 +219,7 @@ static int nfs_set_page_writeback(struct
int ret = test_set_page_writeback(page);
if (!ret) {
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
struct nfs_server *nfss = NFS_SERVER(inode);
if (atomic_long_inc_return(&nfss->writeback) >
@@ -231,7 +231,7 @@ static int nfs_set_page_writeback(struct
static void nfs_end_page_writeback(struct page *page)
{
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
struct nfs_server *nfss = NFS_SERVER(inode);
end_page_writeback(page);
@@ -246,7 +246,7 @@ static void nfs_end_page_writeback(struc
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
struct page *page)
{
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
struct nfs_page *req;
int ret;
@@ -289,12 +289,12 @@ static int nfs_page_async_flush(struct n
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
{
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
- nfs_pageio_cond_complete(pgio, page->index);
+ nfs_pageio_cond_complete(pgio, page_file_index(page));
return nfs_page_async_flush(pgio, page);
}
@@ -306,7 +306,7 @@ static int nfs_writepage_locked(struct p
struct nfs_pageio_descriptor pgio;
int err;
- nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc));
+ nfs_pageio_init_write(&pgio, page_file_mapping(page)->host, wb_priority(wbc));
err = nfs_do_writepage(page, wbc, &pgio);
nfs_pageio_complete(&pgio);
if (err < 0)
@@ -438,7 +438,8 @@ nfs_mark_request_commit(struct nfs_page
NFS_PAGE_TAG_COMMIT);
spin_unlock(&inode->i_lock);
inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
- inc_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_RECLAIMABLE);
+ inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
+ BDI_RECLAIMABLE);
__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
}
@@ -525,7 +526,7 @@ static void nfs_cancel_commit_list(struc
while(!list_empty(head)) {
req = nfs_list_entry(head->next);
dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
- dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
+ dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
BDI_RECLAIMABLE);
nfs_list_remove_request(req);
clear_bit(PG_NEED_COMMIT, &(req)->wb_flags);
@@ -539,7 +540,7 @@ static void nfs_cancel_commit_list(struc
* nfs_scan_commit - Scan an inode for commit requests
* @inode: NFS inode to scan
* @dst: destination list
- * @idx_start: lower bound of page->index to scan.
+ * @idx_start: lower bound of page_file_index(page) to scan.
* @npages: idx_start + npages sets the upper bound to scan.
*
* Moves requests from the inode's 'commit' request list.
@@ -575,7 +576,7 @@ static inline int nfs_scan_commit(struct
static struct nfs_page * nfs_update_request(struct nfs_open_context* ctx,
struct page *page, unsigned int offset, unsigned int bytes)
{
- struct address_space *mapping = page->mapping;
+ struct address_space *mapping = page_file_mapping(page);
struct inode *inode = mapping->host;
struct nfs_page *req, *new = NULL;
pgoff_t rqend, end;
@@ -686,7 +687,7 @@ int nfs_flush_incompatible(struct file *
nfs_release_request(req);
if (!do_flush)
return 0;
- status = nfs_wb_page(page->mapping->host, page);
+ status = nfs_wb_page(page_file_mapping(page)->host, page);
} while (status == 0);
return status;
}
@@ -712,7 +713,7 @@ int nfs_updatepage(struct file *file, st
unsigned int offset, unsigned int count)
{
struct nfs_open_context *ctx = nfs_file_open_context(file);
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
int status = 0;
nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
@@ -980,7 +981,7 @@ static void nfs_writeback_done_partial(s
}
if (nfs_write_need_commit(data)) {
- struct inode *inode = page->mapping->host;
+ struct inode *inode = page_file_mapping(page)->host;
spin_lock(&inode->i_lock);
if (test_bit(PG_NEED_RESCHED, &req->wb_flags)) {
@@ -1231,7 +1232,7 @@ nfs_commit_list(struct inode *inode, str
nfs_list_remove_request(req);
nfs_mark_request_commit(req);
dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
- dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
+ dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
BDI_RECLAIMABLE);
nfs_clear_page_tag_locked(req);
}
@@ -1258,7 +1259,7 @@ static void nfs_commit_done(struct rpc_t
nfs_list_remove_request(req);
clear_bit(PG_NEED_COMMIT, &(req)->wb_flags);
dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
- dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
+ dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
BDI_RECLAIMABLE);
dprintk("NFS: commit (%s/%Ld %d@%Ld)",
@@ -1424,7 +1425,7 @@ int nfs_wb_page_cancel(struct inode *ino
loff_t range_start = page_offset(page);
loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
struct writeback_control wbc = {
- .bdi = page->mapping->backing_dev_info,
+ .bdi = page_file_mapping(page)->backing_dev_info,
.sync_mode = WB_SYNC_ALL,
.nr_to_write = LONG_MAX,
.range_start = range_start,
@@ -1457,7 +1458,7 @@ int nfs_wb_page_cancel(struct inode *ino
}
if (!PagePrivate(page))
return 0;
- ret = nfs_sync_mapping_wait(page->mapping, &wbc, FLUSH_INVALIDATE);
+ ret = nfs_sync_mapping_wait(page_file_mapping(page), &wbc, FLUSH_INVALIDATE);
out:
return ret;
}
@@ -1468,7 +1469,7 @@ static int nfs_wb_page_priority(struct i
loff_t range_start = page_offset(page);
loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
struct writeback_control wbc = {
- .bdi = page->mapping->backing_dev_info,
+ .bdi = page_file_mapping(page)->backing_dev_info,
.sync_mode = WB_SYNC_ALL,
.nr_to_write = LONG_MAX,
.range_start = range_start,
@@ -1484,7 +1485,7 @@ static int nfs_wb_page_priority(struct i
}
if (!PagePrivate(page))
return 0;
- ret = nfs_sync_mapping_wait(page->mapping, &wbc, how);
+ ret = nfs_sync_mapping_wait(page_file_mapping(page), &wbc, how);
if (ret >= 0)
return 0;
out:
Index: linux-2.6/fs/nfs/internal.h
===================================================================
--- linux-2.6.orig/fs/nfs/internal.h
+++ linux-2.6/fs/nfs/internal.h
@@ -252,13 +252,14 @@ void nfs_super_set_maxbytes(struct super
static inline
unsigned int nfs_page_length(struct page *page)
{
- loff_t i_size = i_size_read(page->mapping->host);
+ loff_t i_size = i_size_read(page_file_mapping(page)->host);
if (i_size > 0) {
+ pgoff_t page_index = page_file_index(page);
pgoff_t end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
- if (page->index < end_index)
+ if (page_index < end_index)
return PAGE_CACHE_SIZE;
- if (page->index == end_index)
+ if (page_index == end_index)
return ((i_size - 1) & ~PAGE_CACHE_MASK) + 1;
}
return 0;
--
^ permalink raw reply
* [PATCH 11/28] selinux: tag avc cache alloc as non-critical
From: Peter Zijlstra @ 2008-02-20 14:46 UTC (permalink / raw)
To: Linus Torvalds, Andrew Morton, linux-kernel, linux-mm, netdev,
trond.myklebust
Cc: Peter Zijlstra, James Morris
In-Reply-To: <20080220144610.548202000@chello.nl>
[-- Attachment #1: mm-selinux-emergency.patch --]
[-- Type: text/plain, Size: 768 bytes --]
Failing to allocate a cache entry will only harm performance not correctness.
Do not consume valuable reserve pages for something like that.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: James Morris <jmorris@namei.org>
---
security/selinux/avc.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
Index: linux-2.6/security/selinux/avc.c
===================================================================
--- linux-2.6.orig/security/selinux/avc.c
+++ linux-2.6/security/selinux/avc.c
@@ -334,7 +334,7 @@ static struct avc_node *avc_alloc_node(v
{
struct avc_node *node;
- node = kmem_cache_zalloc(avc_node_cachep, GFP_ATOMIC);
+ node = kmem_cache_zalloc(avc_node_cachep, GFP_ATOMIC|__GFP_NOMEMALLOC);
if (!node)
goto out;
--
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox