From: Pablo Neira Ayuso <pablo@netfilter.org>
To: netfilter-devel@vger.kernel.org
Cc: davem@davemloft.net, netdev@vger.kernel.org
Subject: [PATCH 25/50] netfilter: x_tables: pack percpu counter allocations
Date: Wed, 7 Dec 2016 22:52:31 +0100 [thread overview]
Message-ID: <1481147576-5690-26-git-send-email-pablo@netfilter.org> (raw)
In-Reply-To: <1481147576-5690-1-git-send-email-pablo@netfilter.org>
From: Florian Westphal <fw@strlen.de>
instead of allocating each xt_counter individually, allocate 4k chunks
and then use these for counter allocation requests.
This should speed up rule evaluation by increasing data locality,
also speeds up ruleset loading because we reduce calls to the percpu
allocator.
As Eric points out we can't use PAGE_SIZE, page_allocator would fail on
arches with 64k page size.
Suggested-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
---
include/linux/netfilter/x_tables.h | 7 ++++++-
net/ipv4/netfilter/arp_tables.c | 9 ++++++---
net/ipv4/netfilter/ip_tables.c | 9 ++++++---
net/ipv6/netfilter/ip6_tables.c | 9 ++++++---
net/netfilter/x_tables.c | 33 ++++++++++++++++++++++++---------
5 files changed, 48 insertions(+), 19 deletions(-)
diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h
index 05a94bd32c55..5117e4d2ddfa 100644
--- a/include/linux/netfilter/x_tables.h
+++ b/include/linux/netfilter/x_tables.h
@@ -403,8 +403,13 @@ static inline unsigned long ifname_compare_aligned(const char *_a,
return ret;
}
+struct xt_percpu_counter_alloc_state {
+ unsigned int off;
+ const char __percpu *mem;
+};
-bool xt_percpu_counter_alloc(struct xt_counters *counters);
+bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
+ struct xt_counters *counter);
void xt_percpu_counter_free(struct xt_counters *cnt);
static inline struct xt_counters *
diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
index 808deb275ceb..1258a9ab62ef 100644
--- a/net/ipv4/netfilter/arp_tables.c
+++ b/net/ipv4/netfilter/arp_tables.c
@@ -411,13 +411,14 @@ static inline int check_target(struct arpt_entry *e, const char *name)
}
static inline int
-find_check_entry(struct arpt_entry *e, const char *name, unsigned int size)
+find_check_entry(struct arpt_entry *e, const char *name, unsigned int size,
+ struct xt_percpu_counter_alloc_state *alloc_state)
{
struct xt_entry_target *t;
struct xt_target *target;
int ret;
- if (!xt_percpu_counter_alloc(&e->counters))
+ if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
return -ENOMEM;
t = arpt_get_target(e);
@@ -525,6 +526,7 @@ static inline void cleanup_entry(struct arpt_entry *e)
static int translate_table(struct xt_table_info *newinfo, void *entry0,
const struct arpt_replace *repl)
{
+ struct xt_percpu_counter_alloc_state alloc_state = { 0 };
struct arpt_entry *iter;
unsigned int *offsets;
unsigned int i;
@@ -587,7 +589,8 @@ static int translate_table(struct xt_table_info *newinfo, void *entry0,
/* Finally, each sanity check must pass */
i = 0;
xt_entry_foreach(iter, entry0, newinfo->size) {
- ret = find_check_entry(iter, repl->name, repl->size);
+ ret = find_check_entry(iter, repl->name, repl->size,
+ &alloc_state);
if (ret != 0)
break;
++i;
diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c
index a48430d3420f..308b456723f0 100644
--- a/net/ipv4/netfilter/ip_tables.c
+++ b/net/ipv4/netfilter/ip_tables.c
@@ -531,7 +531,8 @@ static int check_target(struct ipt_entry *e, struct net *net, const char *name)
static int
find_check_entry(struct ipt_entry *e, struct net *net, const char *name,
- unsigned int size)
+ unsigned int size,
+ struct xt_percpu_counter_alloc_state *alloc_state)
{
struct xt_entry_target *t;
struct xt_target *target;
@@ -540,7 +541,7 @@ find_check_entry(struct ipt_entry *e, struct net *net, const char *name,
struct xt_mtchk_param mtpar;
struct xt_entry_match *ematch;
- if (!xt_percpu_counter_alloc(&e->counters))
+ if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
return -ENOMEM;
j = 0;
@@ -676,6 +677,7 @@ static int
translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
const struct ipt_replace *repl)
{
+ struct xt_percpu_counter_alloc_state alloc_state = { 0 };
struct ipt_entry *iter;
unsigned int *offsets;
unsigned int i;
@@ -735,7 +737,8 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
/* Finally, each sanity check must pass */
i = 0;
xt_entry_foreach(iter, entry0, newinfo->size) {
- ret = find_check_entry(iter, net, repl->name, repl->size);
+ ret = find_check_entry(iter, net, repl->name, repl->size,
+ &alloc_state);
if (ret != 0)
break;
++i;
diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c
index a5a92083fd62..d56d8ac09a94 100644
--- a/net/ipv6/netfilter/ip6_tables.c
+++ b/net/ipv6/netfilter/ip6_tables.c
@@ -562,7 +562,8 @@ static int check_target(struct ip6t_entry *e, struct net *net, const char *name)
static int
find_check_entry(struct ip6t_entry *e, struct net *net, const char *name,
- unsigned int size)
+ unsigned int size,
+ struct xt_percpu_counter_alloc_state *alloc_state)
{
struct xt_entry_target *t;
struct xt_target *target;
@@ -571,7 +572,7 @@ find_check_entry(struct ip6t_entry *e, struct net *net, const char *name,
struct xt_mtchk_param mtpar;
struct xt_entry_match *ematch;
- if (!xt_percpu_counter_alloc(&e->counters))
+ if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
return -ENOMEM;
j = 0;
@@ -705,6 +706,7 @@ static int
translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
const struct ip6t_replace *repl)
{
+ struct xt_percpu_counter_alloc_state alloc_state = { 0 };
struct ip6t_entry *iter;
unsigned int *offsets;
unsigned int i;
@@ -764,7 +766,8 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
/* Finally, each sanity check must pass */
i = 0;
xt_entry_foreach(iter, entry0, newinfo->size) {
- ret = find_check_entry(iter, net, repl->name, repl->size);
+ ret = find_check_entry(iter, net, repl->name, repl->size,
+ &alloc_state);
if (ret != 0)
break;
++i;
diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index be5e83047594..f6ce4a7036e6 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -40,6 +40,7 @@ MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
MODULE_DESCRIPTION("{ip,ip6,arp,eb}_tables backend module");
#define SMP_ALIGN(x) (((x) + SMP_CACHE_BYTES-1) & ~(SMP_CACHE_BYTES-1))
+#define XT_PCPU_BLOCK_SIZE 4096
struct compat_delta {
unsigned int offset; /* offset in kernel */
@@ -1618,6 +1619,7 @@ EXPORT_SYMBOL_GPL(xt_proto_fini);
/**
* xt_percpu_counter_alloc - allocate x_tables rule counter
*
+ * @state: pointer to xt_percpu allocation state
* @counter: pointer to counter struct inside the ip(6)/arpt_entry struct
*
* On SMP, the packet counter [ ip(6)t_entry->counters.pcnt ] will then
@@ -1626,21 +1628,34 @@ EXPORT_SYMBOL_GPL(xt_proto_fini);
* Rule evaluation needs to use xt_get_this_cpu_counter() helper
* to fetch the real percpu counter.
*
+ * To speed up allocation and improve data locality, a 4kb block is
+ * allocated.
+ *
+ * xt_percpu_counter_alloc_state contains the base address of the
+ * allocated page and the current sub-offset.
+ *
* returns false on error.
*/
-bool xt_percpu_counter_alloc(struct xt_counters *counter)
+bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
+ struct xt_counters *counter)
{
- void __percpu *res;
+ BUILD_BUG_ON(XT_PCPU_BLOCK_SIZE < (sizeof(*counter) * 2));
if (nr_cpu_ids <= 1)
return true;
- res = __alloc_percpu(sizeof(struct xt_counters),
- sizeof(struct xt_counters));
- if (!res)
- return false;
-
- counter->pcnt = (__force unsigned long)res;
+ if (!state->mem) {
+ state->mem = __alloc_percpu(XT_PCPU_BLOCK_SIZE,
+ XT_PCPU_BLOCK_SIZE);
+ if (!state->mem)
+ return false;
+ }
+ counter->pcnt = (__force unsigned long)(state->mem + state->off);
+ state->off += sizeof(*counter);
+ if (state->off > (XT_PCPU_BLOCK_SIZE - sizeof(*counter))) {
+ state->mem = NULL;
+ state->off = 0;
+ }
return true;
}
EXPORT_SYMBOL_GPL(xt_percpu_counter_alloc);
@@ -1649,7 +1664,7 @@ void xt_percpu_counter_free(struct xt_counters *counters)
{
unsigned long pcnt = counters->pcnt;
- if (nr_cpu_ids > 1)
+ if (nr_cpu_ids > 1 && (pcnt & (XT_PCPU_BLOCK_SIZE - 1)) == 0)
free_percpu((void __percpu *)pcnt);
}
EXPORT_SYMBOL_GPL(xt_percpu_counter_free);
--
2.1.4
next prev parent reply other threads:[~2016-12-07 21:53 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-12-07 21:52 [PATCH 00/50] Netfilter/IPVS updates for net-next Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 01/50] ipvs: Use IS_ERR_OR_NULL(svc) instead of IS_ERR(svc) || svc == NULL Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 02/50] ipvs: Decrement ttl Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 03/50] netfilter: update Arturo Borrero Gonzalez email address Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 04/50] netfilter: built-in NAT support for DCCP Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 05/50] netfilter: built-in NAT support for SCTP Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 06/50] netfilter: built-in NAT support for UDPlite Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 07/50] netfilter: nf_log: do not assume ethernet header in netdev family Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 08/50] netfilter: nfnetlink_log: add "nf-logger-5-1" module alias name Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 09/50] netfilter: nf_conntrack_tuple_common.h: fix #include Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 10/50] netfilter: conntrack: built-in support for DCCP Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 11/50] netfilter: conntrack: built-in support for SCTP Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 12/50] netfilter: conntrack: built-in support for UDPlite Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 13/50] netfilter: conntrack: remove unused init_net hook Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 14/50] netfilter: add and use nf_ct_netns_get/put Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 15/50] netfilter: nat: add dependencies on conntrack module Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 16/50] netfilter: nf_tables: add conntrack dependencies for nat/masq/redir expressions Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 17/50] netfilter: conntrack: register hooks in netns when needed by ruleset Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 18/50] netfilter: conntrack: add nf_conntrack_default_on sysctl Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 19/50] netfilter: defrag: only register defrag functionality if needed Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 20/50] netfilter: introduce accessor functions for hook entries Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 21/50] netfilter: decouple nf_hook_entry and nf_hook_ops Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 22/50] netfilter: convert while loops to for loops Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 23/50] netfilter: x_tables: pass xt_counters struct instead of packet counter Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 24/50] netfilter: x_tables: pass xt_counters struct to counter allocator Pablo Neira Ayuso
2016-12-07 21:52 ` Pablo Neira Ayuso [this message]
2016-12-07 21:52 ` [PATCH 26/50] netfilter: nft_fib: convert htonl to ntohl properly Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 27/50] netfilter: nft_fib_ipv4: initialize *dest to zero Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 28/50] netfilter: nft_payload: layer 4 checksum adjustment for pseudoheader fields Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 29/50] netfilter: xt_multiport: Fix wrong unmatch result with multiple ports Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 30/50] netfilter: ingress: translate 0 nf_hook_slow retval to -1 Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 31/50] netfilter: add and use nf_fwd_netdev_egress Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 32/50] netfilter: nf_tables: add stateful objects Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 33/50] netfilter: nft_counter: add stateful object type Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 34/50] netfilter: nft_quota: " Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 35/50] netfilter: nf_tables: add stateful object reference expression Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 36/50] netfilter: nft_quota: dump consumed quota Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 37/50] netfilter: nf_tables: atomic dump and reset for stateful objects Pablo Neira Ayuso
2016-12-09 0:40 ` Paul Gortmaker
2016-12-09 10:24 ` Pablo Neira Ayuso
2016-12-09 14:24 ` Eric Dumazet
2016-12-09 15:22 ` Eric Dumazet
2016-12-10 12:21 ` Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 38/50] netfilter: nf_tables: notify internal updates of " Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 39/50] netfilter: nft_quota: add depleted flag for objects Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 40/50] netfilter: nf_tables: add stateful object reference to set elements Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 41/50] netfilter: nft_objref: support for stateful object maps Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 42/50] netfilter: nf_tables: allow to filter stateful object dumps by type Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 43/50] netfilter: rpfilter: bypass ipv4 lbcast packets with zeronet source Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 44/50] netfilter: nat: skip checksum on offload SCTP packets Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 45/50] netfilter: nf_tables: constify struct nft_ctx * parameter in nft_trans_alloc() Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 46/50] netfilter: nft_set: introduce nft_{hash, rbtree}_deactivate_one() Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 47/50] netfilter: nf_tables: support for set flushing Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 48/50] netfilter: x_tables: avoid warn and OOM killer on vmalloc call Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 49/50] netfilter: xt_bpf: support ebpf Pablo Neira Ayuso
2016-12-07 21:52 ` [PATCH 50/50] netfilter: nft_quota: allow to restore consumed quota Pablo Neira Ayuso
2016-12-08 0:29 ` [PATCH 00/50] Netfilter/IPVS updates for net-next David Miller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1481147576-5690-26-git-send-email-pablo@netfilter.org \
--to=pablo@netfilter.org \
--cc=davem@davemloft.net \
--cc=netdev@vger.kernel.org \
--cc=netfilter-devel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).