[PATCH 2/3] netfilter : 3 patches to boost ip_tables performance

Eric Dumazet
2005-09-21T21:32:31Z

[-- Attachment #1: Type: text/plain, Size: 330 bytes --]

Patch 2/3 (please apply after Patch 1/3)

2) Loop unrolling

It seems that with current compilers and CFLAGS, the code from
ip_packet_match() is very bad, using lot of mispredicted conditional branches
I made some patches and generated code on i386 and x86_64
is much better.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>

[-- Attachment #2: patch_ip_tables_numa.2 --]
[-- Type: text/plain, Size: 3958 bytes --]

--- linux-2.6/net/ipv4/netfilter/ip_tables.c 2005-09-21 23:55:30.000000000 +0200
+++ linux-2.6-ed/net/ipv4/netfilter/ip_tables.c 2005-09-22 00:39:29.000000000 +0200
@@ -125,6 +125,27 @@
#define up(x) do { printk("UP:%u:" #x "\n", __LINE__); up(x); } while(0)
#endif

+/*
+ * Special macro to compare IFNAMSIZ 'strings', with mask.
+ * Best results if feeded with 3 args pointing to 'unsigned long' types
+ * Loop is manually unrolled for performance reasons.
+ * gcc generates code without branches, IFNAMSIZ being a constant
+ */
+#define compare_if_lstrings(ret, devname, expect, expect_mask) \
+ ret = ((devname)[0] ^ (expect)[0]) & (expect_mask)[0]; \
+ if (IFNAMSIZ > sizeof(*devname)) \
+ ret |= ((devname)[1] ^ (expect)[1]) & (expect_mask)[1]; \
+ if (IFNAMSIZ > 2 * sizeof(*devname)) \
+ ret |= ((devname)[2] ^ (expect)[2]) & (expect_mask)[2]; \
+ if (IFNAMSIZ > 3 * sizeof(*devname)) \
+ ret |= ((devname)[3] ^ (expect)[3]) & (expect_mask)[3]; \
+ /* just in case IFNAMSIZ is enlarged */ \
+ if (IFNAMSIZ > 4 * sizeof(*devname)) { \
+ int i; \
+ for (i = 4 ; (i < IFNAMSIZ/sizeof(*devname)); i++) \
+ ret |= ((devname)[i] ^ (expect)[i]) & (expect_mask)[i]; \
+ }
+
/* Returns whether matches rule or not. */
static inline int
ip_packet_match(const struct iphdr *ip,
@@ -133,16 +154,22 @@
const struct ipt_ip *ipinfo,
int isfrag)
{
- size_t i;
+ int bool1, bool2;
unsigned long ret;

#define FWINV(bool,invflg) ((bool) ^ !!(ipinfo->invflags & invflg))

- if (FWINV((ip->saddr&ipinfo->smsk.s_addr) != ipinfo->src.s_addr,
- IPT_INV_SRCIP)
- || FWINV((ip->daddr&ipinfo->dmsk.s_addr) != ipinfo->dst.s_addr,
- IPT_INV_DSTIP)) {
- dprintf("Source or dest mismatch.\n");
+ bool1 = ((ip->saddr&ipinfo->smsk.s_addr) != ipinfo->src.s_addr);
+ bool1 ^= !!(ipinfo->invflags & IPT_INV_SRCIP);
+
+ bool2 = ((ip->daddr&ipinfo->dmsk.s_addr) != ipinfo->dst.s_addr);
+ bool2 ^= !!(ipinfo->invflags & IPT_INV_DSTIP);
+
+ if ((bool1 | bool2) != 0) {
+ if (bool1)
+ dprintf("Source%s mismatch.\n", bool2 ? " and Dest":"");
+ else
+ dprintf("Dest mismatch.\n");

dprintf("SRC: %u.%u.%u.%u. Mask: %u.%u.%u.%u. Target: %u.%u.%u.%u.%s\n",
NIPQUAD(ip->saddr),
@@ -157,27 +184,26 @@
return 0;
}

- /* Look for ifname matches; this should unroll nicely. */
- for (i = 0, ret = 0; i < IFNAMSIZ/sizeof(unsigned long); i++) {
- ret |= (((const unsigned long *)indev)[i]
- ^ ((const unsigned long *)ipinfo->iniface)[i])
- & ((const unsigned long *)ipinfo->iniface_mask)[i];
- }
+ compare_if_lstrings(ret,
+ (const unsigned long *)indev,
+ (const unsigned long *)ipinfo->iniface,
+ (const unsigned long *)ipinfo->iniface_mask);

- if (FWINV(ret != 0, IPT_INV_VIA_IN)) {
+ bool1 = FWINV(ret != 0, IPT_INV_VIA_IN);
+ if (bool1) {
dprintf("VIA in mismatch (%s vs %s).%s\n",
indev, ipinfo->iniface,
ipinfo->invflags&IPT_INV_VIA_IN ?" (INV)":"");
return 0;
}

- for (i = 0, ret = 0; i < IFNAMSIZ/sizeof(unsigned long); i++) {
- ret |= (((const unsigned long *)outdev)[i]
- ^ ((const unsigned long *)ipinfo->outiface)[i])
- & ((const unsigned long *)ipinfo->outiface_mask)[i];
- }
+ compare_if_lstrings(ret,
+ (const unsigned long *)outdev,
+ (const unsigned long *)ipinfo->outiface,
+ (const unsigned long *)ipinfo->outiface_mask);

- if (FWINV(ret != 0, IPT_INV_VIA_OUT)) {
+ bool1 = FWINV(ret != 0, IPT_INV_VIA_OUT);
+ if (bool1) {
dprintf("VIA out mismatch (%s vs %s).%s\n",
outdev, ipinfo->outiface,
ipinfo->invflags&IPT_INV_VIA_OUT ?" (INV)":"");
@@ -185,8 +211,9 @@
}

/* Check specific protocol */
- if (ipinfo->proto
- && FWINV(ip->protocol != ipinfo->proto, IPT_INV_PROTO)) {
+ bool1 = FWINV(ip->protocol != ipinfo->proto, IPT_INV_PROTO) ;
+ bool1 &= (ipinfo->proto != 0);
+ if (bool1) {
dprintf("Packet protocol %hi does not match %hi.%s\n",
ip->protocol, ipinfo->proto,
ipinfo->invflags&IPT_INV_PROTO ? " (INV)":"");