diff -urN a/net/sched/sch_esfq.c b/net/sched/sch_esfq.c
--- a/net/sched/sch_esfq.c	2004-06-05 15:45:19.000000000 +0200
+++ b/net/sched/sch_esfq.c	2004-06-05 15:47:21.000000000 +0200
@@ -34,6 +34,7 @@
 #include <linux/etherdevice.h>
 #include <linux/notifier.h>
 #include <linux/init.h>
+#include <linux/netfilter_ipv4/ip_conntrack.h>
 #include <net/ip.h>
 #include <linux/ipv6.h>
 #include <net/route.h>
@@ -109,6 +110,18 @@
 	return h & (q->hash_divisor-1);
 }
 
+static inline u32 esfq_get_source(struct sk_buff *skb)
+{
+	struct ip_conntrack *ct;
+	int dir;
+
+	if (skb->nfct == NULL)
+		return skb->nh.iph->saddr;
+	ct = (struct ip_conntrack *)skb->nfct->master;
+	dir = CTINFO2DIR(skb->nfct - ct->infos);
+	return ct->tuplehash[dir].tuple.src.ip;
+}
+
 static unsigned esfq_hash(struct esfq_sched_data *q, struct sk_buff *skb)
 {
 	u32 h, h2;
@@ -119,7 +132,7 @@
 	{
 		struct iphdr *iph = skb->nh.iph;
 		h = iph->daddr;
-		hs = iph->saddr;
+		hs = esfq_get_source(skb);
 		h2 = hs^iph->protocol;
 		if (!(iph->frag_off&htons(IP_MF|IP_OFFSET)) &&
 		    (iph->protocol == IPPROTO_TCP ||