[PATCH iproute2 0/4] support nsec uapi and add netem slotting feature

[PATCH iproute2 0/4] support nsec uapi and add netem slotting feature

[Dave Taht]

This patch series adds support for specifying time in nanoseconds
to tc, updates netem to use such (requires pkt_sched.h from the kernel
patch series to be imported), and lastly adds a new slotting
feature intended to emulate better how LTE and WiFi work.

Dave Taht (4):
  tc: support conversions to or from 64 bit nanosecond-based time
  q_netem: utilize 64 bit nanosecond API for delay and jitter
  q_netem: support delivering packets in delayed time slots
  netem: add documentation for the new slotting feature

 man/man8/tc-netem.8 |  32 +++++++++++++-
 tc/q_netem.c        | 123 +++++++++++++++++++++++++++++++++++++++++-----------
 tc/tc_core.h        |   3 ++
 tc/tc_util.c        |  56 ++++++++++++++++++++++++
 tc/tc_util.h        |   3 ++
 5 files changed, 191 insertions(+), 26 deletions(-)

-- 
2.7.4

[PATCH iproute2 1/4] tc: support conversions to or from 64 bit nanosecond-based time

[Dave Taht]

Using a 32 bit field to represent time in nanoseconds results in a
maximum value of about 4.3 seconds, which is well below many observed
delays in WiFi and LTE, and barely in the ballpark for a trip past the
Earth's moon, Luna.

Using 64 bit time fields in nanoseconds allows us to simulate
network diameters of several hundred light-years. However, only
conversions to and from ns, us, ms, and seconds are provided.

Signed-off-by: Dave Taht <dave.taht@gmail.com>
---
 tc/tc_core.h |  3 +++
 tc/tc_util.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tc/tc_util.h |  3 +++
 3 files changed, 62 insertions(+)

diff --git a/tc/tc_core.h b/tc/tc_core.h
index 8a63b79..e8bda8b 100644
--- a/tc/tc_core.h
+++ b/tc/tc_core.h
@@ -5,6 +5,9 @@
 #include <linux/pkt_sched.h>
 
 #define TIME_UNITS_PER_SEC	1000000
+#define USEC_PER_SEC 1000
+#define MSEC_PER_SEC (1000 * 1000)
+#define NSEC_PER_SEC (1000 * 1000 * 1000)
 
 enum link_layer {
 	LINKLAYER_UNSPEC,
diff --git a/tc/tc_util.c b/tc/tc_util.c
index b39e550..dafbbc3 100644
--- a/tc/tc_util.c
+++ b/tc/tc_util.c
@@ -323,6 +323,62 @@ char *sprint_ticks(__u32 ticks, char *buf)
 	return sprint_time(tc_core_tick2time(ticks), buf);
 }
 
+/* 64 bit times are represented internally in nanoseconds */
+
+int get_time64(__s64 *time, const char *str)
+{
+	double t;
+	char *p;
+
+	t = strtod(str, &p);
+	if (p == str)
+		return -1;
+
+	if (*p) {
+		if (strcasecmp(p, "s") == 0 ||
+		    strcasecmp(p, "sec") == 0 ||
+		    strcasecmp(p, "secs") == 0)
+			t *= NSEC_PER_SEC;
+		else if (strcasecmp(p, "ms") == 0 ||
+			 strcasecmp(p, "msec") == 0 ||
+			 strcasecmp(p, "msecs") == 0)
+			t *= MSEC_PER_SEC;
+		else if (strcasecmp(p, "us") == 0 ||
+			 strcasecmp(p, "usec") == 0 ||
+			 strcasecmp(p, "usecs") == 0)
+			t *= USEC_PER_SEC;
+		else if (strcasecmp(p, "ns") == 0 ||
+			 strcasecmp(p, "nsec") == 0 ||
+			 strcasecmp(p, "nsecs") == 0)
+			t *= 1;
+		else
+			return -1;
+	}
+
+	*time = t;
+	return 0;
+}
+
+void print_time64(char *buf, int len, __s64 time)
+{
+	double tmp = time;
+
+	if (time >= NSEC_PER_SEC)
+		snprintf(buf, len, "%.3fs", tmp/NSEC_PER_SEC);
+	else if (time >= MSEC_PER_SEC)
+		snprintf(buf, len, "%.3fms", tmp/MSEC_PER_SEC);
+	else if (time >= USEC_PER_SEC)
+		snprintf(buf, len, "%.3fus", tmp/USEC_PER_SEC);
+	else
+		snprintf(buf, len, "%lldns", time);
+}
+
+char *sprint_time64(__s64 time, char *buf)
+{
+	print_time64(buf, SPRINT_BSIZE-1, time);
+	return buf;
+}
+
 int get_size(unsigned int *size, const char *str)
 {
 	double sz;
diff --git a/tc/tc_util.h b/tc/tc_util.h
index 583a21a..7a8559b 100644
--- a/tc/tc_util.h
+++ b/tc/tc_util.h
@@ -72,12 +72,14 @@ int get_rate64(__u64 *rate, const char *str);
 int get_size(unsigned int *size, const char *str);
 int get_size_and_cell(unsigned int *size, int *cell_log, char *str);
 int get_time(unsigned int *time, const char *str);
+int get_time64(__s64 *time, const char *str);
 int get_linklayer(unsigned int *val, const char *arg);
 
 void print_rate(char *buf, int len, __u64 rate);
 void print_size(char *buf, int len, __u32 size);
 void print_qdisc_handle(char *buf, int len, __u32 h);
 void print_time(char *buf, int len, __u32 time);
+void print_time64(char *buf, int len, __s64 time);
 void print_linklayer(char *buf, int len, unsigned int linklayer);
 
 char *sprint_rate(__u64 rate, char *buf);
@@ -85,6 +87,7 @@ char *sprint_size(__u32 size, char *buf);
 char *sprint_qdisc_handle(__u32 h, char *buf);
 char *sprint_tc_classid(__u32 h, char *buf);
 char *sprint_time(__u32 time, char *buf);
+char *sprint_time64(__s64 time, char *buf);
 char *sprint_ticks(__u32 ticks, char *buf);
 char *sprint_linklayer(unsigned int linklayer, char *buf);
 
-- 
2.7.4

[PATCH iproute2 2/4] q_netem: utilize 64 bit nanosecond API for delay and jitter

[Dave Taht]

This starts to obsolete the old "ticks" api in favor of well
defined nanoseconds for the kernel/userspace netem interface.

Signed-off-by: Dave Taht <dave.taht@gmail.com>
---
 tc/q_netem.c | 68 +++++++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 44 insertions(+), 24 deletions(-)

diff --git a/tc/q_netem.c b/tc/q_netem.c
index cdaddce..22a5b94 100644
--- a/tc/q_netem.c
+++ b/tc/q_netem.c
@@ -151,24 +151,6 @@ static int get_distribution(const char *type, __s16 *data, int maxdata)
 #define NEXT_IS_SIGNED_NUMBER() \
 	(NEXT_ARG_OK() && (isdigit(argv[1][0]) || argv[1][0] == '-'))
 
-/* Adjust for the fact that psched_ticks aren't always usecs
-   (based on kernel PSCHED_CLOCK configuration */
-static int get_ticks(__u32 *ticks, const char *str)
-{
-	unsigned int t;
-
-	if (get_time(&t, str))
-		return -1;
-
-	if (tc_core_time2big(t)) {
-		fprintf(stderr, "Illegal %u time (too large)\n", t);
-		return -1;
-	}
-
-	*ticks = tc_core_time2tick(t);
-	return 0;
-}
-
 static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 			   struct nlmsghdr *n)
 {
@@ -185,6 +167,8 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	__u16 loss_type = NETEM_LOSS_UNSPEC;
 	int present[__TCA_NETEM_MAX] = {};
 	__u64 rate64 = 0;
+	__s64 latency64 = 0;
+	__s64 jitter64 = 0;
 
 	for ( ; argc > 0; --argc, ++argv) {
 		if (matches(*argv, "limit") == 0) {
@@ -196,14 +180,16 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 		} else if (matches(*argv, "latency") == 0 ||
 			   matches(*argv, "delay") == 0) {
 			NEXT_ARG();
-			if (get_ticks(&opt.latency, *argv)) {
+			present[TCA_NETEM_LATENCY64] = 1;
+			if (get_time64(&latency64, *argv)) {
 				explain1("latency");
 				return -1;
 			}
 
 			if (NEXT_IS_NUMBER()) {
 				NEXT_ARG();
-				if (get_ticks(&opt.jitter, *argv)) {
+				present[TCA_NETEM_JITTER64] = 1;
+				if (get_time64(&jitter64, *argv)) {
 					explain1("latency");
 					return -1;
 				}
@@ -437,7 +423,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	tail = NLMSG_TAIL(n);
 
 	if (reorder.probability) {
-		if (opt.latency == 0) {
+		if (latency64 == 0) {
 			fprintf(stderr, "reordering not possible without specifying some delay\n");
 			explain();
 			return -1;
@@ -458,7 +444,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 		}
 	}
 
-	if (dist_data && (opt.latency == 0 || opt.jitter == 0)) {
+	if (dist_data && (latency64 == 0 || jitter64 == 0)) {
 		fprintf(stderr, "distribution specified but no latency and jitter values\n");
 		explain();
 		return -1;
@@ -467,6 +453,16 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	if (addattr_l(n, 1024, TCA_OPTIONS, &opt, sizeof(opt)) < 0)
 		return -1;
 
+	if (present[TCA_NETEM_LATENCY64] &&
+	    addattr_l(n, 1024, TCA_NETEM_LATENCY64, &latency64,
+		      sizeof(latency64)) < 0)
+		return -1;
+
+	if (present[TCA_NETEM_JITTER64] &&
+	    addattr_l(n, 1024, TCA_NETEM_JITTER64, &jitter64,
+		      sizeof(jitter64)) < 0)
+		return -1;
+
 	if (present[TCA_NETEM_CORR] &&
 	    addattr_l(n, 1024, TCA_NETEM_CORR, &cor, sizeof(cor)) < 0)
 		return -1;
@@ -540,6 +536,8 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 	const struct tc_netem_rate *rate = NULL;
 	int len;
 	__u64 rate64 = 0;
+	__s64 latency64 = 0;
+	__s64 jitter64 = 0;
 
 	SPRINT_BUF(b1);
 
@@ -559,6 +557,18 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 		parse_rtattr(tb, TCA_NETEM_MAX, RTA_DATA(opt) + sizeof(qopt),
 			     len);
 
+		if (tb[TCA_NETEM_LATENCY64]) {
+			if (RTA_PAYLOAD(tb[TCA_NETEM_LATENCY64]) <
+			    sizeof(latency64))
+				return -1;
+			latency64 = rta_getattr_u64(tb[TCA_NETEM_LATENCY64]);
+		}
+		if (tb[TCA_NETEM_JITTER64]) {
+			if (RTA_PAYLOAD(tb[TCA_NETEM_JITTER64]) <
+			    sizeof(jitter64))
+				return -1;
+			jitter64 = rta_getattr_u64(tb[TCA_NETEM_JITTER64]);
+		}
 		if (tb[TCA_NETEM_CORR]) {
 			if (RTA_PAYLOAD(tb[TCA_NETEM_CORR]) < sizeof(*cor))
 				return -1;
@@ -602,13 +612,23 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 
 	fprintf(f, "limit %d", qopt.limit);
 
-	if (qopt.latency) {
+	if (latency64) {
+		fprintf(f, " delay %s", sprint_time64(latency64, b1));
+
+		if (jitter64) {
+			fprintf(f, "  %s", sprint_time64(jitter64, b1));
+			if (cor && cor->delay_corr)
+				fprintf(f, " %s",
+					sprint_percent(cor->delay_corr, b1));
+		}
+	} else if (qopt.latency) {
 		fprintf(f, " delay %s", sprint_ticks(qopt.latency, b1));
 
 		if (qopt.jitter) {
 			fprintf(f, "  %s", sprint_ticks(qopt.jitter, b1));
 			if (cor && cor->delay_corr)
-				fprintf(f, " %s", sprint_percent(cor->delay_corr, b1));
+				fprintf(f, " %s",
+					sprint_percent(cor->delay_corr, b1));
 		}
 	}
 
-- 
2.7.4

[PATCH iproute2 3/4] q_netem: support delivering packets in delayed time slots

[Dave Taht]

Slotting is a crude approximation of the behaviors of shared media such
as cable, wifi, and LTE, which gather up a bunch of packets within a
varying delay window and deliver them, relative to that, nearly all at
once.

It works within the existing loss, duplication, jitter and delay
parameters of netem. Some amount of inherent latency must be specified,
regardless.

The new "slot" parameter specifies a minimum and maximum delay between
transmission attempts.

The "bytes" and "packets" parameters can be used to limit the amount of
information transferred per slot.

Examples of use:

tc qdisc add dev eth0 root netem delay 200us \
        slot 800us 10ms bytes 64k packets 42

A more correct example, using stacked netem instances and a packet limit
to emulate a tail drop wifi queue with slots and variable packet
delivery, with a 200Mbit isochronous underlying rate, and 20ms path
delay:

tc qdisc add dev eth0 root handle 1: netem delay 20ms rate 200mbit \
         limit 10000
tc qdisc add dev eth0 parent 1:1 handle 10:1 netem delay 200us \
         slot 800us 10ms bytes 64k packets 42 limit 512

Signed-off-by: Dave Taht <dave.taht@gmail.com>
---
 tc/q_netem.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/tc/q_netem.c b/tc/q_netem.c
index 22a5b94..c606cfd 100644
--- a/tc/q_netem.c
+++ b/tc/q_netem.c
@@ -40,7 +40,10 @@ static void explain(void)
 "                 [ loss gemodel PERCENT [R [1-H [1-K]]]\n" \
 "                 [ ecn ]\n" \
 "                 [ reorder PRECENT [CORRELATION] [ gap DISTANCE ]]\n" \
-"                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n");
+"                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n" \
+"                 [ slot MIN_DELAY MAX_DELAY [packets MAX_PACKETS]" \
+" [bytes MAX_BYTES]]\n" \
+		);
 }
 
 static void explain1(const char *arg)
@@ -163,6 +166,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	struct tc_netem_gimodel gimodel;
 	struct tc_netem_gemodel gemodel;
 	struct tc_netem_rate rate = {};
+	struct tc_netem_slot slot = {};
 	__s16 *dist_data = NULL;
 	__u16 loss_type = NETEM_LOSS_UNSPEC;
 	int present[__TCA_NETEM_MAX] = {};
@@ -410,6 +414,36 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 					return -1;
 				}
 			}
+		} else if (matches(*argv, "slot") == 0) {
+			NEXT_ARG();
+			present[TCA_NETEM_SLOT] = 1;
+			if (get_time64(&slot.min_delay, *argv)) {
+				explain1("slot min_delay");
+				return -1;
+			}
+			if (NEXT_IS_NUMBER()) {
+				NEXT_ARG();
+				if (get_time64(&slot.max_delay, *argv)) {
+					explain1("slot min_delay max_delay");
+					return -1;
+				}
+			}
+			if (slot.max_delay < slot.min_delay)
+				slot.max_delay = slot.min_delay;
+		} else if (matches(*argv, "packets") == 0) {
+			NEXT_ARG();
+			if (get_s32(&slot.max_packets, *argv, 0)) {
+				explain1("slot packets");
+				return -1;
+			}
+		} else if (matches(*argv, "bytes") == 0) {
+			unsigned int max_bytes;
+			NEXT_ARG();
+			if (get_size(&max_bytes, *argv)) {
+				explain1("slot bytes");
+				return -1;
+			}
+			slot.max_bytes = (int) max_bytes;
 		} else if (strcmp(*argv, "help") == 0) {
 			explain();
 			return -1;
@@ -480,6 +514,10 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	    addattr_l(n, 1024, TCA_NETEM_CORRUPT, &corrupt, sizeof(corrupt)) < 0)
 		return -1;
 
+	if (present[TCA_NETEM_SLOT] &&
+	    addattr_l(n, 1024, TCA_NETEM_SLOT, &slot, sizeof(slot)) < 0)
+		return -1;
+
 	if (loss_type != NETEM_LOSS_UNSPEC) {
 		struct rtattr *start;
 
@@ -534,6 +572,7 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 	int *ecn = NULL;
 	struct tc_netem_qopt qopt;
 	const struct tc_netem_rate *rate = NULL;
+	const struct tc_netem_slot *slot = NULL;
 	int len;
 	__u64 rate64 = 0;
 	__s64 latency64 = 0;
@@ -608,6 +647,11 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 				return -1;
 			rate64 = rta_getattr_u64(tb[TCA_NETEM_RATE64]);
 		}
+		if (tb[TCA_NETEM_SLOT]) {
+			if (RTA_PAYLOAD(tb[TCA_NETEM_SLOT]) < sizeof(*slot))
+				return -1;
+		        slot = RTA_DATA(tb[TCA_NETEM_SLOT]);
+		}
 	}
 
 	fprintf(f, "limit %d", qopt.limit);
@@ -691,6 +735,15 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 			fprintf(f, " celloverhead %d", rate->cell_overhead);
 	}
 
+	if (slot) {
+		fprintf(f, " slot %s", sprint_time64(slot->min_delay, b1));
+		fprintf(f, " %s", sprint_time64(slot->max_delay, b1));
+		if(slot->max_packets)
+			fprintf(f, " packets %d", slot->max_packets);
+		if(slot->max_bytes)
+			fprintf(f, " bytes %d", slot->max_bytes);
+	}
+
 	if (ecn)
 		fprintf(f, " ecn ");
 
-- 
2.7.4

[PATCH iproute2 4/4] netem: add documentation for the new slotting feature

[Dave Taht]

Signed-off-by: Dave Taht <dave.taht@gmail.com>
---
 man/man8/tc-netem.8 | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/man/man8/tc-netem.8 b/man/man8/tc-netem.8
index b31384f..8c41d77 100644
--- a/man/man8/tc-netem.8
+++ b/man/man8/tc-netem.8
@@ -8,7 +8,8 @@ NetEm \- Network Emulator
 .I OPTIONS
 
 .IR OPTIONS " := [ " LIMIT " ] [ " DELAY " ] [ " LOSS \
-" ] [ " CORRUPT " ] [ " DUPLICATION " ] [ " REORDERING " ][ " RATE " ]"
+" ] [ " CORRUPT " ] [ " DUPLICATION " ] [ " REORDERING " ] [ " RATE \
+" ] [ " SLOT " ]"
 
 .IR LIMIT " := "
 .B limit
@@ -51,6 +52,14 @@ NetEm \- Network Emulator
 .B rate
 .IR RATE " [ " PACKETOVERHEAD " [ " CELLSIZE " [ " CELLOVERHEAD " ]]]]"
 
+.IR SLOT " := "
+.BR slot
+.IR MIN_DELAY " [ " MAX_DELAY " ] {["
+.BR packets
+.IR PACKETS " ] [ "
+.BR bytes
+.IR BYTES " ] }]"
+
 
 .SH DESCRIPTION
 NetEm is an enhancement of the Linux traffic control facilities
@@ -162,6 +171,27 @@ granularity avoid a perfect shaping at a specific level. This will show up in
 an artificial packet compression (bursts). Another influence factor are network
 adapter buffers which can also add artificial delay.
 
+.SS slot
+defer delivering accumulated packets to within a slot, with each available slot
+configured with a minimum delay to acquire, and an optional maximum delay.  Slot
+delays can be specified in nanoseconds, microseconds, milliseconds or seconds
+(e.g. 800us). Values for the optional parameters
+.I BYTES
+will limit the number of bytes delivered per slot, and/or
+.I PACKETS
+will limit the number of packets delivered per slot.
+
+These slot options can provide a crude approximation of bursty MACs such as
+DOCSIS, WiFi, and LTE.
+
+Note that slotting is limited by several factors: the kernel clock granularity,
+as with a rate, and attempts to deliver many packets within a slot will be
+smeared by the timer resolution, and by the underlying native bandwidth also.
+
+It is possible to combine slotting with a rate, in which case complex behaviors
+where either the rate, or the slot limits on bytes or packets per slot, govern
+the actual delivered rate.
+
 .SH LIMITATIONS
 The main known limitation of Netem are related to timer granularity, since
 Linux is not a real-time operating system.
-- 
2.7.4