[PATCH iproute2-next 0/3] support delivering packets in delayed

[PATCH iproute2-next 0/3] support delivering packets in delayed

[Yousuk Seung]

This series adds support for the new "slot" netem parameter for
slotting. Slotting is an approximation of shared media that gather up
packets within a varying delay window before delivering them nearly at
once.

Dave Taht (2):
  tc: support conversions to or from 64 bit nanosecond-based time
  q_netem: support delivering packets in delayed time slots

Yousuk Seung (1):
  q_netem: slotting with non-uniform distribution

 man/man8/tc-netem.8 |  40 +++++++++++++++-
 tc/q_netem.c        | 112 +++++++++++++++++++++++++++++++++++++++++++-
 tc/tc_core.h        |   4 ++
 tc/tc_util.c        |  55 ++++++++++++++++++++++
 tc/tc_util.h        |   3 ++
 5 files changed, 212 insertions(+), 2 deletions(-)

-- 
2.18.0.597.ga71716f1ad-goog

[PATCH iproute2-next 1/3] tc: support conversions to or from 64 bit nanosecond-based time

[Yousuk Seung]

From: Dave Taht <dave.taht@gmail.com>

Using a 32 bit field to represent time in nanoseconds results in a
maximum value of about 4.3 seconds, which is well below many observed
delays in WiFi and LTE, and barely in the ballpark for a trip past the
Earth's moon, Luna.

Using 64 bit time fields in nanoseconds allows us to simulate
network diameters of several hundred light-years. However, only
conversions to and from ns, us, ms, and seconds are provided.

Signed-off-by: Yousuk Seung <ysseung@google.com>
Signed-off-by: Dave Taht <dave.taht@gmail.com>
Signed-off-by: Neal Cardwell <ncardwell@google.com>
---
 tc/tc_core.h |  4 ++++
 tc/tc_util.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 tc/tc_util.h |  3 +++
 3 files changed, 62 insertions(+)

diff --git a/tc/tc_core.h b/tc/tc_core.h
index 1dfa9a4f773b..a0fe0923d171 100644
--- a/tc/tc_core.h
+++ b/tc/tc_core.h
@@ -7,6 +7,10 @@
 
 #define TIME_UNITS_PER_SEC	1000000
 
+#define NSEC_PER_USEC 1000
+#define NSEC_PER_MSEC 1000000
+#define NSEC_PER_SEC 1000000000LL
+
 enum link_layer {
 	LINKLAYER_UNSPEC,
 	LINKLAYER_ETHERNET,
diff --git a/tc/tc_util.c b/tc/tc_util.c
index d7578528a31b..c39c9046dcae 100644
--- a/tc/tc_util.c
+++ b/tc/tc_util.c
@@ -385,6 +385,61 @@ char *sprint_ticks(__u32 ticks, char *buf)
 	return sprint_time(tc_core_tick2time(ticks), buf);
 }
 
+/* 64 bit times are represented internally in nanoseconds */
+int get_time64(__s64 *time, const char *str)
+{
+	double nsec;
+	char *p;
+
+	nsec = strtod(str, &p);
+	if (p == str)
+		return -1;
+
+	if (*p) {
+		if (strcasecmp(p, "s") == 0 ||
+		    strcasecmp(p, "sec") == 0 ||
+		    strcasecmp(p, "secs") == 0)
+			nsec *= NSEC_PER_SEC;
+		else if (strcasecmp(p, "ms") == 0 ||
+			 strcasecmp(p, "msec") == 0 ||
+			 strcasecmp(p, "msecs") == 0)
+			nsec *= NSEC_PER_MSEC;
+		else if (strcasecmp(p, "us") == 0 ||
+			 strcasecmp(p, "usec") == 0 ||
+			 strcasecmp(p, "usecs") == 0)
+			nsec *= NSEC_PER_USEC;
+		else if (strcasecmp(p, "ns") == 0 ||
+			 strcasecmp(p, "nsec") == 0 ||
+			 strcasecmp(p, "nsecs") == 0)
+			nsec *= 1;
+		else
+			return -1;
+	}
+
+	*time = nsec;
+	return 0;
+}
+
+void print_time64(char *buf, int len, __s64 time)
+{
+	double nsec = time;
+
+	if (time >= NSEC_PER_SEC)
+		snprintf(buf, len, "%.3fs", nsec/NSEC_PER_SEC);
+	else if (time >= NSEC_PER_MSEC)
+		snprintf(buf, len, "%.3fms", nsec/NSEC_PER_MSEC);
+	else if (time >= NSEC_PER_USEC)
+		snprintf(buf, len, "%.3fus", nsec/NSEC_PER_USEC);
+	else
+		snprintf(buf, len, "%lldns", time);
+}
+
+char *sprint_time64(__s64 time, char *buf)
+{
+	print_time64(buf, SPRINT_BSIZE-1, time);
+	return buf;
+}
+
 int get_size(unsigned int *size, const char *str)
 {
 	double sz;
diff --git a/tc/tc_util.h b/tc/tc_util.h
index 6632c4f9c528..87be951c622d 100644
--- a/tc/tc_util.h
+++ b/tc/tc_util.h
@@ -82,12 +82,14 @@ int get_percent_rate64(__u64 *rate, const char *str, const char *dev);
 int get_size(unsigned int *size, const char *str);
 int get_size_and_cell(unsigned int *size, int *cell_log, char *str);
 int get_time(unsigned int *time, const char *str);
+int get_time64(__s64 *time, const char *str);
 int get_linklayer(unsigned int *val, const char *arg);
 
 void print_rate(char *buf, int len, __u64 rate);
 void print_size(char *buf, int len, __u32 size);
 void print_qdisc_handle(char *buf, int len, __u32 h);
 void print_time(char *buf, int len, __u32 time);
+void print_time64(char *buf, int len, __s64 time);
 void print_linklayer(char *buf, int len, unsigned int linklayer);
 void print_devname(enum output_type type, int ifindex);
 
@@ -96,6 +98,7 @@ char *sprint_size(__u32 size, char *buf);
 char *sprint_qdisc_handle(__u32 h, char *buf);
 char *sprint_tc_classid(__u32 h, char *buf);
 char *sprint_time(__u32 time, char *buf);
+char *sprint_time64(__s64 time, char *buf);
 char *sprint_ticks(__u32 ticks, char *buf);
 char *sprint_linklayer(unsigned int linklayer, char *buf);
 
-- 
2.18.0.597.ga71716f1ad-goog

Re: [PATCH iproute2-next 1/3] tc: support conversions to or from 64 bit nanosecond-based time

[David Ahern]

On 8/6/18 11:09 AM, Yousuk Seung wrote:
> diff --git a/tc/tc_core.h b/tc/tc_core.h
> index 1dfa9a4f773b..a0fe0923d171 100644
> --- a/tc/tc_core.h
> +++ b/tc/tc_core.h
> @@ -7,6 +7,10 @@
>  
>  #define TIME_UNITS_PER_SEC	1000000
>  
> +#define NSEC_PER_USEC 1000
> +#define NSEC_PER_MSEC 1000000
> +#define NSEC_PER_SEC 1000000000LL
> +

These are not specific to tc so a header in include is a better location
(utils.h or a new one)

>  enum link_layer {
>  	LINKLAYER_UNSPEC,
>  	LINKLAYER_ETHERNET,
> diff --git a/tc/tc_util.c b/tc/tc_util.c
> index d7578528a31b..c39c9046dcae 100644
> --- a/tc/tc_util.c
> +++ b/tc/tc_util.c

Similarly for these time functions - not specific to tc so move to
lib/utils.c

> @@ -385,6 +385,61 @@ char *sprint_ticks(__u32 ticks, char *buf)
>  	return sprint_time(tc_core_tick2time(ticks), buf);
>  }
>  
> +/* 64 bit times are represented internally in nanoseconds */
> +int get_time64(__s64 *time, const char *str)

__u64 seems more appropriate than __s64

> +{
> +	double nsec;
> +	char *p;
> +
> +	nsec = strtod(str, &p);
> +	if (p == str)
> +		return -1;
> +
> +	if (*p) {
> +		if (strcasecmp(p, "s") == 0 ||
> +		    strcasecmp(p, "sec") == 0 ||
> +		    strcasecmp(p, "secs") == 0)
> +			nsec *= NSEC_PER_SEC;
> +		else if (strcasecmp(p, "ms") == 0 ||
> +			 strcasecmp(p, "msec") == 0 ||
> +			 strcasecmp(p, "msecs") == 0)
> +			nsec *= NSEC_PER_MSEC;
> +		else if (strcasecmp(p, "us") == 0 ||
> +			 strcasecmp(p, "usec") == 0 ||
> +			 strcasecmp(p, "usecs") == 0)
> +			nsec *= NSEC_PER_USEC;
> +		else if (strcasecmp(p, "ns") == 0 ||
> +			 strcasecmp(p, "nsec") == 0 ||
> +			 strcasecmp(p, "nsecs") == 0)

strncasecmp would be more efficient

> +			nsec *= 1;
> +		else
> +			return -1;
> +	}
> +
> +	*time = nsec;
> +	return 0;
> +}
> +
> +void print_time64(char *buf, int len, __s64 time)
> +{
> +	double nsec = time;
> +
> +	if (time >= NSEC_PER_SEC)
> +		snprintf(buf, len, "%.3fs", nsec/NSEC_PER_SEC);
> +	else if (time >= NSEC_PER_MSEC)
> +		snprintf(buf, len, "%.3fms", nsec/NSEC_PER_MSEC);
> +	else if (time >= NSEC_PER_USEC)
> +		snprintf(buf, len, "%.3fus", nsec/NSEC_PER_USEC);
> +	else
> +		snprintf(buf, len, "%lldns", time);
> +}
> +
> +char *sprint_time64(__s64 time, char *buf)
> +{
> +	print_time64(buf, SPRINT_BSIZE-1, time);
> +	return buf;
> +}
> +
>  int get_size(unsigned int *size, const char *str)
>  {
>  	double sz;

Re: [PATCH iproute2-next 1/3] tc: support conversions to or from 64 bit nanosecond-based time

[Dave Taht]

On Sun, Aug 12, 2018 at 3:09 PM David Ahern <dsahern@gmail.com> wrote:
>
> On 8/6/18 11:09 AM, Yousuk Seung wrote:
> > diff --git a/tc/tc_core.h b/tc/tc_core.h
> > index 1dfa9a4f773b..a0fe0923d171 100644
> > --- a/tc/tc_core.h
> > +++ b/tc/tc_core.h
> > @@ -7,6 +7,10 @@
> >
> >  #define TIME_UNITS_PER_SEC   1000000
> >
> > +#define NSEC_PER_USEC 1000
> > +#define NSEC_PER_MSEC 1000000
> > +#define NSEC_PER_SEC 1000000000LL
> > +
>
> These are not specific to tc so a header in include is a better location
> (utils.h or a new one)
>
> >  enum link_layer {
> >       LINKLAYER_UNSPEC,
> >       LINKLAYER_ETHERNET,
> > diff --git a/tc/tc_util.c b/tc/tc_util.c
> > index d7578528a31b..c39c9046dcae 100644
> > --- a/tc/tc_util.c
> > +++ b/tc/tc_util.c
>
> Similarly for these time functions - not specific to tc so move to
> lib/utils.c
>
> > @@ -385,6 +385,61 @@ char *sprint_ticks(__u32 ticks, char *buf)
> >       return sprint_time(tc_core_tick2time(ticks), buf);
> >  }
> >
> > +/* 64 bit times are represented internally in nanoseconds */
> > +int get_time64(__s64 *time, const char *str)
>
> __u64 seems more appropriate than __s64

The reason why these are signed is to leave room in the API to
print/manage negative values. Wasting the 64th bit thusly would only
matter after extreme uptimes.

There was something of a long debate on this when these patches went
around the first time. We ended up with signed time in the netem code
also.


> > +{
> > +     double nsec;
> > +     char *p;
> > +
> > +     nsec = strtod(str, &p);
> > +     if (p == str)
> > +             return -1;
> > +
> > +     if (*p) {
> > +             if (strcasecmp(p, "s") == 0 ||
> > +                 strcasecmp(p, "sec") == 0 ||
> > +                 strcasecmp(p, "secs") == 0)
> > +                     nsec *= NSEC_PER_SEC;
> > +             else if (strcasecmp(p, "ms") == 0 ||
> > +                      strcasecmp(p, "msec") == 0 ||
> > +                      strcasecmp(p, "msecs") == 0)
> > +                     nsec *= NSEC_PER_MSEC;
> > +             else if (strcasecmp(p, "us") == 0 ||
> > +                      strcasecmp(p, "usec") == 0 ||
> > +                      strcasecmp(p, "usecs") == 0)
> > +                     nsec *= NSEC_PER_USEC;
> > +             else if (strcasecmp(p, "ns") == 0 ||
> > +                      strcasecmp(p, "nsec") == 0 ||
> > +                      strcasecmp(p, "nsecs") == 0)
>
> strncasecmp would be more efficient
>
> > +                     nsec *= 1;
> > +             else
> > +                     return -1;
> > +     }
> > +
> > +     *time = nsec;
> > +     return 0;
> > +}
> > +
> > +void print_time64(char *buf, int len, __s64 time)
> > +{
> > +     double nsec = time;
> > +
> > +     if (time >= NSEC_PER_SEC)
> > +             snprintf(buf, len, "%.3fs", nsec/NSEC_PER_SEC);
> > +     else if (time >= NSEC_PER_MSEC)
> > +             snprintf(buf, len, "%.3fms", nsec/NSEC_PER_MSEC);
> > +     else if (time >= NSEC_PER_USEC)
> > +             snprintf(buf, len, "%.3fus", nsec/NSEC_PER_USEC);
> > +     else
> > +             snprintf(buf, len, "%lldns", time);
> > +}
> > +
> > +char *sprint_time64(__s64 time, char *buf)
> > +{
> > +     print_time64(buf, SPRINT_BSIZE-1, time);
> > +     return buf;
> > +}
> > +
> >  int get_size(unsigned int *size, const char *str)
> >  {
> >       double sz;



-- 

Dave Täht
CEO, TekLibre, LLC
http://www.teklibre.com
Tel: 1-669-226-2619

[PATCH iproute2-next 2/3] q_netem: support delivering packets in delayed time slots

[Yousuk Seung]

From: Dave Taht <dave.taht@gmail.com>

Slotting is a crude approximation of the behaviors of shared media such
as cable, wifi, and LTE, which gather up a bunch of packets within a
varying delay window and deliver them, relative to that, nearly all at
once.

It works within the existing loss, duplication, jitter and delay
parameters of netem. Some amount of inherent latency must be specified,
regardless.

The new "slot" parameter specifies a minimum and maximum delay between
transmission attempts.

The "bytes" and "packets" parameters can be used to limit the amount of
information transferred per slot.

Examples of use:

tc qdisc add dev eth0 root netem delay 200us \
        slot 800us 10ms bytes 64k packets 42

A more correct example, using stacked netem instances and a packet limit
to emulate a tail drop wifi queue with slots and variable packet
delivery, with a 200Mbit isochronous underlying rate, and 20ms path
delay:

tc qdisc add dev eth0 root handle 1: netem delay 20ms rate 200mbit \
         limit 10000
tc qdisc add dev eth0 parent 1:1 handle 10:1 netem delay 200us \
         slot 800us 10ms bytes 64k packets 42 limit 512

Signed-off-by: Yousuk Seung <ysseung@google.com>
Signed-off-by: Dave Taht <dave.taht@gmail.com>
Signed-off-by: Neal Cardwell <ncardwell@google.com>
---
 man/man8/tc-netem.8 | 32 ++++++++++++++++++++++-
 tc/q_netem.c        | 63 ++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 93 insertions(+), 2 deletions(-)

diff --git a/man/man8/tc-netem.8 b/man/man8/tc-netem.8
index f2cd86b6ed8a..8d485b026751 100644
--- a/man/man8/tc-netem.8
+++ b/man/man8/tc-netem.8
@@ -8,7 +8,8 @@ NetEm \- Network Emulator
 .I OPTIONS
 
 .IR OPTIONS " := [ " LIMIT " ] [ " DELAY " ] [ " LOSS \
-" ] [ " CORRUPT " ] [ " DUPLICATION " ] [ " REORDERING " ][ " RATE " ]"
+" ] [ " CORRUPT " ] [ " DUPLICATION " ] [ " REORDERING " ] [ " RATE \
+" ] [ " SLOT " ]"
 
 .IR LIMIT " := "
 .B limit
@@ -51,6 +52,14 @@ NetEm \- Network Emulator
 .B rate
 .IR RATE " [ " PACKETOVERHEAD " [ " CELLSIZE " [ " CELLOVERHEAD " ]]]]"
 
+.IR SLOT " := "
+.BR slot
+.IR MIN_DELAY " [ " MAX_DELAY " ] ["
+.BR packets
+.IR PACKETS " ] [ "
+.BR bytes
+.IR BYTES " ]"
+
 
 .SH DESCRIPTION
 NetEm is an enhancement of the Linux traffic control facilities
@@ -162,6 +171,27 @@ granularity avoid a perfect shaping at a specific level. This will show up in
 an artificial packet compression (bursts). Another influence factor are network
 adapter buffers which can also add artificial delay.
 
+.SS slot
+defer delivering accumulated packets to within a slot, with each available slot
+configured with a minimum delay to acquire, and an optional maximum delay.  Slot
+delays can be specified in nanoseconds, microseconds, milliseconds or seconds
+(e.g. 800us). Values for the optional parameters
+.I BYTES
+will limit the number of bytes delivered per slot, and/or
+.I PACKETS
+will limit the number of packets delivered per slot.
+
+These slot options can provide a crude approximation of bursty MACs such as
+DOCSIS, WiFi, and LTE.
+
+Note that slotting is limited by several factors: the kernel clock granularity,
+as with a rate, and attempts to deliver many packets within a slot will be
+smeared by the timer resolution, and by the underlying native bandwidth also.
+
+It is possible to combine slotting with a rate, in which case complex behaviors
+where either the rate, or the slot limits on bytes or packets per slot, govern
+the actual delivered rate.
+
 .SH LIMITATIONS
 The main known limitation of Netem are related to timer granularity, since
 Linux is not a real-time operating system.
diff --git a/tc/q_netem.c b/tc/q_netem.c
index 9f9a9b3df255..f52a36b6c31c 100644
--- a/tc/q_netem.c
+++ b/tc/q_netem.c
@@ -40,7 +40,10 @@ static void explain(void)
 "                 [ loss gemodel PERCENT [R [1-H [1-K]]]\n" \
 "                 [ ecn ]\n" \
 "                 [ reorder PRECENT [CORRELATION] [ gap DISTANCE ]]\n" \
-"                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n");
+"                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n" \
+"                 [ slot MIN_DELAY [MAX_DELAY] [packets MAX_PACKETS]" \
+" [bytes MAX_BYTES]]\n" \
+		);
 }
 
 static void explain1(const char *arg)
@@ -164,6 +167,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	struct tc_netem_gimodel gimodel;
 	struct tc_netem_gemodel gemodel;
 	struct tc_netem_rate rate = {};
+	struct tc_netem_slot slot = {};
 	__s16 *dist_data = NULL;
 	__u16 loss_type = NETEM_LOSS_UNSPEC;
 	int present[__TCA_NETEM_MAX] = {};
@@ -412,6 +416,44 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 					return -1;
 				}
 			}
+		} else if (matches(*argv, "slot") == 0) {
+			NEXT_ARG();
+			present[TCA_NETEM_SLOT] = 1;
+			if (get_time64(&slot.min_delay, *argv)) {
+				explain1("slot min_delay");
+				return -1;
+			}
+			if (NEXT_IS_NUMBER()) {
+				NEXT_ARG();
+				if (get_time64(&slot.max_delay, *argv)) {
+					explain1("slot min_delay max_delay");
+					return -1;
+				}
+			}
+			if (slot.max_delay < slot.min_delay)
+				slot.max_delay = slot.min_delay;
+			if (NEXT_ARG_OK() &&
+			    matches(*(argv+1), "packets") == 0) {
+				NEXT_ARG();
+				if (!NEXT_ARG_OK() ||
+				    get_s32(&slot.max_packets, *(argv+1), 0)) {
+					explain1("slot packets");
+					return -1;
+				}
+				NEXT_ARG();
+			}
+			if (NEXT_ARG_OK() &&
+			    matches(*(argv+1), "bytes") == 0) {
+				unsigned int max_bytes;
+				NEXT_ARG();
+				if (!NEXT_ARG_OK() ||
+				    get_size(&max_bytes, *(argv+1))) {
+					explain1("slot bytes");
+					return -1;
+				}
+				slot.max_bytes = (int) max_bytes;
+				NEXT_ARG();
+			}
 		} else if (strcmp(*argv, "help") == 0) {
 			explain();
 			return -1;
@@ -472,6 +514,10 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	    addattr_l(n, 1024, TCA_NETEM_CORRUPT, &corrupt, sizeof(corrupt)) < 0)
 		return -1;
 
+	if (present[TCA_NETEM_SLOT] &&
+	    addattr_l(n, 1024, TCA_NETEM_SLOT, &slot, sizeof(slot)) < 0)
+		return -1;
+
 	if (loss_type != NETEM_LOSS_UNSPEC) {
 		struct rtattr *start;
 
@@ -526,6 +572,7 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 	int *ecn = NULL;
 	struct tc_netem_qopt qopt;
 	const struct tc_netem_rate *rate = NULL;
+	const struct tc_netem_slot *slot = NULL;
 	int len;
 	__u64 rate64 = 0;
 
@@ -586,6 +633,11 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 				return -1;
 			rate64 = rta_getattr_u64(tb[TCA_NETEM_RATE64]);
 		}
+		if (tb[TCA_NETEM_SLOT]) {
+			if (RTA_PAYLOAD(tb[TCA_NETEM_SLOT]) < sizeof(*slot))
+				return -1;
+		        slot = RTA_DATA(tb[TCA_NETEM_SLOT]);
+		}
 	}
 
 	fprintf(f, "limit %d", qopt.limit);
@@ -659,6 +711,15 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 			fprintf(f, " celloverhead %d", rate->cell_overhead);
 	}
 
+	if (slot) {
+		fprintf(f, " slot %s", sprint_time64(slot->min_delay, b1));
+		fprintf(f, " %s", sprint_time64(slot->max_delay, b1));
+		if(slot->max_packets)
+			fprintf(f, " packets %d", slot->max_packets);
+		if(slot->max_bytes)
+			fprintf(f, " bytes %d", slot->max_bytes);
+	}
+
 	if (ecn)
 		fprintf(f, " ecn ");
 
-- 
2.18.0.597.ga71716f1ad-goog

Re: [PATCH iproute2-next 2/3] q_netem: support delivering packets in delayed time slots

[David Ahern]

On 8/6/18 11:09 AM, Yousuk Seung wrote:
> diff --git a/tc/q_netem.c b/tc/q_netem.c
> index 9f9a9b3df255..f52a36b6c31c 100644
> --- a/tc/q_netem.c
> +++ b/tc/q_netem.c
> @@ -40,7 +40,10 @@ static void explain(void)
>  "                 [ loss gemodel PERCENT [R [1-H [1-K]]]\n" \
>  "                 [ ecn ]\n" \
>  "                 [ reorder PRECENT [CORRELATION] [ gap DISTANCE ]]\n" \
> -"                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n");
> +"                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n" \
> +"                 [ slot MIN_DELAY [MAX_DELAY] [packets MAX_PACKETS]" \
> +" [bytes MAX_BYTES]]\n" \
> +		);
>  }
>  
>  static void explain1(const char *arg)
> @@ -164,6 +167,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
>  	struct tc_netem_gimodel gimodel;
>  	struct tc_netem_gemodel gemodel;
>  	struct tc_netem_rate rate = {};
> +	struct tc_netem_slot slot = {};
>  	__s16 *dist_data = NULL;
>  	__u16 loss_type = NETEM_LOSS_UNSPEC;
>  	int present[__TCA_NETEM_MAX] = {};
> @@ -412,6 +416,44 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
>  					return -1;
>  				}
>  			}
> +		} else if (matches(*argv, "slot") == 0) {
> +			NEXT_ARG();
> +			present[TCA_NETEM_SLOT] = 1;
> +			if (get_time64(&slot.min_delay, *argv)) {
> +				explain1("slot min_delay");
> +				return -1;
> +			}
> +			if (NEXT_IS_NUMBER()) {
> +				NEXT_ARG();
> +				if (get_time64(&slot.max_delay, *argv)) {
> +					explain1("slot min_delay max_delay");
> +					return -1;
> +				}
> +			}
> +			if (slot.max_delay < slot.min_delay)

If max_delay is user specified, this should generate an error rather
than just overwriting the value.

> +				slot.max_delay = slot.min_delay;
> +			if (NEXT_ARG_OK() &&
> +			    matches(*(argv+1), "packets") == 0) {
> +				NEXT_ARG();
> +				if (!NEXT_ARG_OK() ||
> +				    get_s32(&slot.max_packets, *(argv+1), 0)) {
> +					explain1("slot packets");
> +					return -1;
> +				}
> +				NEXT_ARG();
> +			}
> +			if (NEXT_ARG_OK() &&
> +			    matches(*(argv+1), "bytes") == 0) {
> +				unsigned int max_bytes;
> +				NEXT_ARG();
> +				if (!NEXT_ARG_OK() ||
> +				    get_size(&max_bytes, *(argv+1))) {
> +					explain1("slot bytes");
> +					return -1;
> +				}
> +				slot.max_bytes = (int) max_bytes;
> +				NEXT_ARG();
> +			}
>  		} else if (strcmp(*argv, "help") == 0) {
>  			explain();
>  			return -1;

[PATCH iproute2-next 3/3] q_netem: slotting with non-uniform distribution

[Yousuk Seung]

Extend slotting with support for non-uniform distributions. This is
similar to netem's non-uniform distribution delay feature.

Syntax:
   slot distribution DISTRIBUTION DELAY JITTER [packets MAX_PACKETS] \
      [bytes MAX_BYTES]

The syntax and use of the distribution table is the same as in the
non-uniform distribution delay feature. A file DISTRIBUTION must be
present in TC_LIB_DIR (e.g. /usr/lib/tc) containing numbers scaled by
NETEM_DIST_SCALE. A random value x is selected from the table and it
takes DELAY + ( x * JITTER ) as delay. Correlation between values is not
supported.

Examples:
  Normal distribution delay with mean = 800us and stdev = 100us.
  > tc qdisc add dev eth0 root netem slot distribution normal \
    800us 100us

  Optionally set the max slot size in bytes and/or packets.
  > tc qdisc add dev eth0 root netem slot distribution normal \
    800us 100us bytes 64k packets 42

Signed-off-by: Yousuk Seung <ysseung@google.com>
Signed-off-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: Dave Taht <dave.taht@gmail.com>
---
 man/man8/tc-netem.8 | 20 ++++++++----
 tc/q_netem.c        | 75 +++++++++++++++++++++++++++++++++++++--------
 2 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/man/man8/tc-netem.8 b/man/man8/tc-netem.8
index 8d485b026751..111109cf042f 100644
--- a/man/man8/tc-netem.8
+++ b/man/man8/tc-netem.8
@@ -53,9 +53,13 @@ NetEm \- Network Emulator
 .IR RATE " [ " PACKETOVERHEAD " [ " CELLSIZE " [ " CELLOVERHEAD " ]]]]"
 
 .IR SLOT " := "
-.BR slot
-.IR MIN_DELAY " [ " MAX_DELAY " ] ["
-.BR packets
+.BR slot " { "
+.IR MIN_DELAY " [ " MAX_DELAY " ] |"
+.br
+.RB "               " distribution " { "uniform " | " normal " | " pareto " | " paretonormal " | "
+.IR FILE " } " DELAY " " JITTER " } "
+.br
+.RB "             [ " packets
 .IR PACKETS " ] [ "
 .BR bytes
 .IR BYTES " ]"
@@ -172,9 +176,13 @@ an artificial packet compression (bursts). Another influence factor are network
 adapter buffers which can also add artificial delay.
 
 .SS slot
-defer delivering accumulated packets to within a slot, with each available slot
-configured with a minimum delay to acquire, and an optional maximum delay.  Slot
-delays can be specified in nanoseconds, microseconds, milliseconds or seconds
+defer delivering accumulated packets to within a slot. Each available slot can be
+configured with a minimum delay to acquire, and an optional maximum delay.
+Alternatively it can be configured with the distribution similar to
+.BR distribution
+for
+.BR delay
+option. Slot delays can be specified in nanoseconds, microseconds, milliseconds or seconds
 (e.g. 800us). Values for the optional parameters
 .I BYTES
 will limit the number of bytes delivered per slot, and/or
diff --git a/tc/q_netem.c b/tc/q_netem.c
index f52a36b6c31c..e655e1a82e12 100644
--- a/tc/q_netem.c
+++ b/tc/q_netem.c
@@ -43,7 +43,9 @@ static void explain(void)
 "                 [ rate RATE [PACKETOVERHEAD] [CELLSIZE] [CELLOVERHEAD]]\n" \
 "                 [ slot MIN_DELAY [MAX_DELAY] [packets MAX_PACKETS]" \
 " [bytes MAX_BYTES]]\n" \
-		);
+"                 [ slot distribution" \
+" {uniform|normal|pareto|paretonormal|custom} DELAY JITTER" \
+" [packets MAX_PACKETS] [bytes MAX_BYTES]]\n");
 }
 
 static void explain1(const char *arg)
@@ -159,6 +161,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 			   struct nlmsghdr *n, const char *dev)
 {
 	int dist_size = 0;
+	int slot_dist_size = 0;
 	struct rtattr *tail;
 	struct tc_netem_qopt opt = { .limit = 1000 };
 	struct tc_netem_corr cor = {};
@@ -169,6 +172,7 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	struct tc_netem_rate rate = {};
 	struct tc_netem_slot slot = {};
 	__s16 *dist_data = NULL;
+	__s16 *slot_dist_data = NULL;
 	__u16 loss_type = NETEM_LOSS_UNSPEC;
 	int present[__TCA_NETEM_MAX] = {};
 	__u64 rate64 = 0;
@@ -417,21 +421,53 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 				}
 			}
 		} else if (matches(*argv, "slot") == 0) {
-			NEXT_ARG();
-			present[TCA_NETEM_SLOT] = 1;
-			if (get_time64(&slot.min_delay, *argv)) {
-				explain1("slot min_delay");
-				return -1;
-			}
 			if (NEXT_IS_NUMBER()) {
 				NEXT_ARG();
-				if (get_time64(&slot.max_delay, *argv)) {
-					explain1("slot min_delay max_delay");
+				present[TCA_NETEM_SLOT] = 1;
+				if (get_time64(&slot.min_delay, *argv)) {
+					explain1("slot min_delay");
+					return -1;
+				}
+				if (NEXT_IS_NUMBER()) {
+					NEXT_ARG();
+					if (get_time64(&slot.max_delay, *argv)) {
+						explain1("slot min_delay max_delay");
+						return -1;
+					}
+				}
+				if (slot.max_delay < slot.min_delay)
+					slot.max_delay = slot.min_delay;
+			} else {
+				NEXT_ARG();
+				if (strcmp(*argv, "distribution") == 0) {
+					present[TCA_NETEM_SLOT] = 1;
+					NEXT_ARG();
+					slot_dist_data = calloc(sizeof(slot_dist_data[0]), MAX_DIST);
+					slot_dist_size = get_distribution(*argv, slot_dist_data, MAX_DIST);
+					if (slot_dist_size <= 0) {
+						free(slot_dist_data);
+						return -1;
+					}
+					NEXT_ARG();
+					if (get_time64(&slot.dist_delay, *argv)) {
+						explain1("slot delay");
+						return -1;
+					}
+					NEXT_ARG();
+					if (get_time64(&slot.dist_jitter, *argv)) {
+						explain1("slot jitter");
+						return -1;
+					}
+					if (slot.dist_jitter <= 0) {
+						fprintf(stderr, "Non-positive jitter\n");
+						return -1;
+					}
+				} else {
+					fprintf(stderr, "Unknown slot parameter: %s\n",
+						*argv);
 					return -1;
 				}
 			}
-			if (slot.max_delay < slot.min_delay)
-				slot.max_delay = slot.min_delay;
 			if (NEXT_ARG_OK() &&
 			    matches(*(argv+1), "packets") == 0) {
 				NEXT_ARG();
@@ -558,6 +594,14 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 			return -1;
 		free(dist_data);
 	}
+
+	if (slot_dist_data) {
+		if (addattr_l(n, MAX_DIST * sizeof(slot_dist_data[0]),
+			      TCA_NETEM_SLOT_DIST,
+			      slot_dist_data, slot_dist_size * sizeof(slot_dist_data[0])) < 0)
+			return -1;
+		free(slot_dist_data);
+	}
 	tail->rta_len = (void *) NLMSG_TAIL(n) - (void *) tail;
 	return 0;
 }
@@ -712,8 +756,13 @@ static int netem_print_opt(struct qdisc_util *qu, FILE *f, struct rtattr *opt)
 	}
 
 	if (slot) {
-		fprintf(f, " slot %s", sprint_time64(slot->min_delay, b1));
-		fprintf(f, " %s", sprint_time64(slot->max_delay, b1));
+		if (slot->dist_jitter > 0) {
+		    fprintf(f, " slot distribution %s", sprint_time64(slot->dist_delay, b1));
+		    fprintf(f, " %s", sprint_time64(slot->dist_jitter, b1));
+		} else {
+		    fprintf(f, " slot %s", sprint_time64(slot->min_delay, b1));
+		    fprintf(f, " %s", sprint_time64(slot->max_delay, b1));
+		}
 		if(slot->max_packets)
 			fprintf(f, " packets %d", slot->max_packets);
 		if(slot->max_bytes)
-- 
2.18.0.597.ga71716f1ad-goog

Re: [PATCH iproute2-next 3/3] q_netem: slotting with non-uniform distribution

[David Ahern]

On 8/6/18 11:09 AM, Yousuk Seung wrote:

> @@ -417,21 +421,53 @@ static int netem_parse_opt(struct qdisc_util *qu, int argc, char **argv,
>  				}
>  			}
>  		} else if (matches(*argv, "slot") == 0) {
> -			NEXT_ARG();
> -			present[TCA_NETEM_SLOT] = 1;
> -			if (get_time64(&slot.min_delay, *argv)) {
> -				explain1("slot min_delay");
> -				return -1;
> -			}
>  			if (NEXT_IS_NUMBER()) {
>  				NEXT_ARG();
> -				if (get_time64(&slot.max_delay, *argv)) {
> -					explain1("slot min_delay max_delay");
> +				present[TCA_NETEM_SLOT] = 1;
> +				if (get_time64(&slot.min_delay, *argv)) {
> +					explain1("slot min_delay");
> +					return -1;
> +				}
> +				if (NEXT_IS_NUMBER()) {
> +					NEXT_ARG();
> +					if (get_time64(&slot.max_delay, *argv)) {
> +						explain1("slot min_delay max_delay");
> +						return -1;
> +					}
> +				}
> +				if (slot.max_delay < slot.min_delay)
> +					slot.max_delay = slot.min_delay;
> +			} else {
> +				NEXT_ARG();
> +				if (strcmp(*argv, "distribution") == 0) {
> +					present[TCA_NETEM_SLOT] = 1;
> +					NEXT_ARG();
> +					slot_dist_data = calloc(sizeof(slot_dist_data[0]), MAX_DIST);

if (!slot_dist_data) ...