[PATCH 1/6] Deferred attach of underlying USDT probes

[PATCH 1/6] Deferred attach of underlying USDT probes

[eugene.loh]

From: Eugene Loh <eugene.loh@oracle.com>

We would like dtrace to trace USDT processes that start
even after the dtrace session has been launched.  In that
case, the underlying probe cannot be attached when dtrace
starts up;  rather, the BPF program must be attached once
the USDT process has been detected.

Therefore:

*)  Make dt_bpf_load_prog() callable outside of dt_bpf.c.

*)  Have update_uprobe() call dt_construct(), dt_link(),
    dt_bpf_load_prog(), and attach() for any new underlying
    probes.

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 libdtrace/dt_bpf.c         |  2 +-
 libdtrace/dt_bpf.h         |  3 ++
 libdtrace/dt_prov_uprobe.c | 65 ++++++++++++++++++++++++++++++++++++--
 3 files changed, 67 insertions(+), 3 deletions(-)

diff --git a/libdtrace/dt_bpf.c b/libdtrace/dt_bpf.c
index 90881e398..ab97be2cf 100644
--- a/libdtrace/dt_bpf.c
+++ b/libdtrace/dt_bpf.c
@@ -1134,7 +1134,7 @@ dt_bpf_reloc_prog(dtrace_hdl_t *dtp, const dtrace_difo_t *dp)
  *
  * Note that DTrace generates BPF programs that are licensed under the GPL.
  */
-static int
+int
 dt_bpf_load_prog(dtrace_hdl_t *dtp, const dt_probe_t *prp,
 		 const dtrace_difo_t *dp, uint_t cflags)
 {
diff --git a/libdtrace/dt_bpf.h b/libdtrace/dt_bpf.h
index 5716d2320..115adfbf0 100644
--- a/libdtrace/dt_bpf.h
+++ b/libdtrace/dt_bpf.h
@@ -14,6 +14,7 @@
 #include <dtrace/difo.h>
 #include <dt_btf.h>
 #include <dt_impl.h>
+#include <dt_probe.h>
 
 struct dtrace_hdl;
 
@@ -88,6 +89,8 @@ extern int dt_bpf_prog_load(struct dtrace_hdl *, const struct dt_probe *prp,
 			    size_t sz);
 extern int dt_bpf_raw_tracepoint_open(const void *tp, int fd);
 extern int dt_bpf_make_progs(struct dtrace_hdl *, uint_t);
+extern int dt_bpf_load_prog(dtrace_hdl_t *dtp, const dt_probe_t *prp,
+			    const dtrace_difo_t *dp, uint_t cflags);
 extern int dt_bpf_load_progs(struct dtrace_hdl *, uint_t);
 extern void dt_bpf_init(struct dtrace_hdl *dtp);
 
diff --git a/libdtrace/dt_prov_uprobe.c b/libdtrace/dt_prov_uprobe.c
index cf0a95fba..8fda85532 100644
--- a/libdtrace/dt_prov_uprobe.c
+++ b/libdtrace/dt_prov_uprobe.c
@@ -309,6 +309,10 @@ static int update_uprobe(dtrace_hdl_t *dtp, void *datap)
 	int		i, prid = dtp->dt_probe_id;
 	uint_t		old_strlen = dtp->dt_strlen;
 	dt_pcb_t	pcb;
+	dtrace_optval_t	dest_ok = DTRACEOPT_UNSET;
+
+	/* Determine whether destructive actions are okay. */
+	dtrace_getopt(dtp, "destructive", &dest_ok);
 
 	/* Clear stale pids. */
 	purge_BPFmap(dtp);
@@ -364,6 +368,61 @@ static int update_uprobe(dtrace_hdl_t *dtp, void *datap)
 
 		prp_next = dt_list_next(prp);
 
+		/* Handle underlying probe. */
+		if (prp->prov->impl == &dt_uprobe) {
+			dtrace_difo_t   *dp;
+			int		cflags, fd, rc = -1;
+
+			/*
+			 * Strictly speaking, we want the value passed in to
+			 * dtrace_go().  In practice, its flags pertain to
+			 * compilation and disassembly, which at this stage
+			 * no longer interest us.
+			 */
+			cflags = 0;
+
+			/* Check if the probe is already set up. */
+			if (prp->difo)
+				continue;
+
+			/* Make program. */
+			dp = dt_construct(dtp, prp, cflags, NULL);
+			if (dp == NULL)
+				continue;        // FIXME in dt_bpf_make_progs() this is a fatal error; should we do the same here?
+			prp->difo = dp;
+
+			/* Load program. */
+			if (dt_link(dtp, prp, dp, NULL) == -1)
+				continue;        // FIXME in dt_bpf_load_progs() this is a fatal error; should we do the same here?
+			if (dp->dtdo_flags & DIFOFLG_DESTRUCTIVE &&
+			    dest_ok == DTRACEOPT_UNSET)
+				return dt_set_errno(dtp, EDT_DESTRUCTIVE);
+
+			fd = dt_bpf_load_prog(dtp, prp, dp, cflags);
+			if (fd == -1)
+				continue;        // FIXME in dt_bpf_load_progs() this is a fatal error; should we do the same here?
+
+			if (prp->prov->impl->attach)
+				rc = prp->prov->impl->attach(dtp, prp, fd);
+
+			if (rc == -ENOTSUPP) {
+				char    *s;
+
+				close(fd);
+				if (asprintf(&s, "Failed to enable %s:%s:%s:%s",
+					      prp->desc->prv, prp->desc->mod,
+					      prp->desc->fun, prp->desc->prb) == -1)
+					return dt_set_errno(dtp, EDT_ENABLING_ERR);
+				dt_handle_rawerr(dtp, s);
+				free(s);
+			} else if (rc < 0) {
+				close(fd);
+				return dt_set_errno(dtp, EDT_ENABLING_ERR);
+			}
+
+			continue;
+		}
+
 		/* Make sure it is an overlying USDT probe. */
 		if (prp->prov->impl != &dt_usdt)
 			continue;
@@ -454,9 +513,11 @@ static int update_uprobe(dtrace_hdl_t *dtp, void *datap)
 				dt_bpf_map_update(dtp->dt_usdt_pridsmap_fd, &key, &val);
 			} else if (val.prid != oval.prid || val.mask != oval.mask) {
 				/*
-				 * This can happen when two overlying probes map to the same underlying probe for the same pid.
-				 * E.g., pid:::entry and pid:::0, or pid:::$offset and usdt:::.
+				 * Two different USDT probes should never map to the same
+				 * underlying probe for the same pid.  Nor should the bit
+				 * mask ever change.
 				 */
+				assert(0);
 			} else {
 				/*
 				 * Nothing to do, it already is in the map.
-- 
2.43.5

[PATCH 2/6] test: Correct long-standing dt_flags typo

[eugene.loh]

From: Eugene Loh <eugene.loh@oracle.com>

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 test/unittest/ip/tst.ipv4localudp.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/unittest/ip/tst.ipv4localudp.sh b/test/unittest/ip/tst.ipv4localudp.sh
index 0e357c3d2..4ee584400 100755
--- a/test/unittest/ip/tst.ipv4localudp.sh
+++ b/test/unittest/ip/tst.ipv4localudp.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Oracle Linux DTrace.
-# Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at
 # http://oss.oracle.com/licenses/upl.
 #
@@ -33,7 +33,7 @@ fi
 dtrace=$1
 testdir="$(dirname $_test)"
 local=127.0.0.1
-$dtrace $dt_cflags -c "$testdir/perlping.pl udp $local" -qs /dev/stdin <<EOF
+$dtrace $dt_flags -c "$testdir/perlping.pl udp $local" -qs /dev/stdin <<EOF
 BEGIN
 {
 	send = receive = 0;
-- 
2.43.5

[PATCH 3/6] test: Remove some outdated and unhelpful comments

[eugene.loh]

From: Eugene Loh <eugene.loh@oracle.com>

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 test/triggers/Build | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/triggers/Build b/test/triggers/Build
index 0e8660e59..db80b30ed 100644
--- a/test/triggers/Build
+++ b/test/triggers/Build
@@ -192,18 +192,18 @@ pid-tst-weak1_CFLAGS := -O0
 pid-tst-weak2_CFLAGS := -O0
 profile-tst-ufuncsort_CFLAGS := -O0
 
-# usdt-tst-argmap calls USDT probes (defined in argmap.d) using sys/sdt.h
+# usdt-tst-argmap calls USDT probes using sys/sdt.h
 usdt-tst-argmap_CFLAGS := -Iuts/common
 usdt-tst-argmap_PROV := usdt-tst-argmap-prov.d
 
-# usdt-tst-args calls USDT probes (defined in args.d) using sys/sdt.h
+# usdt-tst-args calls USDT probes using sys/sdt.h
 usdt-tst-args_CFLAGS := -Iuts/common
 usdt-tst-args_PROV := usdt-tst-args-prov.d
 
-# usdt-tst-forker calls USDT probes (defined in args.d) based on dtrace -h
+# usdt-tst-forker calls USDT probes based on dtrace -h
 usdt-tst-forker_PROV := usdt-tst-forker-prov.d
 
-# usdt-tst-special calls USDT probes (defined in prov.d) based on dtrace -h
+# usdt-tst-special calls USDT probes based on dtrace -h
 usdt-tst-special_CFLAGS := -fno-inline -O2
 usdt-tst-special_PROV := usdt-tst-special-prov.d
 
-- 
2.43.5

[PATCH 4/6] test: Add a USDT "deferred" test trigger

[eugene.loh]

From: Eugene Loh <eugene.loh@oracle.com>

After a dtrace session has started, it is possible for new USDT
processes to start and for there to be a delay before dtprobed
and then dtrace becomes aware of the new process.

Add a test trigger with USDT probes that waits for dtrace to
discover it and send it USR1.  Once the USR1 is received, a
short workload is run to completion.

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 test/triggers/Build                      |  5 +-
 test/triggers/usdt-tst-defer-prov.d      | 11 ++++
 test/triggers/usdt-tst-defer.c           | 79 ++++++++++++++++++++++++
 test/unittest/usdt/tst.defer-nodtrace.r  |  1 +
 test/unittest/usdt/tst.defer-nodtrace.sh | 49 +++++++++++++++
 5 files changed, 144 insertions(+), 1 deletion(-)
 create mode 100644 test/triggers/usdt-tst-defer-prov.d
 create mode 100644 test/triggers/usdt-tst-defer.c
 create mode 100644 test/unittest/usdt/tst.defer-nodtrace.r
 create mode 100755 test/unittest/usdt/tst.defer-nodtrace.sh

diff --git a/test/triggers/Build b/test/triggers/Build
index db80b30ed..107b3b4d1 100644
--- a/test/triggers/Build
+++ b/test/triggers/Build
@@ -13,7 +13,7 @@ EXTERNAL_64BIT_TRIGGERS = testprobe readwholedir mmap bogus-ioctl open delaydie
     ustack-tst-spin ustack-tst-mtspin \
     visible-constructor visible-constructor-static visible-constructor-static-unstripped
 
-EXTERNAL_64BIT_SDT_TRIGGERS = usdt-tst-argmap usdt-tst-args usdt-tst-forker usdt-tst-special
+EXTERNAL_64BIT_SDT_TRIGGERS = usdt-tst-argmap usdt-tst-args usdt-tst-forker usdt-tst-defer usdt-tst-special
 EXTERNAL_64BIT_TRIGGERS += $(EXTERNAL_64BIT_SDT_TRIGGERS)
 
 EXTERNAL_32BIT_TRIGGERS := visible-constructor-32
@@ -203,6 +203,9 @@ usdt-tst-args_PROV := usdt-tst-args-prov.d
 # usdt-tst-forker calls USDT probes based on dtrace -h
 usdt-tst-forker_PROV := usdt-tst-forker-prov.d
 
+# usdt-tst-defer calls USDT probes based on dtrace -h
+usdt-tst-defer_PROV := usdt-tst-defer-prov.d
+
 # usdt-tst-special calls USDT probes based on dtrace -h
 usdt-tst-special_CFLAGS := -fno-inline -O2
 usdt-tst-special_PROV := usdt-tst-special-prov.d
diff --git a/test/triggers/usdt-tst-defer-prov.d b/test/triggers/usdt-tst-defer-prov.d
new file mode 100644
index 000000000..a4ef8323d
--- /dev/null
+++ b/test/triggers/usdt-tst-defer-prov.d
@@ -0,0 +1,11 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+provider testprov {
+	probe foo();
+	probe bar(int, int, int);
+};
diff --git a/test/triggers/usdt-tst-defer.c b/test/triggers/usdt-tst-defer.c
new file mode 100644
index 000000000..2df5401b2
--- /dev/null
+++ b/test/triggers/usdt-tst-defer.c
@@ -0,0 +1,79 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+/*
+ * The main characteristic of this trigger code is that it allows deferred
+ * DTrace detection of the trigger.  That is, the trigger spins in "phase 1",
+ * waiting for DTrace to detect it and send it USR1.  Only then does "phase 2"
+ * run a short workload to completion.
+ */
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+#include "usdt-tst-defer-prov.h"
+
+static int phase = 1;
+
+static void
+interrupt(int sig)
+{
+	phase = 2;
+}
+
+int
+main(int argc, char **argv)
+{
+	struct sigaction act;
+	int i;
+	int nphase1 = 0, nphase1foo = 0, nphase1bar = 0;
+	int nphase2 = 0, nphase2foo = 0, nphase2bar = 0;
+
+	/* set the handler to listen for SIGUSR1 */
+	act.sa_handler = interrupt;
+	act.sa_flags = 0;
+	if (sigaction(SIGUSR1, &act, NULL)) {
+		printf("set handler failed\n");
+		return 1;
+	}
+
+	/* in phase 1, loop on probe "foo" to wait on USR1 */
+	while (phase == 1) {
+		nphase1++;
+		if (TESTPROV_FOO_ENABLED()) {
+			nphase1foo++;
+			phase = 2;
+		}
+		if (TESTPROV_BAR_ENABLED()) {
+			nphase1bar++;
+			phase = 2;
+		}
+		TESTPROV_FOO();
+	}
+
+	/* wait for probes to get set up */
+	usleep(100000);
+
+	/* in phase 2, just loop over probe "bar" a fixed number of times */
+	for (i = 0; i < 10; i++) {
+		nphase2++;
+		usleep(2000);
+		if (TESTPROV_FOO_ENABLED())
+			nphase2foo++;
+		usleep(2000);
+		if (TESTPROV_BAR_ENABLED())
+			nphase2bar++;
+		usleep(2000);
+		TESTPROV_BAR(i, i + 2, i * 2);
+	}
+
+	printf("%d: %d %d %d %d %d %d\n", getpid(),
+	    nphase1, nphase1foo, nphase1bar, nphase2, nphase2foo, nphase2bar);
+
+	return 0;
+}
diff --git a/test/unittest/usdt/tst.defer-nodtrace.r b/test/unittest/usdt/tst.defer-nodtrace.r
new file mode 100644
index 000000000..2e9ba477f
--- /dev/null
+++ b/test/unittest/usdt/tst.defer-nodtrace.r
@@ -0,0 +1 @@
+success
diff --git a/test/unittest/usdt/tst.defer-nodtrace.sh b/test/unittest/usdt/tst.defer-nodtrace.sh
new file mode 100755
index 000000000..e5fd2855c
--- /dev/null
+++ b/test/unittest/usdt/tst.defer-nodtrace.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# There are various tests that use the usdt-tst-defer trigger.  As a
+# baseline, let us run the trigger without enabling any USDT probes
+# and check that the counts are as expected.
+
+dtrace=$1
+trigger=`pwd`/test/triggers/usdt-tst-defer
+
+# Set up test directory.
+
+DIRNAME=$tmpdir/defer-nodtrace.$$.$RANDOM
+mkdir -p $DIRNAME
+cd $DIRNAME
+
+# Make a private copy of the trigger executable so that we get our
+# own DOF stash.
+
+cp $trigger main
+
+# Check that the is-enabled probes are false when the USDT probes are not enabled.
+# That is, nphase1foo == nphase1bar == nphase2foo == nphase2bar == 0.
+# Also, nphase2 == 10.
+# Note that nphase1 will be undefined.
+
+./main > main.out &
+pid=$!
+sleep 1
+kill -USR1 $pid
+wait
+
+echo "$pid: undefined 0 0 10 0 0" > main.out.expected
+if ! awk '{ $2 = "undefined"; print }' main.out | diff -q - main.out.expected; then
+	echo program output looks wrong for the no-DTrace case
+	echo === got ===
+	cat main.out
+	echo === expected ===
+	cat main.out.expected
+	exit 1
+fi
+
+echo success
+
+exit 0
-- 
2.43.5

[PATCH 5/6] test: Add USDT tests for deferred detection

[eugene.loh]

From: Eugene Loh <eugene.loh@oracle.com>

When new USDT processes start up, we do not immediately trace
them.  First, dtprobed has to see these processes and then dtrace
has to find them.  Hence, many of our tests XFAIL.

Add some tests that check USDT tracing on processes that start
after the dtrace session has started, allowing for the possibility
that tracing does not start until after the trigger is already
underway.

There are two cases to consider.  One is when there is a USDT
process running when the dtrace session starts.  In this case,
an underlying probe is attached and later USDT processes just
get added on.  Another case is when no USDT processes are running
when the dtrace session starts.  In this case, -Z must be specified,
and the underlying probe will not be attached until one of the USDT
processes is detected.

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 test/unittest/usdt/tst.defer-Z.r  |   1 +
 test/unittest/usdt/tst.defer-Z.sh | 153 +++++++++++++++++++++++++++++
 test/unittest/usdt/tst.defer.r    |   1 +
 test/unittest/usdt/tst.defer.sh   | 158 ++++++++++++++++++++++++++++++
 4 files changed, 313 insertions(+)
 create mode 100644 test/unittest/usdt/tst.defer-Z.r
 create mode 100755 test/unittest/usdt/tst.defer-Z.sh
 create mode 100644 test/unittest/usdt/tst.defer.r
 create mode 100755 test/unittest/usdt/tst.defer.sh

diff --git a/test/unittest/usdt/tst.defer-Z.r b/test/unittest/usdt/tst.defer-Z.r
new file mode 100644
index 000000000..2e9ba477f
--- /dev/null
+++ b/test/unittest/usdt/tst.defer-Z.r
@@ -0,0 +1 @@
+success
diff --git a/test/unittest/usdt/tst.defer-Z.sh b/test/unittest/usdt/tst.defer-Z.sh
new file mode 100755
index 000000000..52e92eb6b
--- /dev/null
+++ b/test/unittest/usdt/tst.defer-Z.sh
@@ -0,0 +1,153 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# This test verifies that USDT will see new processes, even if detection
+# is deferred -- that is, DTrace does not know about a new USDT process
+# until after it's started running.
+#
+# In this test, all processes are started after the DTrace session has started.
+# So the USDT probes will not be recognized at first and -Z must be used.
+
+dtrace=$1
+trigger=`pwd`/test/triggers/usdt-tst-defer
+
+# Set up test directory.
+
+DIRNAME=$tmpdir/defer-Z.$$.$RANDOM
+mkdir -p $DIRNAME
+cd $DIRNAME
+
+# Make a private copy of the trigger executable so that we get our
+# own DOF stash.
+
+cp $trigger main
+
+# Start dtrace.
+
+$dtrace $dt_flags -Zwq -o dtrace.out -n '
+testprov*:::foo
+{
+	raise(SIGUSR1);
+}
+testprov*0:::bar,
+testprov*1:::bar,
+testprov*2:::bar,
+testprov*3:::bar,
+testprov*4:::bar
+{
+	@[pid, 0] = sum(arg0);
+	@[pid, 1] = sum(arg1);
+	@[pid, 2] = sum(arg2);
+	@[pid, 3] = sum(pid % 100);
+}' &
+dtpid=$!
+sleep 2
+if [[ ! -d /proc/$dtpid ]]; then
+	echo ERROR dtrace died
+	exit 1
+fi
+
+# Start processes concurrently.
+
+num=10
+i=0
+while [ $i -lt $num ]; do
+	./main > main.out$i &
+	pids[$i]=$!
+	i=$(($i + 1))
+done
+
+# Confirm that dtrace is still running (otherwise triggers run forever).
+sleep 2
+if [[ ! -d /proc/$dtpid ]]; then
+	echo ERROR dtrace died after triggers started
+	i=0
+	while [ $i -lt $num ]; do
+		kill -USR1 ${pids[$i]}
+		wait       ${pids[$i]}
+		i=$(($i + 1))
+	done
+	exit 1
+fi
+
+# Wait for processes to complete.
+
+i=0
+while [ $i -lt $num ]; do
+	wait ${pids[$i]}
+	i=$(($i + 1))
+done
+
+# Kill the dtrace process.
+
+kill $dtpid
+wait
+
+# Check the program output (main.out$i files).
+
+i=0
+while [ $i -lt $num ]; do
+	if [ $((${pids[$i]} % 10)) -lt 5 ]; then
+		nphase2bar=10
+	else
+		nphase2bar=0
+	fi
+	echo "${pids[$i]}: undefined 0 0 10 10 $nphase2bar" > main.out$i.expected
+	awk '
+	    $3 == "1" { $3 =   0 }   # in phase 1, round 1 down to 0
+	    $4 == "1" { $4 =   0 }   # in phase 1, round 1 down to 0
+	    { $2 = "undefined"; print }' main.out$i > main.out$i.post
+	if ! diff -q main.out$i.post main.out$i.expected; then
+		echo program output looks wrong for DTrace case $i
+		echo === was ===
+		cat main.out$i
+		echo === got ===
+		cat main.out$i.post
+		echo === expected ===
+		cat main.out$i.expected
+		exit 1
+	fi
+	i=$(($i + 1))
+done
+
+# Check the dtrace output.
+
+#     regularize the dtrace output
+awk 'NF != 0 { print $1, $2, $3 }' dtrace.out | sort > dtrace.out.post
+
+#     determine what to expect
+
+i=0
+while [ $i -lt $num ]; do
+	if [ $((${pids[$i]} % 10)) -lt 5 ]; then
+		x=$(((${pids[$i]} % 100) * 10))
+		echo ${pids[$i]} 0 45 >> dtrace.out.expected
+		echo ${pids[$i]} 1 65 >> dtrace.out.expected
+		echo ${pids[$i]} 2 90 >> dtrace.out.expected
+		echo ${pids[$i]} 3 $x >> dtrace.out.expected
+	fi
+
+	i=$(($i + 1))
+done
+
+#     diff
+if ! sort dtrace.out.expected | diff -q - dtrace.out.post; then
+	echo dtrace output looks wrong for DTrace case $i
+	echo === was ===
+	cat dtrace.out
+	echo === got ===
+	cat dtrace.out.post
+	echo === expected ===
+	sort dtrace.out.expected
+	echo === diff ===
+	sort dtrace.out.expected | diff - dtrace.out.post
+	exit 1
+fi
+
+echo success
+
+exit 0
diff --git a/test/unittest/usdt/tst.defer.r b/test/unittest/usdt/tst.defer.r
new file mode 100644
index 000000000..2e9ba477f
--- /dev/null
+++ b/test/unittest/usdt/tst.defer.r
@@ -0,0 +1 @@
+success
diff --git a/test/unittest/usdt/tst.defer.sh b/test/unittest/usdt/tst.defer.sh
new file mode 100755
index 000000000..6a9012b94
--- /dev/null
+++ b/test/unittest/usdt/tst.defer.sh
@@ -0,0 +1,158 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# This test verifies that USDT will see new processes, even if detection
+# is deferred -- that is, DTrace does not know about a new USDT process
+# until after it's started running.
+#
+# In this test, a process is started before the DTrace session has started.
+# So the USDT probes will be recognized at first and -Z need not be used.
+
+dtrace=$1
+trigger=`pwd`/test/triggers/usdt-tst-defer
+
+# Set up test directory.
+
+DIRNAME=$tmpdir/defer.$$.$RANDOM
+mkdir -p $DIRNAME
+cd $DIRNAME
+
+# Make a private copy of the trigger executable so that we get our
+# own DOF stash.
+
+cp $trigger main
+
+# Start one process.
+
+./main > main.out0 &
+pids[0]=$!
+i=1
+
+# Figure out the pid's trailing digit.
+
+lastdigit=$((${pids[0]} % 10))
+
+# Start dtrace.
+
+$dtrace $dt_flags -wq -o dtrace.out -n '
+testprov*:::foo
+{
+	raise(SIGUSR1);
+}
+testprov*'$lastdigit':::bar
+{
+	@[pid, 0] = sum(arg0);
+	@[pid, 1] = sum(arg1);
+	@[pid, 2] = sum(arg2);
+	@[pid, 3] = sum(pid % 100);
+}' &
+dtpid=$!
+sleep 2
+if [[ ! -d /proc/$dtpid ]]; then
+	echo ERROR dtrace died
+	exit 1
+fi
+
+# Start remaining processes.
+
+num=10
+while [ $i -lt $num ]; do
+	./main > main.out$i &
+	pids[$i]=$!
+	i=$(($i + 1))
+done
+
+# Confirm that dtrace is still running (otherwise triggers run forever).
+sleep 2
+if [[ ! -d /proc/$dtpid ]]; then
+	echo ERROR dtrace died after triggers started
+	i=0
+	while [ $i -lt $num ]; do
+		kill -USR1 ${pids[$i]}
+		wait       ${pids[$i]}
+		i=$(($i + 1))
+	done
+	exit 1
+fi
+
+# Wait for processes to complete.
+
+i=0
+while [ $i -lt $num ]; do
+	wait ${pids[$i]}
+	i=$(($i + 1))
+done
+
+# Kill the dtrace process.
+
+kill $dtpid
+wait
+
+# Check the program output (main.out$i files).
+
+i=0
+while [ $i -lt $num ]; do
+	if [ $((${pids[$i]} % 10)) -eq $lastdigit ]; then
+		nphase2bar=10
+	else
+		nphase2bar=0
+	fi
+	echo "${pids[$i]}: undefined 0 0 10 10 $nphase2bar" > main.out$i.expected
+	awk '
+	    $3 == "1" { $3 =   0 }   # in phase 1, round 1 down to 0
+	    $4 == "1" { $4 =   0 }   # in phase 1, round 1 down to 0
+	    { $2 = "undefined"; print }' main.out$i > main.out$i.post
+	if ! diff -q main.out$i.post main.out$i.expected; then
+		echo program output looks wrong for DTrace case $i
+		echo === was ===
+		cat main.out$i
+		echo === got ===
+		cat main.out$i.post
+		echo === expected ===
+		cat main.out$i.expected
+		exit 1
+	fi
+	i=$(($i + 1))
+done
+
+# Check the dtrace output.
+
+#     regularize the dtrace output
+awk 'NF != 0 { print $1, $2, $3 }' dtrace.out | sort > dtrace.out.post
+
+#     determine what to expect
+
+i=0
+while [ $i -lt $num ]; do
+	if [ $((${pids[$i]} % 10)) -eq $lastdigit ]; then
+		x=$(((${pids[$i]} % 100) * 10))
+		echo ${pids[$i]} 0 45 >> dtrace.out.expected
+		echo ${pids[$i]} 1 65 >> dtrace.out.expected
+		echo ${pids[$i]} 2 90 >> dtrace.out.expected
+		echo ${pids[$i]} 3 $x >> dtrace.out.expected
+	fi
+
+	i=$(($i + 1))
+done
+
+#     diff
+if ! sort dtrace.out.expected | diff -q - dtrace.out.post; then
+	echo dtrace output looks wrong for DTrace case $i
+	echo === was ===
+	cat dtrace.out
+	echo === got ===
+	cat dtrace.out.post
+	echo === expected ===
+	sort dtrace.out.expected
+	echo === diff ===
+	sort dtrace.out.expected | diff - dtrace.out.post
+	exit 1
+fi
+
+echo success
+
+exit 0
-- 
2.43.5

[PATCH 6/6] test: Add USDT error tests for -w and -Z

[eugene.loh]

From: Eugene Loh <eugene.loh@oracle.com>

Error checking for USDT probes can be different from other probes.
So add specific USDT tests for -Z and -w.  Specifically:

    err.no-Z  No USDT process yet when dtrace is launched
              and -Z is not specified.  Therefore, dtrace
              will not start up.

    err.no-w  A USDT process is running when dtrace is
              launched, so -Z is not needed.  However, the
              action is destructive and -w is not specified.
              Therefore, dtrace will not start up.

    err.Z_no-w  No USDT process yet when dtrace is launched,
                but -Z is specified.  So, dtrace starts
                successfully.  But the action is destructive
                and -w is not specified.  So when the USDT
                process starts, dtrace fails.

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 test/unittest/usdt/err.Z_no-w.r  |  4 ++
 test/unittest/usdt/err.Z_no-w.sh | 65 ++++++++++++++++++++++++++++++++
 test/unittest/usdt/err.no-Z.r    | 13 +++++++
 test/unittest/usdt/err.no-Z.sh   | 31 +++++++++++++++
 test/unittest/usdt/err.no-w.r    |  3 ++
 test/unittest/usdt/err.no-w.sh   | 31 +++++++++++++++
 6 files changed, 147 insertions(+)
 create mode 100644 test/unittest/usdt/err.Z_no-w.r
 create mode 100755 test/unittest/usdt/err.Z_no-w.sh
 create mode 100644 test/unittest/usdt/err.no-Z.r
 create mode 100755 test/unittest/usdt/err.no-Z.sh
 create mode 100644 test/unittest/usdt/err.no-w.r
 create mode 100755 test/unittest/usdt/err.no-w.sh

diff --git a/test/unittest/usdt/err.Z_no-w.r b/test/unittest/usdt/err.Z_no-w.r
new file mode 100644
index 000000000..b81622481
--- /dev/null
+++ b/test/unittest/usdt/err.Z_no-w.r
@@ -0,0 +1,4 @@
+dtrace is running so start the trigger
+dtrace died as expected after trigger started
+-- @@stderr --
+dtrace: processing aborted: Destructive actions not allowed
diff --git a/test/unittest/usdt/err.Z_no-w.sh b/test/unittest/usdt/err.Z_no-w.sh
new file mode 100755
index 000000000..39b99c691
--- /dev/null
+++ b/test/unittest/usdt/err.Z_no-w.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# This test verifies that dtrace will not run a destructive script
+# for USDT probes if -w is not specified.
+#
+# Specifically, the script is launched with -Z and no USDT processes are
+# initially present.  Only once a USDT process is detected does dtrace
+# fail due to the destructive action.
+
+dtrace=$1
+trigger=`pwd`/test/triggers/usdt-tst-defer
+
+# Set up test directory.
+
+DIRNAME=$tmpdir/Z_no-w.$$.$RANDOM
+mkdir -p $DIRNAME
+cd $DIRNAME
+
+# Make a private copy of the trigger executable so that we get our
+# own DOF stash.
+
+cp $trigger main
+
+# Run dtrace.
+
+$dtrace $dt_flags -Zq -o dtrace.out -n '
+testprov*:::foo
+{
+	raise(SIGUSR1);
+}' &
+dtpid=$!
+sleep 2
+if [[ ! -d /proc/$dtpid ]]; then
+	echo ERROR dtrace died prematurely
+	exit 1
+fi
+
+# Start a trigger process.
+
+echo dtrace is running so start the trigger
+./main > main.out &
+pid=$!
+
+# Check again if dtrace is still running.
+
+sleep 2
+if [[ ! -d /proc/$dtpid ]]; then
+	echo dtrace died as expected after trigger started
+else
+	echo dtrace is unexpectedly still running
+	kill -9 $dtpid
+	wait    $dtpid
+fi
+
+# Tell the trigger to proceed to completion.
+
+kill -USR1 $pid
+wait       $pid
+
+exit 1
diff --git a/test/unittest/usdt/err.no-Z.r b/test/unittest/usdt/err.no-Z.r
new file mode 100644
index 000000000..52d9492ab
--- /dev/null
+++ b/test/unittest/usdt/err.no-Z.r
@@ -0,0 +1,13 @@
+expected failure
+-- @@stderr --
+dtrace: invalid probe specifier 
+BEGIN
+{
+	exit(0);
+}
+
+testprov*:::foo
+{
+	raise(SIGUSR1);
+	exit(0);
+}: probe description testprov*:::foo does not match any probes
diff --git a/test/unittest/usdt/err.no-Z.sh b/test/unittest/usdt/err.no-Z.sh
new file mode 100755
index 000000000..29d63d9ce
--- /dev/null
+++ b/test/unittest/usdt/err.no-Z.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# This test verifies that dtrace will not wait for USDT processes to
+# appear if -Z is not used.
+
+dtrace=$1
+
+$dtrace $dt_flags -qn '
+BEGIN
+{
+	exit(0);
+}
+
+testprov*:::foo
+{
+	raise(SIGUSR1);
+	exit(0);
+}'
+if [ $? -ne 0 ]; then
+	echo expected failure
+	exit 1
+fi
+
+echo unexpected success
+
+exit 0
diff --git a/test/unittest/usdt/err.no-w.r b/test/unittest/usdt/err.no-w.r
new file mode 100644
index 000000000..7a4d60cd0
--- /dev/null
+++ b/test/unittest/usdt/err.no-w.r
@@ -0,0 +1,3 @@
+expected failure
+-- @@stderr --
+dtrace: could not enable tracing: Destructive actions not allowed
diff --git a/test/unittest/usdt/err.no-w.sh b/test/unittest/usdt/err.no-w.sh
new file mode 100755
index 000000000..407654981
--- /dev/null
+++ b/test/unittest/usdt/err.no-w.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# This test verifies that dtrace will not run a destructive script
+# for USDT probes if -w is not specified.
+
+dtrace=$1
+
+$dtrace $dt_flags -c test/triggers/usdt-tst-defer -qn '
+BEGIN
+{
+	exit(0);
+}
+
+testprov*:::foo
+{
+	raise(SIGUSR1);
+	exit(0);
+}'
+if [ $? -ne 0 ]; then
+	echo expected failure
+	exit 1
+fi
+
+echo unexpected success
+
+exit 0
-- 
2.43.5

Re: [PATCH 2/6] test: Correct long-standing dt_flags typo

[Kris Van Hees]

On Fri, Sep 27, 2024 at 10:21:54PM -0400, eugene.loh@oracle.com wrote:
> From: Eugene Loh <eugene.loh@oracle.com>
> 
> Signed-off-by: Eugene Loh <eugene.loh@oracle.com>

Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>

> ---
>  test/unittest/ip/tst.ipv4localudp.sh | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/test/unittest/ip/tst.ipv4localudp.sh b/test/unittest/ip/tst.ipv4localudp.sh
> index 0e357c3d2..4ee584400 100755
> --- a/test/unittest/ip/tst.ipv4localudp.sh
> +++ b/test/unittest/ip/tst.ipv4localudp.sh
> @@ -1,7 +1,7 @@
>  #!/bin/bash
>  #
>  # Oracle Linux DTrace.
> -# Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
> +# Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
>  # Licensed under the Universal Permissive License v 1.0 as shown at
>  # http://oss.oracle.com/licenses/upl.
>  #
> @@ -33,7 +33,7 @@ fi
>  dtrace=$1
>  testdir="$(dirname $_test)"
>  local=127.0.0.1
> -$dtrace $dt_cflags -c "$testdir/perlping.pl udp $local" -qs /dev/stdin <<EOF
> +$dtrace $dt_flags -c "$testdir/perlping.pl udp $local" -qs /dev/stdin <<EOF
>  BEGIN
>  {
>  	send = receive = 0;
> -- 
> 2.43.5
> 

Re: [PATCH 3/6] test: Remove some outdated and unhelpful comments

[Kris Van Hees]

On Fri, Sep 27, 2024 at 10:21:55PM -0400, eugene.loh@oracle.com wrote:
> From: Eugene Loh <eugene.loh@oracle.com>
> 
> Signed-off-by: Eugene Loh <eugene.loh@oracle.com>

Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>

> ---
>  test/triggers/Build | 8 ++++----
>  1 file changed, 4 insertions(+), 4 deletions(-)
> 
> diff --git a/test/triggers/Build b/test/triggers/Build
> index 0e8660e59..db80b30ed 100644
> --- a/test/triggers/Build
> +++ b/test/triggers/Build
> @@ -192,18 +192,18 @@ pid-tst-weak1_CFLAGS := -O0
>  pid-tst-weak2_CFLAGS := -O0
>  profile-tst-ufuncsort_CFLAGS := -O0
>  
> -# usdt-tst-argmap calls USDT probes (defined in argmap.d) using sys/sdt.h
> +# usdt-tst-argmap calls USDT probes using sys/sdt.h
>  usdt-tst-argmap_CFLAGS := -Iuts/common
>  usdt-tst-argmap_PROV := usdt-tst-argmap-prov.d
>  
> -# usdt-tst-args calls USDT probes (defined in args.d) using sys/sdt.h
> +# usdt-tst-args calls USDT probes using sys/sdt.h
>  usdt-tst-args_CFLAGS := -Iuts/common
>  usdt-tst-args_PROV := usdt-tst-args-prov.d
>  
> -# usdt-tst-forker calls USDT probes (defined in args.d) based on dtrace -h
> +# usdt-tst-forker calls USDT probes based on dtrace -h
>  usdt-tst-forker_PROV := usdt-tst-forker-prov.d
>  
> -# usdt-tst-special calls USDT probes (defined in prov.d) based on dtrace -h
> +# usdt-tst-special calls USDT probes based on dtrace -h
>  usdt-tst-special_CFLAGS := -fno-inline -O2
>  usdt-tst-special_PROV := usdt-tst-special-prov.d
>  
> -- 
> 2.43.5
> 

Re: [PATCH 4/6] test: Add a USDT "deferred" test trigger

[Kris Van Hees]

On Fri, Sep 27, 2024 at 10:21:56PM -0400, eugene.loh@oracle.com wrote:
> From: Eugene Loh <eugene.loh@oracle.com>
> 
> After a dtrace session has started, it is possible for new USDT
> processes to start and for there to be a delay before dtprobed
> and then dtrace becomes aware of the new process.
> 
> Add a test trigger with USDT probes that waits for dtrace to
> discover it and send it USR1.  Once the USR1 is received, a
> short workload is run to completion.
> 
> Signed-off-by: Eugene Loh <eugene.loh@oracle.com>

Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>

> ---
>  test/triggers/Build                      |  5 +-
>  test/triggers/usdt-tst-defer-prov.d      | 11 ++++
>  test/triggers/usdt-tst-defer.c           | 79 ++++++++++++++++++++++++
>  test/unittest/usdt/tst.defer-nodtrace.r  |  1 +
>  test/unittest/usdt/tst.defer-nodtrace.sh | 49 +++++++++++++++
>  5 files changed, 144 insertions(+), 1 deletion(-)
>  create mode 100644 test/triggers/usdt-tst-defer-prov.d
>  create mode 100644 test/triggers/usdt-tst-defer.c
>  create mode 100644 test/unittest/usdt/tst.defer-nodtrace.r
>  create mode 100755 test/unittest/usdt/tst.defer-nodtrace.sh
> 
> diff --git a/test/triggers/Build b/test/triggers/Build
> index db80b30ed..107b3b4d1 100644
> --- a/test/triggers/Build
> +++ b/test/triggers/Build
> @@ -13,7 +13,7 @@ EXTERNAL_64BIT_TRIGGERS = testprobe readwholedir mmap bogus-ioctl open delaydie
>      ustack-tst-spin ustack-tst-mtspin \
>      visible-constructor visible-constructor-static visible-constructor-static-unstripped
>  
> -EXTERNAL_64BIT_SDT_TRIGGERS = usdt-tst-argmap usdt-tst-args usdt-tst-forker usdt-tst-special
> +EXTERNAL_64BIT_SDT_TRIGGERS = usdt-tst-argmap usdt-tst-args usdt-tst-forker usdt-tst-defer usdt-tst-special
>  EXTERNAL_64BIT_TRIGGERS += $(EXTERNAL_64BIT_SDT_TRIGGERS)
>  
>  EXTERNAL_32BIT_TRIGGERS := visible-constructor-32
> @@ -203,6 +203,9 @@ usdt-tst-args_PROV := usdt-tst-args-prov.d
>  # usdt-tst-forker calls USDT probes based on dtrace -h
>  usdt-tst-forker_PROV := usdt-tst-forker-prov.d
>  
> +# usdt-tst-defer calls USDT probes based on dtrace -h
> +usdt-tst-defer_PROV := usdt-tst-defer-prov.d
> +
>  # usdt-tst-special calls USDT probes based on dtrace -h
>  usdt-tst-special_CFLAGS := -fno-inline -O2
>  usdt-tst-special_PROV := usdt-tst-special-prov.d
> diff --git a/test/triggers/usdt-tst-defer-prov.d b/test/triggers/usdt-tst-defer-prov.d
> new file mode 100644
> index 000000000..a4ef8323d
> --- /dev/null
> +++ b/test/triggers/usdt-tst-defer-prov.d
> @@ -0,0 +1,11 @@
> +/*
> + * Oracle Linux DTrace.
> + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
> + * Licensed under the Universal Permissive License v 1.0 as shown at
> + * http://oss.oracle.com/licenses/upl.
> + */
> +
> +provider testprov {
> +	probe foo();
> +	probe bar(int, int, int);
> +};
> diff --git a/test/triggers/usdt-tst-defer.c b/test/triggers/usdt-tst-defer.c
> new file mode 100644
> index 000000000..2df5401b2
> --- /dev/null
> +++ b/test/triggers/usdt-tst-defer.c
> @@ -0,0 +1,79 @@
> +/*
> + * Oracle Linux DTrace.
> + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
> + * Licensed under the Universal Permissive License v 1.0 as shown at
> + * http://oss.oracle.com/licenses/upl.
> + */
> +
> +/*
> + * The main characteristic of this trigger code is that it allows deferred
> + * DTrace detection of the trigger.  That is, the trigger spins in "phase 1",
> + * waiting for DTrace to detect it and send it USR1.  Only then does "phase 2"
> + * run a short workload to completion.
> + */
> +
> +#include <stdio.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <signal.h>
> +#include "usdt-tst-defer-prov.h"
> +
> +static int phase = 1;
> +
> +static void
> +interrupt(int sig)
> +{
> +	phase = 2;
> +}
> +
> +int
> +main(int argc, char **argv)
> +{
> +	struct sigaction act;
> +	int i;
> +	int nphase1 = 0, nphase1foo = 0, nphase1bar = 0;
> +	int nphase2 = 0, nphase2foo = 0, nphase2bar = 0;
> +
> +	/* set the handler to listen for SIGUSR1 */
> +	act.sa_handler = interrupt;
> +	act.sa_flags = 0;
> +	if (sigaction(SIGUSR1, &act, NULL)) {
> +		printf("set handler failed\n");
> +		return 1;
> +	}
> +
> +	/* in phase 1, loop on probe "foo" to wait on USR1 */
> +	while (phase == 1) {
> +		nphase1++;
> +		if (TESTPROV_FOO_ENABLED()) {
> +			nphase1foo++;
> +			phase = 2;
> +		}
> +		if (TESTPROV_BAR_ENABLED()) {
> +			nphase1bar++;
> +			phase = 2;
> +		}
> +		TESTPROV_FOO();
> +	}
> +
> +	/* wait for probes to get set up */
> +	usleep(100000);
> +
> +	/* in phase 2, just loop over probe "bar" a fixed number of times */
> +	for (i = 0; i < 10; i++) {
> +		nphase2++;
> +		usleep(2000);
> +		if (TESTPROV_FOO_ENABLED())
> +			nphase2foo++;
> +		usleep(2000);
> +		if (TESTPROV_BAR_ENABLED())
> +			nphase2bar++;
> +		usleep(2000);
> +		TESTPROV_BAR(i, i + 2, i * 2);
> +	}
> +
> +	printf("%d: %d %d %d %d %d %d\n", getpid(),
> +	    nphase1, nphase1foo, nphase1bar, nphase2, nphase2foo, nphase2bar);
> +
> +	return 0;
> +}
> diff --git a/test/unittest/usdt/tst.defer-nodtrace.r b/test/unittest/usdt/tst.defer-nodtrace.r
> new file mode 100644
> index 000000000..2e9ba477f
> --- /dev/null
> +++ b/test/unittest/usdt/tst.defer-nodtrace.r
> @@ -0,0 +1 @@
> +success
> diff --git a/test/unittest/usdt/tst.defer-nodtrace.sh b/test/unittest/usdt/tst.defer-nodtrace.sh
> new file mode 100755
> index 000000000..e5fd2855c
> --- /dev/null
> +++ b/test/unittest/usdt/tst.defer-nodtrace.sh
> @@ -0,0 +1,49 @@
> +#!/bin/bash
> +#
> +# Oracle Linux DTrace.
> +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
> +# Licensed under the Universal Permissive License v 1.0 as shown at
> +# http://oss.oracle.com/licenses/upl.
> +#
> +# There are various tests that use the usdt-tst-defer trigger.  As a
> +# baseline, let us run the trigger without enabling any USDT probes
> +# and check that the counts are as expected.
> +
> +dtrace=$1
> +trigger=`pwd`/test/triggers/usdt-tst-defer
> +
> +# Set up test directory.
> +
> +DIRNAME=$tmpdir/defer-nodtrace.$$.$RANDOM
> +mkdir -p $DIRNAME
> +cd $DIRNAME
> +
> +# Make a private copy of the trigger executable so that we get our
> +# own DOF stash.
> +
> +cp $trigger main
> +
> +# Check that the is-enabled probes are false when the USDT probes are not enabled.
> +# That is, nphase1foo == nphase1bar == nphase2foo == nphase2bar == 0.
> +# Also, nphase2 == 10.
> +# Note that nphase1 will be undefined.
> +
> +./main > main.out &
> +pid=$!
> +sleep 1
> +kill -USR1 $pid
> +wait
> +
> +echo "$pid: undefined 0 0 10 0 0" > main.out.expected
> +if ! awk '{ $2 = "undefined"; print }' main.out | diff -q - main.out.expected; then
> +	echo program output looks wrong for the no-DTrace case
> +	echo === got ===
> +	cat main.out
> +	echo === expected ===
> +	cat main.out.expected
> +	exit 1
> +fi
> +
> +echo success
> +
> +exit 0
> -- 
> 2.43.5
>