[PATCH 18/19] test: Add a lazy USDT test

[eugene.loh@oracle.com]

From: Eugene Loh <eugene.loh@oracle.com>

When new USDT processes start up, we do not immediately know
about their probes.  Our detecting these processes is "lazy."
First, dtprobed has to see these processes and then dtrace has
to find them.  Hence, many of our tests XFAIL.

Introduce a test that accounts for such laziness.  That is,
the target program spins in "phase 1" waiting for USR1.  Our
D script watches for a USDT probe in phase 1 and raises the
anticipated USR1.

Then we expect every USDT probe during phase 2 to fire.

Currently, another limitation is that at least one USDT process
has to be running when the dtrace session starts.  So, start
one process first, then the dtrace session, then many more processes.

We start "many" processes so that we can filter on pids.  Specifically,
we expect the phase 2 USDT probe to match the last digit of the pid.

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
---
 test/unittest/usdt/tst.lazy.r  |   1 +
 test/unittest/usdt/tst.lazy.sh | 245 +++++++++++++++++++++++++++++++++
 2 files changed, 246 insertions(+)
 create mode 100644 test/unittest/usdt/tst.lazy.r
 create mode 100755 test/unittest/usdt/tst.lazy.sh

diff --git a/test/unittest/usdt/tst.lazy.r b/test/unittest/usdt/tst.lazy.r
new file mode 100644
index 00000000..2e9ba477
--- /dev/null
+++ b/test/unittest/usdt/tst.lazy.r
@@ -0,0 +1 @@
+success
diff --git a/test/unittest/usdt/tst.lazy.sh b/test/unittest/usdt/tst.lazy.sh
new file mode 100755
index 00000000..348a70f6
--- /dev/null
+++ b/test/unittest/usdt/tst.lazy.sh
@@ -0,0 +1,245 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+# This test verifies that USDT will see new processes, even if detection
+# is lazy.
+
+dtrace=$1
+
+# Set up test directory.
+
+DIRNAME=$tmpdir/lazy.$$.$RANDOM
+mkdir -p $DIRNAME
+cd $DIRNAME
+
+# Create test source files.
+
+cat > prov.d <<EOF
+provider testprov {
+	probe foo();
+	probe bar(int, int, int);
+};
+EOF
+
+cat > main.c <<EOF
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+#include "prov.h"
+
+static int phase = 1;
+
+static void
+interrupt(int sig)
+{
+	phase = 2;
+}
+
+int
+main(int argc, char **argv)
+{
+	struct sigaction act;
+	int i;
+	int nphase1 = 0, nphase1foo = 0, nphase1bar = 0;
+	int nphase2 = 0, nphase2foo = 0, nphase2bar = 0;
+
+	/* set the handler to listen for SIGUSR1 */
+	act.sa_handler = interrupt;
+	act.sa_flags = 0;
+	if (sigaction(SIGUSR1, &act, NULL)) {
+		printf("set handler failed\n");
+		return 1;
+	}
+
+	/* in phase 1, loop on probe "foo" to wait on USR1 */
+	while (phase == 1) {
+		nphase1++;
+		if (TESTPROV_FOO_ENABLED()) {
+			nphase1foo++;
+			phase = 2;
+		}
+		if (TESTPROV_BAR_ENABLED()) {
+			nphase1bar++;
+			phase = 2;
+		}
+		TESTPROV_FOO();
+	}
+
+	/* in phase 2, just loop over probe "bar" a fixed number of times */
+	for (int i = 0; i < 10; i++) {
+		nphase2++;
+		usleep(2000);
+		if (TESTPROV_FOO_ENABLED())
+			nphase2foo++;
+		usleep(2000);
+		if (TESTPROV_BAR_ENABLED())
+			nphase2bar++;
+		usleep(2000);
+		TESTPROV_BAR(i, i + 2, i * 2);
+	}
+
+	printf("%d: %d %d %d %d %d %d\n", getpid(),
+	    nphase1, nphase1foo, nphase1bar, nphase2, nphase2foo, nphase2bar);
+
+	return 0;
+}
+EOF
+
+# Build the test program.
+
+$dtrace -h -s prov.d
+if [ $? -ne 0 ]; then
+	echo "failed to generate header file" >&2
+	exit 1
+fi
+cc $test_cppflags -c main.c
+if [ $? -ne 0 ]; then
+	echo "failed to compile test" >&2
+	exit 1
+fi
+$dtrace -G -64 -s prov.d main.o
+if [ $? -ne 0 ]; then
+	echo "failed to create DOF" >&2
+	exit 1
+fi
+cc $test_cppflags -o main main.o prov.o
+if [ $? -ne 0 ]; then
+	echo "failed to link final executable" >&2
+	exit 1
+fi
+
+# Check that the is-enabled probes are false when the USDT probes are not enabled.
+# That is, nphase1foo == nphase1bar == nphase2foo == nphase2bar == 0.
+# Also, nphase2 == 10.
+# Note that nphase1 will be undefined.
+
+./main > main.out &
+pid=$!
+sleep 1
+kill -USR1 $pid
+wait
+
+echo "$pid: undefined 0 0 10 0 0" > main.out.expected
+if ! awk '{ $2 = "undefined"; print }' main.out | diff -q - main.out.expected; then
+	echo program output looks wrong for the no-DTrace case
+	echo === got ===
+	cat main.out
+	echo === expected ===
+	cat main.out.expected
+	exit 1
+fi
+
+# Run dtrace.
+
+num=10
+
+#     Start one process.
+./main > main.out0 &
+pids[0]=$!
+i=1
+
+#     Figure out the pid's trailing digit.
+lastdigit=$((${pids[0]} % 10))
+
+#     Start dtrace.
+$dtrace $dt_flags -wq -o dtrace.out -n '
+testprov*:::foo
+{
+	raise(SIGUSR1);
+}
+testprov*'$lastdigit':::bar
+{
+	@[pid, 0] = sum(arg0);
+	@[pid, 1] = sum(arg1);
+	@[pid, 2] = sum(arg2);
+}' &
+dtpid=$!
+sleep 2
+
+#     Start remaining processes.
+while [ $i -lt $num ]; do
+	./main > main.out$i &
+	pids[$i]=$!
+	i=$(($i + 1))
+done
+
+# Wait for processes to complete.
+
+i=0
+while [ $i -lt $num ]; do
+	wait ${pids[$i]}
+	i=$(($i + 1))
+done
+
+# Kill the dtrace process.
+
+kill $dtpid
+wait
+
+# Check the program output (main.out$i files).
+
+i=0
+while [ $i -lt $num ]; do
+	if [ $((${pids[$i]} % 10)) -eq $lastdigit ]; then
+		nphase2bar=10
+	else
+		nphase2bar=0
+	fi
+	echo "${pids[$i]}: undefined 0 0 10 10 $nphase2bar" > main.out$i.expected
+	awk '
+	    $3 == "1" { $3 =   0 }   # in phase 1, round 1 down to 0
+	    $4 == "1" { $4 =   0 }   # in phase 1, round 1 down to 0
+	    { $2 = "undefined"; print }' main.out$i > main.out$i.post
+	if ! diff -q main.out$i.post main.out$i.expected; then
+		echo program output looks wrong for DTrace case $i
+		echo === was ===
+		cat main.out$i
+		echo === got ===
+		cat main.out$i.post
+		echo === expected ===
+		cat main.out$i.expected
+		exit 1
+	fi
+	i=$(($i + 1))
+done
+
+# Check the dtrace output.
+
+#     regularize the dtrace output.
+awk 'NF != 0 { print $1, $2, $3 }' dtrace.out | sort > dtrace.out.post
+
+#     determine what to expect
+
+i=0
+while [ $i -lt $num ]; do
+	if [ $((${pids[$i]} % 10)) -eq $lastdigit ]; then
+		echo ${pids[$i]} 0 45 >> dtrace.out.expected
+		echo ${pids[$i]} 1 65 >> dtrace.out.expected
+		echo ${pids[$i]} 2 90 >> dtrace.out.expected
+	fi
+
+	i=$(($i + 1))
+done
+
+#     diff
+if ! sort dtrace.out.expected | diff -q - dtrace.out.post; then
+	echo dtrace output looks wrong for DTrace case $i
+	echo === was ===
+	cat dtrace.out
+	echo === got ===
+	cat dtrace.out.post
+	echo === expected ===
+	sort dtrace.out.expected
+	echo === diff ===
+	sort dtrace.out.expected | diff - dtrace.out.post
+	exit 1
+fi
+
+echo success
+
+exit 0
-- 
2.43.5