Re: [PATCH v6 00/41] Shadow stacks for userspace

[Pengfei Xu <pengfei.xu@intel.com>]

[-- Attachment #1: Type: text/plain, Size: 4284 bytes --]

Hi Rick,

On 2023-02-18 at 13:13:52 -0800, Rick Edgecombe wrote:
> Hi,
> 
...
> 
> I left tested-by tags in place per discussion with testers. Testers, please
> retest.
> 

1. Tested kself-test from user space shstk on ADL-S, TGL-U without Glibc shstk
support in CentOS 8 stream OS:

// From the test_shadow_stack code in this patch series:
# ./test_shadow_stack
[INFO]  new_ssp = 7f014ac2dff8, *new_ssp = 7f014ac2e001
[INFO]  changing ssp from 7f014a1ffff0 to 7f014ac2dff8
[INFO]  ssp is now 7f014ac2e000
[OK]    Shadow stack pivot
[OK]    Shadow stack faults
[INFO]  Corrupting shadow stack
[INFO]  Generated shadow stack violation successfully
[OK]    Shadow stack violation test
[INFO]  Gup read -> shstk access success
[INFO]  Gup write -> shstk access success
[INFO]  Violation from normal write
[INFO]  Gup read -> write access success
[INFO]  Violation from normal write
[INFO]  Gup write -> write access success
[INFO]  Cow gup write -> write access success
[OK]    Shadow gup test
[INFO]  Violation from shstk access
[OK]    mprotect() test
[OK]    Userfaultfd test
[OK]    32 bit test

// shstk violation without SHSTK glibc support
// Code link: https://github.com/intel/lkvs/blob/main/cet/shstk_cp.c
# ./shstk_cp
[PASS]  Enable SHSTK successfully
[PASS]  Disabling shadow stack successfully
[PASS]  Re-enable shadow stack successfully
[PASS]  SHSTK enabled, ssp:7fa3bfe00000
[INFO]  do_hack() change address for return:
[INFO]  Before,ssp:7fa3bfdffff8,*ssp:40133f,rbp:0x7ffc23b5b440,*rbp:7ffc23b5b480,*(rbp+1):40133f
[INFO]  After, ssp:7fa3bfdffff8,*ssp:40133f,rbp:0x7ffc23b5b440,*rbp:7ffc23b5b480,*(rbp+1):401146
Segmentation fault (core dumped)

Dmesg:
[1117184.518588] shstk_cp[1523882] control protection ip:40122c sp:7ffc23b5b448 ssp:7fa3bfdffff8 error:1(near ret) in shstk_cp[401000+1000]

// shstk ARCH_SHSTK_STATUS read/set test without SHSTK Glibc support
// Code link: https://github.com/intel/lkvs/blob/main/cet/shstk_unlock_test.c
# ./shstk_unlock_test
[PASS]  Parent process enable SHSTK.
[PASS]  Parent pid:1522040, ssp:0x7f57fc400000
[INFO]  pid:1522040, ssp:0x7f57fc3ffff8, *ssp:401799
[PASS]  Unlock CET successfully for pid:1522041
[PASS]  GET CET REG ret:0, err:0, ssp:7f57fc3ffff8
[PASS]  SET CET REG ret:0, err:0, ssp:7f57fc3ffff8
[PASS]  SET ssp -1 failed(expected) ret:-1, errno:22
[PASS]  GET xstate successfully ret:0
[PASS]  SHSTK is enabled in child process
[INFO]  Child:1522041 origin ssp:0x7f57fc400000
[INFO]  Child:1522041, ssp:0x7f57fc400000, bp,0x7ffcf32ba0f0, *bp:401dc0, *(bp+1):7f57fc43ad85
[PASS]  Disabling shadow stack succesfully
[PASS]  SHSTK_STATUS ok, feature:0 is 0, ret:0
[PASS]  Child process re-enable ssp
[PASS]  SHSTK_STATUS ok, feature:1 1st bit is 1, ret:0
[PASS]  Child process enabled wrss
[PASS]  SHSTK_STATUS ok, feature:3 2nd bit is 1, ret:0
[INFO]  Child:1522041, ssp:0x7f57fc400000, bp,0x7ffcf32ba0f0, *bp:401dc0, *(bp+1):7f57fc43ad85
[INFO]  ssp addr:0x7f57fc400000 is same as ssp_verify:0x7f57fc400000
[PASS]  Child process disable shstk successfully.
[PASS]  Parent process disable shadow stack successfully.


2. Tested fedora37 OS + Hongjiu provided user space SHSTK support Glibc:
// shstk with Glibc support:
// Related Glibc support for Fedora37:  http://gnu-4.sc.intel.com/git/?p=hjl/misc.git;a=tree;f=setup/fedora/37;h=63af84a8f28f3d0802f09266e47fb94eb5cdff26;hb=HEAD
# readelf  -n shadow_test_fork | head
readelf: Warning: Gap in build notes detected from 0x4011d7 to 0x4011e4

Displaying notes found in: .note.gnu.property
  Owner                Data size        Description
    GNU                  0x00000040       NT_GNU_PROPERTY_TYPE_0
          Properties: x86 feature: IBT, SHSTK
...
// shadow_test_fork code is in attached
// gcc -fcf-protection=full -mshstk -O0 -fno-stack-check -fno-stack-protector    shadow_test_fork.c   -o shadow_test_fork
# ./shadow_test_fork s2
[INFO]  s2: stack rbp + 1
[INFO]  do_hack() change address for return:
[INFO]  After change, rbp+1 to hacked:0x401296
Segmentation fault (core dumped)

Dmesg:
[418653.591014] shadow_test_for[16529] control protection ip:401367 sp:7fff6ed0a728 ssp:7f661265bfe0 error:1(near ret) in shadow_test_fork[401000+1000]

All above user space SHSTK tests are passed.

Many thanks Rick and all!

Thanks!
BR.
Pengfei

> -- 
> 2.17.1
> 

[-- Attachment #2: shadow_test_fork.c --]
[-- Type: text/plain, Size: 9906 bytes --]

// SPDX-License-Identifier: GPL-2.0
/*
 * Contributors:
 *      Pengfei, Xu <pengfei.xu@intel.com>
 *      - Test CET shadow stack function, should trigger #CP protection
 *      - Add the print, and show stack address and content before and after
 *        changed
 */

#define _GNU_SOURCE
#include <sys/types.h>
#include <sys/wait.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>
#include <sched.h>
#include <immintrin.h>

static long hacked(void)
{
	printf("[INFO]\tAccess hack function\n");
	printf("[FAIL]\tpid=%d Hacked!\n", getpid());
	printf("[WARN]\tYou see this line, which means CET shstk #CP failed!\n");
	return 1;
}

/*
 * stack variable y + 1(1 means 8bytes for 64bit, 4bytes for 32bit) is bp,
 * and here use bp directly, it's bp hacked not sp hacked, so it should not
 * trigger #CP.
 */
static void stack_add1_test(unsigned long changed_bp)
{
	unsigned long *func_bp;

#ifdef __x86_64__
	asm("movq %%rbp,%0" : "=r"(func_bp));
#else
	asm("mov %%ebp,%0" : "=r"(func_bp));
#endif
	printf("[INFO]\tReal add1 function rbp content:%lx for main rbp.\n",
	       *func_bp);
	*func_bp = changed_bp;
	printf("[INFO]\tChange add1 rbp content:%lx, but right main rbp content in it!\n",
	       *func_bp);
}

/* stack base rbp + 1 addr test, which should be hacked and #CP should work */
static unsigned long stack_add2_test(void)
{
	unsigned long y;
	unsigned long *i, *j;

	i = (unsigned long *)_get_ssp();
	j = __builtin_frame_address(0);

	printf("[INFO]\tdo_hack() change address for return:\n");
	printf("[INFO]\tBefore change,y:%lx,&y:%p,j:%p,*j:%lx,*(&j+1):0x%lx, ssp:%p *ssp:0x%lx\n",
	       y, &y, j, *j, *(j+1), i, *i);

	/* j(rbp)+1 is sp address, change rbp+1 to change sp content */
	*(j + 1) = (unsigned long)hacked;

	printf("[INFO]\tAfter change, rbp+1 to hacked:0x%lx\n", *(j+1));
	printf("[INFO]\tAfter hacked  &y:%p, *j:0x%lx,*(&j+1):0x%lx\n",
	       &y, *j, *(j + 1));

	/* Debug purpose: it's not related with ret instruction in objdump. */
	return y;
}

/* stack base y + 3 addr test, which should not be hacked and #CP */
static unsigned long stack_add3_test(void)
{
	unsigned long y;

	printf("[INFO]\tdo_hack() change address for return:\n");
	printf("[INFO]\tBefore change, y:0x%lx, *(&y+2):0x%lx\n", y,
	       *((unsigned long *)&y + 2));
	*((unsigned long *)&y + 3) = (unsigned long)hacked;
	printf("[INFO]\tAfter change, *(&y+3) to change:0x%lx\n", (unsigned long)hacked);
	printf("[INFO]\tAfter change &y+3:%p,*(&x+2):0x%lx\n",
	       (unsigned long *)&y + 3, *((unsigned long *)&y + 3));
	printf("[INFO]\tAfter changed &y:%p, &y+2:%p,*(&y+2):0x%lx\n",
	       &y, (unsigned long *)&y + 2, *((unsigned long *)&y + 2));

	return y;
}

static long stack_long2_test(unsigned long i)
{
	unsigned long *p;


	printf("[INFO]\tuse rbp + long(+8bytes) size to hack:\n");
	/*
	 * Another way to read rbp
	 * asm("movq %%rbp,%0" : "=r"(p));
	 */
	p = __builtin_frame_address(0);

	printf("[INFO]\t*(p+1):%lx will be hacked\n", *(p + 1));
	*(p + 1) = (unsigned long)hacked;

	return 0;
}

/* stack base y + 2 change to random value to do shstk violation */
static unsigned long stack_random(unsigned long j)
{
	unsigned long y;
	unsigned long *p;

	y = j;
	printf("[INFO]\tSHSTK hack with random value:\n");
#ifdef __x86_64__
	asm("movq %%rbp,%0" : "=r"(p));
#else
	asm("mov %%ebp,%0" : "=r"(p));
#endif

	*(p + 1) = j;

	return y;
}

/* stack base y + 2 changed but no return */
static void stack_no_return(void)
{
	unsigned long *p;

	printf("[INFO]\tSHSTK with void no return function:\n");
#ifdef __x86_64__
	asm("movq %%rbp,%0" : "=r"(p));
#else
	asm("mov %%ebp,%0" : "=r"(p));
#endif

	*(p + 1) = (unsigned long)hacked;
}

/* buffer overflow change stack base, which should trigger #CP */
static void stack_buf_impact(void)
{
	char buffer[20];
	int overflow_num = 44;

	printf("[INFO]\tbuffer[20]:%x\n", buffer[20]);
	memset(buffer, 0, overflow_num);
	printf("[INFO]\tbuffer[44]:%x,&buffer[44]:%p\n", buffer[44], &buffer[44]);
	printf("[INFO]\tbuffer[20] after overflow:%x\n", buffer[20]);
}

/* buffer overflow not change stack base, which should not trigger #CP */
static void stack_buf_no_impact(void)
{
	char buf[20];
	int overflow_24 = 24, overflow_28 = 28;

	printf("[INFO]\tbuf[20]:%x\n", buf[20]);
#ifdef __x86_64__
	memset(buf, 0, overflow_28);
#else
	memset(buf, 0, overflow_24);
#endif
	printf("[INFO]\tbuf[20] after overflow:%x\n", buf[20]);
}

/* test hack function */
static int do_hack(void *p)
{
	/*
	 * Ret and then rip will get this value(rbp + 8 bytes in 64 bit OS)
	 * rbp(8 bytes in 64bit OS)
	 * *i, *j and so on variable content
	 */
	unsigned long *i, *j;

	i = (unsigned long *)_get_ssp();
	j = __builtin_frame_address(0);

	printf("[INFO]\tBefore: rbp+8:0x%p content=0x%lx; ssp=0x%p, ssp content=0x%lx\n",
		j + 1, *(j + 1), i, *i);
	*(j+1) = (unsigned long)hacked;
	printf("[INFO]\tAfter: rbp+8:0x%p content=0x%lx; ssp=0x%p, ssp content=0x%lx\n",
		j + 1, *(j + 1), i, *i);

	return 0;
}

/* check shadow stack wo core dump in child pid */
static void stack_wo_core(void)
{
	void *s = malloc(0x100000);

	if (fork() == 0)
		do_hack(s);
}

/* test shstk by clone way */
static int stack_clone(void)
{
	pid_t cid;

	void *child_stack = malloc(0x100000);

	if (child_stack == NULL) {
		printf("[FAIL]\tmalloc child_stack failed!\n");
		return 1;
	}

	cid = clone(
	do_hack, /* function */
	child_stack + 0x100000,
	SIGCHLD,
	0 /*arg*/
	);

	if (cid == -1) {
		printf("[FAIL]\tclone failed!\n");
		free(child_stack);
		return 1;
	}

	printf("[INFO]\tparent=%d, child=%d\n", getpid(), cid);

	if (waitpid(cid, NULL, 0) == -1) {
		printf("[FAIL]\twaitpid() failed!\n");
		return 1;
	}
	printf("[INFO]\tchild exits!\n");

	free(child_stack);
	return 0;
}

/*
 * Check shadow stack address and content and
 * rbp address and protect address content
 */
static int shadow_stack_check(void)
{
	unsigned long y;
	unsigned long *bp_a, *ssp_a;
	unsigned long long size_bp, size_ssp;

	ssp_a = (unsigned long *)_get_ssp();
	bp_a = __builtin_frame_address(0);
	size_bp = sizeof(*(bp_a + 1));
	size_ssp = sizeof(*ssp_a);

	printf("[INFO]\t&y=0x%p\n", &y);
	printf("[INFO]\tbp=%p,bp+1=%p,*(bp+1):0x%lx(size:%lld) ssp=%p *ssp=0x%lx(size:%lld)\n",
		bp_a, bp_a + 1, *(bp_a + 1), size_bp, ssp_a, *ssp_a, size_ssp);
	return 0;
}

static void usage(void)
{
	printf("Usage: [null | s1 | s2 | s3 | sl1 | sr | sn...]\n");
	printf("  null: no parm, stack add 2 test, should trigger #CP\n");
	printf("  s1: stack add 1 test\n");
	printf("  s2: stack add 2 test, should trigger #CP\n");
	printf("  s3: stack add 3 test\n");
	printf("  sl1: stack with long add 2 test\n");
	printf("  sr: stack change to random value\n");
	printf("  sn: stack change but no return\n");
	printf("  buf1: buffer overflow change stack base\n");
	printf("  buf2: buffer overflow not change stack base\n");
	printf("  snc: test shadow stack wo core dump\n");
	printf("  sc: test shadow stack by clone way\n");
	printf("  ssp: check shadow stack addr and content\n");
}

int main(int argc, char *argv[])
{
	char *parm = "";
	unsigned long a = 0, *main_rbp, fake_bp[2];

	a = rand();
	enum {
		e_s1, /* enum stack base, y + 1 */
		e_s2, /* enum stack base + 1 addr content change test */
		e_s3, /* enum stack base y + 3 */
		e_sl1, /* enum stack base with long + 2 */
		e_sr, /* enum stack base change to random value */
		e_sn, /* enum stack base changed but no return */
		e_buf1, /* buffer overflow change stack base */
		e_buf2, /* buffer overflow not change stack base */
		e_snc, /* shadow stack wo core dump */
		e_sc, /* test shstk by stack clone way */
		e_ssp /* check shadow stack addr and content */
	} option;

#ifdef __x86_64__
	asm("movq %%rbp,%0" : "=r"(main_rbp));
#else
	asm("mov %%ebp,%0" : "=r"(main_rbp));
#endif

	/* Use real main rbp address and content to make one fake bp and sp */
	fake_bp[0] = *main_rbp;
	fake_bp[1] = *(main_rbp + 1);

	if (argc == 1) {
		usage();
		stack_add2_test();
	} else {
		parm = argv[1];
		if (strcmp(argv[1], "s1") == 0)
			option = e_s1;
		else if (strcmp(argv[1], "s2") == 0)
			option = e_s2;
		else if (strcmp(argv[1], "s3") == 0)
			option = e_s3;
		else if (strcmp(argv[1], "sl1") == 0)
			option = e_sl1;
		else if (strcmp(argv[1], "sr") == 0)
			option = e_sr;
		else if (strcmp(argv[1], "sn") == 0)
			option = e_sn;
		else if (strcmp(argv[1], "buf1") == 0)
			option = e_buf1;
		else if (strcmp(argv[1], "buf2") == 0)
			option = e_buf2;
		else if (strcmp(argv[1], "snc") == 0)
			option = e_snc;
		else if (strcmp(argv[1], "sc") == 0)
			option = e_sc;
		else if (strcmp(argv[1], "ssp") == 0)
			option = e_ssp;
		else {
			usage();
			exit(1);
		}
	}

	switch (option) {
	case e_s1:
		printf("[INFO]\ts1: stack + 1\n");
		stack_add1_test((unsigned long)&fake_bp[0]);
		break;
	case e_s2:
		printf("[INFO]\ts2: stack rbp + 1\n");
		stack_add2_test();
		break;
	case e_s3:
		printf("[INFO]\ts3: stack + 3\n");
		stack_add3_test();
		break;
	case e_sl1:
		printf("[INFO]\tsl1: stack with long + 2, a:0x%lx\n", a);
		stack_long2_test(a);
		break;
	case e_sr:
		printf("[INFO]\tsr: stack changed to random value a:0x%lx\n", a);
		stack_random(a);
		break;
	case e_sn:
		printf("[INFO]\tsn: stack changed but no return\n");
		stack_no_return();
		break;
	case e_buf1:
		printf("buf1: buffer overflow change stack base\n");
		stack_buf_impact();
		break;
	case e_buf2:
		printf("[INFO]\tbuf2: buffer overflow not change stack base\n");
		stack_buf_no_impact();
		break;
	case e_snc:
		printf("[INFO]\tsnc: test shadow stack wo core dump\n");
		stack_wo_core();
		break;
	case e_sc:
		printf("[INFO]\tsc: test shstk by stack clone way\n");
		stack_clone();
		break;
	case e_ssp:
		printf("[INFO]\tssp: check shadow stack addr and content\n");
		shadow_stack_check();
		break;
	default:
		usage();
		exit(1);
	}

	printf("[RESULTS]\tParent pid=%d is done.\n", getpid());

	return 0;
}