* [PATCH v11 1/7] crypto: ccp - Name -1 return value as SEV_RET_NO_FW_CALL
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
@ 2023-01-11 19:39 ` Dionna Glaze
2023-01-11 19:39 ` [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err Dionna Glaze
` (5 subsequent siblings)
6 siblings, 0 replies; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:39 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Peter Gonda, Thomas Lendacky, Paolo Bonzini, Joerg Roedel,
Ingo Molnar, Andy Lutomirsky, John Allen, Herbert Xu,
David S. Miller, Borislav Petkov, Dionna Glaze
From: Peter Gonda <pgonda@google.com>
The PSP can return a "firmware error" code of -1 in circumstances where
the PSP is not actually called. To make this protocol unambiguous, the
value is named SEV_RET_NO_FW_CALL.
Cc: Thomas Lendacky <Thomas.Lendacky@amd.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Andy Lutomirsky <luto@kernel.org>
Cc: John Allen <john.allen@amd.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Signed-off-by: Peter Gonda <pgonda@google.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
Documentation/virt/coco/sev-guest.rst | 2 +-
drivers/crypto/ccp/sev-dev.c | 22 ++++++++++++++--------
include/uapi/linux/psp-sev.h | 7 +++++++
3 files changed, 22 insertions(+), 9 deletions(-)
diff --git a/Documentation/virt/coco/sev-guest.rst b/Documentation/virt/coco/sev-guest.rst
index bf593e88cfd9..e76393e389eb 100644
--- a/Documentation/virt/coco/sev-guest.rst
+++ b/Documentation/virt/coco/sev-guest.rst
@@ -41,7 +41,7 @@ The guest ioctl should be issued on a file descriptor of the /dev/sev-guest devi
The ioctl accepts struct snp_user_guest_request. The input and output structure is
specified through the req_data and resp_data field respectively. If the ioctl fails
to execute due to a firmware error, then fw_err code will be set otherwise the
-fw_err will be set to 0x00000000000000ff.
+fw_err will be set to 0x00000000ffffffff, i.e., the lower 32-bits are -1.
The firmware checks that the message sequence counter is one greater than
the guests message sequence counter. If guest driver fails to increment message
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index 06fc7156c04f..ac205f78a595 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -440,12 +440,19 @@ static int __sev_init_ex_locked(int *error)
return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
}
+static inline int __sev_do_init_locked(int *psp_ret)
+{
+ if (sev_init_ex_buffer)
+ return __sev_init_ex_locked(psp_ret);
+ else
+ return __sev_init_locked(psp_ret);
+}
+
static int __sev_platform_init_locked(int *error)
{
struct psp_device *psp = psp_master;
struct sev_device *sev;
- int rc = 0, psp_ret = -1;
- int (*init_function)(int *error);
+ int rc = 0, psp_ret = SEV_RET_NO_FW_CALL;
if (!psp || !psp->sev_data)
return -ENODEV;
@@ -456,15 +463,12 @@ static int __sev_platform_init_locked(int *error)
return 0;
if (sev_init_ex_buffer) {
- init_function = __sev_init_ex_locked;
rc = sev_read_init_ex_file();
if (rc)
return rc;
- } else {
- init_function = __sev_init_locked;
}
- rc = init_function(&psp_ret);
+ rc = __sev_do_init_locked(&psp_ret);
if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
/*
* Initialization command returned an integrity check failure
@@ -473,9 +477,11 @@ static int __sev_platform_init_locked(int *error)
* initialization function should succeed by replacing the state
* with a reset state.
*/
- dev_err(sev->dev, "SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
- rc = init_function(&psp_ret);
+ dev_err(sev->dev,
+"SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
+ rc = __sev_do_init_locked(&psp_ret);
}
+
if (error)
*error = psp_ret;
diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h
index 91b4c63d5cbf..e8cfb8bde0d7 100644
--- a/include/uapi/linux/psp-sev.h
+++ b/include/uapi/linux/psp-sev.h
@@ -36,6 +36,13 @@ enum {
* SEV Firmware status code
*/
typedef enum {
+ /*
+ * This error code is not in the SEV spec but is added to convey that
+ * there was an error that prevented the SEV Firmware from being called.
+ * The SEV API error codes are 16 bits, so the -1 value will not overlap
+ * with possible values from the specification.
+ */
+ SEV_RET_NO_FW_CALL = -1,
SEV_RET_SUCCESS = 0,
SEV_RET_INVALID_PLATFORM_STATE,
SEV_RET_INVALID_GUEST_STATE,
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
2023-01-11 19:39 ` [PATCH v11 1/7] crypto: ccp - Name -1 return value as SEV_RET_NO_FW_CALL Dionna Glaze
@ 2023-01-11 19:39 ` Dionna Glaze
2023-01-20 14:09 ` Borislav Petkov
2023-01-11 19:39 ` [PATCH v11 3/7] virt: sev-guest: Remove err in handle_guest_request Dionna Glaze
` (4 subsequent siblings)
6 siblings, 1 reply; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:39 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Dionna Glaze, Tom Lendacky, Paolo Bonzini, Joerg Roedel,
Peter Gonda, Thomas Gleixner, Dave Hansen, Ingo Molnar,
Borislav Petkov, H. Peter Anvin, Venu Busireddy, Michael Roth,
Kirill A. Shutemov, Michael Sterritt
The GHCB specification declares that the firmware error value for a
guest request will be stored in the lower 32 bits of EXIT_INFO_2.
The upper 32 bits are for the VMM's own error code. The fw_err argument
is thus a misnomer, and callers will need access to all 64 bits.
The type of unsigned long also causes problems, since sw_exit_info2 is
u64 (unsigned long long) vs the argument's unsigned long*. A temporary
typedef is introduced for the err argument so it can be changed in a
later patch more cleanly.
The firmware might not even be called, so the call is bookended with
the no firmware call error and clearing the error.
Cc: Tom Lendacky <Thomas.Lendacky@amd.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: Peter Gonda <pgonda@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Venu Busireddy <venu.busireddy@oracle.com>
Cc: Michael Roth <michael.roth@amd.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Michael Sterritt <sterritt@google.com>
Fixes: d5af44dde546 ("x86/sev: Provide support for SNP guest request NAEs")
Reviewed-by: Tom Lendacky <Thomas.Lendacky@amd.com>
Reviewed-by: Borislav Petkov <Borislav.Petkov@amd.com>
Reviewed-by: Peter Gonda <pgonda@google.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
arch/x86/include/asm/sev.h | 10 ++++++++--
arch/x86/kernel/sev.c | 10 ++++++----
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index ebc271bb6d8e..5b03ba18fee7 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -133,6 +133,12 @@ struct snp_secrets_page_layout {
u8 rsvd3[3840];
} __packed;
+/*
+ * Use a type alias temporarily to cleanly change the snp_issue_guest_request
+ * signature cleanly over multiple patches.
+ */
+typedef unsigned long sev_guestreq_err_t;
+
#ifdef CONFIG_AMD_MEM_ENCRYPT
extern struct static_key_false sev_es_enable_key;
extern void __sev_es_ist_enter(struct pt_regs *regs);
@@ -196,7 +202,7 @@ void snp_set_memory_private(unsigned long vaddr, unsigned int npages);
void snp_set_wakeup_secondary_cpu(void);
bool snp_init(struct boot_params *bp);
void __init __noreturn snp_abort(void);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err);
+int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, sev_guestreq_err_t *exitinfo2);
#else
static inline void sev_es_ist_enter(struct pt_regs *regs) { }
static inline void sev_es_ist_exit(void) { }
@@ -217,7 +223,7 @@ static inline void snp_set_wakeup_secondary_cpu(void) { }
static inline bool snp_init(struct boot_params *bp) { return false; }
static inline void snp_abort(void) { }
static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input,
- unsigned long *fw_err)
+ sev_guestreq_err_t *exitinfo2)
{
return -ENOTTY;
}
diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c
index 679026a640ef..d1a6092b1e03 100644
--- a/arch/x86/kernel/sev.c
+++ b/arch/x86/kernel/sev.c
@@ -22,6 +22,7 @@
#include <linux/efi.h>
#include <linux/platform_device.h>
#include <linux/io.h>
+#include <linux/psp-sev.h>
#include <asm/cpu_entry_area.h>
#include <asm/stacktrace.h>
@@ -2175,7 +2176,7 @@ static int __init init_sev_config(char *str)
}
__setup("sev=", init_sev_config);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err)
+int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, sev_guestreq_err_t *exitinfo2)
{
struct ghcb_state state;
struct es_em_ctxt ctxt;
@@ -2186,9 +2187,11 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned
if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
return -ENODEV;
- if (!fw_err)
+ if (!exitinfo2)
return -EINVAL;
+ *exitinfo2 = SEV_RET_NO_FW_CALL;
+
/*
* __sev_get_ghcb() needs to run with IRQs disabled because it is using
* a per-CPU GHCB.
@@ -2212,14 +2215,13 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned
if (ret)
goto e_put;
+ *exitinfo2 = ghcb->save.sw_exit_info_2;
if (ghcb->save.sw_exit_info_2) {
/* Number of expected pages are returned in RBX */
if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST &&
ghcb->save.sw_exit_info_2 == SNP_GUEST_REQ_INVALID_LEN)
input->data_npages = ghcb_get_rbx(ghcb);
- *fw_err = ghcb->save.sw_exit_info_2;
-
ret = -EIO;
}
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err
2023-01-11 19:39 ` [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err Dionna Glaze
@ 2023-01-20 14:09 ` Borislav Petkov
2023-01-20 18:03 ` Dionna Amalie Glaze
0 siblings, 1 reply; 11+ messages in thread
From: Borislav Petkov @ 2023-01-20 14:09 UTC (permalink / raw)
To: Dionna Glaze
Cc: linux-kernel, x86, Tom Lendacky, Paolo Bonzini, Joerg Roedel,
Peter Gonda, Thomas Gleixner, Dave Hansen, Ingo Molnar,
H. Peter Anvin, Venu Busireddy, Michael Roth, Kirill A. Shutemov,
Michael Sterritt
On Wed, Jan 11, 2023 at 07:39:56PM +0000, Dionna Glaze wrote:
> The type of unsigned long also causes problems, since sw_exit_info2 is
> u64 (unsigned long long) vs the argument's unsigned long*. A temporary
> typedef is introduced for the err argument so it can be changed in a
> later patch more cleanly.
Why?
Why not make it a u64 directly and be done with it?
---
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index ebc271bb6d8e..61b0bfc7f131 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -196,7 +196,7 @@ void snp_set_memory_private(unsigned long vaddr, unsigned int npages);
void snp_set_wakeup_secondary_cpu(void);
bool snp_init(struct boot_params *bp);
void __init __noreturn snp_abort(void);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err);
+int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, u64 *exitinfo2);
#else
static inline void sev_es_ist_enter(struct pt_regs *regs) { }
static inline void sev_es_ist_exit(void) { }
@@ -216,8 +216,7 @@ static inline void snp_set_memory_private(unsigned long vaddr, unsigned int npag
static inline void snp_set_wakeup_secondary_cpu(void) { }
static inline bool snp_init(struct boot_params *bp) { return false; }
static inline void snp_abort(void) { }
-static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input,
- unsigned long *fw_err)
+static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, u64 *exitinfo2)
{
return -ENOTTY;
}
diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c
index 679026a640ef..70b4cbd33c45 100644
--- a/arch/x86/kernel/sev.c
+++ b/arch/x86/kernel/sev.c
@@ -22,6 +22,7 @@
#include <linux/efi.h>
#include <linux/platform_device.h>
#include <linux/io.h>
+#include <linux/psp-sev.h>
#include <asm/cpu_entry_area.h>
#include <asm/stacktrace.h>
@@ -2175,7 +2176,7 @@ static int __init init_sev_config(char *str)
}
__setup("sev=", init_sev_config);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err)
+int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, u64 *exitinfo2)
{
struct ghcb_state state;
struct es_em_ctxt ctxt;
@@ -2186,9 +2187,11 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned
if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
return -ENODEV;
- if (!fw_err)
+ if (!exitinfo2)
return -EINVAL;
+ *exitinfo2 = SEV_RET_NO_FW_CALL;
+
/*
* __sev_get_ghcb() needs to run with IRQs disabled because it is using
* a per-CPU GHCB.
@@ -2212,14 +2215,13 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned
if (ret)
goto e_put;
+ *exitinfo2 = ghcb->save.sw_exit_info_2;
if (ghcb->save.sw_exit_info_2) {
/* Number of expected pages are returned in RBX */
if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST &&
ghcb->save.sw_exit_info_2 == SNP_GUEST_REQ_INVALID_LEN)
input->data_npages = ghcb_get_rbx(ghcb);
- *fw_err = ghcb->save.sw_exit_info_2;
-
ret = -EIO;
}
diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c
index 4ec4174e05a3..e15d94785761 100644
--- a/drivers/virt/coco/sev-guest/sev-guest.c
+++ b/drivers/virt/coco/sev-guest/sev-guest.c
@@ -322,8 +322,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, in
u8 type, void *req_buf, size_t req_sz, void *resp_buf,
u32 resp_sz, __u64 *fw_err)
{
- unsigned long err;
- u64 seqno;
+ u64 seqno, err;
int rc;
/* Get message sequence and verify that its a non-zero */
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err
2023-01-20 14:09 ` Borislav Petkov
@ 2023-01-20 18:03 ` Dionna Amalie Glaze
2023-01-20 18:50 ` Borislav Petkov
0 siblings, 1 reply; 11+ messages in thread
From: Dionna Amalie Glaze @ 2023-01-20 18:03 UTC (permalink / raw)
To: Borislav Petkov
Cc: linux-kernel, x86, Tom Lendacky, Paolo Bonzini, Joerg Roedel,
Peter Gonda, Thomas Gleixner, Dave Hansen, Ingo Molnar,
H. Peter Anvin, Venu Busireddy, Michael Roth, Kirill A. Shutemov,
Michael Sterritt
>
> Why not make it a u64 directly and be done with it?
>
> ---
I was operating under the assumption that kvm: sev: patches should
only touch kvm, and virt: coco: sev-guest should only touch sev-guest.
If you're okay with a patch that changes both areas at the same time,
then I could do that.
--
-Dionna Glaze, PhD (she/her)
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err
2023-01-20 18:03 ` Dionna Amalie Glaze
@ 2023-01-20 18:50 ` Borislav Petkov
0 siblings, 0 replies; 11+ messages in thread
From: Borislav Petkov @ 2023-01-20 18:50 UTC (permalink / raw)
To: Dionna Amalie Glaze
Cc: linux-kernel, x86, Tom Lendacky, Paolo Bonzini, Joerg Roedel,
Peter Gonda, Thomas Gleixner, Dave Hansen, Ingo Molnar,
H. Peter Anvin, Venu Busireddy, Michael Roth, Kirill A. Shutemov,
Michael Sterritt
On Fri, Jan 20, 2023 at 10:03:33AM -0800, Dionna Amalie Glaze wrote:
> I was operating under the assumption that kvm: sev: patches should
> only touch kvm, and virt: coco: sev-guest should only touch sev-guest.
No, we pretty-much never do that. If a patch touches multiple subsystems, the
relevant maintainers agree on its path upstream.
> If you're okay with a patch that changes both areas at the same time,
> then I could do that.
Yes pls.
Thx.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v11 3/7] virt: sev-guest: Remove err in handle_guest_request
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
2023-01-11 19:39 ` [PATCH v11 1/7] crypto: ccp - Name -1 return value as SEV_RET_NO_FW_CALL Dionna Glaze
2023-01-11 19:39 ` [PATCH v11 2/7] x86/sev: Change snp_guest_issue_request's fw_err Dionna Glaze
@ 2023-01-11 19:39 ` Dionna Glaze
2023-01-11 19:39 ` [PATCH v11 4/7] virt: sev-guest: interpret VMM errors from guest request Dionna Glaze
` (3 subsequent siblings)
6 siblings, 0 replies; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:39 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Dionna Glaze, Tom Lendacky, Paolo Bonzini, Joerg Roedel,
Peter Gonda, Thomas Gleixner, Dave Hansen, Borislav Petkov,
Haowen Bai, Liam Merwick, Yang Yingliang
The err variable may not be set in the call to snp_issue_guest_request,
yet it is unconditionally written back to fw_err if fw_err is non-null.
This is undefined behavior, and currently returns uninitialized kernel
stack memory to user space.
The fw_err argument is better to just pass through to
snp_issue_guest_request, so that's done by passing along the ioctl
argument. This removes the need for an argument to handle_guest_request.
Instead of using the fw_err field of the argument, which is a bit of a
misnomer, instead change to exitinfo2. The exitinfo2 field type is a
temporary typedef that will be removed.
Cc: Tom Lendacky <Thomas.Lendacky@amd.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: Peter Gonda <pgonda@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Cc: Haowen Bai <baihaowen@meizu.com>
Cc: Liam Merwick <liam.merwick@oracle.com>
Cc: Yang Yingliang <yangyingliang@huawei.com>
Fixes: fce96cf04430 ("virt: Add SEV-SNP guest driver")
Reviewed-by: Tom Lendacky <Thomas.Lendacky@amd.com>
Reviewed-by: Borislav Petkov <Borislav.Petkov@amd.com>
Reviewed-by: Peter Gonda <pgonda@google.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
drivers/virt/coco/sev-guest/sev-guest.c | 43 ++++++++++++-------------
include/uapi/linux/sev-guest.h | 12 +++++--
2 files changed, 31 insertions(+), 24 deletions(-)
diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c
index 4ec4174e05a3..e2fcb5215630 100644
--- a/drivers/virt/coco/sev-guest/sev-guest.c
+++ b/drivers/virt/coco/sev-guest/sev-guest.c
@@ -318,11 +318,11 @@ static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8
return __enc_payload(snp_dev, req, payload, sz);
}
-static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, int msg_ver,
+static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code,
+ struct snp_guest_request_ioctl *arg,
u8 type, void *req_buf, size_t req_sz, void *resp_buf,
- u32 resp_sz, __u64 *fw_err)
+ u32 resp_sz)
{
- unsigned long err;
u64 seqno;
int rc;
@@ -334,7 +334,8 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, in
memset(snp_dev->response, 0, sizeof(struct snp_guest_msg));
/* Encrypt the userspace provided payload */
- rc = enc_payload(snp_dev, seqno, msg_ver, type, req_buf, req_sz);
+ rc = enc_payload(snp_dev, seqno, arg->msg_version, type, req_buf,
+ req_sz);
if (rc)
return rc;
@@ -344,7 +345,8 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, in
* sequence number must be incremented or the VMPCK must be deleted to
* prevent reuse of the IV.
*/
- rc = snp_issue_guest_request(exit_code, &snp_dev->input, &err);
+ rc = snp_issue_guest_request(exit_code, &snp_dev->input,
+ &arg->exitinfo2);
/*
* If the extended guest request fails due to having too small of a
@@ -366,24 +368,22 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, in
* of the VMPCK and the error code being propagated back to the
* user as an ioctl() return code.
*/
- rc = snp_issue_guest_request(exit_code, &snp_dev->input, &err);
+ rc = snp_issue_guest_request(exit_code, &snp_dev->input,
+ &arg->exitinfo2);
/*
* Override the error to inform callers the given extended
* request buffer size was too small and give the caller the
* required buffer size.
*/
- err = SNP_GUEST_REQ_INVALID_LEN;
+ arg->exitinfo2 = SNP_GUEST_REQ_INVALID_LEN;
snp_dev->input.data_npages = certs_npages;
}
- if (fw_err)
- *fw_err = err;
-
if (rc) {
dev_alert(snp_dev->dev,
- "Detected error from ASP request. rc: %d, fw_err: %llu\n",
- rc, *fw_err);
+ "Detected error from ASP request. rc: %d, exitinfo2: %llu\n",
+ rc, (u64)arg->exitinfo2);
goto disable_vmpck;
}
@@ -430,9 +430,9 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io
if (!resp)
return -ENOMEM;
- rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg->msg_version,
+ rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg,
SNP_MSG_REPORT_REQ, &req, sizeof(req), resp->data,
- resp_len, &arg->fw_err);
+ resp_len);
if (rc)
goto e_free;
@@ -470,9 +470,8 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque
if (copy_from_user(&req, (void __user *)arg->req_data, sizeof(req)))
return -EFAULT;
- rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg->msg_version,
- SNP_MSG_KEY_REQ, &req, sizeof(req), buf, resp_len,
- &arg->fw_err);
+ rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg,
+ SNP_MSG_KEY_REQ, &req, sizeof(req), buf, resp_len);
if (rc)
return rc;
@@ -532,12 +531,12 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
return -ENOMEM;
snp_dev->input.data_npages = npages;
- ret = handle_guest_request(snp_dev, SVM_VMGEXIT_EXT_GUEST_REQUEST, arg->msg_version,
+ ret = handle_guest_request(snp_dev, SVM_VMGEXIT_EXT_GUEST_REQUEST, arg,
SNP_MSG_REPORT_REQ, &req.data,
- sizeof(req.data), resp->data, resp_len, &arg->fw_err);
+ sizeof(req.data), resp->data, resp_len);
/* If certs length is invalid then copy the returned length */
- if (arg->fw_err == SNP_GUEST_REQ_INVALID_LEN) {
+ if (arg->exitinfo2 == SNP_GUEST_REQ_INVALID_LEN) {
req.certs_len = snp_dev->input.data_npages << PAGE_SHIFT;
if (copy_to_user((void __user *)arg->req_data, &req, sizeof(req)))
@@ -572,7 +571,7 @@ static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long
if (copy_from_user(&input, argp, sizeof(input)))
return -EFAULT;
- input.fw_err = 0xff;
+ input.exitinfo2 = 0xff;
/* Message version must be non-zero */
if (!input.msg_version)
@@ -603,7 +602,7 @@ static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long
mutex_unlock(&snp_cmd_mutex);
- if (input.fw_err && copy_to_user(argp, &input, sizeof(input)))
+ if (input.exitinfo2 && copy_to_user(argp, &input, sizeof(input)))
return -EFAULT;
return ret;
diff --git a/include/uapi/linux/sev-guest.h b/include/uapi/linux/sev-guest.h
index 256aaeff7e65..52e994b68d90 100644
--- a/include/uapi/linux/sev-guest.h
+++ b/include/uapi/linux/sev-guest.h
@@ -52,8 +52,16 @@ struct snp_guest_request_ioctl {
__u64 req_data;
__u64 resp_data;
- /* firmware error code on failure (see psp-sev.h) */
- __u64 fw_err;
+ /* bits[63:32]: VMM error code, bits[31:0] firmware error code (see psp-sev.h) */
+
+ union {
+ __u64 fw_err; /* Name deprecated in favor of others */
+ sev_guestreq_err_t exitinfo2;
+ struct {
+ __u32 fw_error;
+ __u32 vmm_error;
+ };
+ };
};
struct snp_ext_report_req {
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v11 4/7] virt: sev-guest: interpret VMM errors from guest request
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
` (2 preceding siblings ...)
2023-01-11 19:39 ` [PATCH v11 3/7] virt: sev-guest: Remove err in handle_guest_request Dionna Glaze
@ 2023-01-11 19:39 ` Dionna Glaze
2023-01-11 19:39 ` [PATCH v11 5/7] x86/sev: Change sev_guestreq_err_t to u64 Dionna Glaze
` (2 subsequent siblings)
6 siblings, 0 replies; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:39 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Dionna Glaze, Tom Lendacky, Peter Gonda, Borislav Petkov,
Tom Lendacky, Liam Merwick, Yang Yingliang, Haowen Bai
The GHCB specification states that the upper 32 bits of exitinfo2 are
for the VMM's error codes. The sev-guest ABI has already locked in
that the fw_err status of the input will be 64 bits, and that
BIT_ULL(32) means that the extended guest request's data buffer was too
small. Add BIT_ULL(33) for "host refused the request for throttling
reasons".
The driver interprets the upper 32 bits of exitinfo2 for the user
anyway in case the request gets throttled. For safety, since the
encryption algorithm in GHCBv2 is AES_GCM, control must remain in the
kernel to complete the request with the current sequence number.
Returning without finishing the request allows the the guest to make
another request but with different message contents. This is IV reuse,
and breaks cryptographic protections.
When throttled, the driver will reschedule itself and then try
again after sleeping half its ratelimit time to avoid a big wait queue.
The ioctl may block indefinitely, but that has always been the case
when deferring these requests to the host.
Cc: Tom Lendacky <Thomas.Lendacky@amd.com>
Cc: Peter Gonda <pgonda@google.com>
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Liam Merwick <liam.merwick@oracle.com>
Cc: Yang Yingliang <yangyingliang@huawei.com>
Cc: Haowen Bai <baihaowen@meizu.com>
Reviewed-by: Tom Lendacky <Thomas.Lendacky@amd.com>
Reviewed-by: Peter Gonda <pgonda@google.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
Documentation/virt/coco/sev-guest.rst | 21 +++++++++-----
drivers/virt/coco/sev-guest/sev-guest.c | 37 +++++++++++++++++++++++--
include/uapi/linux/sev-guest.h | 7 +++++
3 files changed, 55 insertions(+), 10 deletions(-)
diff --git a/Documentation/virt/coco/sev-guest.rst b/Documentation/virt/coco/sev-guest.rst
index e76393e389eb..fd8bf833f1cd 100644
--- a/Documentation/virt/coco/sev-guest.rst
+++ b/Documentation/virt/coco/sev-guest.rst
@@ -37,11 +37,12 @@ along with a description:
the return value. General error numbers (-ENOMEM, -EINVAL)
are not detailed, but errors with specific meanings are.
-The guest ioctl should be issued on a file descriptor of the /dev/sev-guest device.
-The ioctl accepts struct snp_user_guest_request. The input and output structure is
-specified through the req_data and resp_data field respectively. If the ioctl fails
-to execute due to a firmware error, then fw_err code will be set otherwise the
-fw_err will be set to 0x00000000ffffffff, i.e., the lower 32-bits are -1.
+The guest ioctl should be issued on a file descriptor of the
+/dev/sev-guest device. The ioctl accepts struct
+snp_user_guest_request. The input and output structure is specified
+through the req_data and resp_data field respectively. If the ioctl
+fails to execute due to a firmware error, then the fw_error code will
+be set, otherwise fw_error will be set to -1.
The firmware checks that the message sequence counter is one greater than
the guests message sequence counter. If guest driver fails to increment message
@@ -57,8 +58,14 @@ counter (e.g. counter overflow), then -EIO will be returned.
__u64 req_data;
__u64 resp_data;
- /* firmware error code on failure (see psp-sev.h) */
- __u64 fw_err;
+ /* bits[63:32]: VMM error code, bits[31:0] firmware error code (see psp-sev.h) */
+ union {
+ __u64 exitinfo2;
+ struct {
+ __u32 fw_error;
+ __u32 vmm_error;
+ };
+ };
};
2.1 SNP_GET_REPORT
diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c
index e2fcb5215630..04c30e4da642 100644
--- a/drivers/virt/coco/sev-guest/sev-guest.c
+++ b/drivers/virt/coco/sev-guest/sev-guest.c
@@ -14,6 +14,7 @@
#include <linux/io.h>
#include <linux/platform_device.h>
#include <linux/miscdevice.h>
+#include <linux/ratelimit.h>
#include <linux/set_memory.h>
#include <linux/fs.h>
#include <crypto/aead.h>
@@ -48,12 +49,22 @@ struct snp_guest_dev {
struct snp_req_data input;
u32 *os_area_msg_seqno;
u8 *vmpck;
+
+ struct ratelimit_state rs;
};
static u32 vmpck_id;
module_param(vmpck_id, uint, 0444);
MODULE_PARM_DESC(vmpck_id, "The VMPCK ID to use when communicating with the PSP.");
+static int rate_s = 1;
+module_param(rate_s, int, 0444);
+MODULE_PARM_DESC(rate_s, "The rate limit interval in seconds to limit requests to.");
+
+static int rate_burst = 2;
+module_param(rate_burst, int, 0444);
+MODULE_PARM_DESC(rate_burst, "The rate limit burst amount to limit requests to.");
+
/* Mutex to serialize the shared buffer access and command handling. */
static DEFINE_MUTEX(snp_cmd_mutex);
@@ -339,6 +350,16 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code,
if (rc)
return rc;
+retry:
+ /*
+ * Rate limit commands internally since the host can also throttle, and
+ * the guest shouldn't create a tight request spin that could end up
+ * getting this VM throttled more heavily.
+ */
+ if (!__ratelimit(&snp_dev->rs)) {
+ schedule_timeout_interruptible((rate_s * HZ) / 2);
+ goto retry;
+ }
/*
* Call firmware to process the request. In this function the encrypted
* message enters shared memory with the host. So after this call the
@@ -348,6 +369,14 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code,
rc = snp_issue_guest_request(exit_code, &snp_dev->input,
&arg->exitinfo2);
+ /*
+ * The host may return EBUSY if the request has been throttled.
+ * We retry in the driver to avoid returning and reusing the message
+ * sequence number on a different message.
+ */
+ if (arg->vmm_error == SNP_GUEST_VMM_ERR_BUSY)
+ goto retry;
+
/*
* If the extended guest request fails due to having too small of a
* certificate data buffer, retry the same guest request without the
@@ -355,7 +384,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code,
* and thus avoid IV reuse.
*/
if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST &&
- err == SNP_GUEST_REQ_INVALID_LEN) {
+ arg->vmm_error == SNP_GUEST_VMM_ERR_INVALID_LEN) {
const unsigned int certs_npages = snp_dev->input.data_npages;
exit_code = SVM_VMGEXIT_GUEST_REQUEST;
@@ -376,7 +405,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code,
* request buffer size was too small and give the caller the
* required buffer size.
*/
- arg->exitinfo2 = SNP_GUEST_REQ_INVALID_LEN;
+ arg->vmm_error = SNP_GUEST_VMM_ERR_INVALID_LEN;
snp_dev->input.data_npages = certs_npages;
}
@@ -536,7 +565,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
sizeof(req.data), resp->data, resp_len);
/* If certs length is invalid then copy the returned length */
- if (arg->exitinfo2 == SNP_GUEST_REQ_INVALID_LEN) {
+ if (arg->vmm_error == SNP_GUEST_VMM_ERR_INVALID_LEN) {
req.certs_len = snp_dev->input.data_npages << PAGE_SHIFT;
if (copy_to_user((void __user *)arg->req_data, &req, sizeof(req)))
@@ -752,6 +781,8 @@ static int __init sev_guest_probe(struct platform_device *pdev)
if (ret)
goto e_free_cert_data;
+ ratelimit_state_init(&snp_dev->rs, rate_s * HZ, rate_burst);
+
dev_info(dev, "Initialized SEV guest driver (using vmpck_id %d)\n", vmpck_id);
return 0;
diff --git a/include/uapi/linux/sev-guest.h b/include/uapi/linux/sev-guest.h
index 52e994b68d90..49fc4341ac93 100644
--- a/include/uapi/linux/sev-guest.h
+++ b/include/uapi/linux/sev-guest.h
@@ -85,4 +85,11 @@ struct snp_ext_report_req {
/* Get SNP extended report as defined in the GHCB specification version 2. */
#define SNP_GET_EXT_REPORT _IOWR(SNP_GUEST_REQ_IOC_TYPE, 0x2, struct snp_guest_request_ioctl)
+/* Guest message request EXIT_INFO_2 constants */
+#define SNP_GUEST_FW_ERR_MASK GENMASK_ULL(31, 0)
+#define SNP_GUEST_VMM_ERR_SHIFT 32
+
+#define SNP_GUEST_VMM_ERR_INVALID_LEN 1
+#define SNP_GUEST_VMM_ERR_BUSY 2
+
#endif /* __UAPI_LINUX_SEV_GUEST_H_ */
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v11 5/7] x86/sev: Change sev_guestreq_err_t to u64
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
` (3 preceding siblings ...)
2023-01-11 19:39 ` [PATCH v11 4/7] virt: sev-guest: interpret VMM errors from guest request Dionna Glaze
@ 2023-01-11 19:39 ` Dionna Glaze
2023-01-11 19:40 ` [PATCH v11 6/7] virt/coco/sev-guest: Remove dependence on sev_guestreq_t Dionna Glaze
2023-01-11 19:40 ` [PATCH v11 7/7] x86/sev: Remove temporary typedef Dionna Glaze
6 siblings, 0 replies; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:39 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Dionna Glaze, Borislav Petkov, Peter Gonda, Thomas Lendacky
As part of changing the signature across modules, this step ensures the
new type will continue to compile before the typedef is removed.
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Cc: Peter Gonda <pgonda@google.com>
Cc: Thomas Lendacky <Thomas.Lendacky@amd.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
arch/x86/include/asm/sev.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 5b03ba18fee7..301e3b69f477 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -137,7 +137,7 @@ struct snp_secrets_page_layout {
* Use a type alias temporarily to cleanly change the snp_issue_guest_request
* signature cleanly over multiple patches.
*/
-typedef unsigned long sev_guestreq_err_t;
+typedef u64 sev_guestreq_err_t;
#ifdef CONFIG_AMD_MEM_ENCRYPT
extern struct static_key_false sev_es_enable_key;
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v11 6/7] virt/coco/sev-guest: Remove dependence on sev_guestreq_t
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
` (4 preceding siblings ...)
2023-01-11 19:39 ` [PATCH v11 5/7] x86/sev: Change sev_guestreq_err_t to u64 Dionna Glaze
@ 2023-01-11 19:40 ` Dionna Glaze
2023-01-11 19:40 ` [PATCH v11 7/7] x86/sev: Remove temporary typedef Dionna Glaze
6 siblings, 0 replies; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:40 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Dionna Glaze, Borislav Petkov, Peter Gonda, Thomas Lendacky
Next step in changing the cross-module signature change is to remove
this dependency in sev-guest in favor of the concrete type, u64.
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Cc: Peter Gonda <pgonda@google.com>
Cc: Thomas Lendacky <Thomas.Lendacky@amd.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
drivers/virt/coco/sev-guest/sev-guest.c | 2 +-
include/uapi/linux/sev-guest.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c
index 04c30e4da642..1204939d4f2b 100644
--- a/drivers/virt/coco/sev-guest/sev-guest.c
+++ b/drivers/virt/coco/sev-guest/sev-guest.c
@@ -412,7 +412,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code,
if (rc) {
dev_alert(snp_dev->dev,
"Detected error from ASP request. rc: %d, exitinfo2: %llu\n",
- rc, (u64)arg->exitinfo2);
+ rc, arg->exitinfo2);
goto disable_vmpck;
}
diff --git a/include/uapi/linux/sev-guest.h b/include/uapi/linux/sev-guest.h
index 49fc4341ac93..ef910e605ef0 100644
--- a/include/uapi/linux/sev-guest.h
+++ b/include/uapi/linux/sev-guest.h
@@ -56,7 +56,7 @@ struct snp_guest_request_ioctl {
union {
__u64 fw_err; /* Name deprecated in favor of others */
- sev_guestreq_err_t exitinfo2;
+ __u64 exitinfo2;
struct {
__u32 fw_error;
__u32 vmm_error;
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v11 7/7] x86/sev: Remove temporary typedef
2023-01-11 19:39 [PATCH v11 0/7] Add throttling detection to sev-guest Dionna Glaze
` (5 preceding siblings ...)
2023-01-11 19:40 ` [PATCH v11 6/7] virt/coco/sev-guest: Remove dependence on sev_guestreq_t Dionna Glaze
@ 2023-01-11 19:40 ` Dionna Glaze
6 siblings, 0 replies; 11+ messages in thread
From: Dionna Glaze @ 2023-01-11 19:40 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Dionna Glaze, Borislav Petkov, Peter Gonda, Thomas Lendacky
The sev_guestreq_err_t typedef has served its purpose for cleanly
changing the sev_issue_guest_request signature, so it's no longer
needed.
Cc: Borislav Petkov <Borislav.Petkov@amd.com>
Cc: Peter Gonda <pgonda@google.com>
Cc: Thomas Lendacky <Thomas.Lendacky@amd.com>
Signed-off-by: Dionna Glaze <dionnaglaze@google.com>
---
arch/x86/include/asm/sev.h | 10 ++--------
arch/x86/kernel/sev.c | 2 +-
2 files changed, 3 insertions(+), 9 deletions(-)
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 301e3b69f477..05de34d10d89 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -133,12 +133,6 @@ struct snp_secrets_page_layout {
u8 rsvd3[3840];
} __packed;
-/*
- * Use a type alias temporarily to cleanly change the snp_issue_guest_request
- * signature cleanly over multiple patches.
- */
-typedef u64 sev_guestreq_err_t;
-
#ifdef CONFIG_AMD_MEM_ENCRYPT
extern struct static_key_false sev_es_enable_key;
extern void __sev_es_ist_enter(struct pt_regs *regs);
@@ -202,7 +196,7 @@ void snp_set_memory_private(unsigned long vaddr, unsigned int npages);
void snp_set_wakeup_secondary_cpu(void);
bool snp_init(struct boot_params *bp);
void __init __noreturn snp_abort(void);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, sev_guestreq_err_t *exitinfo2);
+int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, u64 *exitinfo2);
#else
static inline void sev_es_ist_enter(struct pt_regs *regs) { }
static inline void sev_es_ist_exit(void) { }
@@ -223,7 +217,7 @@ static inline void snp_set_wakeup_secondary_cpu(void) { }
static inline bool snp_init(struct boot_params *bp) { return false; }
static inline void snp_abort(void) { }
static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input,
- sev_guestreq_err_t *exitinfo2)
+ u64 *exitinfo2)
{
return -ENOTTY;
}
diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c
index d1a6092b1e03..70b4cbd33c45 100644
--- a/arch/x86/kernel/sev.c
+++ b/arch/x86/kernel/sev.c
@@ -2176,7 +2176,7 @@ static int __init init_sev_config(char *str)
}
__setup("sev=", init_sev_config);
-int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, sev_guestreq_err_t *exitinfo2)
+int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, u64 *exitinfo2)
{
struct ghcb_state state;
struct es_em_ctxt ctxt;
--
2.39.0.314.g84b9a713c41-goog
^ permalink raw reply related [flat|nested] 11+ messages in thread