* [PATCH 1/3] virtio-gpu: set hostaddr for virtio iGPU
2024-12-07 10:55 [PATCH 0/3] Support getting p2pdma_distance Julia Zhang
@ 2024-12-07 10:55 ` Julia Zhang
2024-12-07 10:55 ` [PATCH 2/3] pci: introduce a function to get PCIDevice Julia Zhang
2024-12-07 10:55 ` [PATCH 3/3] virtio-gpu: add a new command to get p2pdma_distance Julia Zhang
2 siblings, 0 replies; 5+ messages in thread
From: Julia Zhang @ 2024-12-07 10:55 UTC (permalink / raw)
To: Stefano Stabellini, Anthony PERARD, Paul Durrant,
Edgar E . Iglesias, Michael S . Tsirkin, Marcel Apfelbaum,
xen-devel
Cc: Alex Deucher, Christian König, Xenia Ragiadakou, Julia Zhang,
Chen Jiqian, Huang Rui, Penny Zheng, Zhu Lingshan,
Roger Pau Monné, Jan Beulich, Juergen Gross,
Oleksandr Tyshchenko, qemu-devel
Get physical pci notation from hvm file and set hostaddr for virtio
iGPU.
Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
hw/display/virtio-gpu.c | 6 ++++++
hw/xen/xen_pt.c | 10 +++++-----
hw/xen/xen_pt.h | 1 -
include/hw/pci/pci_device.h | 1 +
include/hw/virtio/virtio-gpu.h | 2 ++
5 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
index c0570ef856..d68b64ac22 100644
--- a/hw/display/virtio-gpu.c
+++ b/hw/display/virtio-gpu.c
@@ -26,6 +26,7 @@
#include "hw/virtio/virtio-gpu-bswap.h"
#include "hw/virtio/virtio-gpu-pixman.h"
#include "hw/virtio/virtio-bus.h"
+#include "hw/virtio/virtio-pci.h"
#include "hw/qdev-properties.h"
#include "qemu/log.h"
#include "qemu/memfd.h"
@@ -1458,6 +1459,8 @@ void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
{
VirtIODevice *vdev = VIRTIO_DEVICE(qdev);
VirtIOGPU *g = VIRTIO_GPU(qdev);
+ BusState *qbus = BUS(qdev_get_parent_bus(qdev));
+ VirtIOPCIProxy *proxy = VIRTIO_PCI(qbus->parent);
if (virtio_gpu_blob_enabled(g->parent_obj.conf)) {
if (!virtio_gpu_rutabaga_enabled(g->parent_obj.conf) &&
@@ -1508,6 +1511,8 @@ void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
QTAILQ_INIT(&g->reslist);
QTAILQ_INIT(&g->cmdq);
QTAILQ_INIT(&g->fenceq);
+
+ proxy->pci_dev.hostaddr = g->parent_obj.conf.hostaddr;
}
static void virtio_gpu_device_unrealize(DeviceState *qdev)
@@ -1669,6 +1674,7 @@ static Property virtio_gpu_properties[] = {
VIRTIO_GPU_FLAG_BLOB_ENABLED, false),
DEFINE_PROP_SIZE("hostmem", VirtIOGPU, parent_obj.conf.hostmem, 0),
DEFINE_PROP_UINT8("x-scanout-vmstate-version", VirtIOGPU, scanout_vmstate_version, 2),
+ DEFINE_PROP_PCI_HOST_DEVADDR("hostaddr", VirtIOGPU, parent_obj.conf.hostaddr),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/hw/xen/xen_pt.c b/hw/xen/xen_pt.c
index c0042b3515..8e5ff09b66 100644
--- a/hw/xen/xen_pt.c
+++ b/hw/xen/xen_pt.c
@@ -828,9 +828,9 @@ static void xen_pt_realize(PCIDevice *d, Error **errp)
/* register real device */
XEN_PT_LOG(d, "Assigning real physical device %02x:%02x.%d"
- " to devfn 0x%x\n",
+ " to devfn 0x%x, %02x:%02x.%d\n",
s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function,
- s->dev.devfn);
+ s->dev.devfn, pci_bus_num(pci_get_bus(&s->dev)), PCI_SLOT(s->dev.devfn), PCI_FUNC(s->dev.devfn));
s->is_virtfn = s->real_device.is_virtfn;
if (s->is_virtfn) {
@@ -991,7 +991,7 @@ static void xen_pt_unregister_device(PCIDevice *d)
}
static Property xen_pci_passthrough_properties[] = {
- DEFINE_PROP_PCI_HOST_DEVADDR("hostaddr", XenPCIPassthroughState, hostaddr),
+ DEFINE_PROP_PCI_HOST_DEVADDR("hostaddr", XenPCIPassthroughState, dev.hostaddr),
DEFINE_PROP_BOOL("permissive", XenPCIPassthroughState, permissive, false),
DEFINE_PROP_END_OF_LIST(),
};
@@ -1022,8 +1022,8 @@ static void xen_igd_clear_slot(DeviceState *qdev, Error **errp)
PCIBus *pci_bus = pci_get_bus(pci_dev);
xen_host_pci_device_get(&s->real_device,
- s->hostaddr.domain, s->hostaddr.bus,
- s->hostaddr.slot, s->hostaddr.function,
+ s->dev.hostaddr.domain, s->dev.hostaddr.bus,
+ s->dev.hostaddr.slot, s->dev.hostaddr.function,
errp);
if (*errp) {
error_append_hint(errp, "Failed to \"open\" the real pci device");
diff --git a/hw/xen/xen_pt.h b/hw/xen/xen_pt.h
index 095a0f0365..f61a134ff5 100644
--- a/hw/xen/xen_pt.h
+++ b/hw/xen/xen_pt.h
@@ -233,7 +233,6 @@ typedef struct XenPTMSIX {
struct XenPCIPassthroughState {
PCIDevice dev;
- PCIHostDeviceAddress hostaddr;
bool is_virtfn;
bool permissive;
bool permissive_warned;
diff --git a/include/hw/pci/pci_device.h b/include/hw/pci/pci_device.h
index 8eaf0d58bb..b7acf3822d 100644
--- a/include/hw/pci/pci_device.h
+++ b/include/hw/pci/pci_device.h
@@ -56,6 +56,7 @@ typedef struct PCIReqIDCache PCIReqIDCache;
struct PCIDevice {
DeviceState qdev;
+ PCIHostDeviceAddress hostaddr;
bool partially_hotplugged;
bool has_power;
diff --git a/include/hw/virtio/virtio-gpu.h b/include/hw/virtio/virtio-gpu.h
index 553799b8cc..e385794a06 100644
--- a/include/hw/virtio/virtio-gpu.h
+++ b/include/hw/virtio/virtio-gpu.h
@@ -24,6 +24,7 @@
#include "standard-headers/linux/virtio_gpu.h"
#include "standard-headers/linux/virtio_ids.h"
#include "qom/object.h"
+#include "hw/pci/pci.h"
#define TYPE_VIRTIO_GPU_BASE "virtio-gpu-base"
OBJECT_DECLARE_TYPE(VirtIOGPUBase, VirtIOGPUBaseClass,
@@ -125,6 +126,7 @@ struct virtio_gpu_base_conf {
uint32_t xres;
uint32_t yres;
uint64_t hostmem;
+ PCIHostDeviceAddress hostaddr;
};
struct virtio_gpu_ctrl_command {
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 2/3] pci: introduce a function to get PCIDevice
2024-12-07 10:55 [PATCH 0/3] Support getting p2pdma_distance Julia Zhang
2024-12-07 10:55 ` [PATCH 1/3] virtio-gpu: set hostaddr for virtio iGPU Julia Zhang
@ 2024-12-07 10:55 ` Julia Zhang
2024-12-07 10:55 ` [PATCH 3/3] virtio-gpu: add a new command to get p2pdma_distance Julia Zhang
2 siblings, 0 replies; 5+ messages in thread
From: Julia Zhang @ 2024-12-07 10:55 UTC (permalink / raw)
To: Stefano Stabellini, Anthony PERARD, Paul Durrant,
Edgar E . Iglesias, Michael S . Tsirkin, Marcel Apfelbaum,
xen-devel
Cc: Alex Deucher, Christian König, Xenia Ragiadakou, Julia Zhang,
Chen Jiqian, Huang Rui, Penny Zheng, Zhu Lingshan,
Roger Pau Monné, Jan Beulich, Juergen Gross,
Oleksandr Tyshchenko, qemu-devel
Introduce a helper function to get PCIDevice from qdev pci notation.
Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
hw/pci/pci.c | 22 ++++++++++++++++++++++
include/hw/pci/pci.h | 2 ++
2 files changed, 24 insertions(+)
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index 1416ae202c..95806ead4f 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -2617,6 +2617,28 @@ static int pci_qdev_find_recursive(PCIBus *bus,
return -EINVAL;
}
+int pci_qdev_get_device(uint32_t virt_bus, uint32_t virt_slot, uint32_t virt_func,
+ PCIDevice **pci_dev)
+{
+ PCIHostState *host_bridge;
+ PCIDevice *d;
+ int devfn;
+ int rc = -ENODEV;
+
+ QLIST_FOREACH(host_bridge, &pci_host_bridges, next) {
+ for(devfn = 0; devfn < ARRAY_SIZE(host_bridge->bus->devices); devfn++) {
+ d = host_bridge->bus->devices[devfn];
+ if (d && d->devfn == PCI_DEVFN(virt_slot, virt_func) &&
+ pci_bus_num(pci_get_bus(d)) == virt_bus) {
+ *pci_dev = d;
+ rc = 0;
+ break;
+ }
+ }
+ }
+ return rc;
+}
+
int pci_qdev_find_device(const char *id, PCIDevice **pdev)
{
PCIHostState *host_bridge;
diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h
index 463d9984b3..1b493ab95e 100644
--- a/include/hw/pci/pci.h
+++ b/include/hw/pci/pci.h
@@ -366,6 +366,8 @@ const char *pci_root_bus_path(PCIDevice *dev);
bool pci_bus_bypass_iommu(PCIBus *bus);
PCIDevice *pci_find_device(PCIBus *bus, int bus_num, uint8_t devfn);
int pci_qdev_find_device(const char *id, PCIDevice **pdev);
+int pci_qdev_get_device(uint32_t virt_bus, uint32_t virt_slot,
+ uint32_t virt_func, PCIDevice **pci_dev);
void pci_bus_get_w64_range(PCIBus *bus, Range *range);
void pci_device_deassert_intx(PCIDevice *dev);
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 3/3] virtio-gpu: add a new command to get p2pdma_distance
2024-12-07 10:55 [PATCH 0/3] Support getting p2pdma_distance Julia Zhang
2024-12-07 10:55 ` [PATCH 1/3] virtio-gpu: set hostaddr for virtio iGPU Julia Zhang
2024-12-07 10:55 ` [PATCH 2/3] pci: introduce a function to get PCIDevice Julia Zhang
@ 2024-12-07 10:55 ` Julia Zhang
2025-01-08 14:14 ` Michael S. Tsirkin
2 siblings, 1 reply; 5+ messages in thread
From: Julia Zhang @ 2024-12-07 10:55 UTC (permalink / raw)
To: Stefano Stabellini, Anthony PERARD, Paul Durrant,
Edgar E . Iglesias, Michael S . Tsirkin, Marcel Apfelbaum,
xen-devel
Cc: Alex Deucher, Christian König, Xenia Ragiadakou, Julia Zhang,
Chen Jiqian, Huang Rui, Penny Zheng, Zhu Lingshan,
Roger Pau Monné, Jan Beulich, Juergen Gross,
Oleksandr Tyshchenko, qemu-devel
To implement passthrough dGPU prime in guest, virtio-gpu need to check
p2pdma_distance of two GPUs. This adds a new command for guest to pass
virtual pci notations of two pci devices to host and send xen privcmd to
calculate physical p2pdma_distance.
Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
hw/display/virtio-gpu-virgl.c | 47 +++++++++++++++++++++
hw/i386/xen/xen-hvm.c | 6 +++
include/hw/virtio/virtio-gpu-bswap.h | 12 ++++++
include/hw/xen/xen.h | 3 ++
include/standard-headers/linux/virtio_gpu.h | 19 +++++++++
5 files changed, 87 insertions(+)
diff --git a/hw/display/virtio-gpu-virgl.c b/hw/display/virtio-gpu-virgl.c
index 07faeb1834..eb9b193ade 100644
--- a/hw/display/virtio-gpu-virgl.c
+++ b/hw/display/virtio-gpu-virgl.c
@@ -16,6 +16,12 @@
#include "qemu/iov.h"
#include "trace.h"
#include "hw/virtio/virtio.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/virtio/virtio-pci.h"
+#include "hw/virtio/virtio-bus.h"
+#include "hw/xen/xen.h"
+
#include "hw/virtio/virtio-gpu.h"
#include "hw/virtio/virtio-gpu-bswap.h"
#include "hw/virtio/virtio-gpu-pixman.h"
@@ -188,6 +194,44 @@ virtio_gpu_virgl_unmap_resource_blob(VirtIOGPU *g,
return 0;
}
+
+static void virgl_cmd_p2pdma_distance(VirtIOGPU *g,
+ struct virtio_gpu_ctrl_command *cmd)
+{
+ struct virtio_gpu_device_p2pdma_distance cmd_p;
+ struct virtio_gpu_resp_distance resp;
+ PCIDevice *client = NULL, *provider = NULL;
+ int ret;
+
+ VIRTIO_GPU_FILL_CMD(cmd_p);
+ virtio_gpu_p2pdma_distance_bswap(&cmd_p);
+
+ ret = pci_qdev_get_device(cmd_p.provider_bus, cmd_p.provider_slot, cmd_p.provider_func, &provider);
+
+ if (ret) {
+ qemu_log_mask(LOG_GUEST_ERROR, "%s: virgl get physical device error: %s\n",
+ __func__, strerror(-ret));
+ cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+ return;
+ }
+
+ ret = pci_qdev_get_device(cmd_p.client_bus, cmd_p.client_slot, cmd_p.client_func, &client);
+ if (ret) {
+ qemu_log_mask(LOG_GUEST_ERROR, "%s: virgl get physical device error: %s\n",
+ __func__, strerror(-ret));
+ cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+ return;
+ }
+
+ int distance = xen_p2pdma_distance(provider->hostaddr.bus, provider->hostaddr.slot,
+ provider->hostaddr.function,client->hostaddr.bus,
+ client->hostaddr.slot, client->hostaddr.function);
+
+ memset(&resp, 0, sizeof(resp));
+ resp.hdr.type = VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE;
+ resp.distance = distance;
+ virtio_gpu_ctrl_response(g, cmd, &resp.hdr, sizeof(resp));
+}
#endif
static void virgl_cmd_create_resource_2d(VirtIOGPU *g,
@@ -913,6 +957,9 @@ void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
case VIRTIO_GPU_CMD_SUBMIT_3D:
virgl_cmd_submit_3d(g, cmd);
break;
+ case VIRTIO_GPU_CMD_P2PDMA_DISTANCE:
+ virgl_cmd_p2pdma_distance(g, cmd);
+ break;
case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D:
virgl_cmd_transfer_to_host_2d(g, cmd);
break;
diff --git a/hw/i386/xen/xen-hvm.c b/hw/i386/xen/xen-hvm.c
index d3df488c48..c8dd27dad7 100644
--- a/hw/i386/xen/xen-hvm.c
+++ b/hw/i386/xen/xen-hvm.c
@@ -604,6 +604,12 @@ static bool xen_check_stubdomain(struct xs_handle *xsh)
return is_stubdom;
}
+int xen_p2pdma_distance(uint32_t bus, uint32_t slot, uint32_t func,
+ uint32_t c_bus, uint32_t c_slot, uint32_t c_func)
+{
+ return xc_physdev_p2pdma_distance(xen_xc, bus, slot, func, c_bus, c_slot, c_func);
+}
+
void xen_hvm_init_pc(PCMachineState *pcms, MemoryRegion **ram_memory)
{
MachineState *ms = MACHINE(pcms);
diff --git a/include/hw/virtio/virtio-gpu-bswap.h b/include/hw/virtio/virtio-gpu-bswap.h
index dd1975e2d4..b5c0c0adcd 100644
--- a/include/hw/virtio/virtio-gpu-bswap.h
+++ b/include/hw/virtio/virtio-gpu-bswap.h
@@ -78,6 +78,18 @@ virtio_gpu_map_blob_bswap(struct virtio_gpu_resource_map_blob *mblob)
le64_to_cpus(&mblob->offset);
}
+static inline void
+virtio_gpu_p2pdma_distance_bswap(struct virtio_gpu_device_p2pdma_distance *p2p_dist)
+{
+ virtio_gpu_ctrl_hdr_bswap(&p2p_dist->hdr);
+ le32_to_cpus(&p2p_dist->provider_bus);
+ le32_to_cpus(&p2p_dist->provider_slot);
+ le32_to_cpus(&p2p_dist->provider_func);
+ le32_to_cpus(&p2p_dist->client_bus);
+ le32_to_cpus(&p2p_dist->client_bus);
+ le32_to_cpus(&p2p_dist->client_bus);
+}
+
static inline void
virtio_gpu_unmap_blob_bswap(struct virtio_gpu_resource_unmap_blob *ublob)
{
diff --git a/include/hw/xen/xen.h b/include/hw/xen/xen.h
index ecb89ecfc1..fe1d628327 100644
--- a/include/hw/xen/xen.h
+++ b/include/hw/xen/xen.h
@@ -48,4 +48,7 @@ qemu_irq *xen_interrupt_controller_init(void);
void xen_register_framebuffer(struct MemoryRegion *mr);
+int xen_p2pdma_distance(uint32_t bus, uint32_t slot, uint32_t func,
+ uint32_t c_bus, uint32_t c_slot, uint32_t c_func);
+
#endif /* QEMU_HW_XEN_H */
diff --git a/include/standard-headers/linux/virtio_gpu.h b/include/standard-headers/linux/virtio_gpu.h
index 6459fdb9fb..2e55dcc2fe 100644
--- a/include/standard-headers/linux/virtio_gpu.h
+++ b/include/standard-headers/linux/virtio_gpu.h
@@ -95,6 +95,7 @@ enum virtio_gpu_ctrl_type {
VIRTIO_GPU_CMD_SUBMIT_3D,
VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB,
VIRTIO_GPU_CMD_RESOURCE_UNMAP_BLOB,
+ VIRTIO_GPU_CMD_P2PDMA_DISTANCE,
/* cursor commands */
VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
@@ -108,6 +109,7 @@ enum virtio_gpu_ctrl_type {
VIRTIO_GPU_RESP_OK_EDID,
VIRTIO_GPU_RESP_OK_RESOURCE_UUID,
VIRTIO_GPU_RESP_OK_MAP_INFO,
+ VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE,
/* error responses */
VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
@@ -429,6 +431,23 @@ struct virtio_gpu_set_scanout_blob {
uint32_t offsets[4];
};
+/* VIRTIO_GPU_CMD_P2PDMA_DISTANCE */
+struct virtio_gpu_device_p2pdma_distance {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 provider_bus;
+ __le32 provider_slot;
+ __le32 provider_func;
+ __le32 client_bus;
+ __le32 client_slot;
+ __le32 client_func;
+};
+
+/* VIRTIO_GPU_RESP_DISTANCE */
+struct virtio_gpu_resp_distance {
+ struct virtio_gpu_ctrl_hdr hdr;
+ __le32 distance;
+};
+
/* VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB */
struct virtio_gpu_resource_map_blob {
struct virtio_gpu_ctrl_hdr hdr;
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH 3/3] virtio-gpu: add a new command to get p2pdma_distance
2024-12-07 10:55 ` [PATCH 3/3] virtio-gpu: add a new command to get p2pdma_distance Julia Zhang
@ 2025-01-08 14:14 ` Michael S. Tsirkin
0 siblings, 0 replies; 5+ messages in thread
From: Michael S. Tsirkin @ 2025-01-08 14:14 UTC (permalink / raw)
To: Julia Zhang
Cc: Stefano Stabellini, Anthony PERARD, Paul Durrant,
Edgar E . Iglesias, Marcel Apfelbaum, xen-devel, Alex Deucher,
Christian König, Xenia Ragiadakou, Chen Jiqian, Huang Rui,
Penny Zheng, Zhu Lingshan, Roger Pau Monné, Jan Beulich,
Juergen Gross, Oleksandr Tyshchenko, qemu-devel
On Sat, Dec 07, 2024 at 06:55:40PM +0800, Julia Zhang wrote:
> diff --git a/include/standard-headers/linux/virtio_gpu.h b/include/standard-headers/linux/virtio_gpu.h
> index 6459fdb9fb..2e55dcc2fe 100644
> --- a/include/standard-headers/linux/virtio_gpu.h
> +++ b/include/standard-headers/linux/virtio_gpu.h
> @@ -95,6 +95,7 @@ enum virtio_gpu_ctrl_type {
> VIRTIO_GPU_CMD_SUBMIT_3D,
> VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB,
> VIRTIO_GPU_CMD_RESOURCE_UNMAP_BLOB,
> + VIRTIO_GPU_CMD_P2PDMA_DISTANCE,
>
> /* cursor commands */
> VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
> @@ -108,6 +109,7 @@ enum virtio_gpu_ctrl_type {
> VIRTIO_GPU_RESP_OK_EDID,
> VIRTIO_GPU_RESP_OK_RESOURCE_UUID,
> VIRTIO_GPU_RESP_OK_MAP_INFO,
> + VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE,
>
> /* error responses */
> VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
> @@ -429,6 +431,23 @@ struct virtio_gpu_set_scanout_blob {
> uint32_t offsets[4];
> };
>
> +/* VIRTIO_GPU_CMD_P2PDMA_DISTANCE */
> +struct virtio_gpu_device_p2pdma_distance {
> + struct virtio_gpu_ctrl_hdr hdr;
> + __le32 provider_bus;
> + __le32 provider_slot;
> + __le32 provider_func;
> + __le32 client_bus;
> + __le32 client_slot;
> + __le32 client_func;
> +};
> +
> +/* VIRTIO_GPU_RESP_DISTANCE */
> +struct virtio_gpu_resp_distance {
> + struct virtio_gpu_ctrl_hdr hdr;
> + __le32 distance;
> +};
> +
> /* VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB */
> struct virtio_gpu_resource_map_blob {
> struct virtio_gpu_ctrl_hdr hdr;
> --
> 2.34.1
This is not how we change this header.
you need a spec patch, an UAPI change to be accepted into linux.
--
MST
^ permalink raw reply [flat|nested] 5+ messages in thread