[PATCH 13/15] KVM: arm64: vgic-its: Protect cached vgic_irq pointers with RCU

Oliver Upton
2024-01-24T20:49:48Z

RCU readers of the LPI translation cache will be able to run in parallel
with a cache invalidation, which clears the RCU pointer. Start using RCU
protection on the cached irq pointer in light of this.

Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
---
arch/arm64/kvm/vgic/vgic-its.c | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c
index ed0c6c333a6c..79b35fdaa1cd 100644
--- a/arch/arm64/kvm/vgic/vgic-its.c
+++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -153,7 +153,7 @@ struct vgic_translation_cache_entry {
phys_addr_t db;
u32 devid;
u32 eventid;
- struct vgic_irq *irq;
+ struct vgic_irq __rcu *irq;
atomic64_t usage_count;
};

@@ -571,7 +571,7 @@ static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,
* If we hit a NULL entry, there is nothing after this
* point.
*/
- if (!cte->irq)
+ if (!rcu_access_pointer(cte->irq))
break;

if (cte->db != db || cte->devid != devid ||
@@ -579,7 +579,7 @@ static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,
continue;

atomic64_inc(&cte->usage_count);
- return cte->irq;
+ return rcu_dereference(cte->irq);
}

return NULL;
@@ -622,7 +622,7 @@ static struct vgic_translation_cache_entry *vgic_its_cache_victim(struct vgic_di
* deliberately non-atomic, so this is all best-effort.
*/
list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
- if (!cte->irq)
+ if (!rcu_access_pointer(cte->irq))
return cte;

tmp = atomic64_xchg_relaxed(&cte->usage_count, 0);
@@ -653,6 +653,7 @@ static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
return;

raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
+ rcu_read_lock();

/*
* We could have raced with another CPU caching the same
@@ -686,12 +687,13 @@ static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
new->db = db;
new->devid = devid;
new->eventid = eventid;
- new->irq = irq;
+ rcu_assign_pointer(new->irq, irq);

/* Move the new translation to the head of the list */
list_add(&new->entry, &dist->lpi_translation_cache);

out:
+ rcu_read_unlock();
raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);

/*
@@ -712,19 +714,21 @@ void vgic_its_invalidate_cache(struct kvm *kvm)
unsigned long flags;

raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
+ rcu_read_lock();

list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
/*
* If we hit a NULL entry, there is nothing after this
* point.
*/
- if (!cte->irq)
+ if (!rcu_access_pointer(cte->irq))
break;

vgic_put_irq(kvm, cte->irq);
- cte->irq = NULL;
+ rcu_assign_pointer(cte->irq, NULL);
}

+ rcu_read_unlock();
raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
}

--
2.43.0.429.g432eaa2c6b-goog