@@ -47,6 +47,9 @@
static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
long pte_index, unsigned long pteh,
unsigned long ptel, unsigned long *pte_idx_ret);
+static void kvmppc_virtmode_do_h_remove(struct kvm *kvm, unsigned long flags,
+ unsigned long pte_index, unsigned long avpn,
+ unsigned long *hpret);
struct kvm_resize_hpt {
/* These fields read-only after init */
@@ -308,6 +311,22 @@ static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
}
+/*
+ * Virtual-mode H_REMOVE. kvmppc_do_h_remove() is also called from real mode,
+ * where preempt_disable() is not usable, so the guard stays here. The function
+ * spins on HPTE_V_HVLOCK and then lock_rmap() (via remove_revmap_chain());
+ * preemption must be disabled across both bit-locks so that spinning callers
+ * holding preempt_disable() can make forward progress.
+ */
+static void kvmppc_virtmode_do_h_remove(struct kvm *kvm, unsigned long flags,
+ unsigned long pte_index, unsigned long avpn,
+ unsigned long *hpret)
+{
+ preempt_disable();
+ kvmppc_do_h_remove(kvm, flags, pte_index, avpn, hpret);
+ preempt_enable();
+}
+
static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
gva_t eaddr)
{
@@ -810,9 +829,11 @@ static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
for (;;) {
+ preempt_disable();
lock_rmap(rmapp);
if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
unlock_rmap(rmapp);
+ preempt_enable();
break;
}
@@ -826,6 +847,7 @@ static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
/* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp);
+ preempt_enable();
while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
cpu_relax();
continue;
@@ -834,6 +856,7 @@ static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);
unlock_rmap(rmapp);
__unlock_hpte(hptep, be64_to_cpu(hptep[0]));
+ preempt_enable();
}
}
@@ -890,6 +913,7 @@ static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
retry:
+ preempt_disable();
lock_rmap(rmapp);
if (*rmapp & KVMPPC_RMAP_REFERENCED) {
*rmapp &= ~KVMPPC_RMAP_REFERENCED;
@@ -897,6 +921,7 @@ static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
}
if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
unlock_rmap(rmapp);
+ preempt_enable();
return ret;
}
@@ -912,6 +937,7 @@ static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
/* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp);
+ preempt_enable();
while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
cpu_relax();
goto retry;
@@ -931,6 +957,7 @@ static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
} while ((i = j) != head);
unlock_rmap(rmapp);
+ preempt_enable();
return ret;
}
@@ -1219,6 +1246,7 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
return 0; /* nothing to do */
+ preempt_disable();
while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
cpu_relax();
@@ -1346,6 +1374,7 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
out:
unlock_hpte(hptep, vpte);
+ preempt_enable();
return ret;
}
@@ -1868,7 +1897,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
nb += HPTE_SIZE;
if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
- kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
+ kvmppc_virtmode_do_h_remove(kvm, 0, i, 0, tmp);
err = -EIO;
ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
tmp);
@@ -1897,7 +1926,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
for (j = 0; j < hdr.n_invalid; ++j) {
if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
- kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
+ kvmppc_virtmode_do_h_remove(kvm, 0, i, 0, tmp);
++i;
hptp += 2;
}
@@ -1159,6 +1159,10 @@ static long kvmppc_h_rpt_invalidate(struct kvm_vcpu *vcpu,
return H_SUCCESS;
}
+/*
+ * Must be called with preemption disabled. The HPT hcall handlers spin
+ * on HPTE bit-locks and cannot make any blocking/sleeping calls.
+ */
static long kvmppc_pseries_do_hpt_hcall(struct kvm_vcpu *vcpu, unsigned long req)
{
switch (req) {
@@ -1212,9 +1216,11 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
case H_CLEAR_REF:
case H_PROTECT:
case H_BULK_REMOVE:
+ preempt_disable();
idx = srcu_read_lock(&kvm->srcu);
ret = kvmppc_pseries_do_hpt_hcall(vcpu, req);
srcu_read_unlock(&kvm->srcu, idx);
+ preempt_enable();
if (ret == H_TOO_HARD)
return RESUME_HOST;
break;
@@ -1834,8 +1840,10 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
else
vsid = vcpu->arch.fault_gpa;
+ preempt_disable();
err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar,
vsid, vcpu->arch.fault_dsisr, true);
+ preempt_enable();
if (err == 0) {
r = RESUME_GUEST;
} else if (err == -1 || err == -2) {
@@ -1881,8 +1889,10 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
else
vsid = vcpu->arch.fault_gpa;
+ preempt_disable();
err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar,
vsid, vcpu->arch.fault_dsisr, false);
+ preempt_enable();
if (err == 0) {
r = RESUME_GUEST;
} else if (err == -1) {
kvmppc_hv_find_lock_hpte() requires virtual-mode callers to run with preemption disabled, because it can return with HPTE_V_HVLOCK still held until the caller later unlocks the HPTE. Existing virtual-mode callers in book3s_64_mmu_hv.c already follow that rule, but several paths do not. kvmppc_handle_exit_hv() calls kvmppc_hpte_hv_fault() for hash-mode data-side and instruction-side faults after guest exit with preemption enabled. kvmppc_pseries_do_hcall() executes virtual-mode HPT hcall handlers via kvmppc_pseries_do_hpt_hcall() with preemption enabled; the handlers for H_ENTER, H_REMOVE, H_READ, H_CLEAR_MOD, H_CLEAR_REF, H_PROTECT, and H_BULK_REMOVE all spin on try_lock_hpte() or lock_rmap(). H_ENTER also reaches kvmppc_do_h_enter(), which uses arch_spin_lock() on kvm->mmu_lock. That raw lock choice is intentional because kvmppc_do_h_enter() is also called from real-mode paths, so the correct fix is to establish the proper preemption context at the virtual-mode caller boundary. kvm_htab_write(), the HPT migration restore path, also called kvmppc_do_h_remove() directly with preemption enabled from two sites: once in the n_valid loop to evict any occupied slot before inserting a new HPTE, and once in the n_invalid loop to evict any occupied slot that the restore marks as invalid. This was inconsistent: the insert in the same function already used the guarded kvmppc_virtmode_do_h_enter() helper. On the host side, kvm_unmap_rmapp(), kvm_age_rmapp(), kvm_test_clear_dirty_npages(), and resize_hpt_rehash_hpte() also acquire HPTE_V_HVLOCK via try_lock_hpte() in process context with preemption enabled, serving MMU notifier callbacks, dirty-log harvesting, and HPT resize respectively. If any of these threads is preempted while holding HPTE_V_HVLOCK, any other thread on the same CPU spinning on the same bit-lock can never make progress, as the lock owner cannot be rescheduled to release it. This is particularly acute when the spinning thread has preemption disabled: it will never yield, causing a permanent CPU hang. Fix this by adding preempt_disable()/preempt_enable() pairs around the two kvmppc_hpte_hv_fault() call sites in kvmppc_handle_exit_hv() and around the kvmppc_pseries_do_hpt_hcall() invocation in kvmppc_pseries_do_hcall(). For kvm_htab_write(), introduce kvmppc_virtmode_do_h_remove(), a thin wrapper that brackets kvmppc_do_h_remove() with preempt_disable()/ preempt_enable(), mirroring the existing kvmppc_virtmode_do_h_enter() pattern. The wrapper retains the full flags and avpn parameters so that future virtual-mode callers can use the H_AVPN and H_ANDCOND conditional guards if needed. Note that kvmppc_do_h_remove() spins on both HPTE_V_HVLOCK and lock_rmap() (via remove_revmap_chain()); preemption must be disabled across both bit-locks. For kvm_unmap_rmapp() and kvm_age_rmapp(), place preempt_disable() before lock_rmap() so that both the rmap chain lock and the subsequent HPTE_V_HVLOCK bit-lock are held under a single non-preemptible window. There is an ABBA ordering constraint between the two locks: the rmap chain lock must be dropped before spinning on the HPTE bit-lock (documented in the comment above the try_lock_hpte() call in kvm_unmap_rmapp()). To preserve this, preempt_enable() is called after unlock_rmap() on the failed try_lock_hpte() retry path and on any early-exit path, before the cpu_relax() spin, so the HPTE lock owner can be scheduled. For kvm_test_clear_dirty_npages(), remove the per-iteration preempt_disable()/preempt_enable() pairs: this function has a single call site, kvmppc_hv_get_dirty_log_hpt(), which already holds preempt_disable() across the entire loop, making the inner guards redundant. For resize_hpt_rehash_hpte(), place preempt_disable() before the unconditional try_lock_hpte() spin loop and preempt_enable() after unlock_hpte() at the single exit point. This function is called from kvm_vm_ioctl_resize_hpt_commit(), which first quiesces all vCPUs by clearing kvm->arch.mmu_ready and calling on_each_cpu() to flush any vCPU currently running in guest mode back to host. With all vCPUs out of the guest, no vCPU thread can hold HPTE_V_HVLOCK; any remaining lock holder (an MMU notifier callback or dirty-log walker) runs on a separate CPU and is not preempted, so the spin always makes forward progress. Fixes: 6165d5dd99db ("KVM: PPC: Book3S HV: add virtual mode handlers for HPT hcalls and page faults") Cc: stable@vger.kernel.org # v5.14+ Signed-off-by: Amit Machhiwal <amachhiw@linux.ibm.com> --- Changes in v4: - Introduced kvmppc_virtmode_do_h_remove(), a thin wrapper bracketing kvmppc_do_h_remove() with preempt_disable()/preempt_enable(), mirroring kvmppc_virtmode_do_h_enter(). Replaced both raw kvmppc_do_h_remove() call sites in kvm_htab_write() with the new wrapper. - Updated commit message to document the kvm_htab_write() fix. - Dropped Reviewed-by from Shrikanth as the patch was materially extended. arch/powerpc/kvm/book3s_64_mmu_hv.c | 33 +++++++++++++++++++++++++++-- arch/powerpc/kvm/book3s_hv.c | 10 +++++++++ 2 files changed, 41 insertions(+), 2 deletions(-)