From patchwork Fri Nov 9 05:44:57 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexey Kardashevskiy X-Patchwork-Id: 995319 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [203.11.71.2]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 42rpz45bKxz9sBk for ; Fri, 9 Nov 2018 16:45:36 +1100 (AEDT) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=ozlabs.ru Received: from lists.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 42rpz42m1JzF3X3 for ; Fri, 9 Nov 2018 16:45:36 +1100 (AEDT) Authentication-Results: lists.ozlabs.org; dmarc=none (p=none dis=none) header.from=ozlabs.ru X-Original-To: skiboot@lists.ozlabs.org Delivered-To: skiboot@lists.ozlabs.org Authentication-Results: lists.ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=ozlabs.ru (client-ip=107.173.13.209; helo=ozlabs.ru; envelope-from=aik@ozlabs.ru; receiver=) Authentication-Results: lists.ozlabs.org; dmarc=none (p=none dis=none) header.from=ozlabs.ru Received: from ozlabs.ru (unknown [107.173.13.209]) by lists.ozlabs.org (Postfix) with ESMTP id 42rpz03Mn6zF3Wt for ; Fri, 9 Nov 2018 16:45:32 +1100 (AEDT) Received: from fstn1-p1.ozlabs.ibm.com (localhost [IPv6:::1]) by ozlabs.ru (Postfix) with ESMTP id 84DBAAE8003F; Fri, 9 Nov 2018 00:44:59 -0500 (EST) From: Alexey Kardashevskiy To: skiboot@lists.ozlabs.org Date: Fri, 9 Nov 2018 16:44:57 +1100 Message-Id: <20181109054457.34000-1-aik@ozlabs.ru> X-Mailer: git-send-email 2.17.1 Subject: [Skiboot] [PATCH skiboot] npu2: Add XTS_BDF_MAP wildcard refcount X-BeenThere: skiboot@lists.ozlabs.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Mailing list for skiboot development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Alistair Popple , Reza Arbab MIME-Version: 1.0 Errors-To: skiboot-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org Sender: "Skiboot" Currently PID wildcard is programmed into the NPU once and never cleared up. This works for the bare metal as MSR does not change while the host OS is running. However with the device virtualization, we need to keep track of wildcard entries use and clear them up before switching a GPU from a host to a guest or vice versa. This adds refcount to a NPU2, one counter per wildcard entry. The index is a short lparid which is allocated in opal_npu_map_lpar() and should be smaller than NPU2_XTS_BDF_MAP_SIZE. Signed-off-by: Alexey Kardashevskiy --- This is a rework of [PATCH skiboot] npu2: Clear XTS_BDF_MAP when destroying context for next init_context --- include/npu2.h | 2 ++ hw/npu2.c | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/include/npu2.h b/include/npu2.h index 1de963d1..d05c0011 100644 --- a/include/npu2.h +++ b/include/npu2.h @@ -19,6 +19,7 @@ #include #include +#include /* Debugging options */ #define NPU2DBG(p, fmt, a...) prlog(PR_DEBUG, "NPU%d: " fmt, \ @@ -162,6 +163,7 @@ struct npu2 { uint32_t total_devices; struct npu2_dev *devices; enum phys_map_type gpu_map_type; + int ctx_ref[NPU2_XTS_BDF_MAP_SIZE]; /* IODA cache */ uint64_t lxive_cache[8]; diff --git a/hw/npu2.c b/hw/npu2.c index ba1264be..e69a7dc7 100644 --- a/hw/npu2.c +++ b/hw/npu2.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -2097,6 +2096,11 @@ static int64_t opal_npu_init_context(uint64_t phb_id, int pasid __unused, } id = GETFIELD(NPU2_XTS_BDF_MAP_LPARSHORT, xts_bdf); + if (id >= NPU2_XTS_BDF_MAP_SIZE) { + NPU2ERR(p, "Out of bounds id=%d\n", id); + id = OPAL_INTERNAL_ERROR; + goto out; + } NPU2DBG(p, "Found LPARSHORT = 0x%x for BDF = 0x%03llx\n", id, bdf); /* Enable this mapping for both real and virtual addresses */ @@ -2129,14 +2133,20 @@ static int64_t opal_npu_init_context(uint64_t phb_id, int pasid __unused, GETFIELD(NPU2_XTS_PID_MAP_MSR, xts_bdf_pid)) { NPU2ERR(p, "%s: Unexpected MSR value\n", __func__); id = OPAL_PARAMETER; + } else if (!p->ctx_ref[id]) { + NPU2ERR(p, "%s: Unexpected mapping\n", __func__); + id = OPAL_INTERNAL_ERROR; } goto out; } /* Write the entry */ - NPU2DBG(p, "XTS_PID_MAP[%03d] = 0x%08llx\n", id, xts_bdf_pid); - npu2_write(p, NPU2_XTS_PID_MAP + id*0x20, xts_bdf_pid); + if (!p->ctx_ref[id]) { + NPU2DBG(p, "XTS_PID_MAP[%03d] = 0x%08llx\n", id, xts_bdf_pid); + npu2_write(p, NPU2_XTS_PID_MAP + id*0x20, xts_bdf_pid); + } + ++p->ctx_ref[id]; if (!GETFIELD(NPU2_XTS_BDF_MAP_VALID, xts_bdf)) { xts_bdf = SETFIELD(NPU2_XTS_BDF_MAP_VALID, xts_bdf, 1); @@ -2155,7 +2165,7 @@ static int opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused, struct phb *phb = pci_get_phb(phb_id); struct npu2 *p; uint64_t xts_bdf; - int rc = 0; + int rc = 0, id; if (!phb || phb->phb_type != phb_type_npu_v2) return OPAL_PARAMETER; @@ -2172,10 +2182,24 @@ static int opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused, } /* - * The bdf/pid table only contains wildcard entries, so we don't - * need to remove anything here. + * The bdf/pid table contains wildcard entries and MSR bits which + * we need to clear between switching a device from a host to a guest + * or vice versa. */ - + id = GETFIELD(NPU2_XTS_BDF_MAP_LPARSHORT, xts_bdf); + if (id >= NPU2_XTS_BDF_MAP_SIZE) { + NPU2ERR(p, "Out of bounds id=%d\n", id); + rc = OPAL_INTERNAL_ERROR; + } else { + --p->ctx_ref[id]; + rc = p->ctx_ref[id]; /* For debug */ + if (p->ctx_ref[id] < 0) { + rc = OPAL_INTERNAL_ERROR; /* Sanity check */ + } else if (!p->ctx_ref[id]) { + NPU2DBG(p, "XTS_PID_MAP[%03d] = 0 (destroy)\n", id); + npu2_write(p, NPU2_XTS_PID_MAP + id*0x20, 0); + } + } unlock(&p->lock); return rc; }