Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/807334/?format=api
{ "id": 807334, "url": "http://patchwork.ozlabs.org/api/patches/807334/?format=api", "web_url": "http://patchwork.ozlabs.org/project/qemu-devel/patch/20170829220547.31136.57696.stgit@gimli.home/", "project": { "id": 14, "url": "http://patchwork.ozlabs.org/api/projects/14/?format=api", "name": "QEMU Development", "link_name": "qemu-devel", "list_id": "qemu-devel.nongnu.org", "list_email": "qemu-devel@nongnu.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20170829220547.31136.57696.stgit@gimli.home>", "list_archive_url": null, "date": "2017-08-29T22:05:47", "name": "[3/3] vfio/pci: Add NVIDIA GPUDirect Cliques support", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "5208062f16c5c0effeb90e5b8020ed82ce5e4c9c", "submitter": { "id": 4123, "url": "http://patchwork.ozlabs.org/api/people/4123/?format=api", "name": "Alex Williamson", "email": "alex.williamson@redhat.com" }, "delegate": null, "mbox": "http://patchwork.ozlabs.org/project/qemu-devel/patch/20170829220547.31136.57696.stgit@gimli.home/mbox/", "series": [ { "id": 492, "url": "http://patchwork.ozlabs.org/api/series/492/?format=api", "web_url": "http://patchwork.ozlabs.org/project/qemu-devel/list/?series=492", "date": "2017-08-29T22:05:24", "name": "vfio/pci: Add NVIDIA GPUDirect P2P clique support", "version": 1, "mbox": "http://patchwork.ozlabs.org/series/492/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/807334/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/807334/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org>", "X-Original-To": "incoming@patchwork.ozlabs.org", "Delivered-To": "patchwork-incoming@bilbo.ozlabs.org", "Authentication-Results": [ "ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=nongnu.org\n\t(client-ip=2001:4830:134:3::11; helo=lists.gnu.org;\n\tenvelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org;\n\treceiver=<UNKNOWN>)", "ext-mx08.extmail.prod.ext.phx2.redhat.com;\n\tdmarc=none (p=none dis=none) header.from=redhat.com", "ext-mx08.extmail.prod.ext.phx2.redhat.com;\n\tspf=fail smtp.mailfrom=alex.williamson@redhat.com" ], "Received": [ "from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11])\n\t(using TLSv1 with cipher AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 3xhjTg4BLZz9s9Y\n\tfor <incoming@patchwork.ozlabs.org>;\n\tWed, 30 Aug 2017 08:09:11 +1000 (AEST)", "from localhost ([::1]:47243 helo=lists.gnu.org)\n\tby lists.gnu.org with esmtp (Exim 4.71) (envelope-from\n\t<qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org>)\n\tid 1dmoh3-0001Du-DW\n\tfor incoming@patchwork.ozlabs.org; Tue, 29 Aug 2017 18:09:09 -0400", "from eggs.gnu.org ([2001:4830:134:3::10]:39230)\n\tby lists.gnu.org with esmtp (Exim 4.71)\n\t(envelope-from <alex.williamson@redhat.com>) id 1dmods-00083R-TB\n\tfor qemu-devel@nongnu.org; Tue, 29 Aug 2017 18:05:54 -0400", "from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71)\n\t(envelope-from <alex.williamson@redhat.com>) id 1dmodr-0001nS-7C\n\tfor qemu-devel@nongnu.org; Tue, 29 Aug 2017 18:05:52 -0400", "from mx1.redhat.com ([209.132.183.28]:30014)\n\tby eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32)\n\t(Exim 4.71) (envelope-from <alex.williamson@redhat.com>)\n\tid 1dmodq-0001lm-UZ\n\tfor qemu-devel@nongnu.org; Tue, 29 Aug 2017 18:05:51 -0400", "from smtp.corp.redhat.com\n\t(int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11])\n\t(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby mx1.redhat.com (Postfix) with ESMTPS id EE9C3C0587E8;\n\tTue, 29 Aug 2017 22:05:49 +0000 (UTC)", "from gimli.home (ovpn-116-27.phx2.redhat.com [10.3.116.27])\n\tby smtp.corp.redhat.com (Postfix) with ESMTP id A4F8119EE7;\n\tTue, 29 Aug 2017 22:05:47 +0000 (UTC)" ], "DMARC-Filter": "OpenDMARC Filter v1.3.2 mx1.redhat.com EE9C3C0587E8", "From": "Alex Williamson <alex.williamson@redhat.com>", "To": "qemu-devel@nongnu.org", "Date": "Tue, 29 Aug 2017 16:05:47 -0600", "Message-ID": "<20170829220547.31136.57696.stgit@gimli.home>", "In-Reply-To": "<20170829214929.31136.21144.stgit@gimli.home>", "References": "<20170829214929.31136.21144.stgit@gimli.home>", "User-Agent": "StGit/0.17.1-dirty", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=\"utf-8\"", "Content-Transfer-Encoding": "7bit", "X-Scanned-By": "MIMEDefang 2.79 on 10.5.11.11", "X-Greylist": "Sender IP whitelisted, not delayed by milter-greylist-4.5.16\n\t(mx1.redhat.com [10.5.110.32]);\n\tTue, 29 Aug 2017 22:05:50 +0000 (UTC)", "X-detected-operating-system": "by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic]\n\t[fuzzy]", "X-Received-From": "209.132.183.28", "Subject": "[Qemu-devel] [PATCH 3/3] vfio/pci: Add NVIDIA GPUDirect Cliques\n\tsupport", "X-BeenThere": "qemu-devel@nongnu.org", "X-Mailman-Version": "2.1.21", "Precedence": "list", "List-Id": "<qemu-devel.nongnu.org>", "List-Unsubscribe": "<https://lists.nongnu.org/mailman/options/qemu-devel>,\n\t<mailto:qemu-devel-request@nongnu.org?subject=unsubscribe>", "List-Archive": "<http://lists.nongnu.org/archive/html/qemu-devel/>", "List-Post": "<mailto:qemu-devel@nongnu.org>", "List-Help": "<mailto:qemu-devel-request@nongnu.org?subject=help>", "List-Subscribe": "<https://lists.nongnu.org/mailman/listinfo/qemu-devel>,\n\t<mailto:qemu-devel-request@nongnu.org?subject=subscribe>", "Cc": "a175818323@gmail.com", "Errors-To": "qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org", "Sender": "\"Qemu-devel\"\n\t<qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org>" }, "content": "NVIDIA has defined a specification for creating GPUDirect \"cliques\",\nwhere devices with the same clique ID support direct peer-to-peer DMA.\nWhen running on bare-metal, tools like NVIDIA's p2pBandwidthLatencyTest\n(part of cuda-samples) determine which GPUs can support peer-to-peer\nbased on chipset and topology. When running in a VM, these tools have\nno visibility to the physical hardware support or topology. This\noption allows the user to specify hints via a vendor defined\ncapability. For instance:\n\n <qemu:commandline>\n <qemu:arg value='-set'/>\n <qemu:arg value='device.hostdev0.x-nv-gpudirect-clique=0'/>\n <qemu:arg value='-set'/>\n <qemu:arg value='device.hostdev1.x-nv-gpudirect-clique=1'/>\n <qemu:arg value='-set'/>\n <qemu:arg value='device.hostdev2.x-nv-gpudirect-clique=1'/>\n </qemu:commandline>\n\nThis enables two cliques. The first is a singleton clique with ID 0,\nfor the first hostdev defined in the XML (note that since cliques\ndefine peer-to-peer sets, singleton clique offer no benefit). The\nsubsequent two hostdevs are both added to clique ID 1, indicating\npeer-to-peer is possible between these devices.\n\nQEMU only provides validation that the clique ID is valid and applied\nto an NVIDIA graphics device, any validation that the resulting\ncliques are functional and valid is the user's responsibility. The\nNVIDIA specification allows a 4-bit clique ID, thus valid values are\n0-15.\n\nSigned-off-by: Alex Williamson <alex.williamson@redhat.com>\n---\n hw/vfio/pci-quirks.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++\n hw/vfio/pci.c | 5 ++\n hw/vfio/pci.h | 3 +\n 3 files changed, 118 insertions(+)", "diff": "diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c\nindex 40aaae76feb9..14291c2a16b2 100644\n--- a/hw/vfio/pci-quirks.c\n+++ b/hw/vfio/pci-quirks.c\n@@ -14,6 +14,7 @@\n #include \"qemu/error-report.h\"\n #include \"qemu/range.h\"\n #include \"qapi/error.h\"\n+#include \"qapi/visitor.h\"\n #include \"hw/nvram/fw_cfg.h\"\n #include \"pci.h\"\n #include \"trace.h\"\n@@ -1850,7 +1851,116 @@ void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev)\n break;\n }\n }\n+\n+/*\n+ * The NVIDIA GPUDirect P2P Vendor capability allows the user to specify\n+ * devices as a member of a clique. Devices within the same clique ID\n+ * are capable of direct P2P. It's the user's responsibility that this\n+ * is correct. The spec says that this may reside at any unused config\n+ * offset, but reserves and recommends hypervisors place this at C8h.\n+ * The spec also states that the hypervisor should place this capability\n+ * at the end of the capability list, thus next is defined as 0h.\n+ *\n+ * +----------------+----------------+----------------+----------------+\n+ * | sig 7:0 ('P') | vndr len (8h) | next (0h) | cap id (9h) |\n+ * +----------------+----------------+----------------+----------------+\n+ * | rsvd 15:7(0h),id 6:3,ver 2:0(0h)| sig 23:8 ('P2') |\n+ * +---------------------------------+---------------------------------+\n+ *\n+ * https://lists.gnu.org/archive/html/qemu-devel/2017-08/pdfUda5iEpgOS.pdf\n+ */\n+static void get_nv_gpudirect_clique_id(Object *obj, Visitor *v,\n+ const char *name, void *opaque,\n+ Error **errp)\n+{\n+ DeviceState *dev = DEVICE(obj);\n+ Property *prop = opaque;\n+ uint8_t *ptr = qdev_get_prop_ptr(dev, prop);\n+\n+ visit_type_uint8(v, name, ptr, errp);\n+}\n+\n+static void set_nv_gpudirect_clique_id(Object *obj, Visitor *v,\n+ const char *name, void *opaque,\n+ Error **errp)\n+{\n+ DeviceState *dev = DEVICE(obj);\n+ Property *prop = opaque;\n+ uint8_t value, *ptr = qdev_get_prop_ptr(dev, prop);\n+ Error *local_err = NULL;\n+\n+ if (dev->realized) {\n+ qdev_prop_set_after_realize(dev, name, errp);\n+ return;\n+ }\n+\n+ visit_type_uint8(v, name, &value, &local_err);\n+ if (local_err) {\n+ error_propagate(errp, local_err);\n+ return;\n+ }\n+\n+ if (value & ~0xF) {\n+ error_setg(errp, \"Property %s: valid range 0-15\", name);\n+ return;\n+ }\n+\n+ *ptr = value;\n+}\n+\n+const PropertyInfo qdev_prop_nv_gpudirect_clique = {\n+ .name = \"uint4\",\n+ .description = \"NVIDIA GPUDirect Clique ID (0 - 15)\",\n+ .get = get_nv_gpudirect_clique_id,\n+ .set = set_nv_gpudirect_clique_id,\n+};\n+\n+static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp)\n+{\n+ PCIDevice *pdev = &vdev->pdev;\n+ int ret, pos = 0xC8;\n+\n+ if (vdev->nv_gpudirect_clique == 0xFF) {\n+ return 0;\n+ }\n+\n+ if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID)) {\n+ error_setg(errp, \"NVIDIA GPUDirect Clique ID: invalid device vendor\");\n+ return -EINVAL;\n+ }\n+\n+ if (pci_get_byte(pdev->config + PCI_CLASS_DEVICE + 1) !=\n+ PCI_BASE_CLASS_DISPLAY) {\n+ error_setg(errp, \"NVIDIA GPUDirect Clique ID: unsupported PCI class\");\n+ return -EINVAL;\n+ }\n+\n+ ret = pci_add_capability(pdev, PCI_CAP_ID_VNDR, pos, 8, errp);\n+ if (ret < 0) {\n+ error_prepend(errp, \"Failed to add NVIDIA GPUDirect cap: \");\n+ return ret;\n+ }\n+\n+ memset(vdev->emulated_config_bits + pos, 0xFF, 8);\n+ pos += PCI_CAP_FLAGS;\n+ pci_set_byte(pdev->config + pos++, 8);\n+ pci_set_byte(pdev->config + pos++, 'P');\n+ pci_set_byte(pdev->config + pos++, '2');\n+ pci_set_byte(pdev->config + pos++, 'P');\n+ pci_set_byte(pdev->config + pos++, vdev->nv_gpudirect_clique << 3);\n+ pci_set_byte(pdev->config + pos, 0);\n+\n+ return 0;\n+}\n+\n int vfio_add_virt_caps(VFIOPCIDevice *vdev, Error **errp)\n {\n+ int ret;\n+\n+ ret = vfio_add_nv_gpudirect_cap(vdev, errp);\n+ if (ret) {\n+ return ret;\n+ }\n+\n return 0;\n }\ndiff --git a/hw/vfio/pci.c b/hw/vfio/pci.c\nindex bfeaaef22d00..9e86db7c3b6d 100644\n--- a/hw/vfio/pci.c\n+++ b/hw/vfio/pci.c\n@@ -2970,6 +2970,8 @@ static void vfio_instance_init(Object *obj)\n vdev->host.bus = ~0U;\n vdev->host.slot = ~0U;\n vdev->host.function = ~0U;\n+\n+ vdev->nv_gpudirect_clique = 0xFF;\n }\n \n static Property vfio_pci_dev_properties[] = {\n@@ -2994,6 +2996,9 @@ static Property vfio_pci_dev_properties[] = {\n DEFINE_PROP_UINT32(\"x-pci-sub-device-id\", VFIOPCIDevice,\n sub_device_id, PCI_ANY_ID),\n DEFINE_PROP_UINT32(\"x-igd-gms\", VFIOPCIDevice, igd_gms, 0),\n+ DEFINE_PROP_UNSIGNED_NODEFAULT(\"x-nv-gpudirect-clique\", VFIOPCIDevice,\n+ nv_gpudirect_clique,\n+ qdev_prop_nv_gpudirect_clique, uint8_t),\n /*\n * TODO - support passed fds... is this necessary?\n * DEFINE_PROP_STRING(\"vfiofd\", VFIOPCIDevice, vfiofd_name),\ndiff --git a/hw/vfio/pci.h b/hw/vfio/pci.h\nindex 958cee058b3b..502a5755b944 100644\n--- a/hw/vfio/pci.h\n+++ b/hw/vfio/pci.h\n@@ -135,6 +135,7 @@ typedef struct VFIOPCIDevice {\n int32_t bootindex;\n uint32_t igd_gms;\n uint8_t pm_cap;\n+ uint8_t nv_gpudirect_clique;\n bool pci_aer;\n bool req_enabled;\n bool has_flr;\n@@ -162,6 +163,8 @@ void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr);\n void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev);\n int vfio_add_virt_caps(VFIOPCIDevice *vdev, Error **errp);\n \n+extern const PropertyInfo qdev_prop_nv_gpudirect_clique;\n+\n int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp);\n \n int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev,\n", "prefixes": [ "3/3" ] }