Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/807357/?format=api
{ "id": 807357, "url": "http://patchwork.ozlabs.org/api/patches/807357/?format=api", "web_url": "http://patchwork.ozlabs.org/project/sparclinux/patch/1504049763-51918-3-git-send-email-rob.gardner@oracle.com/", "project": { "id": 10, "url": "http://patchwork.ozlabs.org/api/projects/10/?format=api", "name": "Linux SPARC Development ", "link_name": "sparclinux", "list_id": "sparclinux.vger.kernel.org", "list_email": "sparclinux@vger.kernel.org", "web_url": null, "scm_url": null, "webscm_url": null, "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<1504049763-51918-3-git-send-email-rob.gardner@oracle.com>", "list_archive_url": null, "date": "2017-08-29T23:36:03", "name": "[RFC,2/2] sparc64: Oracle DAX driver", "commit_ref": null, "pull_url": null, "state": "rfc", "archived": false, "hash": "0b76bdecc642d4b22f673e842fb2a6bcf1ad2514", "submitter": { "id": 65883, "url": "http://patchwork.ozlabs.org/api/people/65883/?format=api", "name": "Rob Gardner", "email": "rob.gardner@oracle.com" }, "delegate": { "id": 34, "url": "http://patchwork.ozlabs.org/api/users/34/?format=api", "username": "davem", "first_name": "David", "last_name": "Miller", "email": "davem@davemloft.net" }, "mbox": "http://patchwork.ozlabs.org/project/sparclinux/patch/1504049763-51918-3-git-send-email-rob.gardner@oracle.com/mbox/", "series": [ { "id": 505, "url": "http://patchwork.ozlabs.org/api/series/505/?format=api", "web_url": "http://patchwork.ozlabs.org/project/sparclinux/list/?series=505", "date": "2017-08-29T23:36:02", "name": "[RFC,1/2] sparc64: Oracle DAX infrastructure", "version": 1, "mbox": "http://patchwork.ozlabs.org/series/505/mbox/" } ], "comments": "http://patchwork.ozlabs.org/api/patches/807357/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/807357/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<sparclinux-owner@vger.kernel.org>", "X-Original-To": "patchwork-incoming@ozlabs.org", "Delivered-To": "patchwork-incoming@ozlabs.org", "Authentication-Results": "ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=sparclinux-owner@vger.kernel.org;\n\treceiver=<UNKNOWN>)", "Received": [ "from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xhlR03XhJz9sMN\n\tfor <patchwork-incoming@ozlabs.org>;\n\tWed, 30 Aug 2017 09:37:00 +1000 (AEST)", "(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1751318AbdH2XhA (ORCPT <rfc822;patchwork-incoming@ozlabs.org>);\n\tTue, 29 Aug 2017 19:37:00 -0400", "from aserp1040.oracle.com ([141.146.126.69]:50576 \"EHLO\n\taserp1040.oracle.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1751272AbdH2Xg6 (ORCPT\n\t<rfc822; sparclinux@vger.kernel.org>); Tue, 29 Aug 2017 19:36:58 -0400", "from aserv0022.oracle.com (aserv0022.oracle.com [141.146.126.234])\n\tby aserp1040.oracle.com (Sentrion-MTA-4.3.2/Sentrion-MTA-4.3.2)\n\twith ESMTP id v7TNawLS014110\n\t(version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256\n\tverify=OK)\n\tfor <sparclinux@vger.kernel.org>; Tue, 29 Aug 2017 23:36:58 GMT", "from ca-qasparc12.us.oracle.com (ca-qasparc12.us.oracle.com\n\t[10.147.25.210])\n\tby aserv0022.oracle.com (8.14.4/8.14.4) with ESMTP id v7TNajof023935; \n\tTue, 29 Aug 2017 23:36:57 GMT" ], "From": "Rob Gardner <rob.gardner@oracle.com>", "To": "sparclinux@vger.kernel.org", "Cc": "Rob Gardner <rob.gardner@oracle.com>,\n\tJonathan Helman <jonathan.helman@oracle.com>,\n\tSanath Kumar <sanath.s.kumar@oracle.com>", "Subject": "[RFC 2/2] sparc64: Oracle DAX driver", "Date": "Tue, 29 Aug 2017 17:36:03 -0600", "Message-Id": "<1504049763-51918-3-git-send-email-rob.gardner@oracle.com>", "X-Mailer": "git-send-email 1.7.1", "In-Reply-To": "<1504049763-51918-1-git-send-email-rob.gardner@oracle.com>", "References": "<1504049763-51918-1-git-send-email-rob.gardner@oracle.com>", "X-Source-IP": "aserv0022.oracle.com [141.146.126.234]", "Sender": "sparclinux-owner@vger.kernel.org", "Precedence": "bulk", "List-ID": "<sparclinux.vger.kernel.org>", "X-Mailing-List": "sparclinux@vger.kernel.org" }, "content": "DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8\n(DAX2) processor chips, and has direct access to the CPU's L3 caches\nas well as physical memory. It can perform several operations on data\nstreams with various input and output formats. This driver provides a\ntransport mechanism and does not have knowledge of the various opcodes\nand data formats. A user space library provides high level services\nand translates these into low level commands which are then passed\ninto the driver and subsequently the hypervisor and the coprocessor.\nThe library is the recommended way for applications to use the\ncoprocessor, and the driver interface is not intended for general use.\n\nSigned-off-by: Rob Gardner <rob.gardner@oracle.com>\nSigned-off-by: Jonathan Helman <jonathan.helman@oracle.com>\nSigned-off-by: Sanath Kumar <sanath.s.kumar@oracle.com>\n---\n Documentation/sparc/oracle_dax.txt | 210 ++++++++\n arch/sparc/include/uapi/asm/oradax.h | 91 ++++\n drivers/sbus/char/Kconfig | 8 +\n drivers/sbus/char/Makefile | 1 +\n drivers/sbus/char/oradax.c | 951 ++++++++++++++++++++++++++++++++++\n 5 files changed, 1261 insertions(+), 0 deletions(-)\n create mode 100644 Documentation/sparc/oracle_dax.txt\n create mode 100644 arch/sparc/include/uapi/asm/oradax.h\n create mode 100644 drivers/sbus/char/oradax.c", "diff": "diff --git a/Documentation/sparc/oracle_dax.txt b/Documentation/sparc/oracle_dax.txt\nnew file mode 100644\nindex 0000000..3265081\n--- /dev/null\n+++ b/Documentation/sparc/oracle_dax.txt\n@@ -0,0 +1,210 @@\n+Oracle Data Analytics Accelerator (DAX)\n+---------------------------------------\n+\n+DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8\n+(DAX2) processor chips, and has direct access to the CPU's L3 caches\n+as well as physical memory. It can perform several operations on data\n+streams with various input and output formats. A driver provides a\n+transport mechanism and does not have knowledge of the various opcodes\n+and data formats. A user space library (insert link here) provides\n+high level services and translates these into low level commands which\n+are then passed into the driver and subsequently the Hypervisor and\n+the coprocessor. The library is the recommended way for applications\n+to use the coprocessor, and the driver interface is not intended for\n+general use. This document describes the general flow of the driver,\n+its structures, and its programmatic interface.\n+\n+\n+High Level Overview\n+-------------------\n+\n+A coprocessor request is described by a Command Control Block\n+(CCB). The CCB contains an opcode and various parameters. The opcode\n+specifies what operation is to be done, and the parameters specify\n+options, flags, sizes, and addresses. The CCB (or an array of CCBs)\n+is passed to the Hypervisor, which handles queueing and scheduling of\n+requests to the available coprocessor execution units. A status code\n+returned indicates if the request was submitted successfully or if\n+there was an error. One of the addresses given in each CCB is a\n+pointer to a \"completion area\", which is a 128 byte memory block that\n+is written by the coprocessor to provide execution status. No\n+interrupt is generated upon completion; the completion area must be\n+polled by software to find out when a transaction has finished, but\n+the M7 and later processors provide a mechanism to pause the virtual\n+processor until the completion status has been updated by the\n+coprocessor. This is done using the monitored load and mwait\n+instructions, which are described in more detail later. The DAX\n+coprocessor was designed so that after a request is submitted, the\n+kernel is no longer involved in the processing of it. The polling is\n+done at the user level, which results in almost zero latency between\n+completion of a request and resumption of execution of the requesting\n+thread.\n+\n+\n+Addressing Memory\n+-----------------\n+\n+The kernel does not have access to physical memory in the Sun4v\n+architecture, as there is an additional level of memory virtualization\n+present. This intermediate level is called \"real\" memory, and the\n+kernel treats this as if it were physical. The Hypervisor handles the\n+translations between real memory and physical so that each logical\n+domain (LDOM) can have a partition of physical memory that is isolated\n+from that of other LDOMs. When the kernel sets up a virtual mapping,\n+it specifies a virtual address and the real address to which it should\n+be mapped.\n+\n+The DAX coprocessor can only operate on physical memory, so before a\n+request can be fed to the coprocessor, all the addresses in a CCB must\n+be converted into physical addresses. The kernel cannot do this since\n+it has no visibility into physical addresses. So a CCB may contain\n+either the virtual or real addresses of the buffers or a combination\n+of them. An \"address type\" field is available for each address that\n+may be given in the CCB. In all cases, the Hypervisor will translate\n+all the addresses to physical before dispatching to hardware.\n+\n+\n+The Driver API\n+--------------\n+\n+An application makes requests to the driver via the write() system\n+call, and gets results (if any) via read(). The completion areas are\n+made accessible via mmap(), and are read-only for the application.\n+\n+The request may either be an immediate command or an array of CCBs to\n+be submitted to the hardware.\n+\n+Each open instance of the device is exclusive to the thread that\n+opened it, and must be used by that thread for all subsequent\n+operations. The driver open function creates a new context for the\n+thread and initializes it for use. This context contains pointers and\n+values used internally by the driver to keep track of submitted\n+requests. The completion area buffer is also allocated, and this is\n+large enough to contain the completion areas for many concurrent\n+requests. When the device is closed, any outstanding transactions are\n+flushed and the context is cleaned up.\n+\n+On a DAX1 system (M7), the device will be called \"oradax1\", while on a\n+DAX2 system (M8) it will be \"oradax2\". If an application requires one\n+or the other, it should simply attempt to open the appropriate\n+device. Only one of the devices will exist on any given system, so the\n+name can be used to determine what the platform supports.\n+\n+The immediate commands are CCB_DEQUEUE, CCB_KILL, and CCB_INFO. For\n+all of these, success is indicated by a return value from write()\n+equal to the number of bytes given in the call. Otherwise -1 is\n+returned and errno is set.\n+\n+CCB_DEQUEUE\n+\n+Tells the driver to clean up resources associated with past\n+requests. Since no interrupt is generated upon the completion of a\n+request, the driver must be told when it may reclaim resources. No\n+further status information is returned, so the user should not\n+subsequently call read().\n+\n+CCB_KILL\n+\n+Kills a CCB during execution. The CCB is guaranteed to not continue\n+executing once this call returns successfully. On success, read() must\n+be called to retrieve the result of the action.\n+\n+CCB_INFO\n+\n+Retrieves information about a currently executing CCB. Note that some\n+Hypervisors might return 'notfound' when the CCB is in 'inprogress'\n+state. To ensure a CCB in the 'notfound' state will never be executed,\n+CCB_KILL must be invoked on that CCB. Upon success, read() must be\n+called to retrieve the details of the action.\n+\n+Submission of an array of CCBs for execution\n+\n+A write() whose length is a multiple of the CCB size is treated as a\n+submit operation. The file offset is treated as the index of the\n+completion area to use, and may be set via lseek() or using the\n+pwrite() system call. If -1 is returned then errno is set to indicate\n+the error. Otherwise, the return value is the length of the array that\n+was actually accepted by the coprocessor. If the accepted length is\n+equal to the requested length, then the operation was completely\n+successful and there is no further status needed; hence, the user\n+should not subsequently call read(). Partial acceptance of the CCB\n+array is indicated by a return value less than the requested length,\n+and read() must be called to retrieve further status information. The\n+status will reflect the error caused by the first CCB that was not\n+accepted, and status_data will provide additional data in some cases.\n+\n+MMAP\n+\n+The mmap() function provides access to the completion area allocated\n+in the driver. Note that the completion area is not writeable by the\n+user process.\n+\n+\n+Completion of a Request\n+-----------------------\n+\n+The first byte in each completion area is the command status which is\n+updated by the coprocessor hardware. Software may take advantage of\n+new M7/M8 processor capabilities to efficiently poll this status byte.\n+First, a \"monitored load\" is achieved via a Load from Alternate Space\n+(ldxa, lduba, etc.) with ASI 0x84 (ASI_MONITOR_PRIMARY). Second, a\n+\"monitored wait\" is achieved via the mwait instruction. This\n+instruction is like pause in that it suspends execution of the virtual\n+processor, but in addition will terminate early when one of several\n+events occur. If the block of data containing the monitored location\n+is modified, then the mwait terminates. This allows software to resume\n+execution immediately (without a context switch or kernel to user\n+transition) after a transaction completes. Thus the latency between\n+transaction completion and resumption of execution may be just a few\n+nanoseconds.\n+\n+\n+Application Life Cycle of a DAX Submission\n+------------------------------------------\n+\n+ - open dax device\n+ - call mmap() to get the completion area address\n+ - allocate a CCB and fill in the opcode, flags, parameter, addresses, etc.\n+ - submit CCB via write()\n+ - go into a loop executing monitored load + monitored wait and\n+ terminate when the command status indicates the request is complete\n+ (CCB_KILL or CCB_INFO may be used any time as necessary)\n+ - perform a CCB_DEQUEUE\n+ - call munmap() for completion area\n+ - close the dax device\n+\n+\n+Memory Constraints\n+------------------\n+\n+The DAX hardware operates only on physical addresses. Therefore, it is\n+not aware of virtual memory mappings and the discontiguities that may\n+exist in the physical memory that a virtual buffer maps to. There is\n+no I/O TLB or any scatter/gather mechanism. All buffers, whether input\n+or output, must reside in a physically contiguous region of memory.\n+\n+The Hypervisor translates all addresses within a CCB to physical\n+before handing off the CCB to DAX. The Hypervisor determines the\n+virtual page size for each virtual address given, and uses this to\n+program a size limit for each address. This prevents the coprocessor\n+from reading or writing beyond the bound of the virtual page, even\n+though it is accessing physical memory directly. A simpler way of\n+saying this is that a DAX operation will never \"cross\" a virtual page\n+boundary. If an 8k virtual page is used, then the data is strictly\n+limited to 8k. If a user's buffer is larger than 8k, then a larger\n+page size must be used, or the transaction size will be truncated to\n+8k.\n+\n+Huge pages. A user may allocate huge pages using standard\n+interfaces. Memory buffers residing on huge pages may be used to\n+achieve much larger DAX transaction sizes, but the rules must still be\n+followed, and no transaction will cross a page boundary, even a huge\n+page. A major caveat is that Linux on Sparc presents 8Mb as one of\n+the huge page sizes. Sparc does not actually provide a 8Mb hardware\n+page size, and this size is synthesized by pasting together two 4Mb\n+pages. The reasons for this are historical, and it creates an issue\n+because only half of this 8Mb page can actually be used for any given\n+buffer in a DAX request, and it must be either the first half or the\n+second half; it cannot be a 4Mb chunk in the middle, since that\n+crosses a (hardware) page boundary. Note that this entire issue may be\n+hidden by higher level libraries.\ndiff --git a/arch/sparc/include/uapi/asm/oradax.h b/arch/sparc/include/uapi/asm/oradax.h\nnew file mode 100644\nindex 0000000..7229519\n--- /dev/null\n+++ b/arch/sparc/include/uapi/asm/oradax.h\n@@ -0,0 +1,91 @@\n+/*\n+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.\n+ *\n+ * This program is free software: you can redistribute it and/or modify\n+ * it under the terms of the GNU General Public License as published by\n+ * the Free Software Foundation, either version 3 of the License, or\n+ * (at your option) any later version.\n+ *\n+ * This program is distributed in the hope that it will be useful,\n+ * but WITHOUT ANY WARRANTY; without even the implied warranty of\n+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n+ * GNU General Public License for more details.\n+ *\n+ * You should have received a copy of the GNU General Public License\n+ * along with this program. If not, see <http://www.gnu.org/licenses/>.\n+ */\n+\n+/*\n+ * Oracle DAX driver API definitions\n+ */\n+\n+#ifndef _ORADAX_H\n+#define\t_ORADAX_H\n+\n+#include <linux/types.h>\n+\n+#define\tCCB_KILL 0\n+#define\tCCB_INFO 1\n+#define\tCCB_DEQUEUE 2\n+\n+struct dax_command {\n+\t__u16 command;\t\t/* CCB_KILL/INFO/DEQUEUE */\n+\t__u16 ca_offset;\t/* offset into mmapped completion area */\n+};\n+\n+struct ccb_kill_result {\n+\t__u16 action;\t\t/* action taken to kill ccb */\n+};\n+\n+struct ccb_info_result {\n+\t__u16 state;\t\t/* state of enqueued ccb */\n+\t__u16 inst_num;\t\t/* dax instance number of enqueued ccb */\n+\t__u16 q_num;\t\t/* queue number of enqueued ccb */\n+\t__u16 q_pos;\t\t/* ccb position in queue */\n+};\n+\n+struct ccb_exec_result {\n+\t__u64\tstatus_data;\t/* additional status data (e.g. bad VA) */\n+\t__u32\tstatus;\t\t/* one of DAX_SUBMIT_* */\n+};\n+\n+union ccb_result {\n+\tstruct ccb_exec_result exec;\n+\tstruct ccb_info_result info;\n+\tstruct ccb_kill_result kill;\n+};\n+\n+#define\tDAX_MMAP_LEN\t\t(16 * 1024)\n+#define\tDAX_MAX_CCBS\t\t15\n+#define\tDAX_CCB_BUF_MAXLEN\t(DAX_MAX_CCBS * 64)\n+#define\tDAX_NAME\t\t\"oradax\"\n+\n+/* CCB_EXEC status */\n+#define\tDAX_SUBMIT_OK\t\t\t0\n+#define\tDAX_SUBMIT_ERR_RETRY\t\t1\n+#define\tDAX_SUBMIT_ERR_WOULDBLOCK\t2\n+#define\tDAX_SUBMIT_ERR_BUSY\t\t3\n+#define\tDAX_SUBMIT_ERR_THR_INIT\t\t4\n+#define\tDAX_SUBMIT_ERR_ARG_INVAL\t5\n+#define\tDAX_SUBMIT_ERR_CCB_INVAL\t6\n+#define\tDAX_SUBMIT_ERR_NO_CA_AVAIL\t7\n+#define\tDAX_SUBMIT_ERR_CCB_ARR_MMU_MISS\t8\n+#define\tDAX_SUBMIT_ERR_NOMAP\t\t9\n+#define\tDAX_SUBMIT_ERR_NOACCESS\t\t10\n+#define\tDAX_SUBMIT_ERR_TOOMANY\t\t11\n+#define\tDAX_SUBMIT_ERR_UNAVAIL\t\t12\n+#define\tDAX_SUBMIT_ERR_INTERNAL\t\t13\n+\n+/* CCB_INFO states - must match HV_CCB_STATE_* definitions */\n+#define\tDAX_CCB_COMPLETED\t0\n+#define\tDAX_CCB_ENQUEUED\t1\n+#define\tDAX_CCB_INPROGRESS\t2\n+#define\tDAX_CCB_NOTFOUND\t3\n+\n+/* CCB_KILL actions - must match HV_CCB_KILL_* definitions */\n+#define\tDAX_KILL_COMPLETED\t0\n+#define\tDAX_KILL_DEQUEUED\t1\n+#define\tDAX_KILL_KILLED\t\t2\n+#define\tDAX_KILL_NOTFOUND\t3\n+\n+#endif /* _ORADAX_H */\ndiff --git a/drivers/sbus/char/Kconfig b/drivers/sbus/char/Kconfig\nindex 5ba684f..a785aa7 100644\n--- a/drivers/sbus/char/Kconfig\n+++ b/drivers/sbus/char/Kconfig\n@@ -70,5 +70,13 @@ config DISPLAY7SEG\n \t another UltraSPARC-IIi-cEngine boardset with a 7-segment display,\n \t you should say N to this option.\n \n+config ORACLE_DAX\n+\ttristate \"Oracle Data Analytics Accelerator\"\n+\tdefault m if SPARC64\n+\thelp\n+\t Driver for Oracle Data Analytics Accelerator, which is\n+\t a coprocessor that performs database operations in hardware.\n+\t It is available on M7 and M8 based systems only.\n+\n endmenu\n \ndiff --git a/drivers/sbus/char/Makefile b/drivers/sbus/char/Makefile\nindex 78b6183..cdb5565 100644\n--- a/drivers/sbus/char/Makefile\n+++ b/drivers/sbus/char/Makefile\n@@ -16,3 +16,4 @@ obj-$(CONFIG_SUN_OPENPROMIO)\t\t+= openprom.o\n obj-$(CONFIG_TADPOLE_TS102_UCTRL)\t+= uctrl.o\n obj-$(CONFIG_SUN_JSFLASH)\t\t+= jsflash.o\n obj-$(CONFIG_BBC_I2C)\t\t\t+= bbc.o\n+obj-$(CONFIG_ORACLE_DAX) \t\t+= oradax.o\ndiff --git a/drivers/sbus/char/oradax.c b/drivers/sbus/char/oradax.c\nnew file mode 100644\nindex 0000000..2edfffc\n--- /dev/null\n+++ b/drivers/sbus/char/oradax.c\n@@ -0,0 +1,951 @@\n+/*\n+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.\n+ *\n+ * This program is free software: you can redistribute it and/or modify\n+ * it under the terms of the GNU General Public License as published by\n+ * the Free Software Foundation, either version 3 of the License, or\n+ * (at your option) any later version.\n+ *\n+ * This program is distributed in the hope that it will be useful,\n+ * but WITHOUT ANY WARRANTY; without even the implied warranty of\n+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n+ * GNU General Public License for more details.\n+ *\n+ * You should have received a copy of the GNU General Public License\n+ * along with this program. If not, see <http://www.gnu.org/licenses/>.\n+ */\n+\n+/*\n+ * Oracle Data Analytics Accelerator (DAX)\n+ *\n+ * DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8\n+ * (DAX2) processor chips, and has direct access to the CPU's L3\n+ * caches as well as physical memory. It can perform several\n+ * operations on data streams with various input and output formats.\n+ * The driver provides a transport mechanism and does not have\n+ * knowledge of the various opcodes and data formats. A user space\n+ * library provides high level services and translates these into low\n+ * level commands which are then passed into the driver and\n+ * subsequently the hypervisor and the coprocessor. The library is\n+ * the recommended way for applications to use the coprocessor, and\n+ * the driver interface is not intended for general use.\n+ *\n+ * See Documentation/sparc/oracle_dax.txt for more details.\n+ */\n+\n+#include <linux/uaccess.h>\n+#include <linux/module.h>\n+#include <linux/delay.h>\n+#include <linux/cdev.h>\n+#include <linux/slab.h>\n+#include <linux/mm.h>\n+\n+#include <asm/hypervisor.h>\n+#include <asm/mdesc.h>\n+#include <asm/oradax.h>\n+\n+MODULE_LICENSE(\"GPL\");\n+MODULE_DESCRIPTION(\"Driver for Oracle Data Analytics Accelerator\");\n+\n+#define\tDAX_DBG_FLG_BASIC\t0x01\n+#define\tDAX_DBG_FLG_STAT\t0x02\n+#define\tDAX_DBG_FLG_INFO\t0x04\n+#define\tDAX_DBG_FLG_ALL\t\t0xff\n+\n+#define\tdax_err(fmt, ...) pr_err(\"%s: \" fmt \"\\n\", __func__, ##__VA_ARGS__)\n+#define\tdax_info(fmt, ...) pr_info(\"%s: \" fmt \"\\n\", __func__, ##__VA_ARGS__)\n+\n+#define\tdax_dbg(fmt, ...)\tdo {\t\t\t\t\t\\\n+\t\t\t\t\tif (dax_debug & DAX_DBG_FLG_BASIC)\\\n+\t\t\t\t\t\tdax_info(fmt, ##__VA_ARGS__); \\\n+\t\t\t\t} while (0)\n+#define\tdax_stat_dbg(fmt, ...)\tdo {\t\t\t\t\t\\\n+\t\t\t\t\tif (dax_debug & DAX_DBG_FLG_STAT) \\\n+\t\t\t\t\t\tdax_info(fmt, ##__VA_ARGS__); \\\n+\t\t\t\t} while (0)\n+#define\tdax_info_dbg(fmt, ...)\tdo { \\\n+\t\t\t\t\tif (dax_debug & DAX_DBG_FLG_INFO) \\\n+\t\t\t\t\t\tdax_info(fmt, ##__VA_ARGS__); \\\n+\t\t\t\t} while (0)\n+\n+#define\tDAX1_MINOR\t\t1\n+#define\tDAX1_MAJOR\t\t1\n+#define\tDAX2_MINOR\t\t0\n+#define\tDAX2_MAJOR\t\t2\n+\n+#define\tDAX1_STR \"ORCL,sun4v-dax\"\n+#define\tDAX2_STR \"ORCL,sun4v-dax2\"\n+\n+#define\tDAX_CA_ELEMS\t\t(DAX_MMAP_LEN / sizeof(struct compl_area))\n+\n+#define\tDAX_CCB_USEC\t\t100\n+#define\tDAX_CCB_RETRIES\t\t10000\n+\n+/* stream types */\n+enum {\n+\tDEST,\n+\tSRC0,\n+\tSRC1,\n+\tTBL,\n+\tNUM_STREAM_TYPES\n+};\n+\n+/* CCB address types */\n+#define\tCCB_AT_IMM\t\t0\t/* immediate */\n+#define\tCCB_AT_VA_ALT\t\t1\t/* secondary context */\n+#define\tCCB_AT_RA\t\t2\t/* real address */\n+#define\tCCB_AT_VA\t\t3\t/* virtual address */\n+\n+/* CCB dword index values */\n+#define\tCCB_DWORD_CTL\t\t0\n+#define\tCCB_DWORD_COMPL\t\t1\n+#define\tCCB_DWORD_INPUT\t\t2\n+#define\tCCB_DWORD_DAC\t\t3\n+#define\tCCB_DWORD_SEC_INPUT\t4\n+#define\tCCB_DWORD_RSVD\t\t5\n+#define\tCCB_DWORD_OUTPUT\t6\n+#define\tCCB_DWORD_TBL\t\t7\n+#define\tDWORDS_PER_CCB\t\t8\n+\n+/* CCB header sync flags */\n+#define\tCCB_SYNC_SERIAL\t\tBIT(0)\n+#define\tCCB_SYNC_COND\t\tBIT(1)\n+#define\tCCB_SYNC_LONGCCB\tBIT(2)\n+\n+/* Completion area cmd_status */\n+#define\tCCB_CMD_STAT_IN_PROGRESS\t0\n+#define\tCCB_CMD_STAT_COMPLETED\t\t1\n+#define\tCCB_CMD_STAT_FAILED\t\t2\n+#define\tCCB_CMD_STAT_KILLED\t\t3\n+#define\tCCB_CMD_STAT_NOT_RUN\t\t4\n+#define\tCCB_CMD_STAT_NO_OUTPUT\t\t5\n+#define\tCCB_CMD_STAT_PIPE_ERR_SRC\t6\n+#define\tCCB_CMD_STAT_PIPE_ERR_DST\t7\n+\n+/* Completion area err_mask of user visible errors */\n+#define\tCCB_CMD_ERR_BOF\t\t\t0x1\t/* buffer overflow */\n+#define\tCCB_CMD_ERR_DECODE\t\t0x2\t/* CCB decode error */\n+#define\tCCB_CMD_ERR_POF\t\t\t0x3\t/* page overflow */\n+#define\tCCB_CMD_ERR_KILL\t\t0x7\t/* command was killed */\n+#define\tCCB_CMD_ERR_TO\t\t\t0x8\t/* command timeout */\n+#define\tCCB_CMD_ERR_MCD\t\t\t0x9\t/* MCD error */\n+#define\tCCB_CMD_ERR_DATA_FMT\t\t0xA\t/* data format error */\n+#define\tCCB_CMD_ERR_NO_RETRY\t\t0xE\t/* other error, don't retry */\n+#define\tCCB_CMD_ERR_RETRY\t\t0xF\t/* other error, do retry */\n+#define\tCCB_CMD_ERR_PARTIAL_SYM\t\t0x80\t/* partial symbol warning */\n+\n+#define\tIS_LONG_CCB(ccb)\t\\\n+\t((ccb)->hdr.sync_flags & CCB_SYNC_LONGCCB)\n+\n+struct ccb_hdr {\n+\tu32\trsvd:4;\n+\tu32\tsync_flags:4;\n+\tu32\trsvd2:11;\n+\t/* use CCB_AT_* macros for at_* fields */\n+\tu32\tat_tbl:2;\n+\tu32\tat_dst:3;\n+\tu32\tat_src1:3;\n+\tu32\tat_src0:3;\n+\tu32\tat_cmpl:2;\n+};\n+\n+union ccb {\n+\tu64\t\t\tdwords[DWORDS_PER_CCB];\n+\tstruct ccb_hdr\t\thdr;\n+};\n+\n+struct compl_area {\n+\tu8\tcmd_status;\t/* user may mwait on this address */\n+\tu8\terr_mask;\t/* user visible error notification */\n+\tu8\trsvd[2];\t/* reserved */\n+\tu32\trsvd2;\t\t/* reserved */\n+\tu32\toutput_sz;\t/* Bytes of output */\n+\tu32\trsvd3;\t\t/* reserved */\n+\tu64\trun_time;\t/* run time in OCND2 cycles */\n+\tu64\trun_stats;\t/* nothing reported for DAX1 nor in DAX2 */\n+\tu32\tn_processed;\t/* input elements processed */\n+\tu32\trsvd4[5];\t/* reserved */\n+\tu64\tcommand_rv;\t/* command return value */\n+\tu64\trsvd5[8];\t/* reserved */\n+};\n+\n+/* per thread CCB context */\n+struct dax_ctx {\n+\tunion ccb\t\t*ccb_buf;\n+\tu64\t\t\tccb_buf_ra;\t/* cached RA of ccb_buf */\n+\tstruct compl_area\t*ca_buf;\n+\tu64\t\t\tca_buf_ra;\t/* cached RA of ca_buf */\n+\tstruct page\t\t*pages[DAX_CA_ELEMS][NUM_STREAM_TYPES];\n+\t\t\t\t\t\t/* array of locked pages */\n+\tstruct task_struct\t*owner;\t\t/* thread that owns ctx */\n+\tstruct task_struct\t*client;\t/* requesting thread */\n+\tunion ccb_result\tresult;\n+\tu32\t\t\tccb_count;\n+\tu32\t\t\tfail_count;\n+};\n+\n+/* driver public entry points */\n+static int dax_open(struct inode *inode, struct file *file);\n+static ssize_t dax_read(struct file *filp, char __user *buf,\n+\t\t\tsize_t count, loff_t *ppos);\n+static ssize_t dax_write(struct file *filp, const char __user *buf,\n+\t\t\t size_t count, loff_t *ppos);\n+static int dax_devmap(struct file *f, struct vm_area_struct *vma);\n+static int dax_close(struct inode *i, struct file *f);\n+\n+static const struct file_operations dax_fops = {\n+\t.owner\t=\tTHIS_MODULE,\n+\t.open\t=\tdax_open,\n+\t.read\t=\tdax_read,\n+\t.write\t=\tdax_write,\n+\t.mmap\t=\tdax_devmap,\n+\t.release =\tdax_close,\n+};\n+\n+static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,\n+\t\t\tsize_t count, loff_t *ppos);\n+static int dax_ccb_info(u64 ca, struct ccb_info_result *info);\n+static int dax_ccb_kill(u64 ca, u16 *kill_res);\n+\n+static struct cdev c_dev;\n+static struct class *cl;\n+static dev_t first;\n+\n+static int dax_debug;\n+module_param(dax_debug, int, 0644);\n+MODULE_PARM_DESC(dax_debug, \"Debug flags\");\n+\n+static int __init dax_attach(void)\n+{\n+\tunsigned long dummy, hv_rv, major, minor, minor_requested, max_ccbs;\n+\tstruct mdesc_handle *hp = mdesc_grab();\n+\tchar *prop, *dax_name;\n+\tbool found = false;\n+\tint len, ret = 0;\n+\tu64 pn;\n+\n+\tif (hp == NULL) {\n+\t\tdax_err(\"Unable to grab mdesc\");\n+\t\treturn -ENODEV;\n+\t}\n+\n+\tmdesc_for_each_node_by_name(hp, pn, \"virtual-device\") {\n+\t\tprop = (char *)mdesc_get_property(hp, pn, \"name\", &len);\n+\t\tif (prop == NULL)\n+\t\t\tcontinue;\n+\t\tif (strncmp(prop, \"dax\", strlen(\"dax\")))\n+\t\t\tcontinue;\n+\t\tdax_dbg(\"Found node 0x%llx = %s\", pn, prop);\n+\n+\t\tprop = (char *)mdesc_get_property(hp, pn, \"compatible\", &len);\n+\t\tif (prop == NULL)\n+\t\t\tcontinue;\n+\t\tdax_dbg(\"Found node 0x%llx = %s\", pn, prop);\n+\t\tfound = true;\n+\t\tbreak;\n+\t}\n+\n+\tif (!found) {\n+\t\tdax_err(\"No DAX device found\");\n+\t\tret = -ENODEV;\n+\t\tgoto done;\n+\t}\n+\n+\tif (strncmp(prop, DAX2_STR, strlen(DAX2_STR)) == 0) {\n+\t\tdax_name = DAX_NAME \"2\";\n+\t\tmajor = DAX2_MAJOR;\n+\t\tminor_requested = DAX2_MINOR;\n+\t\tdax_dbg(\"MD indicates DAX2 coprocessor\");\n+\t} else if (strncmp(prop, DAX1_STR, strlen(DAX1_STR)) == 0) {\n+\t\tdax_name = DAX_NAME \"1\";\n+\t\tmajor = DAX1_MAJOR;\n+\t\tminor_requested = DAX1_MINOR;\n+\t\tdax_dbg(\"MD indicates DAX1 coprocessor\");\n+\t} else {\n+\t\tdax_err(\"Unknown dax type: %s\", prop);\n+\t\tret = -ENODEV;\n+\t\tgoto done;\n+\t}\n+\n+\tminor = minor_requested;\n+\tdax_dbg(\"Registering DAX HV api with major %ld minor %ld\", major,\n+\t\tminor);\n+\tif (sun4v_hvapi_register(HV_GRP_DAX, major, &minor)) {\n+\t\tdax_err(\"hvapi_register failed\");\n+\t\tret = -ENODEV;\n+\t\tgoto done;\n+\t} else {\n+\t\tdax_dbg(\"Max minor supported by HV = %ld (major %ld)\", minor,\n+\t\t\tmajor);\n+\t\tminor = min(minor, minor_requested);\n+\t\tdax_dbg(\"registered DAX major %ld minor %ld\", major, minor);\n+\t}\n+\n+\t/* submit a zero length ccb array to query coprocessor queue size */\n+\thv_rv = sun4v_ccb_submit(0, 0, HV_CCB_QUERY_CMD, 0, &max_ccbs, &dummy);\n+\tif (hv_rv != 0) {\n+\t\tdax_err(\"get_hwqueue_size failed with status=%ld and max_ccbs=%ld\",\n+\t\t\thv_rv, max_ccbs);\n+\t\tret = -ENODEV;\n+\t\tgoto done;\n+\t}\n+\n+\tif (max_ccbs != DAX_MAX_CCBS) {\n+\t\tdax_err(\"HV reports unsupported max_ccbs=%ld\", max_ccbs);\n+\t\tret = -ENODEV;\n+\t\tgoto done;\n+\t}\n+\n+\tif (alloc_chrdev_region(&first, 0, 1, DAX_NAME) < 0) {\n+\t\tdax_err(\"alloc_chrdev_region failed\");\n+\t\tret = -ENXIO;\n+\t\tgoto done;\n+\t}\n+\n+\tcl = class_create(THIS_MODULE, DAX_NAME);\n+\tif (cl == NULL) {\n+\t\tdax_err(\"class_create failed\");\n+\t\tret = -ENXIO;\n+\t\tgoto class_error;\n+\t}\n+\n+\tif (device_create(cl, NULL, first, NULL, dax_name) == NULL) {\n+\t\tdax_err(\"device_create failed\");\n+\t\tret = -ENXIO;\n+\t\tgoto device_error;\n+\t}\n+\n+\tcdev_init(&c_dev, &dax_fops);\n+\tif (cdev_add(&c_dev, first, 1) == -1) {\n+\t\tdax_err(\"cdev_add failed\");\n+\t\tret = -ENXIO;\n+\t\tgoto cdev_error;\n+\t}\n+\n+\tpr_info(\"Attached DAX module\\n\");\n+\tgoto done;\n+\n+cdev_error:\n+\tdevice_destroy(cl, first);\n+device_error:\n+\tclass_destroy(cl);\n+class_error:\n+\tunregister_chrdev_region(first, 1);\n+done:\n+\tmdesc_release(hp);\n+\treturn ret;\n+}\n+module_init(dax_attach);\n+\n+static void __exit dax_detach(void)\n+{\n+\tpr_info(\"Cleaning up DAX module\\n\");\n+\tcdev_del(&c_dev);\n+\tdevice_destroy(cl, first);\n+\tclass_destroy(cl);\n+\tunregister_chrdev_region(first, 1);\n+}\n+module_exit(dax_detach);\n+\n+/* map completion area */\n+static int dax_devmap(struct file *f, struct vm_area_struct *vma)\n+{\n+\tstruct dax_ctx *ctx = (struct dax_ctx *)f->private_data;\n+\tsize_t len = vma->vm_end - vma->vm_start;\n+\n+\tdax_dbg(\"len=0x%lx, flags=0x%lx\", len, vma->vm_flags);\n+\n+\tif (ctx->owner != current) {\n+\t\tdax_dbg(\"devmap called from wrong thread\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (len != DAX_MMAP_LEN) {\n+\t\tdax_dbg(\"len(%lu) != DAX_MMAP_LEN(%d)\", len, DAX_MMAP_LEN);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* completion area is mapped read-only for user */\n+\tif (vma->vm_flags & VM_WRITE)\n+\t\treturn -EPERM;\n+\tvma->vm_flags &= ~VM_MAYWRITE;\n+\n+\tif (remap_pfn_range(vma, vma->vm_start, ctx->ca_buf_ra >> PAGE_SHIFT,\n+\t\t\t len, vma->vm_page_prot))\n+\t\treturn -EAGAIN;\n+\n+\tdax_dbg(\"mmapped completion area at uva 0x%lx\", vma->vm_start);\n+\treturn 0;\n+}\n+\n+/* Unlock user pages. Called during dequeue or device close */\n+static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem)\n+{\n+\tint i, j;\n+\n+\tfor (i = ccb_index; i < ccb_index + nelem; i++) {\n+\t\tfor (j = 0; j < NUM_STREAM_TYPES; j++) {\n+\t\t\tstruct page *p = ctx->pages[i][j];\n+\n+\t\t\tif (p) {\n+\t\t\t\tdax_dbg(\"freeing page %p\", p);\n+\t\t\t\tif (j == DEST)\n+\t\t\t\t\tset_page_dirty(p);\n+\t\t\t\tput_page(p);\n+\t\t\t\tctx->pages[i][j] = NULL;\n+\t\t\t}\n+\t\t}\n+\t}\n+}\n+\n+static int dax_lock_page(unsigned long va, struct page **p)\n+{\n+\tint ret;\n+\n+\tdax_dbg(\"uva 0x%lx\", va);\n+\n+\tret = get_user_pages_fast(va, 1, 1, p);\n+\tif (ret == 1) {\n+\t\tdax_dbg(\"locked page %p, for VA 0x%lx\", *p, va);\n+\t\treturn 0;\n+\t}\n+\n+\tdax_dbg(\"get_user_pages failed, va=0x%lx, ret=%d\", va, ret);\n+\treturn -1;\n+}\n+\n+static int dax_lock_pages(struct dax_ctx *ctx, int idx,\n+\t\t\t int nelem, u64 *err_va)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < nelem; i++) {\n+\t\tunion ccb *ccbp = &ctx->ccb_buf[i];\n+\n+\t\t/*\n+\t\t * For each address in the CCB whose type is virtual,\n+\t\t * lock the page and change the type to virtual alternate\n+\t\t * context. On error, return the offending address in\n+\t\t * err_va.\n+\t\t */\n+\t\tif (ccbp->hdr.at_dst == CCB_AT_VA) {\n+\t\t\tdax_dbg(\"output\");\n+\t\t\tif (dax_lock_page(ccbp->dwords[CCB_DWORD_OUTPUT],\n+\t\t\t\t\t &ctx->pages[i + idx][DEST]) != 0) {\n+\t\t\t\t*err_va = ccbp->dwords[CCB_DWORD_OUTPUT];\n+\t\t\t\tgoto error;\n+\t\t\t}\n+\t\t\tccbp->hdr.at_dst = CCB_AT_VA_ALT;\n+\t\t}\n+\n+\t\tif (ccbp->hdr.at_src0 == CCB_AT_VA) {\n+\t\t\tdax_dbg(\"input\");\n+\t\t\tif (dax_lock_page(ccbp->dwords[CCB_DWORD_INPUT],\n+\t\t\t\t\t &ctx->pages[i + idx][SRC0]) != 0) {\n+\t\t\t\t*err_va = ccbp->dwords[CCB_DWORD_INPUT];\n+\t\t\t\tgoto error;\n+\t\t\t}\n+\t\t\tccbp->hdr.at_src0 = CCB_AT_VA_ALT;\n+\t\t}\n+\n+\t\tif (ccbp->hdr.at_src1 == CCB_AT_VA) {\n+\t\t\tdax_dbg(\"sec input\");\n+\t\t\tif (dax_lock_page(ccbp->dwords[CCB_DWORD_SEC_INPUT],\n+\t\t\t\t\t &ctx->pages[i + idx][SRC1]) != 0) {\n+\t\t\t\t*err_va = ccbp->dwords[CCB_DWORD_SEC_INPUT];\n+\t\t\t\tgoto error;\n+\t\t\t}\n+\t\t\tccbp->hdr.at_src1 = CCB_AT_VA_ALT;\n+\t\t}\n+\n+\t\tif (ccbp->hdr.at_tbl == CCB_AT_VA) {\n+\t\t\tdax_dbg(\"tbl\");\n+\t\t\tif (dax_lock_page(ccbp->dwords[CCB_DWORD_TBL],\n+\t\t\t\t\t &ctx->pages[i + idx][TBL]) != 0) {\n+\t\t\t\t*err_va = ccbp->dwords[CCB_DWORD_TBL];\n+\t\t\t\tgoto error;\n+\t\t\t}\n+\t\t\tccbp->hdr.at_tbl = CCB_AT_VA_ALT;\n+\t\t}\n+\n+\t\t/* skip over 2nd 64 bytes of long CCB */\n+\t\tif (IS_LONG_CCB(ccbp))\n+\t\t\ti++;\n+\t}\n+\treturn DAX_SUBMIT_OK;\n+\n+error:\n+\tdax_unlock_pages(ctx, idx, nelem);\n+\treturn DAX_SUBMIT_ERR_NOACCESS;\n+}\n+\n+static void dax_ccb_wait(struct dax_ctx *ctx, int idx)\n+{\n+\tint ret, nretries;\n+\tu16 kill_res;\n+\n+\tdax_dbg(\"idx=%d\", idx);\n+\n+\tfor (nretries = 0; nretries < DAX_CCB_RETRIES; nretries++) {\n+\t\tif (ctx->ca_buf[idx].cmd_status == CCB_CMD_STAT_IN_PROGRESS)\n+\t\t\tudelay(DAX_CCB_USEC);\n+\t\telse\n+\t\t\treturn;\n+\t}\n+\tdax_dbg(\"ctx (%p): CCB[%d] timed out, wait usec=%d, retries=%d. Killing ccb\",\n+\t\t(void *)ctx, idx, DAX_CCB_USEC, DAX_CCB_RETRIES);\n+\n+\tret = dax_ccb_kill(ctx->ca_buf_ra + idx * sizeof(struct compl_area),\n+\t\t\t &kill_res);\n+\tdax_dbg(\"Kill CCB[%d] %s\", idx, ret ? \"failed\" : \"succeeded\");\n+}\n+\n+static int dax_close(struct inode *ino, struct file *f)\n+{\n+\tstruct dax_ctx *ctx = (struct dax_ctx *)f->private_data;\n+\tint i;\n+\n+\tf->private_data = NULL;\n+\n+\tfor (i = 0; i < DAX_CA_ELEMS; i++) {\n+\t\tif (ctx->ca_buf[i].cmd_status == CCB_CMD_STAT_IN_PROGRESS) {\n+\t\t\tdax_dbg(\"CCB[%d] not completed\", i);\n+\t\t\tdax_ccb_wait(ctx, i);\n+\t\t}\n+\t\tdax_unlock_pages(ctx, i, 1);\n+\t}\n+\n+\tkfree(ctx->ccb_buf);\n+\tkfree(ctx->ca_buf);\n+\tdax_stat_dbg(\"CCBs: %d good, %d bad\", ctx->ccb_count, ctx->fail_count);\n+\tkfree(ctx);\n+\n+\treturn 0;\n+}\n+\n+static ssize_t dax_read(struct file *f, char __user *buf,\n+\t\t\tsize_t count, loff_t *ppos)\n+{\n+\tstruct dax_ctx *ctx = f->private_data;\n+\n+\tif (ctx->client != current)\n+\t\treturn -EUSERS;\n+\n+\tctx->client = NULL;\n+\n+\tif (count != sizeof(union ccb_result))\n+\t\treturn -EINVAL;\n+\tif (copy_to_user(buf, &ctx->result, sizeof(union ccb_result)))\n+\t\treturn -EFAULT;\n+\treturn count;\n+}\n+\n+static ssize_t dax_write(struct file *f, const char __user *buf,\n+\t\t\t size_t count, loff_t *ppos)\n+{\n+\tstruct dax_ctx *ctx = f->private_data;\n+\tstruct dax_command hdr;\n+\tunsigned long ca;\n+\tint i, idx, ret;\n+\n+\tif (ctx->client != NULL)\n+\t\treturn -EINVAL;\n+\n+\tif (count == 0 || count > DAX_MAX_CCBS * sizeof(union ccb))\n+\t\treturn -EINVAL;\n+\n+\tif (count % sizeof(union ccb) == 0)\n+\t\treturn dax_ccb_exec(ctx, buf, count, ppos); /* CCB EXEC */\n+\n+\tif (count != sizeof(struct dax_command))\n+\t\treturn -EINVAL;\n+\n+\t/* immediate command */\n+\tif (ctx->owner != current)\n+\t\treturn -EUSERS;\n+\n+\tif (copy_from_user(&hdr, buf, sizeof(hdr)))\n+\t\treturn -EFAULT;\n+\n+\tca = ctx->ca_buf_ra + hdr.ca_offset;\n+\n+\tswitch (hdr.command) {\n+\tcase CCB_KILL:\n+\t\tif (hdr.ca_offset >= DAX_MMAP_LEN) {\n+\t\t\tdax_dbg(\"invalid ca_offset (%d) >= ca_buflen (%d)\",\n+\t\t\t\thdr.ca_offset, DAX_MMAP_LEN);\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\tret = dax_ccb_kill(ca, &ctx->result.kill.action);\n+\t\tif (ret != 0) {\n+\t\t\tdax_dbg(\"dax_ccb_kill failed (ret=%d)\", ret);\n+\t\t\treturn ret;\n+\t\t}\n+\n+\t\tdax_info_dbg(\"killed (ca_offset %d)\", hdr.ca_offset);\n+\t\tidx = hdr.ca_offset / sizeof(struct compl_area);\n+\t\tctx->ca_buf[idx].cmd_status = CCB_CMD_STAT_KILLED;\n+\t\tctx->ca_buf[idx].err_mask = CCB_CMD_ERR_KILL;\n+\t\tctx->client = current;\n+\t\treturn count;\n+\n+\tcase CCB_INFO:\n+\t\tif (hdr.ca_offset >= DAX_MMAP_LEN) {\n+\t\t\tdax_dbg(\"invalid ca_offset (%d) >= ca_buflen (%d)\",\n+\t\t\t\thdr.ca_offset, DAX_MMAP_LEN);\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\tret = dax_ccb_info(ca, &ctx->result.info);\n+\t\tif (ret != 0) {\n+\t\t\tdax_dbg(\"dax_ccb_info failed (ret=%d)\", ret);\n+\t\t\treturn ret;\n+\t\t}\n+\n+\t\tdax_info_dbg(\"info succeeded on ca_offset %d\", hdr.ca_offset);\n+\t\tctx->client = current;\n+\t\treturn count;\n+\n+\tcase CCB_DEQUEUE:\n+\t\tfor (i = 0; i < DAX_CA_ELEMS; i++) {\n+\t\t\tif (ctx->ca_buf[i].cmd_status !=\n+\t\t\t CCB_CMD_STAT_IN_PROGRESS)\n+\t\t\t\tdax_unlock_pages(ctx, i, 1);\n+\t\t}\n+\t\treturn count;\n+\n+\tdefault:\n+\t\treturn -EINVAL;\n+\t}\n+}\n+\n+static int dax_open(struct inode *inode, struct file *f)\n+{\n+\tstruct dax_ctx *ctx = NULL;\n+\tint i;\n+\n+\tctx = kzalloc(sizeof(*ctx), GFP_KERNEL);\n+\tif (ctx == NULL)\n+\t\tgoto done;\n+\n+\tctx->ccb_buf = kcalloc(DAX_MAX_CCBS, sizeof(union ccb), GFP_KERNEL);\n+\tif (ctx->ccb_buf == NULL)\n+\t\tgoto done;\n+\n+\tctx->ccb_buf_ra = virt_to_phys(ctx->ccb_buf);\n+\tdax_dbg(\"ctx->ccb_buf=0x%p, ccb_buf_ra=0x%llx\",\n+\t\t(void *)ctx->ccb_buf, ctx->ccb_buf_ra);\n+\n+\t/* allocate CCB completion area buffer */\n+\tctx->ca_buf = kzalloc(DAX_MMAP_LEN, GFP_KERNEL);\n+\tif (ctx->ca_buf == NULL)\n+\t\tgoto alloc_error;\n+\tfor (i = 0; i < DAX_CA_ELEMS; i++)\n+\t\tctx->ca_buf[i].cmd_status = CCB_CMD_STAT_COMPLETED;\n+\n+\tctx->ca_buf_ra = virt_to_phys(ctx->ca_buf);\n+\tdax_dbg(\"ctx=0x%p, ctx->ca_buf=0x%p, ca_buf_ra=0x%llx\",\n+\t\t(void *)ctx, (void *)ctx->ca_buf, ctx->ca_buf_ra);\n+\n+\tctx->owner = current;\n+\tf->private_data = ctx;\n+\treturn 0;\n+\n+alloc_error:\n+\tkfree(ctx->ccb_buf);\n+done:\n+\tif (ctx != NULL)\n+\t\tkfree(ctx);\n+\treturn -ENOMEM;\n+}\n+\n+static char *dax_hv_errno(unsigned long hv_ret, int *ret)\n+{\n+\tswitch (hv_ret) {\n+\tcase HV_EBADALIGN:\n+\t\t*ret = -EFAULT;\n+\t\treturn \"HV_EBADALIGN\";\n+\tcase HV_ENORADDR:\n+\t\t*ret = -EFAULT;\n+\t\treturn \"HV_ENORADDR\";\n+\tcase HV_EINVAL:\n+\t\t*ret = -EINVAL;\n+\t\treturn \"HV_EINVAL\";\n+\tcase HV_EWOULDBLOCK:\n+\t\t*ret = -EAGAIN;\n+\t\treturn \"HV_EWOULDBLOCK\";\n+\tcase HV_ENOACCESS:\n+\t\t*ret = -EPERM;\n+\t\treturn \"HV_ENOACCESS\";\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\t*ret = -EIO;\n+\treturn \"UNKNOWN\";\n+}\n+\n+static int dax_ccb_kill(u64 ca, u16 *kill_res)\n+{\n+\tunsigned long hv_ret;\n+\tint count, ret = 0;\n+\tchar *err_str;\n+\n+\tfor (count = 0; count < DAX_CCB_RETRIES; count++) {\n+\t\tdax_dbg(\"attempting kill on ca_ra 0x%llx\", ca);\n+\t\thv_ret = sun4v_ccb_kill(ca, kill_res);\n+\n+\t\tif (hv_ret == HV_EOK) {\n+\t\t\tdax_info_dbg(\"HV_EOK (ca_ra 0x%llx): %d\", ca,\n+\t\t\t\t *kill_res);\n+\t\t} else {\n+\t\t\terr_str = dax_hv_errno(hv_ret, &ret);\n+\t\t\tdax_dbg(\"%s (ca_ra 0x%llx)\", err_str, ca);\n+\t\t}\n+\n+\t\tif (ret != -EAGAIN)\n+\t\t\treturn ret;\n+\t\tdax_info_dbg(\"ccb_kill count = %d\", count);\n+\t\tudelay(DAX_CCB_USEC);\n+\t}\n+\n+\treturn -EAGAIN;\n+}\n+\n+static int dax_ccb_info(u64 ca, struct ccb_info_result *info)\n+{\n+\tunsigned long hv_ret;\n+\tchar *err_str;\n+\tint ret = 0;\n+\n+\tdax_dbg(\"attempting info on ca_ra 0x%llx\", ca);\n+\thv_ret = sun4v_ccb_info(ca, info);\n+\n+\tif (hv_ret == HV_EOK) {\n+\t\tdax_info_dbg(\"HV_EOK (ca_ra 0x%llx): %d\", ca, info->state);\n+\t\tif (info->state == DAX_CCB_ENQUEUED) {\n+\t\t\tdax_info_dbg(\"dax_unit %d, queue_num %d, queue_pos %d\",\n+\t\t\t\t info->inst_num, info->q_num, info->q_pos);\n+\t\t}\n+\t} else {\n+\t\terr_str = dax_hv_errno(hv_ret, &ret);\n+\t\tdax_dbg(\"%s (ca_ra 0x%llx)\", err_str, ca);\n+\t}\n+\n+\treturn ret;\n+}\n+\n+static void dax_prt_ccbs(union ccb *ccb, int nelem)\n+{\n+\tint i, j;\n+\n+\tdax_dbg(\"ccb buffer:\");\n+\tfor (i = 0; i < nelem; i++) {\n+\t\tdax_dbg(\" %sccb[%d]\", IS_LONG_CCB(&ccb[i]) ? \"long \" : \"\", i);\n+\t\tfor (j = 0; j < DWORDS_PER_CCB; j++)\n+\t\t\tdax_dbg(\"\\tccb[%d].dwords[%d]=0x%llx\",\n+\t\t\t\ti, j, ccb[i].dwords[j]);\n+\t}\n+}\n+\n+/*\n+ * Validates user CCB content. Also sets completion address and address types\n+ * for all addresses contained in CCB.\n+ */\n+static int dax_preprocess_usr_ccbs(struct dax_ctx *ctx, int idx, int nelem)\n+{\n+\tint i;\n+\n+\t/*\n+\t * The user is not allowed to specify real address types in\n+\t * the CCB header. This must be enforced by the kernel before\n+\t * submitting the CCBs to HV. The only allowed values for all\n+\t * address fields are VA or IMM\n+\t */\n+\tfor (i = 0; i < nelem; i++) {\n+\t\tunion ccb *ccbp = &ctx->ccb_buf[i];\n+\t\tunsigned long ca_offset;\n+\n+\t\tif (ccbp->hdr.at_dst != CCB_AT_VA &&\n+\t\t ccbp->hdr.at_dst != CCB_AT_IMM) {\n+\t\t\tdax_dbg(\"invalid at_dst in user CCB[%d]\", i);\n+\t\t\treturn DAX_SUBMIT_ERR_CCB_INVAL;\n+\t\t}\n+\n+\t\tif (ccbp->hdr.at_src0 != CCB_AT_VA &&\n+\t\t ccbp->hdr.at_src0 != CCB_AT_IMM) {\n+\t\t\tdax_dbg(\"invalid at_src0 in user CCB[%d]\", i);\n+\t\t\treturn DAX_SUBMIT_ERR_CCB_INVAL;\n+\t\t}\n+\n+\t\tif (ccbp->hdr.at_src1 != CCB_AT_VA &&\n+\t\t ccbp->hdr.at_src1 != CCB_AT_IMM) {\n+\t\t\tdax_dbg(\"invalid at_src1 in user CCB[%d]\", i);\n+\t\t\treturn DAX_SUBMIT_ERR_CCB_INVAL;\n+\t\t}\n+\n+\t\tif (ccbp->hdr.at_tbl != CCB_AT_VA &&\n+\t\t ccbp->hdr.at_tbl != CCB_AT_IMM) {\n+\t\t\tdax_dbg(\"invalid at_tbl in user CCB[%d]\", i);\n+\t\t\treturn DAX_SUBMIT_ERR_CCB_INVAL;\n+\t\t}\n+\n+\t\t/* set completion (real) address and address type */\n+\t\tccbp->hdr.at_cmpl = CCB_AT_RA;\n+\t\tca_offset = (idx + i) * sizeof(struct compl_area);\n+\t\tccbp->dwords[CCB_DWORD_COMPL] = ctx->ca_buf_ra + ca_offset;\n+\t\tmemset(&ctx->ca_buf[idx + i], 0, sizeof(struct compl_area));\n+\n+\t\tdax_dbg(\"ccb[%d]=%p, ca_offset=0x%lx, compl RA=0x%llx\",\n+\t\t\ti, ccbp, ca_offset, ctx->ca_buf_ra + ca_offset);\n+\n+\t\t/* skip over 2nd 64 bytes of long CCB */\n+\t\tif (IS_LONG_CCB(ccbp))\n+\t\t\ti++;\n+\t}\n+\n+\treturn DAX_SUBMIT_OK;\n+}\n+\n+static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,\n+\t\t\tsize_t count, loff_t *ppos)\n+{\n+\tunsigned long accepted_len, hv_rv;\n+\tint i, idx, nccbs, naccepted;\n+\n+\tctx->client = current;\n+\tidx = *ppos;\n+\tnccbs = count / sizeof(union ccb);\n+\n+\tif (ctx->owner != current) {\n+\t\tdax_dbg(\"wrong thread\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_THR_INIT;\n+\t\treturn 0;\n+\t}\n+\tdax_dbg(\"args: ccb_buf_len=%ld, idx=%d\", count, idx);\n+\n+\t/* for given index and length, verify ca_buf range exists */\n+\tif (idx + nccbs >= DAX_CA_ELEMS) {\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;\n+\t\treturn 0;\n+\t}\n+\n+\t/*\n+\t * Copy CCBs into kernel buffer to prevent modification by the\n+\t * user in between validation and submission.\n+\t */\n+\tif (copy_from_user(ctx->ccb_buf, buf, count)) {\n+\t\tdax_dbg(\"copyin of user CCB buffer failed\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS;\n+\t\treturn 0;\n+\t}\n+\n+\t/* check to see if ca_buf[idx] .. ca_buf[idx + nccbs] are available */\n+\tfor (i = idx; i < idx + nccbs; i++) {\n+\t\tif (ctx->ca_buf[i].cmd_status == CCB_CMD_STAT_IN_PROGRESS) {\n+\t\t\tdax_dbg(\"CA range not available, dequeue needed\");\n+\t\t\tctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;\n+\t\t\treturn 0;\n+\t\t}\n+\t}\n+\tdax_unlock_pages(ctx, idx, nccbs);\n+\n+\tctx->result.exec.status = dax_preprocess_usr_ccbs(ctx, idx, nccbs);\n+\tif (ctx->result.exec.status != DAX_SUBMIT_OK)\n+\t\treturn 0;\n+\n+\tctx->result.exec.status = dax_lock_pages(ctx, idx, nccbs,\n+\t\t\t\t\t\t &ctx->result.exec.status_data);\n+\tif (ctx->result.exec.status != DAX_SUBMIT_OK)\n+\t\treturn 0;\n+\n+\tif (dax_debug & DAX_DBG_FLG_BASIC)\n+\t\tdax_prt_ccbs(ctx->ccb_buf, nccbs);\n+\n+\thv_rv = sun4v_ccb_submit(ctx->ccb_buf_ra, count,\n+\t\t\t\t HV_CCB_QUERY_CMD | HV_CCB_VA_SECONDARY, 0,\n+\t\t\t\t &accepted_len, &ctx->result.exec.status_data);\n+\n+\tswitch (hv_rv) {\n+\tcase HV_EOK:\n+\t\t/*\n+\t\t * Hcall succeeded with no errors but the accepted\n+\t\t * length may be less than the requested length. The\n+\t\t * only way the driver can resubmit the remainder is\n+\t\t * to wait for completion of the submitted CCBs since\n+\t\t * there is no way to guarantee the ordering semantics\n+\t\t * required by the client applications. Therefore we\n+\t\t * let the user library deal with resubmissions.\n+\t\t */\n+\t\tctx->result.exec.status = DAX_SUBMIT_OK;\n+\t\tbreak;\n+\tcase HV_EWOULDBLOCK:\n+\t\t/*\n+\t\t * This is a transient HV API error. The user library\n+\t\t * can retry.\n+\t\t */\n+\t\tdax_dbg(\"hcall returned HV_EWOULDBLOCK\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_WOULDBLOCK;\n+\t\tbreak;\n+\tcase HV_ENOMAP:\n+\t\t/*\n+\t\t * HV was unable to translate a VA. The VA it could\n+\t\t * not translate is returned in the status_data param.\n+\t\t */\n+\t\tdax_dbg(\"hcall returned HV_ENOMAP\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_NOMAP;\n+\t\tbreak;\n+\tcase HV_EINVAL:\n+\t\t/*\n+\t\t * This is the result of an invalid user CCB as HV is\n+\t\t * validating some of the user CCB fields. Pass this\n+\t\t * error back to the user. There is no supporting info\n+\t\t * to isolate the invalid field.\n+\t\t */\n+\t\tdax_dbg(\"hcall returned HV_EINVAL\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_CCB_INVAL;\n+\t\tbreak;\n+\tcase HV_ENOACCESS:\n+\t\t/*\n+\t\t * HV found a VA that did not have the appropriate\n+\t\t * permissions (such as the w bit). The VA in question\n+\t\t * is returned in status_data param.\n+\t\t */\n+\t\tdax_dbg(\"hcall returned HV_ENOACCESS\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_NOACCESS;\n+\t\tbreak;\n+\tcase HV_EUNAVAILABLE:\n+\t\t/*\n+\t\t * The requested CCB operation could not be performed\n+\t\t * at this time. Return the specific unavailable code\n+\t\t * in the status_data field.\n+\t\t */\n+\t\tdax_dbg(\"hcall returned HV_EUNAVAILABLE\");\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_UNAVAIL;\n+\t\tbreak;\n+\tdefault:\n+\t\tctx->result.exec.status = DAX_SUBMIT_ERR_INTERNAL;\n+\t\tdax_dbg(\"unknown hcall return value (%ld)\", hv_rv);\n+\t\tbreak;\n+\t}\n+\n+\t/* unlock pages associated with the unaccepted CCBs */\n+\tnaccepted = accepted_len / sizeof(union ccb);\n+\tdax_unlock_pages(ctx, idx + naccepted, nccbs - naccepted);\n+\n+\t/* mark unaccepted CCBs as not completed */\n+\tfor (i = idx + naccepted; i < idx + nccbs; i++)\n+\t\tctx->ca_buf[i].cmd_status = CCB_CMD_STAT_COMPLETED;\n+\n+\tctx->ccb_count += naccepted;\n+\tctx->fail_count += nccbs - naccepted;\n+\n+\tdax_dbg(\"hcall rv=%ld, accepted_len=%ld, status_data=0x%llx, ret status=%d\",\n+\t\thv_rv, accepted_len, ctx->result.exec.status_data,\n+\t\tctx->result.exec.status);\n+\n+\tif (count == accepted_len)\n+\t\tctx->client = NULL; /* no read needed to complete protocol */\n+\treturn accepted_len;\n+}\n", "prefixes": [ "RFC", "2/2" ] }