diff mbox series

[RFC,3/6] pcie/cxl_timeout: Add CXL.mem timeout range programming

Message ID 20240215194048.141411-4-Benjamin.Cheatham@amd.com
State New
Headers show
Series Implement initial CXL Timeout & Isolation support | expand

Commit Message

Ben Cheatham Feb. 15, 2024, 7:40 p.m. UTC 
Add programming of CXL.mem transaction timeout range (CXL 3.0 8.2.4.23.1
bits 4 & 11:8) through sysfs. To program the device, read the ranges
supported from "available_timeout_ranges" in the PCIe service device
directory, then write the desired value to "timeout_range". The current
value can be found by reading from "timeout_range".

Example for CXL-enabled PCIe port 0000:0c:00.0, with CXL timeout
service as 020:
 # cd /sys/bus/pci_express/devices/0000:0c:00.0:pcie020
 # cat available_timeout_ranges
 0x0	Default range (50us-10ms)
 0x1	Range A (50us-100us)
 0x2	Range A (1ms-10ms)
 # cat timeout_range
 0
 # echo 0x2 > timeout_range
 # cat timeout_range
 2

Signed-off-by: Ben Cheatham <Benjamin.Cheatham@amd.com>
---
 drivers/cxl/cxl.h              |  13 +++
 drivers/pci/pcie/cxl_timeout.c | 185 +++++++++++++++++++++++++++++++++
 2 files changed, 198 insertions(+)

Comments

Bjorn Helgaas Feb. 15, 2024, 9:35 p.m. UTC | #1 
On Thu, Feb 15, 2024 at 01:40:45PM -0600, Ben Cheatham wrote:
> Add programming of CXL.mem transaction timeout range (CXL 3.0 8.2.4.23.1
> bits 4 & 11:8) through sysfs. To program the device, read the ranges
> supported from "available_timeout_ranges" in the PCIe service device
> directory, then write the desired value to "timeout_range". The current
> value can be found by reading from "timeout_range".
> 
> Example for CXL-enabled PCIe port 0000:0c:00.0, with CXL timeout
> service as 020:
>  # cd /sys/bus/pci_express/devices/0000:0c:00.0:pcie020

I would really, really like to avoid adding sysfs dependences on
portdrv.  Is there any chance these files could go in the normal
/sys/bus/pci/devices/ hierarchy instead?

> +/* CXL 3.0 8.2.4.23.2 CXL Timeout and Isolation Control Register, bits 3:0 */
> +#define CXL_TIMEOUT_TIMEOUT_RANGE_DEFAULT 0x0

> +#define CXL_TIMEOUT_TIMEOUT_RANGE_D2 0xE

Looks like the single other example in this file of hex constants
using A-F uses lower-case.

Does "TIMEOUT_TIMEOUT" add information over just "TIMEOUT"?

> +#define NUM_CXL_TIMEOUT_RANGES 9

I don't think we actually need this constant, do we?

> +static bool cxl_validate_timeout_range(struct cxl_timeout *cxlt, u8 range)

"validate" is not a very name for a function returning "bool" because
you can't tell what true/false means from the name.  "valid" would be
fine.

> +	pci_dbg(cxlt->dev->port,
> +		 "Timeout & isolation timeout set to range 0x%x\n", range);

I don't know CXL, but "timeout ... timeout" reads sort of strange.  Is
it actually a timeout for a timeout?  Maybe it was supposed to be
"transaction timeout"?

> +const struct cxl_timeout_range {
> +	const char *str;
> +	u8 range_val;
> +} cxl_timeout_ranges[NUM_CXL_TIMEOUT_RANGES] = {

Static?

Bjorn
Ben Cheatham Feb. 15, 2024, 10:21 p.m. UTC | #2 
On 2/15/24 3:35 PM, Bjorn Helgaas wrote:
> On Thu, Feb 15, 2024 at 01:40:45PM -0600, Ben Cheatham wrote:
>> Add programming of CXL.mem transaction timeout range (CXL 3.0 8.2.4.23.1
>> bits 4 & 11:8) through sysfs. To program the device, read the ranges
>> supported from "available_timeout_ranges" in the PCIe service device
>> directory, then write the desired value to "timeout_range". The current
>> value can be found by reading from "timeout_range".
>>
>> Example for CXL-enabled PCIe port 0000:0c:00.0, with CXL timeout
>> service as 020:
>>  # cd /sys/bus/pci_express/devices/0000:0c:00.0:pcie020
> 
> I would really, really like to avoid adding sysfs dependences on
> portdrv.  Is there any chance these files could go in the normal
> /sys/bus/pci/devices/ hierarchy instead?
> 

This was just the first place I tried putting it that seemed ok. I'll take a look
at moving them over to pci/devices instead.

>> +/* CXL 3.0 8.2.4.23.2 CXL Timeout and Isolation Control Register, bits 3:0 */
>> +#define CXL_TIMEOUT_TIMEOUT_RANGE_DEFAULT 0x0
> 
>> +#define CXL_TIMEOUT_TIMEOUT_RANGE_D2 0xE
> 
> Looks like the single other example in this file of hex constants
> using A-F uses lower-case.
> 
> Does "TIMEOUT_TIMEOUT" add information over just "TIMEOUT"?
> 

TIMEOUT_TIMEOUT is supposed to specify transaction timeout, but I'll agree it is
confusing. I could try changing to CXL_TRANSACTION_TIMEOUT_RANGE_* instead, but
it would break the convention of CXL timeout & isolation defines starting with
CXL_TIMEOUT.

>> +#define NUM_CXL_TIMEOUT_RANGES 9
> 
> I don't think we actually need this constant, do we?
> 

Not really, just gets rid of a magic number (as well as makes sure anyone who
changes that array remembers to update the array size).

>> +static bool cxl_validate_timeout_range(struct cxl_timeout *cxlt, u8 range)
> 
> "validate" is not a very name for a function returning "bool" because
> you can't tell what true/false means from the name.  "valid" would be
> fine.
> 

Will change.

>> +	pci_dbg(cxlt->dev->port,
>> +		 "Timeout & isolation timeout set to range 0x%x\n", range);
> 
> I don't know CXL, but "timeout ... timeout" reads sort of strange.  Is
> it actually a timeout for a timeout?  Maybe it was supposed to be
> "transaction timeout"?
> 

Yeah it's the same as above. I'll change it to "Timeout & isolation transaction timeout ..." instead.

>> +const struct cxl_timeout_range {
>> +	const char *str;
>> +	u8 range_val;
>> +} cxl_timeout_ranges[NUM_CXL_TIMEOUT_RANGES] = {
> 
> Static?
> 

Yep, I'll change it.

> Bjorn
>
Bjorn Helgaas Feb. 15, 2024, 10:29 p.m. UTC | #3 
On Thu, Feb 15, 2024 at 04:21:48PM -0600, Ben Cheatham wrote:
> On 2/15/24 3:35 PM, Bjorn Helgaas wrote:
> > On Thu, Feb 15, 2024 at 01:40:45PM -0600, Ben Cheatham wrote:

> >> +#define NUM_CXL_TIMEOUT_RANGES 9
> > 
> > I don't think we actually need this constant, do we?
> 
> Not really, just gets rid of a magic number (as well as makes sure
> anyone who changes that array remembers to update the array size).

My point was that I don't think you need to specify the size of the
array at all since it's initialized, and you already use ARRAY_SIZE()
elsewhere.
Ben Cheatham Feb. 15, 2024, 10:30 p.m. UTC | #4 
On 2/15/24 4:29 PM, Bjorn Helgaas wrote:
> On Thu, Feb 15, 2024 at 04:21:48PM -0600, Ben Cheatham wrote:
>> On 2/15/24 3:35 PM, Bjorn Helgaas wrote:
>>> On Thu, Feb 15, 2024 at 01:40:45PM -0600, Ben Cheatham wrote:
> 
>>>> +#define NUM_CXL_TIMEOUT_RANGES 9
>>>
>>> I don't think we actually need this constant, do we?
>>
>> Not really, just gets rid of a magic number (as well as makes sure
>> anyone who changes that array remembers to update the array size).
> 
> My point was that I don't think you need to specify the size of the
> array at all since it's initialized, and you already use ARRAY_SIZE()
> elsewhere.
> 

Sorry about that, I just completely misinterpreted that. I agree, I'll remove it.

Thanks,
Ben
diff mbox series

Patch

diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 0c65f4ec7aae..4aa5fecc43bd 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -129,11 +129,24 @@  static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
 
 /* CXL 3.0 8.2.4.23 CXL Timeout and Isolation Capability Structure */
 #define CXL_TIMEOUT_CAPABILITY_OFFSET 0x0
+#define   CXL_TIMEOUT_CAP_MEM_TIMEOUT_MASK GENMASK(3, 0)
 #define   CXL_TIMEOUT_CAP_MEM_TIMEOUT_SUPP BIT(4)
 #define CXL_TIMEOUT_CONTROL_OFFSET 0x8
+#define   CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_MASK GENMASK(3, 0)
 #define   CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_ENABLE BIT(4)
 #define CXL_TIMEOUT_CAPABILITY_LENGTH 0x10
 
+/* CXL 3.0 8.2.4.23.2 CXL Timeout and Isolation Control Register, bits 3:0 */
+#define CXL_TIMEOUT_TIMEOUT_RANGE_DEFAULT 0x0
+#define CXL_TIMEOUT_TIMEOUT_RANGE_A1 0x1
+#define CXL_TIMEOUT_TIMEOUT_RANGE_A2 0x2
+#define CXL_TIMEOUT_TIMEOUT_RANGE_B1 0x5
+#define CXL_TIMEOUT_TIMEOUT_RANGE_B2 0x6
+#define CXL_TIMEOUT_TIMEOUT_RANGE_C1 0x9
+#define CXL_TIMEOUT_TIMEOUT_RANGE_C2 0xA
+#define CXL_TIMEOUT_TIMEOUT_RANGE_D1 0xD
+#define CXL_TIMEOUT_TIMEOUT_RANGE_D2 0xE
+
 /* RAS Registers CXL 2.0 8.2.5.9 CXL RAS Capability Structure */
 #define CXL_RAS_UNCORRECTABLE_STATUS_OFFSET 0x0
 #define   CXL_RAS_UNCORRECTABLE_STATUS_MASK (GENMASK(16, 14) | GENMASK(11, 0))
diff --git a/drivers/pci/pcie/cxl_timeout.c b/drivers/pci/pcie/cxl_timeout.c
index 84f2df0e0397..916dbaf2bb58 100644
--- a/drivers/pci/pcie/cxl_timeout.c
+++ b/drivers/pci/pcie/cxl_timeout.c
@@ -20,6 +20,8 @@ 
 #include "../../cxl/cxlpci.h"
 #include "portdrv.h"
 
+#define NUM_CXL_TIMEOUT_RANGES 9
+
 struct cxl_timeout {
 	struct pcie_device *dev;
 	void __iomem *regs;
@@ -130,6 +132,57 @@  static struct pcie_cxlt_data *cxlt_create_pdata(struct pcie_device *dev)
 	return data;
 }
 
+static bool cxl_validate_timeout_range(struct cxl_timeout *cxlt, u8 range)
+{
+	u8 timeout_ranges = FIELD_GET(CXL_TIMEOUT_CAP_MEM_TIMEOUT_MASK,
+				      cxlt->cap);
+
+	if (!timeout_ranges)
+		return false;
+
+	switch (range) {
+	case CXL_TIMEOUT_TIMEOUT_RANGE_DEFAULT:
+		return true;
+	case CXL_TIMEOUT_TIMEOUT_RANGE_A1:
+	case CXL_TIMEOUT_TIMEOUT_RANGE_A2:
+		return timeout_ranges & BIT(0);
+	case CXL_TIMEOUT_TIMEOUT_RANGE_B1:
+	case CXL_TIMEOUT_TIMEOUT_RANGE_B2:
+		return timeout_ranges & BIT(1);
+	case CXL_TIMEOUT_TIMEOUT_RANGE_C1:
+	case CXL_TIMEOUT_TIMEOUT_RANGE_C2:
+		return timeout_ranges & BIT(2);
+	case CXL_TIMEOUT_TIMEOUT_RANGE_D1:
+	case CXL_TIMEOUT_TIMEOUT_RANGE_D2:
+		return timeout_ranges & BIT(3);
+	default:
+		pci_info(cxlt->dev->port, "Invalid timeout range: %d\n",
+			 range);
+		return false;
+	}
+}
+
+static int cxl_set_mem_timeout_range(struct cxl_timeout *cxlt, u8 range)
+{
+	u32 cntrl;
+
+	if (!cxlt)
+		return -ENXIO;
+
+	if (!FIELD_GET(CXL_TIMEOUT_CAP_MEM_TIMEOUT_MASK, cxlt->cap)
+	    || !cxl_validate_timeout_range(cxlt, range))
+		return -ENXIO;
+
+	cntrl = readl(cxlt->regs + CXL_TIMEOUT_CONTROL_OFFSET);
+	cntrl &= ~CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_MASK;
+	cntrl |= CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_MASK & range;
+	writel(cntrl, cxlt->regs + CXL_TIMEOUT_CONTROL_OFFSET);
+
+	pci_dbg(cxlt->dev->port,
+		 "Timeout & isolation timeout set to range 0x%x\n", range);
+	return 0;
+}
+
 static void cxl_disable_timeout(void *data)
 {
 	struct cxl_timeout *cxlt = data;
@@ -154,6 +207,135 @@  static int cxl_enable_timeout(struct pcie_device *dev, struct cxl_timeout *cxlt)
 					cxlt);
 }
 
+static ssize_t timeout_range_show(struct device *dev,
+				  struct device_attribute *attr,
+				  char *buf)
+{
+	struct pcie_device *pdev = to_pcie_device(dev);
+	struct pcie_cxlt_data *pdata = get_service_data(pdev);
+	u32 cntrl, range;
+
+	if (!pdata || !pdata->cxlt)
+		return -ENXIO;
+
+	cntrl = readl(pdata->cxlt->regs + CXL_TIMEOUT_CONTROL_OFFSET);
+
+	range = FIELD_GET(CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_MASK, cntrl);
+	return sysfs_emit(buf, "%u\n", range);
+}
+
+static ssize_t timeout_range_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	struct pcie_device *pdev = to_pcie_device(dev);
+	struct pcie_cxlt_data *pdata = get_service_data(pdev);
+	u8 range;
+	int rc;
+
+	if (kstrtou8(buf, 16, &range) < 0)
+		return -EINVAL;
+
+	if (!pdata || !pdata->cxlt)
+		return -ENXIO;
+
+	rc = cxl_set_mem_timeout_range(pdata->cxlt, range);
+	if (rc)
+		return rc;
+
+	return count;
+}
+static DEVICE_ATTR_RW(timeout_range);
+
+const struct cxl_timeout_range {
+	const char *str;
+	u8 range_val;
+} cxl_timeout_ranges[NUM_CXL_TIMEOUT_RANGES] = {
+		{ "Default range (50us-10ms)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_DEFAULT },
+		{ "Range A (50us-100us)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_A1 },
+		{ "Range A (1ms-10ms)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_A2 },
+		{ "Range B (16ms-55ms)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_B1 },
+		{ "Range B (65ms-210ms)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_B2 },
+		{ "Range C (260ms-900ms)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_C1 },
+		{ "Range C (1s-3.5s)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_C2 },
+		{ "Range D (4s-13s)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_D1 },
+		{ "Range D (17s-64s)",
+			CXL_TIMEOUT_TIMEOUT_RANGE_D2 },
+};
+
+static ssize_t available_timeout_ranges_show(struct device *dev,
+					     struct device_attribute *attr,
+					     char *buf)
+{
+	struct pcie_device *pdev = to_pcie_device(dev);
+	struct pcie_cxlt_data *pdata = get_service_data(pdev);
+	ssize_t count = 0;
+	u8 range;
+
+	if (!pdata || !pdata->cxlt)
+		return -ENXIO;
+
+	for (int i = 0; i < ARRAY_SIZE(cxl_timeout_ranges); i++) {
+		range = cxl_timeout_ranges[i].range_val;
+
+		if (cxl_validate_timeout_range(pdata->cxlt, range)) {
+			count += sysfs_emit_at(buf, count, "0x%x\t%s\n",
+					       cxl_timeout_ranges[i].range_val,
+					       cxl_timeout_ranges[i].str);
+		}
+	}
+
+	return count;
+}
+static DEVICE_ATTR_RO(available_timeout_ranges);
+
+static umode_t cxl_timeout_is_visible(struct kobject *kobj,
+				       struct attribute *attr, int val)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct pcie_device *pdev = to_pcie_device(dev);
+	struct pcie_cxlt_data *pdata = get_service_data(pdev);
+	u32 cap;
+
+	if (!pdata || !pdata->cxlt)
+		return 0;
+
+	cap = pdata->cxlt->cap;
+
+	if ((attr == &dev_attr_timeout_range.attr) &&
+	    cap & CXL_TIMEOUT_CAP_MEM_TIMEOUT_SUPP)
+		return attr->mode;
+
+	if ((attr == &dev_attr_available_timeout_ranges.attr) &&
+	    (FIELD_GET(CXL_TIMEOUT_CAP_MEM_TIMEOUT_MASK, cap)))
+		return attr->mode;
+
+	return 0;
+}
+static struct attribute *cxl_timeout_timeout_attributes[] = {
+	&dev_attr_timeout_range.attr,
+	&dev_attr_available_timeout_ranges.attr,
+	NULL,
+};
+
+static struct attribute_group cxl_timeout_timeout_group = {
+	.attrs = cxl_timeout_timeout_attributes,
+	.is_visible = cxl_timeout_is_visible,
+};
+
+static const struct attribute_group *cxl_timeout_attribute_groups[] = {
+	&cxl_timeout_timeout_group,
+	NULL,
+};
+
 static int cxl_timeout_probe(struct pcie_device *dev)
 {
 	struct pci_dev *port = dev->port;
@@ -187,6 +369,9 @@  static struct pcie_port_service_driver cxltdriver = {
 	.service	= PCIE_PORT_SERVICE_CXLT,
 
 	.probe		= cxl_timeout_probe,
+	.driver		= {
+		.dev_groups = cxl_timeout_attribute_groups,
+	},
 };
 
 int __init pcie_cxlt_init(void)