diff mbox series

[6/6] platforms/astbmc/witherspoon: Implement OpenCAPI support

Message ID 9d2afa345b903e7b3af63c378f0742f8e05c34e8.1534495426.git-series.andrew.donnellan@au1.ibm.com
State Superseded
Headers show
Series OpenCAPI support for Witherspoon | expand

Checks

Context Check Description
snowpatch_ozlabs/apply_patch success master/apply_patch Successfully applied
snowpatch_ozlabs/make_check success Test make_check on branch master

Commit Message

Andrew Donnellan Aug. 17, 2018, 8:44 a.m. UTC 
OpenCAPI on Witherspoon is slightly more involved than on Zaius and ZZ, due
to the OpenCAPI links using the SXM2 connectors that are used for NVLink
GPUs.

This patch adds the regular OpenCAPI platform information, and also a
Witherspoon-specific presence detection callback that uses the previously
added OCC GPU presence detection to figure out the device types plugged
into each SXM2 socket.

The SXM2 connectors are capable of carrying 2 OpenCAPI links, and future
OpenCAPI devices are expected to make use of this. However, we don't yet
support ganged links and the various implications that has for handling
things like device reset, so for now, we only enable 1 brick per device.

Signed-off-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
---
 platforms/astbmc/witherspoon.c | 216 +++++++++++++++++++++++++++++++++-
 1 file changed, 213 insertions(+), 3 deletions(-)

Comments

Reza Arbab Aug. 20, 2018, 5:05 p.m. UTC | #1 
On Fri, Aug 17, 2018 at 06:44:41PM +1000, Andrew Donnellan wrote:
>@@ -63,6 +98,26 @@ static bool witherspoon_probe(void)
>
> 	vpd_dt_fixup();
>
>+	witherspoon_create_ocapi_i2c_bus();
>+
>+	dt_for_each_compatible(dt_root, np, "ibm,npu-link") {
>+		gpu_group_id = dt_prop_get_u32(np, "ibm,npu-group-id");
>+		if (gpu_group_id > highest_gpu_group_id)
>+			highest_gpu_group_id = gpu_group_id;
>+	};
>+
>+	switch (highest_gpu_group_id) {
>+	case 1:
>+		witherspoon_type = WITHERSPOON_TYPE_REDBUD;
>+		break;
>+	case 2:
>+		witherspoon_type = WITHERSPOON_TYPE_SEQUOIA;
>+		break;
>+	default:
>+		witherspoon_type = WITHERSPOON_TYPE_UNKNOWN;
>+		prlog(PR_NOTICE, "PLAT: Unknown Witherspoon variant detected\n");
>+	}
>+
> 	return true;
> }
>

Nice! Figuring this out was also the motivation for creating 
npu2_links_per_gpu(); 3 links for Redbud, 2 for Sequoia.

Alexey will also need to know which we are [1]. Do you think it's worth 
finding a way to represent this that we can all leverage? Just a 
thought. Regardless,

Acked-by: Reza Arbab <arbab@linux.ibm.com>

[1] http://patchwork.ozlabs.org/patch/957813/
Andrew Donnellan Aug. 21, 2018, 1:47 a.m. UTC | #2 
On 21/08/18 03:05, Reza Arbab wrote:
> Nice! Figuring this out was also the motivation for creating 
> npu2_links_per_gpu(); 3 links for Redbud, 2 for Sequoia.
> 
> Alexey will also need to know which we are [1]. Do you think it's worth 
> finding a way to represent this that we can all leverage? Just a 
> thought. Regardless,

Hmm. I'd seen Alexey's patch on the list, but I didn't notice 
npu2_links_per_gpu()...

It would be nice to not have to repeat ourselves here, but I'm also not 
sure there's a nice way to do it. I'm open to suggestions though.
Alistair Popple Aug. 21, 2018, 6:59 a.m. UTC | #3 
<snip>

> +	gpu0_present = occ_get_gpu_presence(chip, 0);
> +	if (gpu0_present) {
> +		prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 slot present\n", chip->id);
> +	}
> +
> +	gpu1_present = occ_get_gpu_presence(chip, 1);
> +	if (gpu1_present) {
> +		prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 slot present\n", chip->id);
> +	}
> +
> +	/* Set pins to input */
> +	state = 0xff;
> +	rc = i2c_request_send(i2c_port_id,
> +			      platform.ocapi->i2c_presence_addr, SMBUS_WRITE, 3,
> +			      1, &state, 1, 120);
> +	if (rc)
> +		goto i2c_failed;
> +
> +	/* Read the presence value */
> +	state = 0x00;
> +	rc = i2c_request_send(i2c_port_id,
> +			      platform.ocapi->i2c_presence_addr, SMBUS_READ, 0,
> +			      1, &state, 1, 120);
> +	if (rc)
> +		goto i2c_failed;
> +
> +	if (gpu0_present) {
> +		if (state & (1 << 0)) {
> +			prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 is OpenCAPI\n",
> +			      chip->id);
> +			/*
> +			 * On witherspoon, bricks 2 and 3 are connected to
> +			 * the lanes matching links 1 and 0 in OpenCAPI mode.
> +			 */
> +			dev = find_link_index(npu, 0);
> +			if (dev) {
> +				dev->type = NPU2_DEV_TYPE_OPENCAPI;
> +				dev->brick_index = 3;
> +			}
> +
> +			dev = find_link_index(npu, 1);
> +			if (dev) {
> +				/* dev->type = NPU2_DEV_TYPE_OPENCAPI; */
> +				/* We current don't support using the second link */
> +				dev->type = NPU2_DEV_TYPE_UNKNOWN;
> +				dev->brick_index = 2;
> +			}
> +		} else {
> +			prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 is NVLink\n",
> +			      chip->id);
> +			dev = find_link_index(npu, 0);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_NVLINK;
> +			dev = find_link_index(npu, 1);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_NVLINK;
> +			dev = find_link_index(npu, 2);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_NVLINK;
> +		}
> +	}
> +
> +	if (gpu1_present) {
> +		if (state & (1 << 1)) {
> +			prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 is OpenCAPI\n",
> +			      chip->id);
> +			dev = find_link_index(npu, 4);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_OPENCAPI;
> +			dev = find_link_index(npu, 5);
> +			if (dev)
> +				/* dev->type = NPU2_DEV_TYPE_OPENCAPI; */
> +				/* We current don't support using the second link */
> +				dev->type = NPU2_DEV_TYPE_UNKNOWN;
> +		} else {
> +			prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 is NVLink\n",
> +			      chip->id);
> +			dev = find_link_index(npu, 3);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_NVLINK;
> +			dev = find_link_index(npu, 4);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_NVLINK;
> +			dev = find_link_index(npu, 5);
> +			if (dev)
> +				dev->type = NPU2_DEV_TYPE_NVLINK;
> +		}
> +	}

There looks to be a reasonable amount of code duplication here which might
benefit from a little refactoring but it looks correct.

Reviewed-By: Alistair Popple <alistair@popple.id.au>
Alistair Popple Aug. 21, 2018, 7:02 a.m. UTC | #4 
> Alexey will also need to know which we are [1]. Do you think it's worth 
> finding a way to represent this that we can all leverage? Just a 
> thought. Regardless,

Ideally the information Alexey needs should come from the MRW (machine readable
workbook). However that's "hard" so I guess he could use Redbud vs. Sequoia to
populate the device tree with appropriate nvlink-peer nodes.

- Alistair

> Acked-by: Reza Arbab <arbab@linux.ibm.com>
> 
> [1] http://patchwork.ozlabs.org/patch/957813/
> 
>
Alexey Kardashevskiy Aug. 21, 2018, 7:43 a.m. UTC | #5 
On 21/08/2018 17:02, Alistair Popple wrote:
>> Alexey will also need to know which we are [1]. Do you think it's worth 
>> finding a way to represent this that we can all leverage? Just a 
>> thought. Regardless,
> 
> Ideally the information Alexey needs should come from the MRW (machine readable
> workbook). However that's "hard" so I guess he could use Redbud vs. Sequoia to
> populate the device tree with appropriate nvlink-peer nodes.


I have to walk through GPUs anyway to get their labels/numbers and this
gives me the number of GPUs so this patch is not particularly useful for
my task.


> 
> - Alistair
> 
>> Acked-by: Reza Arbab <arbab@linux.ibm.com>
>>
>> [1] http://patchwork.ozlabs.org/patch/957813/
diff mbox series

Patch

diff --git a/platforms/astbmc/witherspoon.c b/platforms/astbmc/witherspoon.c
index ce83ff9701d3..0d97c46760a7 100644
--- a/platforms/astbmc/witherspoon.c
+++ b/platforms/astbmc/witherspoon.c
@@ -28,10 +28,18 @@ 
 #include <pci-slot.h>
 #include <phb4.h>
 #include <npu2.h>
+#include <occ.h>
+#include <i2c.h>
 
 #include "astbmc.h"
 #include "ast.h"
 
+static enum {
+	WITHERSPOON_TYPE_UNKNOWN,
+	WITHERSPOON_TYPE_SEQUOIA,
+	WITHERSPOON_TYPE_REDBUD
+} witherspoon_type;
+
 /*
  * HACK: Hostboot doesn't export the correct data for the system VPD EEPROM
  *       for this system. So we need to work around it here.
@@ -50,8 +58,35 @@  static void vpd_dt_fixup(void)
 	}
 }
 
+static void witherspoon_create_ocapi_i2c_bus(void)
+{
+	struct dt_node *xscom, *i2cm, *i2c_bus;
+	prlog(PR_DEBUG, "OCAPI: Adding I2C bus device node for OCAPI reset\n");
+	dt_for_each_compatible(dt_root, xscom, "ibm,xscom") {
+		i2cm = dt_find_by_name(xscom, "i2cm@a1000");
+		if (!i2cm) {
+			prlog(PR_ERR, "OCAPI: Failed to add I2C bus device node\n");
+			continue;
+		}
+
+		if (dt_find_by_name(i2cm, "i2c-bus@4"))
+			continue;
+
+		i2c_bus = dt_new_addr(i2cm, "i2c-bus", 4);
+		dt_add_property_cells(i2c_bus, "reg", 4);
+		dt_add_property_cells(i2c_bus, "bus-frequency", 0x61a80);
+		dt_add_property_strings(i2c_bus, "compatible",
+					"ibm,opal-i2c", "ibm,power8-i2c-port",
+					"ibm,power9-i2c-port");
+	}
+}
+
 static bool witherspoon_probe(void)
 {
+	struct dt_node *np;
+	int highest_gpu_group_id = 0;
+	int gpu_group_id;
+
 	if (!dt_node_is_compatible(dt_root, "ibm,witherspoon"))
 		return false;
 
@@ -63,6 +98,26 @@  static bool witherspoon_probe(void)
 
 	vpd_dt_fixup();
 
+	witherspoon_create_ocapi_i2c_bus();
+
+	dt_for_each_compatible(dt_root, np, "ibm,npu-link") {
+		gpu_group_id = dt_prop_get_u32(np, "ibm,npu-group-id");
+		if (gpu_group_id > highest_gpu_group_id)
+			highest_gpu_group_id = gpu_group_id;
+	};
+
+	switch (highest_gpu_group_id) {
+	case 1:
+		witherspoon_type = WITHERSPOON_TYPE_REDBUD;
+		break;
+	case 2:
+		witherspoon_type = WITHERSPOON_TYPE_SEQUOIA;
+		break;
+	default:
+		witherspoon_type = WITHERSPOON_TYPE_UNKNOWN;
+		prlog(PR_NOTICE, "PLAT: Unknown Witherspoon variant detected\n");
+	}
+
 	return true;
 }
 
@@ -154,14 +209,168 @@  static void witherspoon_pre_pci_fixup(void)
 	phb4_pre_pci_fixup_witherspoon();
 }
 
-static void witherspoon_npu2_device_detect(struct npu2 *npu)
+static struct npu2_dev *find_link_index(struct npu2 *npu, uint32_t link_index)
 {
-        /* Stub until we implement real device detection */
 	for (int i = 0; i < npu->total_devices; i++) {
-		npu->devices[i].type = NPU2_DEV_TYPE_NVLINK;
+		if (npu->devices[i].link_index == link_index)
+			return &npu->devices[i];
 	}
+	prlog(PR_ERR, "PLAT: Could not find NPU link index %d\n", link_index);
+	return NULL;
 }
 
+static void witherspoon_npu2_device_detect(struct npu2 *npu)
+{
+	struct proc_chip *chip;
+	struct npu2_dev *dev;
+	uint8_t state;
+	uint64_t i2c_port_id = 0;
+	char port_name[17];
+	struct dt_node *dn;
+	int rc;
+
+	bool gpu0_present, gpu1_present;
+
+	if (witherspoon_type != WITHERSPOON_TYPE_REDBUD) {
+		prlog(PR_DEBUG, "PLAT: Setting all NPU links to NVLink, OpenCAPI only supported on Redbud\n");
+	        for (int i = 0; i < npu->total_devices; i++) {
+			npu->devices[i].type = NPU2_DEV_TYPE_NVLINK;
+		}
+		return;
+	}
+	assert(npu->total_devices == 6);
+
+	chip = get_chip(npu->chip_id);
+
+	/* Find I2C port */
+	snprintf(port_name, sizeof(port_name), "p8_%08x_e%dp%d",
+		 chip->id, platform.ocapi->i2c_engine,
+		 platform.ocapi->i2c_port);
+	dt_for_each_compatible(dt_root, dn, "ibm,power9-i2c-port") {
+		if (streq(port_name, dt_prop_get(dn, "ibm,port-name"))) {
+			i2c_port_id = dt_prop_get_u32(dn, "ibm,opal-id");
+			break;
+		}
+	}
+
+	if (!i2c_port_id) {
+		prlog(PR_ERR, "PLAT: Could not find NPU presence I2C port\n");
+		return;
+	}
+
+	gpu0_present = occ_get_gpu_presence(chip, 0);
+	if (gpu0_present) {
+		prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 slot present\n", chip->id);
+	}
+
+	gpu1_present = occ_get_gpu_presence(chip, 1);
+	if (gpu1_present) {
+		prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 slot present\n", chip->id);
+	}
+
+	/* Set pins to input */
+	state = 0xff;
+	rc = i2c_request_send(i2c_port_id,
+			      platform.ocapi->i2c_presence_addr, SMBUS_WRITE, 3,
+			      1, &state, 1, 120);
+	if (rc)
+		goto i2c_failed;
+
+	/* Read the presence value */
+	state = 0x00;
+	rc = i2c_request_send(i2c_port_id,
+			      platform.ocapi->i2c_presence_addr, SMBUS_READ, 0,
+			      1, &state, 1, 120);
+	if (rc)
+		goto i2c_failed;
+
+	if (gpu0_present) {
+		if (state & (1 << 0)) {
+			prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 is OpenCAPI\n",
+			      chip->id);
+			/*
+			 * On witherspoon, bricks 2 and 3 are connected to
+			 * the lanes matching links 1 and 0 in OpenCAPI mode.
+			 */
+			dev = find_link_index(npu, 0);
+			if (dev) {
+				dev->type = NPU2_DEV_TYPE_OPENCAPI;
+				dev->brick_index = 3;
+			}
+
+			dev = find_link_index(npu, 1);
+			if (dev) {
+				/* dev->type = NPU2_DEV_TYPE_OPENCAPI; */
+				/* We current don't support using the second link */
+				dev->type = NPU2_DEV_TYPE_UNKNOWN;
+				dev->brick_index = 2;
+			}
+		} else {
+			prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 is NVLink\n",
+			      chip->id);
+			dev = find_link_index(npu, 0);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_NVLINK;
+			dev = find_link_index(npu, 1);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_NVLINK;
+			dev = find_link_index(npu, 2);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_NVLINK;
+		}
+	}
+
+	if (gpu1_present) {
+		if (state & (1 << 1)) {
+			prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 is OpenCAPI\n",
+			      chip->id);
+			dev = find_link_index(npu, 4);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_OPENCAPI;
+			dev = find_link_index(npu, 5);
+			if (dev)
+				/* dev->type = NPU2_DEV_TYPE_OPENCAPI; */
+				/* We current don't support using the second link */
+				dev->type = NPU2_DEV_TYPE_UNKNOWN;
+		} else {
+			prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 is NVLink\n",
+			      chip->id);
+			dev = find_link_index(npu, 3);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_NVLINK;
+			dev = find_link_index(npu, 4);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_NVLINK;
+			dev = find_link_index(npu, 5);
+			if (dev)
+				dev->type = NPU2_DEV_TYPE_NVLINK;
+		}
+	}
+
+	return;
+
+i2c_failed:
+	prlog(PR_ERR, "PLAT: NPU device type detection failed, rc=%d\n", rc);
+	return;
+}
+
+const struct platform_ocapi witherspoon_ocapi = {
+       .i2c_engine     = 1,
+       .i2c_port       = 4,
+       .odl_phy_swap   = false,
+       .i2c_reset_addr = 0x20,
+       .i2c_reset_pin2 = 1 << 0,
+       .i2c_reset_pin3 = 1 << 0,
+       .i2c_reset_pin4 = 1 << 1,
+       .i2c_reset_pin5 = 1 << 1,
+       .i2c_presence_addr = 0x20,
+       /* unused, we do this in custom presence detect */
+       .i2c_presence_pin2 = 0,
+       .i2c_presence_pin3 = 0,
+       .i2c_presence_pin4 = 0,
+       .i2c_presence_pin5 = 0,
+};
+
 /* The only difference between these is the PCI slot handling */
 
 DECLARE_PLATFORM(witherspoon) = {
@@ -179,5 +388,6 @@  DECLARE_PLATFORM(witherspoon) = {
 	.terminate		= ipmi_terminate,
 
 	.pci_get_slot_info	= dt_slot_get_slot_info,
+	.ocapi                  = &witherspoon_ocapi,
 	.npu2_device_detect	= witherspoon_npu2_device_detect,
 };