[RFC,4/8] pef: Ultravisor initialisation

From: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>

The ultravisor image after is start on each CPU after being loaded from
the flash/fsp.  It is copied to secure memory and run.

Signed-off-by: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
Signed-off-by: Santosh Sivaraj <santosh@fossix.org>
[ grimm: Add init_uv comments, logging, and logic cleanups ]
[ grimm: Increase UV image max size to 2MB ]
[ grimm: Redfine the OPAL UV shared data structure ]
[ grimm: Remove Hostboot regions from secure range 0 ]
Signed-off-by: Ryan Grimm <grimm@linux.ibm.com>
[ andmike: Split init and start of ultravisor ]
Signed-off-by: Michael Anderson <andmike@linux.ibm.com>
---
 core/init.c              |  11 +-
 hw/ultravisor.c          | 256 +++++++++++++++++++++++++++++++++++++--
 include/ultravisor-api.h |  31 +++++
 include/ultravisor.h     |  13 +-
 4 files changed, 294 insertions(+), 17 deletions(-)
 create mode 100644 include/ultravisor-api.h

On Thu, Sep 5, 2019, at 6:29 AM, Ryan Grimm wrote:
> From: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
> 
> The ultravisor image after is start on each CPU after being loaded from
> the flash/fsp.  It is copied to secure memory and run.
> 
> Signed-off-by: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
> Signed-off-by: Santosh Sivaraj <santosh@fossix.org>
> [ grimm: Add init_uv comments, logging, and logic cleanups ]
> [ grimm: Increase UV image max size to 2MB ]
> [ grimm: Redfine the OPAL UV shared data structure ]
> [ grimm: Remove Hostboot regions from secure range 0 ]
> Signed-off-by: Ryan Grimm <grimm@linux.ibm.com>
> [ andmike: Split init and start of ultravisor ]
> Signed-off-by: Michael Anderson <andmike@linux.ibm.com>
> ---
>  core/init.c              |  11 +-
>  hw/ultravisor.c          | 256 +++++++++++++++++++++++++++++++++++++--
>  include/ultravisor-api.h |  31 +++++
>  include/ultravisor.h     |  13 +-
>  4 files changed, 294 insertions(+), 17 deletions(-)
>  create mode 100644 include/ultravisor-api.h
> 
> diff --git a/core/init.c b/core/init.c
> index 08989b2d..756ee211 100644
> --- a/core/init.c
> +++ b/core/init.c
> @@ -544,6 +544,10 @@ void __noreturn load_and_boot_kernel(bool is_reboot)
>  
>  	trustedboot_exit_boot_services();
>  
> +	if (!is_reboot) {
> +		start_ultravisor();
> +	}
> +

I don't think this should live in a function that is claled load_and_boot_kernel(), have it be a separate step in the main init flow.

It represents a fairly significant step in boot, and completely changes how we have to talk to things.

>  	ipmi_set_fw_progress_sensor(IPMI_FW_OS_BOOT);
>  
>  
> @@ -1265,12 +1269,7 @@ void __noreturn __nomcount main_cpu_entry(const 
> void *fdt)
>  	add_opal_interrupts();
>  
>  	/* Init uiltravisor software */
> -	fdt = create_dtb(dt_root, false);
> -	if (!fdt) {
> -		op_display(OP_FATAL, OP_MOD_INIT, 2);
> -		abort();
> -	}
> -	init_uv(fdt);
> +	init_uv();
>  
>  	/* Now release parts of memory nodes we haven't used ourselves... */
>  	mem_region_release_unused();
> diff --git a/hw/ultravisor.c b/hw/ultravisor.c
> index 4f049254..e3d7d42a 100644
> --- a/hw/ultravisor.c
> +++ b/hw/ultravisor.c
> @@ -23,10 +23,14 @@
>  #include <string.h>
>  #include <inttypes.h>
>  #include <ultravisor.h>
> +#include <mem_region.h>
> +#include <ultravisor-api.h>
> +#include <libfdt/libfdt.h>
>  
>  static char *uv_image = NULL;
>  static size_t uv_image_size;
>  struct xz_decompress *uv_xz = NULL;
> +static struct uv_opal *uv_opal;
>  
>  static struct dt_node *add_uv_dt_node(void)
>  {
> @@ -63,32 +67,266 @@ static struct dt_node *find_uv_node(void)
>  	return uv_node;
>  }
>  
> -void init_uv(const void *fdt)
> +static bool find_secure_mem_to_copy(uint64_t *target, uint64_t *sz)
> +{
> +	struct dt_node *uv_node = find_uv_node();
> +	const struct dt_property *ranges;
> +	uint64_t uv_pef_reg;
> +	uint64_t *range, sm_size, img_size = UV_LOAD_MAX_SIZE;
> +
> +	/*
> +	 * "uv-secure-memory" property could have multiple
> +	 * secure memory blocks. Pick first to load
> +	 * ultravisor in it.
> +	 */
> +	ranges = dt_find_property(uv_node, "secure-memory-ranges");
> +	if (!ranges)
> +		return false;
> +
> +	range = (void *)ranges->prop;
> +	do {
> +		uv_pef_reg = dt_get_number(range, 2);
> +		if (!uv_pef_reg)
> +			return false;
> +
> +		sm_size = dt_get_number(range + 1, 2);
> +		if (sm_size > img_size)
> +			break;
> +		range += 2;
> +	} while (range);
> +
> +	*target = uv_pef_reg;
> +	*sz = sm_size;
> +	return true;
> +}
> +
> +static uint64_t find_uv_fw_base_addr(struct dt_node *uv_node)
> +{
> +	uint64_t base_addr = 0;
> +
> +	if (dt_has_node_property(uv_node, "uv-base-address", NULL))
> +		base_addr = dt_prop_get_u64(uv_node, "uv-base-address");
> +
> +	return base_addr;
> +}
> +
> +static void reserve_secure_memory_region(void)
> +{
> +	struct dt_node *uv_node = find_uv_node();
> +	const struct dt_property *ranges;
> +	uint64_t *range, *rangesp, sm_size, addr;
> +	char buf[128];
> +	int i=0;
> +
> +	ranges = dt_find_property(uv_node, "secure-memory-ranges");
> +	if (!ranges)
> +		return;
> +
> +	for (rangesp = (uint64_t *)(ranges->prop + ranges->len),
> +			range = (uint64_t *)ranges->prop;
> +			range < rangesp;
> +			range += 2) {
> +		addr = dt_get_number(range, 2);
> +		if (!addr)
> +			break;
> +
> +		sm_size = dt_get_number(range + 1, 2);
> +		if (!sm_size)
> +			break;
> +
> +		/* Remove Hostboot regions from secure memory 0 so we don't abort
> +		 * on overlapping regions */
> +		if (i == 0) {
> +			prlog(PR_INFO, "Secure region 0, removing HB region\n");
> +			/* TODO: Check with Hostboot for memory map */
> +			sm_size = sm_size - UV_HB_RESERVE_SIZE;
> +		}
> +
> +		snprintf(buf, 128, "ibm,secure-region-%d",i++);
> +		mem_reserve_fw(strdup(buf), addr, sm_size);
> +	}
> +
> +	return;
> +}
> +
> +static void reserve_uv_memory(struct uv_opal *uv_opal)
> +{
> +	if (uv_opal->uv_base_addr == UV_LOAD_BASE) {
> +		mem_reserve_fw("ibm,uv-code", UV_LOAD_BASE, UV_LOAD_MAX_SIZE);
> +	} else {
> +		reserve_secure_memory_region();
> +	}
> +}
> +
> +static void cpu_start_ultravisor(void *data)
> +{
> +	struct uv_opal *ptr = (struct uv_opal *)data;
> +	start_uv(ptr->uv_base_addr, ptr);
> +}
> +
> +int start_ultravisor(void)
> +{
> +	struct cpu_thread *cpu;
> +	struct cpu_job **jobs;
> +	int i=0;
> +
> +	prlog(PR_NOTICE, "UV: Starting Ultravisor at 0x%llx sys_fdt 0x%llx 
> uv_fdt 0x%0llx\n",
> +				uv_opal->uv_base_addr, uv_opal->sys_fdt, uv_opal->uv_fdt);
> +
> +	/* Alloc memory for Jobs */
> +	jobs = zalloc(sizeof(struct cpu_job*) * cpu_max_pir);
> +
> +	for_each_available_cpu(cpu) {
> +		if (cpu == this_cpu())
> +			continue;
> +		jobs[i++] = cpu_queue_job(cpu, "start_ultravisor",
> +					cpu_start_ultravisor, (void *)uv_opal);
> +	}
> +
> +	cpu_start_ultravisor((void *)uv_opal);
> +
> +	/* wait for everyone to sync back */
> +	while (i > 0) {
> +		cpu_wait_job(jobs[--i], true);
> +	}
> +
> +	/* free used stuff */
> +	free(jobs);
> +
> +	/* Check everything is fine */
> +	if (uv_opal->uv_ret_code) {
> +		return OPAL_HARDWARE;
> +	}
> +
> +	return OPAL_SUCCESS;
> +}
> +
> +static int create_dtb_uv(void *uv_fdt)
> +{
> +	if (fdt_create(uv_fdt, UV_FDT_MAX_SIZE)) {
> +		prerror("UV: Failed to create uv_fdt\n");
> +		return 1;
> +	}
> +
> +	fdt_finish_reservemap(uv_fdt);
> +	fdt_begin_node(uv_fdt, "");
> +	fdt_property_string(uv_fdt, "description", "UV dt");
> +	fdt_begin_node(uv_fdt, "ibm,uv-firmware");
> +	fdt_property_string(uv_fdt, "compatible", "ibm,uv-v1");
> +	fdt_end_node(uv_fdt);
> +	fdt_end_node(uv_fdt);
> +	fdt_finish(uv_fdt);
> +
> +	return OPAL_SUCCESS;
> +}

why are you manually manipulating the FDT rather than just using the normal functions and flattening it?

> +/* We could be running on Mambo, Cronus, or Hostboot
> + *
> + * Detect Mambo via chip quirk.  Mambo writes the uncompressed UV 
> images
> + * directly to secure memory and passes secure memory location via 
> device tree.
> + *
> + * Detect Cronus when HB decompress fails.  Cronus writes the 
> uncompressed UV
> + * image to insecure memory and init_uv will copy from insecure to 
> secure.
> + *
> + * Assume HB by waiting for decompress.  UV should have been loaded 
> from FSP
> + * and decompressed earlier via uv_preload_image and 
> uv_decompress_image.  The
> + * secure location of the UV provided by those functions in xz struct. 
> */
> +void init_uv()
>  {
>  	struct dt_node *node;
>  	const struct dt_property *base;
> +	uint64_t uv_src_addr, uv_pef_reg, uv_pef_size;
> +	void *uv_fdt;
>  
> -	assert(fdt);
> +	prlog(PR_NOTICE, "UV: Init starting\n");
>  
> -	if (!is_msr_bit_set(MSR_S))
> +	uv_opal = zalloc(sizeof(struct uv_opal));
> +	if (!uv_opal) {
> +		prerror("UV: Failed to allocate uv_opal\n");
>  		return;
> +	}
> +
> +	if (!is_msr_bit_set(MSR_S)) {
> +		prlog(PR_DEBUG, "UV: S bit not set\n");
> +		return;
> +	}
>  
> -	if (!uv_xz)
> +	node = find_uv_node();
> +	if (!node) {
> +		prlog(PR_DEBUG, "UV: Device tree node not found\n");
>  		return;
> +	}
> +
> +	if (proc_chip_quirks & QUIRK_MAMBO_CALLOUTS) {
> +		prlog(PR_INFO, "UV: Mambo simulator detected\n");
> +
> +		if (!find_secure_mem_to_copy(&uv_pef_reg, &uv_pef_size)) {
> +			prlog(PR_DEBUG, "UV: No secure memory configured, exiting\n");
> +			goto load_error;
> +		}
> +
> +		goto start;
> +	}
> +
> +	/* This would be null in case we are on Mambo or Cronus */
> +	if (!uv_xz) {
> +
> +		prlog(PR_INFO, "UV: Platform load failed, detecting UV image via 
> device tree\n");
> +
> +		if (!find_secure_mem_to_copy(&uv_pef_reg, &uv_pef_size)) {
> +			prlog(PR_DEBUG, "UV: No secure memory configured, exiting\n");
> +			goto load_error;
> +		}
> +
> +		uv_src_addr = find_uv_fw_base_addr(node);
> +		if (!uv_src_addr) {
> +			prlog(PR_DEBUG, "UV: Couldn't find UV base address in device 
> tree\n");
> +			goto load_error;
> +		}
>  
> +		prlog(PR_INFO, "UV: Copying Ultravisor to protected memory 0x%llx 
> from 0x%llx\n", uv_pef_reg, uv_src_addr);
> +
> +		memcpy((void *)uv_pef_reg, (void *)uv_src_addr, UV_LOAD_MAX_SIZE);
> +
> +		goto start;
> +	}
> +
> +	/* Hostboot path */
>  	wait_xz_decompress(uv_xz);
> -	if (uv_xz->status)
> +	if (uv_xz->status) {
> +		prlog(PR_INFO, "UV: Compressed Ultravisor image failed to 
> decompress");
>  		goto load_error;
> +	}

IMNSHO in all of these situations the secure and trusted boot dance should be done.

> -	/* the uncompressed location will be the base address of ultravisor */
> -	node = find_uv_node();
> +	/* the uncompressed location will be the base address of ultravisor
> +	 * so fix up if it's already there */
>  	base = dt_find_property(node, "uv-base-address");
>  	if (base)
>  		dt_del_property(node, (struct dt_property *)base);
>  
>  	dt_add_property_u64(node, "uv-base-address", (uint64_t)uv_xz->dst);
>  
> -	/* TODO start ultravisor */
> +	uv_pef_reg = (uint64_t)uv_xz->dst;
> +	uv_pef_size = (uint64_t)uv_xz->dst_size;
> +
> +start:
> +	uv_opal->uv_base_addr = uv_pef_reg;
> +
> +	uv_opal->sys_fdt = (__be64)create_dtb(dt_root, false);
> +	if (!uv_opal->sys_fdt) {
> +		prerror("UV: Failed to create system fdt\n");
> +		goto load_error;
> +	}
> +
> +	uv_fdt = (void *)(uv_pef_reg + UV_LOAD_MAX_SIZE);
> +	if (create_dtb_uv(uv_fdt)) {
> +		prerror("UV: Failed to create uv fdt\n");
> +		goto load_error;
> +	}
> +	uv_opal->uv_fdt = (__be64)uv_fdt;
> +
> +	reserve_uv_memory(uv_opal);
> +
>  load_error:
>  	free(uv_image);
>  	free(uv_xz);
> @@ -190,7 +428,7 @@ void uv_preload_image(void)
>  	uv_image_size = MAX_COMPRESSED_UV_IMAGE_SIZE;
>  	uv_image = malloc(MAX_COMPRESSED_UV_IMAGE_SIZE);
>  	if (!uv_image) {
> -		prerror("Memory allocation for ultravisor failed\n");
> +		prerror("UV: Memory allocation for Ultravisor failed\n");
>  		return;
>  	}
>  
> diff --git a/include/ultravisor-api.h b/include/ultravisor-api.h
> new file mode 100644
> index 00000000..52d86024
> --- /dev/null
> +++ b/include/ultravisor-api.h
> @@ -0,0 +1,31 @@
> +/* Copyright 2013-2018 IBM Corp.

nothing in 2019?

Also, should be SPDX headers.

> + *
> + * Licensed under the Apache License, Version 2.0 (the "License");
> + * you may not use this file except in compliance with the License.
> + * You may obtain a copy of the License at
> + *
> + * 	http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> + * implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +#ifndef __ULTRAVISOR_API_H
> +#define __ULTRAVISOR_API_H
> +
> +struct uv_opal {
> +	__be32 magic;		/**< 'OPUV' 0x4F505556 OPUV_MAGIC */
> +	__be32 version;		/**< uv_opal struct version */
> +	__be32 uv_ret_code;	/**< 0 - Success, <0> : error. */
> +	__be32 uv_api_ver;	/**< Current uv api version. */
> +	__be64 uv_base_addr;	/**< Base address of UV in secure memory. */
> +	__be64 sys_fdt;		/**< System FDT. */
> +	__be64 uv_fdt;		/**< UV FDT in secure memory. */
> +	__be64 uv_mem;		/**< struct memcons */
> +};
> +
> +#endif /* __ULTRAVISOR_API_H */
> diff --git a/include/ultravisor.h b/include/ultravisor.h
> index 08f10da4..976b6323 100644
> --- a/include/ultravisor.h
> +++ b/include/ultravisor.h
> @@ -17,15 +17,24 @@
>  #ifndef __ULTRAVISOR_H
>  #define __ULTRAVISOR_H

needs spdx header.