@@ -913,8 +913,7 @@ opal_entry:
std %r1,STACK_GPR1(%r12)
mr %r1,%r12
- /* May save arguments for tracing */
-#ifdef OPAL_TRACE_ENTRY
+ /* Save arguments because we call C */
std %r3,STACK_GPR3(%r1)
std %r4,STACK_GPR4(%r1)
std %r5,STACK_GPR5(%r1)
@@ -923,7 +922,7 @@ opal_entry:
std %r8,STACK_GPR8(%r1)
std %r9,STACK_GPR9(%r1)
std %r10,STACK_GPR10(%r1)
-#endif
+
/* Save Token (r0), LR and r13 */
mflr %r12
std %r0,STACK_GPR0(%r1)
@@ -944,15 +943,10 @@ opal_entry:
addis %r2,%r2,(__toc_start - __head)@ha
addi %r2,%r2,(__toc_start - __head)@l
- /* Check for a reboot in progress */
- LOAD_ADDR_FROM_TOC(%r12, reboot_in_progress)
- lbz %r12,0(%r12)
- cmpwi %r12,0
- bne 3f
-
-#ifdef OPAL_TRACE_ENTRY
mr %r3,%r1
- bl opal_trace_entry
+ bl opal_entry_check
+ cmpdi %r3,0
+ bne 1f
ld %r0,STACK_GPR0(%r1)
ld %r3,STACK_GPR3(%r1)
ld %r4,STACK_GPR4(%r1)
@@ -962,39 +956,25 @@ opal_entry:
ld %r8,STACK_GPR8(%r1)
ld %r9,STACK_GPR9(%r1)
ld %r10,STACK_GPR10(%r1)
-#endif /* OPAL_TRACE_ENTRY */
/* Convert our token into a table entry and get the
* function pointer. Also check the token.
* For ELFv2 ABI, the local entry point is used so no need for r12.
*/
- cmpldi %r0,OPAL_LAST
- bgt- 2f
sldi %r0,%r0,3
LOAD_ADDR_FROM_TOC(%r12, opal_branch_table)
ldx %r0,%r12,%r0
- cmpldi %r0,0
- beq- 2f
mtctr %r0
/* Jump ! */
bctrl
-
+ bl opal_exit_check
1: ld %r12,STACK_LR(%r1)
mtlr %r12
ld %r13,STACK_GPR13(%r1)
ld %r1,STACK_GPR1(%r1)
blr
-2: /* Bad token */
- ld %r3,STACK_GPR0(%r1)
- bl opal_bad_token
- b 1b
-
-3: /* Reboot in progress, reject all calls */
- li %r3,OPAL_BUSY
- b 1b
-
.global start_kernel
start_kernel:
sync
@@ -35,7 +35,6 @@
/* Flag tested by the OPAL entry code */
-uint8_t reboot_in_progress;
static volatile bool fast_boot_release;
static struct cpu_thread *last_man_standing;
static struct lock reset_lock = LOCK_UNLOCKED;
@@ -333,19 +332,17 @@ void fast_reboot(void)
}
unlock(&fast_reboot_disabled_lock);
+ /*
+ * Ensure all other CPUs have left OPAL calls.
+ */
+ if (!opal_shutdown(SHUTDOWN_QUIESCE, -1)) {
+ prlog(PR_DEBUG, "RESET: Fast reboot disabled because OPAL calls not quiescing\n");
+ return;
+ }
+
prlog(PR_NOTICE, "RESET: Initiating fast reboot %d...\n", ++fast_reboot_count);
free(fdt);
- /* XXX We need a way to ensure that no other CPU is in skiboot
- * holding locks (via the OPAL APIs) and if they are, we need
- * for them to get out. Hopefully that isn't happening, but...
- *
- * To fix this properly, we want to keep track of OPAL entry/exit
- * on all CPUs.
- */
- reboot_in_progress = 1;
- time_wait_ms(200);
-
/* Lock so the new guys coming don't reset us */
lock(&reset_lock);
@@ -552,7 +549,7 @@ void __noreturn fast_reboot_entry(void)
/* Clear release flag for next time */
fast_boot_release = false;
- reboot_in_progress = 0;
+ opal_shutdown(SHUTDOWN_RESUME, -1);
/* Cleanup ourselves */
cleanup_cpu_state();
@@ -93,6 +93,9 @@ bool try_lock(struct lock *l)
{
struct cpu_thread *cpu = this_cpu();
+ if (bust_locks)
+ return true;
+
if (__try_lock(cpu, l)) {
if (l->in_con_path)
cpu->con_suspend++;
@@ -112,8 +115,13 @@ void lock(struct lock *l)
if (try_lock(l))
break;
smt_lowest();
- while (l->lock_val)
+ while (l->lock_val) {
+ if (bust_locks) {
+ smt_medium();
+ return;
+ }
barrier();
+ }
smt_medium();
}
}
@@ -92,19 +92,12 @@ long opal_bad_token(uint64_t token)
return OPAL_PARAMETER;
}
-/* Called from head.S, thus no prototype */
-void opal_trace_entry(struct stack_frame *eframe);
-
-void opal_trace_entry(struct stack_frame *eframe)
+static void opal_trace_entry(struct stack_frame *eframe __unused)
{
+#ifdef OPAL_TRACE_ENTRY
union trace t;
unsigned nargs, i;
- if (this_cpu()->pir != mfspr(SPR_PIR)) {
- printf("CPU MISMATCH ! PIR=%04lx cpu @%p -> pir=%04x\n",
- mfspr(SPR_PIR), this_cpu(), this_cpu()->pir);
- abort();
- }
if (eframe->gpr[0] > OPAL_LAST)
nargs = 0;
else
@@ -117,7 +110,167 @@ void opal_trace_entry(struct stack_frame *eframe)
t.opal.r3_to_11[i] = cpu_to_be64(eframe->gpr[3+i]);
trace_add(&t, TRACE_OPAL, offsetof(struct trace_opal, r3_to_11[nargs]));
+#endif
+}
+
+/* Shutdown state has PIR in top 32-bits, 0/1 in bottom bit */
+static uint64_t opal_shutdown_state = 0;
+
+/* Called from head.S, thus no prototype */
+int64_t opal_entry_check(struct stack_frame *eframe);
+
+static int64_t opal_check_token(uint64_t token);
+
+int64_t opal_entry_check(struct stack_frame *eframe)
+{
+ struct cpu_thread *cpu = this_cpu();
+ uint64_t token = eframe->gpr[0];
+
+ if (cpu->pir != mfspr(SPR_PIR)) {
+ printf("CPU MISMATCH ! PIR=%04lx cpu @%p -> pir=%04x\n",
+ mfspr(SPR_PIR), cpu, cpu->pir);
+ abort();
+ }
+
+ opal_trace_entry(eframe);
+
+ if (!opal_check_token(token))
+ return opal_bad_token(token);
+
+ if (!opal_shutdown_state && cpu->in_opal_call) {
+ printf("CPU ATTEMPT TO RE-ENTER FIRMWARE! PIR=%04lx cpu @%p -> pir=%04x\n",
+ mfspr(SPR_PIR), cpu, cpu->pir);
+ return OPAL_BUSY;
+ }
+
+again:
+ cpu->in_opal_call++;
+ sync(); /* Store in_opal_call vs load opal_quiesce */
+ if (cpu->quiesce_opal_call) {
+ cpu->in_opal_call--;
+ smt_lowest();
+ while (cpu->quiesce_opal_call)
+ barrier();
+ smt_medium();
+ goto again;
+ }
+
+ return OPAL_SUCCESS;
+}
+
+void opal_exit_check(void);
+
+void opal_exit_check(void)
+{
+ struct cpu_thread *cpu = this_cpu();
+
+ if (!cpu->in_opal_call) {
+ printf("CPU UN-ACCOUNTED FIRMWARE ENTRY! PIR=%04lx cpu @%p -> pir=%04x\n",
+ mfspr(SPR_PIR), cpu, cpu->pir);
+ } else {
+ cpu->in_opal_call--;
+ }
+}
+
+int64_t opal_shutdown(uint64_t shutdown_type, int cpu_target)
+{
+ struct cpu_thread *cpu = this_cpu();
+ struct cpu_thread *c = NULL;
+ uint64_t end;
+ bool stuck = false;
+
+ if (cpu_target >= 0) {
+ c = find_cpu_by_server(cpu_target);
+ if (!c)
+ return OPAL_PARAMETER;
+ } else if (cpu_target != -1) {
+ return OPAL_PARAMETER;
+ }
+
+ if (shutdown_type == SHUTDOWN_QUIESCE) {
+ if (cmpxchg64(&opal_shutdown_state, 0,
+ ((uint64_t)cpu->pir << 32) | 1) != 0) {
+ printf("opal_shutdown another CPU already in shutdown\n");
+ return OPAL_BUSY;
+ }
+ }
+
+ if (opal_shutdown_state != (((uint64_t)cpu->pir << 32) | 1)) {
+ printf("opal_shutdown CPU does not own shutdown state (must call SHUTDOWN_QUIESCE)\n");
+ return OPAL_BUSY;
+ }
+
+ /* Okay now we own the shutdown state */
+
+ if (shutdown_type == SHUTDOWN_RESUME) {
+ bust_locks = false;
+ sync();
+ if (c) {
+ c->quiesce_opal_call = false;
+ } else {
+ for_each_cpu(c) {
+ if (c == cpu)
+ continue;
+ c->quiesce_opal_call = false;
+ }
+ }
+ sync();
+ opal_shutdown_state = 0;
+ return OPAL_SUCCESS;
+ }
+
+ if (shutdown_type == SHUTDOWN_SINGLE_THREADED) {
+ bust_locks = true;
+ return OPAL_SUCCESS;
+ }
+
+ if (c) {
+ c->quiesce_opal_call = true;
+ } else {
+ for_each_cpu(c) {
+ if (c == cpu)
+ continue;
+ c->quiesce_opal_call = true;
+ }
+ }
+ sync();
+
+ end = mftb() + msecs_to_tb(1000);
+
+ smt_lowest();
+ if (c) {
+ while (c->in_opal_call) {
+ if (tb_compare(mftb(), end) == TB_AAFTERB) {
+ printf("OPAL shutdown CPU:%04x stuck in OPAL\n", c->pir);
+ stuck = true;
+ break;
+ }
+ barrier();
+ }
+ } else {
+ for_each_cpu(c) {
+ if (c == cpu)
+ continue;
+ while (c->in_opal_call) {
+ if (tb_compare(mftb(), end) == TB_AAFTERB) {
+ printf("OPAL shutdown CPU:%04x stuck in OPAL\n", c->pir);
+ stuck = true;
+ break;
+ }
+ barrier();
+ }
+ }
+ }
+ smt_medium();
+
+ if (stuck) {
+ printf("OPAL shutdown could not kick all CPUs out of OPAL\n");
+ return OPAL_PARTIAL;
+ }
+
+ return OPAL_SUCCESS;
}
+opal_call(OPAL_SHUTDOWN, opal_shutdown, 2);
void __opal_register(uint64_t token, void *func, unsigned int nargs)
{
@@ -54,6 +54,8 @@ static int64_t opal_cec_reboot(void)
{
prlog(PR_NOTICE, "OPAL: Reboot request...\n");
+ opal_shutdown(SHUTDOWN_QUIESCE, -1);
+
console_complete_flush();
/* Try fast-reset unless explicitly disabled */
@@ -71,6 +73,8 @@ static int64_t opal_cec_reboot2(uint32_t reboot_type, char *diag)
{
struct errorlog *buf;
+ opal_shutdown(SHUTDOWN_QUIESCE, -1);
+
switch (reboot_type) {
case OPAL_REBOOT_NORMAL:
return opal_cec_reboot();
@@ -54,9 +54,11 @@ struct cpu_thread {
struct trace_info *trace;
uint64_t save_r1;
void *icp_regs;
+ uint32_t in_opal_call;
uint32_t lock_depth;
uint32_t con_suspend;
bool con_need_flush;
+ bool quiesce_opal_call;
bool in_mcount;
bool in_poller;
bool in_reinit;
@@ -18,6 +18,7 @@
#define __LOCK_H
#include <stdbool.h>
+#include <processor.h>
struct lock {
/* Lock value has bit 63 as lock bit and the PIR of the owner
@@ -86,6 +87,17 @@ static inline uint64_t __cmpxchg64(uint64_t *mem, uint64_t old, uint64_t new)
return prev;
}
+static inline uint64_t cmpxchg64(uint64_t *mem, uint64_t old, uint64_t new)
+{
+ uint64_t prev;
+
+ sync();
+ prev = __cmpxchg64(mem, old,new);
+ sync();
+
+ return prev;
+}
+
extern bool try_lock(struct lock *l);
extern void lock(struct lock *l);
extern void unlock(struct lock *l);
@@ -214,7 +214,12 @@
#define OPAL_SET_POWER_SHIFT_RATIO 155
#define OPAL_SENSOR_GROUP_CLEAR 156
#define OPAL_PCI_SET_P2P 157
-#define OPAL_LAST 157
+#define OPAL_SHUTDOWN 158
+#define OPAL_LAST 158
+
+#define SHUTDOWN_QUIESCE 1 /* CPUs out of OPAL calls */
+#define SHUTDOWN_SINGLE_THREADED 2 /* Call after smp_send_stop() */
+#define SHUTDOWN_RESUME 3 /* Un-quiesce */
/* Device tree flags */
@@ -61,6 +61,8 @@ extern void add_opal_node(void);
(func), (nargs))
extern void __opal_register(uint64_t token, void *func, unsigned num_args);
+int64_t opal_shutdown(uint64_t shutdown_type, int cpu);
+
/* Warning: no locking at the moment, do at init time only
*
* XXX TODO: Add the big RCU-ish "opal API lock" to protect us here