@@ -42,6 +42,7 @@
* IMC Core engine expects 8K bytes of memory for counter collection.
*/
#define IMC_CORE_COUNTER_MEM 8192
+#define IMC_THREAD_COUNTER_MEM 8192
/*
*Compatbility macros for IMC devices
@@ -51,6 +52,9 @@
#define IMC_DTB_CORE_COMPAT "ibm,imc-counters-core"
#define IMC_DTB_THREAD_COMPAT "ibm,imc-counters-thread"
+#define THREAD_IMC_LDBAR_MASK 0x0003ffffffffe000
+#define THREAD_IMC_ENABLE 0x8000000000000000
+
/*
* Structure to hold per chip specific memory address
* information for nest pmus. Nest Counter data are exported
@@ -110,4 +114,5 @@ extern struct imc_pmu *per_nest_pmu_arr[IMC_MAX_PMUS];
extern struct imc_pmu *core_imc_pmu;
extern int __init init_imc_pmu(struct imc_events *events,int idx, struct imc_pmu *pmu_ptr);
void core_imc_disable(void);
+void thread_imc_disable(void);
#endif /* PPC_POWERNV_IMC_PMU_DEF_H */
@@ -38,6 +38,9 @@ static u64 per_core_pdbar_add[IMC_MAX_CHIPS][IMC_MAX_CORES];
static cpumask_t core_imc_cpumask;
struct imc_pmu *core_imc_pmu;
+/* Maintains base address for all the cpus */
+static u64 per_cpu_add[NR_CPUS];
+
/* Needed for sanity check */
extern u64 nest_max_offset;
extern u64 core_max_offset;
@@ -482,6 +485,56 @@ static int core_imc_event_init(struct perf_event *event)
return 0;
}
+static int thread_imc_event_init(struct perf_event *event)
+{
+ struct task_struct *target;
+
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ /* Sampling not supported */
+ if (event->hw.sample_period)
+ return -EINVAL;
+
+ event->hw.idx = -1;
+
+ /* Sanity check for config (event offset) */
+ if (event->attr.config > thread_max_offset)
+ return -EINVAL;
+
+ target = event->hw.target;
+
+ if (!target)
+ return -EINVAL;
+
+ event->pmu->task_ctx_nr = perf_sw_context;
+ return 0;
+}
+
+static void thread_imc_read_counter(struct perf_event *event)
+{
+ u64 *addr, data;
+ int cpu_id = smp_processor_id();
+
+ addr = (u64 *)(per_cpu_add[cpu_id] + event->attr.config);
+ data = __be64_to_cpu(READ_ONCE(*addr));
+ local64_set(&event->hw.prev_count, data);
+}
+
+static void thread_imc_perf_event_update(struct perf_event *event)
+{
+ u64 counter_prev, counter_new, final_count, *addr;
+ int cpu_id = smp_processor_id();
+
+ addr = (u64 *)(per_cpu_add[cpu_id] + event->attr.config);
+ counter_prev = local64_read(&event->hw.prev_count);
+ counter_new = __be64_to_cpu(READ_ONCE(*addr));
+ final_count = counter_new - counter_prev;
+
+ local64_set(&event->hw.prev_count, counter_new);
+ local64_add(final_count, &event->count);
+}
+
static void imc_read_counter(struct perf_event *event)
{
u64 *addr, data;
@@ -723,6 +776,84 @@ static int core_imc_event_add(struct perf_event *event, int flags)
}
+static void thread_imc_event_start(struct perf_event *event, int flags)
+{
+ int rc;
+
+ /*
+ * Core pmu units are enabled only when it is used.
+ * See if this is triggered for the first time.
+ * If yes, take the mutex lock and enable the core counters.
+ * If not, just increment the count in core_events.
+ */
+ if (atomic_inc_return(&core_events) == 1) {
+ mutex_lock(&imc_core_reserve);
+ rc = core_imc_control(IMC_COUNTER_ENABLE);
+ mutex_unlock(&imc_core_reserve);
+ if (rc)
+ pr_err("IMC: Unbale to start the counters\n");
+ }
+ thread_imc_read_counter(event);
+}
+
+static void thread_imc_event_stop(struct perf_event *event, int flags)
+{
+ int rc;
+
+ thread_imc_perf_event_update(event);
+ /*
+ * See if we need to disable the IMC PMU.
+ * If no events are currently in use, then we have to take a
+ * mutex to ensure that we don't race with another task doing
+ * enable or disable the core counters.
+ */
+ if (atomic_dec_return(&core_events) == 0) {
+ mutex_lock(&imc_core_reserve);
+ rc = core_imc_control(IMC_COUNTER_DISABLE);
+ mutex_unlock(&imc_core_reserve);
+ if (rc)
+ pr_err("IMC: Disable counters failed\n");
+
+ }
+}
+
+static void thread_imc_event_del(struct perf_event *event, int flags)
+{
+ thread_imc_perf_event_update(event);
+}
+
+static int thread_imc_event_add(struct perf_event *event, int flags)
+{
+ thread_imc_event_start(event, flags);
+
+ return 0;
+}
+
+static void thread_imc_pmu_start_txn(struct pmu *pmu,
+ unsigned int txn_flags)
+{
+ if (txn_flags & ~PERF_PMU_TXN_ADD)
+ return;
+ perf_pmu_disable(pmu);
+}
+
+static void thread_imc_pmu_cancel_txn(struct pmu *pmu)
+{
+ perf_pmu_enable(pmu);
+}
+
+static int thread_imc_pmu_commit_txn(struct pmu *pmu)
+{
+ perf_pmu_enable(pmu);
+ return 0;
+}
+
+static void thread_imc_pmu_sched_task(struct perf_event_context *ctx,
+ bool sched_in)
+{
+ return;
+}
+
/* update_pmu_ops : Populate the appropriate operations for "pmu" */
static int update_pmu_ops(struct imc_pmu *pmu)
{
@@ -749,6 +880,19 @@ static int update_pmu_ops(struct imc_pmu *pmu)
pmu->attr_groups[IMC_FORMAT_ATTR] = &imc_format_group;
pmu->pmu.attr_groups = pmu->attr_groups;
+ if (pmu->domain == IMC_DOMAIN_THREAD) {
+ pmu->pmu.event_init = thread_imc_event_init;
+ pmu->pmu.start = thread_imc_event_start;
+ pmu->pmu.add = thread_imc_event_add;
+ pmu->pmu.del = thread_imc_event_del;
+ pmu->pmu.stop = thread_imc_event_stop;
+ pmu->pmu.read = thread_imc_perf_event_update;
+ pmu->pmu.start_txn = thread_imc_pmu_start_txn;
+ pmu->pmu.cancel_txn = thread_imc_pmu_cancel_txn;
+ pmu->pmu.commit_txn = thread_imc_pmu_commit_txn;
+ pmu->pmu.sched_task = thread_imc_pmu_sched_task;
+ }
+
return 0;
}
@@ -809,6 +953,56 @@ static int update_events_in_group(struct imc_events *events,
return 0;
}
+static void thread_imc_ldbar_disable(void *dummy)
+{
+ /* LDBAR spr is a per-thread */
+ mtspr(SPRN_LDBAR, 0);
+}
+
+void thread_imc_disable(void)
+{
+ on_each_cpu(thread_imc_ldbar_disable, NULL, 1);
+}
+
+static void cleanup_thread_imc_memory(void *dummy)
+{
+ int cpu_id = smp_processor_id();
+ u64 addr = per_cpu_add[cpu_id];
+
+ /* Only if the address is non-zero, shall we free it */
+ if (addr)
+ free_pages(addr, 0);
+}
+
+static void cleanup_all_thread_imc_memory(void)
+{
+ on_each_cpu(cleanup_thread_imc_memory, NULL, 1);
+}
+
+/*
+ * Allocates a page of memory for each of the online cpus, and, writes the
+ * physical base address of that page to the LDBAR for that cpu. This starts
+ * the thread IMC counters.
+ */
+static void thread_imc_mem_alloc(void *dummy)
+{
+ u64 ldbar_addr, ldbar_value;
+ int cpu_id = smp_processor_id();
+ int phys_id = topology_physical_package_id(smp_processor_id());
+
+ per_cpu_add[cpu_id] = (u64)alloc_pages_exact_nid(phys_id,
+ (size_t)IMC_THREAD_COUNTER_MEM, GFP_KERNEL | __GFP_ZERO);
+ ldbar_addr = (u64)virt_to_phys((void *)per_cpu_add[cpu_id]);
+ ldbar_value = (ldbar_addr & (u64)THREAD_IMC_LDBAR_MASK) |
+ (u64)THREAD_IMC_ENABLE;
+ mtspr(SPRN_LDBAR, ldbar_value);
+}
+
+void thread_imc_cpu_init(void)
+{
+ on_each_cpu(thread_imc_mem_alloc, NULL, 1);
+}
+
/*
* init_imc_pmu : Setup and register the IMC pmu device.
*
@@ -836,6 +1030,9 @@ int __init init_imc_pmu(struct imc_events *events, int idx,
if (ret)
return ret;
break;
+ case IMC_DOMAIN_THREAD:
+ thread_imc_cpu_init();
+ break;
default:
return -1; /* Unknown domain */
}
@@ -868,5 +1065,9 @@ int __init init_imc_pmu(struct imc_events *events, int idx,
if (pmu_ptr->domain == IMC_DOMAIN_CORE)
cleanup_all_core_imc_memory();
+ /* For thread_imc, we have allocated memory, we need to free it */
+ if (pmu_ptr->domain == IMC_DOMAIN_THREAD)
+ cleanup_all_thread_imc_memory();
+
return ret;
}
@@ -582,6 +582,9 @@ static void opal_imc_counters_shutdown(struct platform_device *pdev)
{
/* Disable the IMC Core functions */
core_imc_disable();
+
+ /* Disable the IMC Thread functions */
+ thread_imc_disable();
}
static const struct of_device_id opal_imc_match[] = {