@@ -35,18 +35,8 @@ struct mm_iommu_table_group_mem_t {
atomic64_t mapped;
unsigned int pageshift;
u64 ua; /* userspace address */
- u64 entries; /* number of entries in hpas/hpages[] */
- /*
- * in mm_iommu_get we temporarily use this to store
- * struct page address.
- *
- * We need to convert ua to hpa in real mode. Make it
- * simpler by storing physical address.
- */
- union {
- struct page **hpages; /* vmalloc'ed */
- phys_addr_t *hpas;
- };
+ u64 entries; /* number of entries in hpas */
+ phys_addr_t *hpas;
#define MM_IOMMU_TABLE_INVALID_HPA ((uint64_t)-1)
u64 dev_hpa; /* Device memory base address */
};
@@ -91,26 +81,36 @@ bool mm_iommu_preregistered(struct mm_struct *mm)
}
EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
+/* Must be called with &mem_list_mutex held */
+static bool mm_iommu_find(struct mm_struct *mm, unsigned long ua,
+ unsigned long entries)
+{
+ struct mm_iommu_table_group_mem_t *mem;
+
+ list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
+ /* Overlap? */
+ if ((mem->ua < (ua + (entries << PAGE_SHIFT))) &&
+ (ua < (mem->ua + (mem->entries << PAGE_SHIFT))))
+ return true;
+ }
+ return false;
+}
+
static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
unsigned long entries, unsigned long dev_hpa,
struct mm_iommu_table_group_mem_t **pmem)
{
struct mm_iommu_table_group_mem_t *mem;
long i, ret, locked_entries = 0;
- unsigned int pageshift;
+ unsigned int pageshift, mem_pageshift;
+ struct page **hpages;
+ phys_addr_t *hpas;
mutex_lock(&mem_list_mutex);
- list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list,
- next) {
- /* Overlap? */
- if ((mem->ua < (ua + (entries << PAGE_SHIFT))) &&
- (ua < (mem->ua +
- (mem->entries << PAGE_SHIFT)))) {
- ret = -EINVAL;
- goto unlock_exit;
- }
-
+ if (mm_iommu_find(mm, ua, entries)) {
+ ret = -EINVAL;
+ goto unlock_exit;
}
if (dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) {
@@ -128,58 +128,60 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
}
if (dev_hpa != MM_IOMMU_TABLE_INVALID_HPA) {
- mem->pageshift = __ffs(dev_hpa | (entries << PAGE_SHIFT));
+ mem_pageshift = __ffs(dev_hpa | (entries << PAGE_SHIFT));
+ hpas = NULL;
mem->dev_hpa = dev_hpa;
goto good_exit;
}
mem->dev_hpa = MM_IOMMU_TABLE_INVALID_HPA;
- /*
- * For a starting point for a maximum page size calculation
- * we use @ua and @entries natural alignment to allow IOMMU pages
- * smaller than huge pages but still bigger than PAGE_SIZE.
- */
- mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT));
- mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0])));
- if (!mem->hpas) {
+ hpages = vzalloc(array_size(entries, sizeof(hpages[0])));
+ if (!hpages) {
kfree(mem);
ret = -ENOMEM;
goto unlock_exit;
}
down_read(&mm->mmap_sem);
- ret = get_user_pages_longterm(ua, entries, FOLL_WRITE, mem->hpages, NULL);
+ ret = get_user_pages_longterm(ua, entries, FOLL_WRITE, hpages, NULL);
up_read(&mm->mmap_sem);
if (ret != entries) {
/* free the reference taken */
for (i = 0; i < ret; i++)
- put_page(mem->hpages[i]);
+ put_page(hpages[i]);
- vfree(mem->hpas);
+ vfree(hpages);
kfree(mem);
ret = -EFAULT;
goto unlock_exit;
}
+ /*
+ * For a starting point for a maximum page size calculation
+ * we use @ua and @entries natural alignment to allow IOMMU pages
+ * smaller than huge pages but still bigger than PAGE_SIZE.
+ */
+ mem_pageshift = __ffs(ua | (entries << PAGE_SHIFT));
+ hpas = (phys_addr_t *) hpages;
pageshift = PAGE_SHIFT;
for (i = 0; i < entries; ++i) {
- struct page *page = mem->hpages[i];
+ struct page *page = hpages[i];
/*
* Allow to use larger than 64k IOMMU pages. Only do that
* if we are backed by hugetlb.
*/
- if ((mem->pageshift > PAGE_SHIFT) && PageHuge(page)) {
+ if ((mem_pageshift > PAGE_SHIFT) && PageHuge(page)) {
struct page *head = compound_head(page);
pageshift = compound_order(head) + PAGE_SHIFT;
}
- mem->pageshift = min(mem->pageshift, pageshift);
+ mem_pageshift = min(mem_pageshift, pageshift);
/*
* We don't need struct page reference any more, switch
* to physical address.
*/
- mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
+ hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
}
good_exit:
@@ -188,6 +190,8 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
mem->used = 1;
mem->ua = ua;
mem->entries = entries;
+ mem->hpas = hpas;
+ mem->pageshift = mem_pageshift;
*pmem = mem;
list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
The next patch will reduce amount of time spent under locks. This adds mm_iommu_find() to see if the region is already registered. This removes a rather ugly union from the mm_iommu_table_group_mem_t struct and keeps the hack local in mm_iommu_do_alloc(). This makes pageshift and hpas local and assigns them late as soon this moves to a helper. This should cause no behavioral change. Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru> --- arch/powerpc/mm/mmu_context_iommu.c | 82 +++++++++++++++-------------- 1 file changed, 43 insertions(+), 39 deletions(-)