Patchwork [v4,08/12] memory-hotplug: remove memmap of sparse-vmemmap

login
register
mail settings
Submitter Jianguo Wu
Date Nov. 28, 2012, 9:40 a.m.
Message ID <50B5DC00.20103@huawei.com>
Download mbox | patch
Permalink /patch/202407/
State Not Applicable
Headers show

Comments

Jianguo Wu - Nov. 28, 2012, 9:40 a.m.
Hi Congyang,

I think vmemmap's pgtable pages should be freed after all entries are cleared, I have a patch to do this.
The code logic is the same as [Patch v4 09/12] memory-hotplug: remove page table of x86_64 architecture.

How do you think about this?

Signed-off-by: Jianguo Wu <wujianguo@huawei.com>
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
---
 include/linux/mm.h  |    1 +
 mm/sparse-vmemmap.c |  214 +++++++++++++++++++++++++++++++++++++++++++++++++++
 mm/sparse.c         |    5 +-
 3 files changed, 218 insertions(+), 2 deletions(-)
Wen Congyang - Nov. 30, 2012, 1:45 a.m.
At 11/28/2012 05:40 PM, Jianguo Wu Wrote:
> Hi Congyang,
> 
> I think vmemmap's pgtable pages should be freed after all entries are cleared, I have a patch to do this.
> The code logic is the same as [Patch v4 09/12] memory-hotplug: remove page table of x86_64 architecture.
> 
> How do you think about this?
> 
> Signed-off-by: Jianguo Wu <wujianguo@huawei.com>
> Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
> ---
>  include/linux/mm.h  |    1 +
>  mm/sparse-vmemmap.c |  214 +++++++++++++++++++++++++++++++++++++++++++++++++++
>  mm/sparse.c         |    5 +-
>  3 files changed, 218 insertions(+), 2 deletions(-)
> 
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index 5657670..1f26af5 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1642,6 +1642,7 @@ int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
>  void vmemmap_populate_print_last(void);
>  void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
>  				  unsigned long size);
> +void vmemmap_free(struct page *memmap, unsigned long nr_pages);
>  
>  enum mf_flags {
>  	MF_COUNT_INCREASED = 1 << 0,
> diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
> index 1b7e22a..242cb28 100644
> --- a/mm/sparse-vmemmap.c
> +++ b/mm/sparse-vmemmap.c
> @@ -29,6 +29,10 @@
>  #include <asm/pgalloc.h>
>  #include <asm/pgtable.h>
>  
> +#ifdef CONFIG_MEMORY_HOTREMOVE
> +#include <asm/tlbflush.h>
> +#endif
> +
>  /*
>   * Allocate a block of memory to be used to back the virtual memory map
>   * or to back the page tables that are used to create the mapping.
> @@ -224,3 +228,213 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
>  		vmemmap_buf_end = NULL;
>  	}
>  }
> +
> +#ifdef CONFIG_MEMORY_HOTREMOVE
> +static void vmemmap_free_pages(struct page *page, int order)
> +{
> +	struct zone *zone;
> +	unsigned long magic;
> +
> +	magic = (unsigned long) page->lru.next;
> +	if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) {
> +		put_page_bootmem(page);
> +
> +		zone = page_zone(page);
> +		zone_span_writelock(zone);
> +		zone->present_pages++;
> +		zone_span_writeunlock(zone);
> +		totalram_pages++;
> +	} else {
> +		if (is_vmalloc_addr(page_address(page)))
> +			vfree(page_address(page));

Hmm, vmemmap doesn't use vmalloc() to allocate memory.

> +		else
> +			free_pages((unsigned long)page_address(page), order);
> +	}
> +}
> +
> +static void free_pte_table(pmd_t *pmd)
> +{
> +	pte_t *pte, *pte_start;
> +	int i;
> +
> +	pte_start = (pte_t *)pmd_page_vaddr(*pmd);
> +	for (i = 0; i < PTRS_PER_PTE; i++) {
> +		pte = pte_start + i;
> +		if (pte_val(*pte))
> +			return;
> +	}
> +
> +	/* free a pte talbe */
> +	vmemmap_free_pages(pmd_page(*pmd), 0);
> +	spin_lock(&init_mm.page_table_lock);
> +	pmd_clear(pmd);
> +	spin_unlock(&init_mm.page_table_lock);
> +}
> +
> +static void free_pmd_table(pud_t *pud)
> +{
> +	pmd_t *pmd, *pmd_start;
> +	int i;
> +
> +	pmd_start = (pmd_t *)pud_page_vaddr(*pud);
> +	for (i = 0; i < PTRS_PER_PMD; i++) {
> +		pmd = pmd_start + i;
> +		if (pmd_val(*pmd))
> +			return;
> +	}
> +
> +	/* free a pmd talbe */
> +	vmemmap_free_pages(pud_page(*pud), 0);
> +	spin_lock(&init_mm.page_table_lock);
> +	pud_clear(pud);
> +	spin_unlock(&init_mm.page_table_lock);
> +}
> +
> +static void free_pud_table(pgd_t *pgd)
> +{
> +	pud_t *pud, *pud_start;
> +	int i;
> +
> +	pud_start = (pud_t *)pgd_page_vaddr(*pgd);
> +	for (i = 0; i < PTRS_PER_PUD; i++) {
> +		pud = pud_start + i;
> +		if (pud_val(*pud))
> +			return;
> +	}
> +
> +	/* free a pud table */
> +	vmemmap_free_pages(pgd_page(*pgd), 0);
> +	spin_lock(&init_mm.page_table_lock);
> +	pgd_clear(pgd);
> +	spin_unlock(&init_mm.page_table_lock);
> +}
> +
> +static int split_large_page(pte_t *kpte, unsigned long address, pte_t *pbase)
> +{
> +	struct page *page = pmd_page(*(pmd_t *)kpte);
> +	int i = 0;
> +	unsigned long magic;
> +	unsigned long section_nr;
> +
> +	__split_large_page(kpte, address, pbase);
> +	__flush_tlb_all();
> +
> +	magic = (unsigned long) page->lru.next;
> +	if (magic == SECTION_INFO) {
> +		section_nr = pfn_to_section_nr(page_to_pfn(page));
> +		while (i < PTRS_PER_PMD) {
> +			page++;
> +			i++;
> +			get_page_bootmem(section_nr, page, SECTION_INFO);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +static void vmemmap_pte_remove(pmd_t *pmd, unsigned long addr, unsigned long end)
> +{
> +	pte_t *pte;
> +	unsigned long next;
> +
> +	pte = pte_offset_kernel(pmd, addr);
> +	for (; addr < end; pte++, addr += PAGE_SIZE) {
> +		next = (addr + PAGE_SIZE) & PAGE_MASK;
> +		if (next > end)
> +			next = end;
> +
> +		if (pte_none(*pte))
> +			continue;
> +		if (IS_ALIGNED(addr, PAGE_SIZE) &&
> +		    IS_ALIGNED(end, PAGE_SIZE)) {
> +			vmemmap_free_pages(pte_page(*pte), 0);
> +			spin_lock(&init_mm.page_table_lock);
> +			pte_clear(&init_mm, addr, pte);
> +			spin_unlock(&init_mm.page_table_lock);

If addr or end is not alianed with PAGE_SIZE, you may leak some
memory.

> +		}
> +	}
> +
> +	free_pte_table(pmd);
> +	__flush_tlb_all();
> +}
> +
> +static void vmemmap_pmd_remove(pud_t *pud, unsigned long addr, unsigned long end)
> +{
> +	unsigned long next;
> +	pmd_t *pmd;
> +
> +	pmd = pmd_offset(pud, addr);
> +	for (; addr < end; addr = next, pmd++) {
> +		next = pmd_addr_end(addr, end);
> +		if (pmd_none(*pmd))
> +			continue;
> +
> +		if (cpu_has_pse) {
> +			unsigned long pte_base;
> +
> +			if (IS_ALIGNED(addr, PMD_SIZE) &&
> +			    IS_ALIGNED(next, PMD_SIZE)) {
> +				vmemmap_free_pages(pmd_page(*pmd),
> +						   get_order(PMD_SIZE));
> +				spin_lock(&init_mm.page_table_lock);
> +				pmd_clear(pmd);
> +				spin_unlock(&init_mm.page_table_lock);
> +				continue;
> +			}
> +
> +			/*
> +			 * We use 2M page, but we need to remove part of them,
> +			 * so split 2M page to 4K page.
> +			 */
> +			pte_base = get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);

get_zeored_page() may fail. You should handle this error.

> +			split_large_page((pte_t *)pmd, addr, (pte_t *)pte_base);
> +			__flush_tlb_all();
> +
> +			spin_lock(&init_mm.page_table_lock);
> +			pmd_populate_kernel(&init_mm, pmd, (pte_t *)pte_base);
> +			spin_unlock(&init_mm.page_table_lock);
> +		}
> +
> +		vmemmap_pte_remove(pmd, addr, next);
> +	}
> +
> +	free_pmd_table(pud);
> +	__flush_tlb_all();
> +}
> +
> +static void vmemmap_pud_remove(pgd_t *pgd, unsigned long addr, unsigned long end)
> +{
> +	unsigned long next;
> +	pud_t *pud;
> +
> +	pud = pud_offset(pgd, addr);
> +	for (; addr < end; addr = next, pud++) {
> +		next = pud_addr_end(addr, end);
> +		if (pud_none(*pud))
> +			continue;
> +
> +		vmemmap_pmd_remove(pud, addr, next);
> +	}
> +
> +	free_pud_table(pgd);
> +	__flush_tlb_all();
> +}
> +
> +void vmemmap_free(struct page *memmap, unsigned long nr_pages)
> +{
> +	unsigned long addr = (unsigned long)memmap;
> +	unsigned long end = (unsigned long)(memmap + nr_pages);
> +	unsigned long next;
> +
> +	for (; addr < end; addr = next) {
> +		pgd_t *pgd = pgd_offset_k(addr);
> +
> +		next = pgd_addr_end(addr, end);
> +		if (!pgd_present(*pgd))
> +			continue;
> +
> +		vmemmap_pud_remove(pgd, addr, next);
> +		sync_global_pgds(addr, next);

The parameter for sync_global_pgds() is [start, end], not
[start, end)

> +	}
> +}
> +#endif
> diff --git a/mm/sparse.c b/mm/sparse.c
> index fac95f2..3a16d68 100644
> --- a/mm/sparse.c
> +++ b/mm/sparse.c
> @@ -613,12 +613,13 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
>  	/* This will make the necessary allocations eventually. */
>  	return sparse_mem_map_populate(pnum, nid);
>  }
> -static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
> +static void __kfree_section_memmap(struct page *page, unsigned long nr_pages)
Why do you change this line?

>  {
> -	return; /* XXX: Not implemented yet */
> +	vmemmap_free(page, nr_pages);
>  }
>  static void free_map_bootmem(struct page *page, unsigned long nr_pages)
>  {
> +	vmemmap_free(page, nr_pages);
>  }
>  #else
>  static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
Jianguo Wu - Nov. 30, 2012, 2:47 a.m.
Hi Congyang,

Thanks for your review and comments.

On 2012/11/30 9:45, Wen Congyang wrote:

> At 11/28/2012 05:40 PM, Jianguo Wu Wrote:
>> Hi Congyang,
>>
>> I think vmemmap's pgtable pages should be freed after all entries are cleared, I have a patch to do this.
>> The code logic is the same as [Patch v4 09/12] memory-hotplug: remove page table of x86_64 architecture.
>>
>> How do you think about this?
>>
>> Signed-off-by: Jianguo Wu <wujianguo@huawei.com>
>> Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
>> ---
>>  include/linux/mm.h  |    1 +
>>  mm/sparse-vmemmap.c |  214 +++++++++++++++++++++++++++++++++++++++++++++++++++
>>  mm/sparse.c         |    5 +-
>>  3 files changed, 218 insertions(+), 2 deletions(-)
>>
>> diff --git a/include/linux/mm.h b/include/linux/mm.h
>> index 5657670..1f26af5 100644
>> --- a/include/linux/mm.h
>> +++ b/include/linux/mm.h
>> @@ -1642,6 +1642,7 @@ int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
>>  void vmemmap_populate_print_last(void);
>>  void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
>>  				  unsigned long size);
>> +void vmemmap_free(struct page *memmap, unsigned long nr_pages);
>>  
>>  enum mf_flags {
>>  	MF_COUNT_INCREASED = 1 << 0,
>> diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
>> index 1b7e22a..242cb28 100644
>> --- a/mm/sparse-vmemmap.c
>> +++ b/mm/sparse-vmemmap.c
>> @@ -29,6 +29,10 @@
>>  #include <asm/pgalloc.h>
>>  #include <asm/pgtable.h>
>>  
>> +#ifdef CONFIG_MEMORY_HOTREMOVE
>> +#include <asm/tlbflush.h>
>> +#endif
>> +
>>  /*
>>   * Allocate a block of memory to be used to back the virtual memory map
>>   * or to back the page tables that are used to create the mapping.
>> @@ -224,3 +228,213 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
>>  		vmemmap_buf_end = NULL;
>>  	}
>>  }
>> +
>> +#ifdef CONFIG_MEMORY_HOTREMOVE
>> +static void vmemmap_free_pages(struct page *page, int order)
>> +{
>> +	struct zone *zone;
>> +	unsigned long magic;
>> +
>> +	magic = (unsigned long) page->lru.next;
>> +	if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) {
>> +		put_page_bootmem(page);
>> +
>> +		zone = page_zone(page);
>> +		zone_span_writelock(zone);
>> +		zone->present_pages++;
>> +		zone_span_writeunlock(zone);
>> +		totalram_pages++;
>> +	} else {
>> +		if (is_vmalloc_addr(page_address(page)))
>> +			vfree(page_address(page));
> 
> Hmm, vmemmap doesn't use vmalloc() to allocate memory.
> 

yes, this can be removed.

>> +		else
>> +			free_pages((unsigned long)page_address(page), order);
>> +	}
>> +}
>> +
>> +static void free_pte_table(pmd_t *pmd)
>> +{
>> +	pte_t *pte, *pte_start;
>> +	int i;
>> +
>> +	pte_start = (pte_t *)pmd_page_vaddr(*pmd);
>> +	for (i = 0; i < PTRS_PER_PTE; i++) {
>> +		pte = pte_start + i;
>> +		if (pte_val(*pte))
>> +			return;
>> +	}
>> +
>> +	/* free a pte talbe */
>> +	vmemmap_free_pages(pmd_page(*pmd), 0);
>> +	spin_lock(&init_mm.page_table_lock);
>> +	pmd_clear(pmd);
>> +	spin_unlock(&init_mm.page_table_lock);
>> +}
>> +
>> +static void free_pmd_table(pud_t *pud)
>> +{
>> +	pmd_t *pmd, *pmd_start;
>> +	int i;
>> +
>> +	pmd_start = (pmd_t *)pud_page_vaddr(*pud);
>> +	for (i = 0; i < PTRS_PER_PMD; i++) {
>> +		pmd = pmd_start + i;
>> +		if (pmd_val(*pmd))
>> +			return;
>> +	}
>> +
>> +	/* free a pmd talbe */
>> +	vmemmap_free_pages(pud_page(*pud), 0);
>> +	spin_lock(&init_mm.page_table_lock);
>> +	pud_clear(pud);
>> +	spin_unlock(&init_mm.page_table_lock);
>> +}
>> +
>> +static void free_pud_table(pgd_t *pgd)
>> +{
>> +	pud_t *pud, *pud_start;
>> +	int i;
>> +
>> +	pud_start = (pud_t *)pgd_page_vaddr(*pgd);
>> +	for (i = 0; i < PTRS_PER_PUD; i++) {
>> +		pud = pud_start + i;
>> +		if (pud_val(*pud))
>> +			return;
>> +	}
>> +
>> +	/* free a pud table */
>> +	vmemmap_free_pages(pgd_page(*pgd), 0);
>> +	spin_lock(&init_mm.page_table_lock);
>> +	pgd_clear(pgd);
>> +	spin_unlock(&init_mm.page_table_lock);
>> +}
>> +
>> +static int split_large_page(pte_t *kpte, unsigned long address, pte_t *pbase)
>> +{
>> +	struct page *page = pmd_page(*(pmd_t *)kpte);
>> +	int i = 0;
>> +	unsigned long magic;
>> +	unsigned long section_nr;
>> +
>> +	__split_large_page(kpte, address, pbase);
>> +	__flush_tlb_all();
>> +
>> +	magic = (unsigned long) page->lru.next;
>> +	if (magic == SECTION_INFO) {
>> +		section_nr = pfn_to_section_nr(page_to_pfn(page));
>> +		while (i < PTRS_PER_PMD) {
>> +			page++;
>> +			i++;
>> +			get_page_bootmem(section_nr, page, SECTION_INFO);
>> +		}
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +static void vmemmap_pte_remove(pmd_t *pmd, unsigned long addr, unsigned long end)
>> +{
>> +	pte_t *pte;
>> +	unsigned long next;
>> +
>> +	pte = pte_offset_kernel(pmd, addr);
>> +	for (; addr < end; pte++, addr += PAGE_SIZE) {
>> +		next = (addr + PAGE_SIZE) & PAGE_MASK;
>> +		if (next > end)
>> +			next = end;
>> +
>> +		if (pte_none(*pte))
>> +			continue;
>> +		if (IS_ALIGNED(addr, PAGE_SIZE) &&
>> +		    IS_ALIGNED(end, PAGE_SIZE)) {
>> +			vmemmap_free_pages(pte_page(*pte), 0);
>> +			spin_lock(&init_mm.page_table_lock);
>> +			pte_clear(&init_mm, addr, pte);
>> +			spin_unlock(&init_mm.page_table_lock);
> 
> If addr or end is not alianed with PAGE_SIZE, you may leak some
> memory.
> 

yes, I think we can handle this situation with the method you mentioned in the change log:
1. When removing memory, the page structs of the revmoved memory are filled
   with 0xFD.
2. All page structs are filled with 0xFD on PT/PMD, PT/PMD can be cleared.
   In this case, the page used as PT/PMD can be freed.

By the way, why is 0xFD?

>> +		}
>> +	}
>> +
>> +	free_pte_table(pmd);
>> +	__flush_tlb_all();
>> +}
>> +
>> +static void vmemmap_pmd_remove(pud_t *pud, unsigned long addr, unsigned long end)
>> +{
>> +	unsigned long next;
>> +	pmd_t *pmd;
>> +
>> +	pmd = pmd_offset(pud, addr);
>> +	for (; addr < end; addr = next, pmd++) {
>> +		next = pmd_addr_end(addr, end);
>> +		if (pmd_none(*pmd))
>> +			continue;
>> +
>> +		if (cpu_has_pse) {
>> +			unsigned long pte_base;
>> +
>> +			if (IS_ALIGNED(addr, PMD_SIZE) &&
>> +			    IS_ALIGNED(next, PMD_SIZE)) {
>> +				vmemmap_free_pages(pmd_page(*pmd),
>> +						   get_order(PMD_SIZE));
>> +				spin_lock(&init_mm.page_table_lock);
>> +				pmd_clear(pmd);
>> +				spin_unlock(&init_mm.page_table_lock);
>> +				continue;
>> +			}
>> +
>> +			/*
>> +			 * We use 2M page, but we need to remove part of them,
>> +			 * so split 2M page to 4K page.
>> +			 */
>> +			pte_base = get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
> 
> get_zeored_page() may fail. You should handle this error.
> 

That means system is out of memory, I will trigger a bug_on.

>> +			split_large_page((pte_t *)pmd, addr, (pte_t *)pte_base);
>> +			__flush_tlb_all();
>> +
>> +			spin_lock(&init_mm.page_table_lock);
>> +			pmd_populate_kernel(&init_mm, pmd, (pte_t *)pte_base);
>> +			spin_unlock(&init_mm.page_table_lock);
>> +		}
>> +
>> +		vmemmap_pte_remove(pmd, addr, next);
>> +	}
>> +
>> +	free_pmd_table(pud);
>> +	__flush_tlb_all();
>> +}
>> +
>> +static void vmemmap_pud_remove(pgd_t *pgd, unsigned long addr, unsigned long end)
>> +{
>> +	unsigned long next;
>> +	pud_t *pud;
>> +
>> +	pud = pud_offset(pgd, addr);
>> +	for (; addr < end; addr = next, pud++) {
>> +		next = pud_addr_end(addr, end);
>> +		if (pud_none(*pud))
>> +			continue;
>> +
>> +		vmemmap_pmd_remove(pud, addr, next);
>> +	}
>> +
>> +	free_pud_table(pgd);
>> +	__flush_tlb_all();
>> +}
>> +
>> +void vmemmap_free(struct page *memmap, unsigned long nr_pages)
>> +{
>> +	unsigned long addr = (unsigned long)memmap;
>> +	unsigned long end = (unsigned long)(memmap + nr_pages);
>> +	unsigned long next;
>> +
>> +	for (; addr < end; addr = next) {
>> +		pgd_t *pgd = pgd_offset_k(addr);
>> +
>> +		next = pgd_addr_end(addr, end);
>> +		if (!pgd_present(*pgd))
>> +			continue;
>> +
>> +		vmemmap_pud_remove(pgd, addr, next);
>> +		sync_global_pgds(addr, next);
> 
> The parameter for sync_global_pgds() is [start, end], not
> [start, end)
> 

yes, thanks.

>> +	}
>> +}
>> +#endif
>> diff --git a/mm/sparse.c b/mm/sparse.c
>> index fac95f2..3a16d68 100644
>> --- a/mm/sparse.c
>> +++ b/mm/sparse.c
>> @@ -613,12 +613,13 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
>>  	/* This will make the necessary allocations eventually. */
>>  	return sparse_mem_map_populate(pnum, nid);
>>  }
>> -static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
>> +static void __kfree_section_memmap(struct page *page, unsigned long nr_pages)
> Why do you change this line?
> 

0k, it is no need to change.

>>  {
>> -	return; /* XXX: Not implemented yet */
>> +	vmemmap_free(page, nr_pages);
>>  }
>>  static void free_map_bootmem(struct page *page, unsigned long nr_pages)
>>  {
>> +	vmemmap_free(page, nr_pages);
>>  }
>>  #else
>>  static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
> 
> 
> .
>
Yasuaki Ishimatsu - Nov. 30, 2012, 2:55 a.m.
Hi Jianguo,

2012/11/30 11:47, Jianguo Wu wrote:
> Hi Congyang,
>
> Thanks for your review and comments.
>
> On 2012/11/30 9:45, Wen Congyang wrote:
>
>> At 11/28/2012 05:40 PM, Jianguo Wu Wrote:
>>> Hi Congyang,
>>>
>>> I think vmemmap's pgtable pages should be freed after all entries are cleared, I have a patch to do this.
>>> The code logic is the same as [Patch v4 09/12] memory-hotplug: remove page table of x86_64 architecture.
>>>
>>> How do you think about this?
>>>
>>> Signed-off-by: Jianguo Wu <wujianguo@huawei.com>
>>> Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
>>> ---
>>>   include/linux/mm.h  |    1 +
>>>   mm/sparse-vmemmap.c |  214 +++++++++++++++++++++++++++++++++++++++++++++++++++
>>>   mm/sparse.c         |    5 +-
>>>   3 files changed, 218 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/include/linux/mm.h b/include/linux/mm.h
>>> index 5657670..1f26af5 100644
>>> --- a/include/linux/mm.h
>>> +++ b/include/linux/mm.h
>>> @@ -1642,6 +1642,7 @@ int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
>>>   void vmemmap_populate_print_last(void);
>>>   void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
>>>   				  unsigned long size);
>>> +void vmemmap_free(struct page *memmap, unsigned long nr_pages);
>>>
>>>   enum mf_flags {
>>>   	MF_COUNT_INCREASED = 1 << 0,
>>> diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
>>> index 1b7e22a..242cb28 100644
>>> --- a/mm/sparse-vmemmap.c
>>> +++ b/mm/sparse-vmemmap.c
>>> @@ -29,6 +29,10 @@
>>>   #include <asm/pgalloc.h>
>>>   #include <asm/pgtable.h>
>>>
>>> +#ifdef CONFIG_MEMORY_HOTREMOVE
>>> +#include <asm/tlbflush.h>
>>> +#endif
>>> +
>>>   /*
>>>    * Allocate a block of memory to be used to back the virtual memory map
>>>    * or to back the page tables that are used to create the mapping.
>>> @@ -224,3 +228,213 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
>>>   		vmemmap_buf_end = NULL;
>>>   	}
>>>   }
>>> +
>>> +#ifdef CONFIG_MEMORY_HOTREMOVE
>>> +static void vmemmap_free_pages(struct page *page, int order)
>>> +{
>>> +	struct zone *zone;
>>> +	unsigned long magic;
>>> +
>>> +	magic = (unsigned long) page->lru.next;
>>> +	if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) {
>>> +		put_page_bootmem(page);
>>> +
>>> +		zone = page_zone(page);
>>> +		zone_span_writelock(zone);
>>> +		zone->present_pages++;
>>> +		zone_span_writeunlock(zone);
>>> +		totalram_pages++;
>>> +	} else {
>>> +		if (is_vmalloc_addr(page_address(page)))
>>> +			vfree(page_address(page));
>>
>> Hmm, vmemmap doesn't use vmalloc() to allocate memory.
>>
>
> yes, this can be removed.
>
>>> +		else
>>> +			free_pages((unsigned long)page_address(page), order);
>>> +	}
>>> +}
>>> +
>>> +static void free_pte_table(pmd_t *pmd)
>>> +{
>>> +	pte_t *pte, *pte_start;
>>> +	int i;
>>> +
>>> +	pte_start = (pte_t *)pmd_page_vaddr(*pmd);
>>> +	for (i = 0; i < PTRS_PER_PTE; i++) {
>>> +		pte = pte_start + i;
>>> +		if (pte_val(*pte))
>>> +			return;
>>> +	}
>>> +
>>> +	/* free a pte talbe */
>>> +	vmemmap_free_pages(pmd_page(*pmd), 0);
>>> +	spin_lock(&init_mm.page_table_lock);
>>> +	pmd_clear(pmd);
>>> +	spin_unlock(&init_mm.page_table_lock);
>>> +}
>>> +
>>> +static void free_pmd_table(pud_t *pud)
>>> +{
>>> +	pmd_t *pmd, *pmd_start;
>>> +	int i;
>>> +
>>> +	pmd_start = (pmd_t *)pud_page_vaddr(*pud);
>>> +	for (i = 0; i < PTRS_PER_PMD; i++) {
>>> +		pmd = pmd_start + i;
>>> +		if (pmd_val(*pmd))
>>> +			return;
>>> +	}
>>> +
>>> +	/* free a pmd talbe */
>>> +	vmemmap_free_pages(pud_page(*pud), 0);
>>> +	spin_lock(&init_mm.page_table_lock);
>>> +	pud_clear(pud);
>>> +	spin_unlock(&init_mm.page_table_lock);
>>> +}
>>> +
>>> +static void free_pud_table(pgd_t *pgd)
>>> +{
>>> +	pud_t *pud, *pud_start;
>>> +	int i;
>>> +
>>> +	pud_start = (pud_t *)pgd_page_vaddr(*pgd);
>>> +	for (i = 0; i < PTRS_PER_PUD; i++) {
>>> +		pud = pud_start + i;
>>> +		if (pud_val(*pud))
>>> +			return;
>>> +	}
>>> +
>>> +	/* free a pud table */
>>> +	vmemmap_free_pages(pgd_page(*pgd), 0);
>>> +	spin_lock(&init_mm.page_table_lock);
>>> +	pgd_clear(pgd);
>>> +	spin_unlock(&init_mm.page_table_lock);
>>> +}
>>> +
>>> +static int split_large_page(pte_t *kpte, unsigned long address, pte_t *pbase)
>>> +{
>>> +	struct page *page = pmd_page(*(pmd_t *)kpte);
>>> +	int i = 0;
>>> +	unsigned long magic;
>>> +	unsigned long section_nr;
>>> +
>>> +	__split_large_page(kpte, address, pbase);
>>> +	__flush_tlb_all();
>>> +
>>> +	magic = (unsigned long) page->lru.next;
>>> +	if (magic == SECTION_INFO) {
>>> +		section_nr = pfn_to_section_nr(page_to_pfn(page));
>>> +		while (i < PTRS_PER_PMD) {
>>> +			page++;
>>> +			i++;
>>> +			get_page_bootmem(section_nr, page, SECTION_INFO);
>>> +		}
>>> +	}
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static void vmemmap_pte_remove(pmd_t *pmd, unsigned long addr, unsigned long end)
>>> +{
>>> +	pte_t *pte;
>>> +	unsigned long next;
>>> +
>>> +	pte = pte_offset_kernel(pmd, addr);
>>> +	for (; addr < end; pte++, addr += PAGE_SIZE) {
>>> +		next = (addr + PAGE_SIZE) & PAGE_MASK;
>>> +		if (next > end)
>>> +			next = end;
>>> +
>>> +		if (pte_none(*pte))
>>> +			continue;
>>> +		if (IS_ALIGNED(addr, PAGE_SIZE) &&
>>> +		    IS_ALIGNED(end, PAGE_SIZE)) {
>>> +			vmemmap_free_pages(pte_page(*pte), 0);
>>> +			spin_lock(&init_mm.page_table_lock);
>>> +			pte_clear(&init_mm, addr, pte);
>>> +			spin_unlock(&init_mm.page_table_lock);
>>
>> If addr or end is not alianed with PAGE_SIZE, you may leak some
>> memory.
>>
>
> yes, I think we can handle this situation with the method you mentioned in the change log:
> 1. When removing memory, the page structs of the revmoved memory are filled
>     with 0xFD.
> 2. All page structs are filled with 0xFD on PT/PMD, PT/PMD can be cleared.
>     In this case, the page used as PT/PMD can be freed.
>
> By the way, why is 0xFD?

There is no reason. I just filled the page with unique number.

Thanks,
Yasuaki Ishimatsu

>
>>> +		}
>>> +	}
>>> +
>>> +	free_pte_table(pmd);
>>> +	__flush_tlb_all();
>>> +}
>>> +
>>> +static void vmemmap_pmd_remove(pud_t *pud, unsigned long addr, unsigned long end)
>>> +{
>>> +	unsigned long next;
>>> +	pmd_t *pmd;
>>> +
>>> +	pmd = pmd_offset(pud, addr);
>>> +	for (; addr < end; addr = next, pmd++) {
>>> +		next = pmd_addr_end(addr, end);
>>> +		if (pmd_none(*pmd))
>>> +			continue;
>>> +
>>> +		if (cpu_has_pse) {
>>> +			unsigned long pte_base;
>>> +
>>> +			if (IS_ALIGNED(addr, PMD_SIZE) &&
>>> +			    IS_ALIGNED(next, PMD_SIZE)) {
>>> +				vmemmap_free_pages(pmd_page(*pmd),
>>> +						   get_order(PMD_SIZE));
>>> +				spin_lock(&init_mm.page_table_lock);
>>> +				pmd_clear(pmd);
>>> +				spin_unlock(&init_mm.page_table_lock);
>>> +				continue;
>>> +			}
>>> +
>>> +			/*
>>> +			 * We use 2M page, but we need to remove part of them,
>>> +			 * so split 2M page to 4K page.
>>> +			 */
>>> +			pte_base = get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
>>
>> get_zeored_page() may fail. You should handle this error.
>>
>
> That means system is out of memory, I will trigger a bug_on.
>
>>> +			split_large_page((pte_t *)pmd, addr, (pte_t *)pte_base);
>>> +			__flush_tlb_all();
>>> +
>>> +			spin_lock(&init_mm.page_table_lock);
>>> +			pmd_populate_kernel(&init_mm, pmd, (pte_t *)pte_base);
>>> +			spin_unlock(&init_mm.page_table_lock);
>>> +		}
>>> +
>>> +		vmemmap_pte_remove(pmd, addr, next);
>>> +	}
>>> +
>>> +	free_pmd_table(pud);
>>> +	__flush_tlb_all();
>>> +}
>>> +
>>> +static void vmemmap_pud_remove(pgd_t *pgd, unsigned long addr, unsigned long end)
>>> +{
>>> +	unsigned long next;
>>> +	pud_t *pud;
>>> +
>>> +	pud = pud_offset(pgd, addr);
>>> +	for (; addr < end; addr = next, pud++) {
>>> +		next = pud_addr_end(addr, end);
>>> +		if (pud_none(*pud))
>>> +			continue;
>>> +
>>> +		vmemmap_pmd_remove(pud, addr, next);
>>> +	}
>>> +
>>> +	free_pud_table(pgd);
>>> +	__flush_tlb_all();
>>> +}
>>> +
>>> +void vmemmap_free(struct page *memmap, unsigned long nr_pages)
>>> +{
>>> +	unsigned long addr = (unsigned long)memmap;
>>> +	unsigned long end = (unsigned long)(memmap + nr_pages);
>>> +	unsigned long next;
>>> +
>>> +	for (; addr < end; addr = next) {
>>> +		pgd_t *pgd = pgd_offset_k(addr);
>>> +
>>> +		next = pgd_addr_end(addr, end);
>>> +		if (!pgd_present(*pgd))
>>> +			continue;
>>> +
>>> +		vmemmap_pud_remove(pgd, addr, next);
>>> +		sync_global_pgds(addr, next);
>>
>> The parameter for sync_global_pgds() is [start, end], not
>> [start, end)
>>
>
> yes, thanks.
>
>>> +	}
>>> +}
>>> +#endif
>>> diff --git a/mm/sparse.c b/mm/sparse.c
>>> index fac95f2..3a16d68 100644
>>> --- a/mm/sparse.c
>>> +++ b/mm/sparse.c
>>> @@ -613,12 +613,13 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
>>>   	/* This will make the necessary allocations eventually. */
>>>   	return sparse_mem_map_populate(pnum, nid);
>>>   }
>>> -static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
>>> +static void __kfree_section_memmap(struct page *page, unsigned long nr_pages)
>> Why do you change this line?
>>
>
> 0k, it is no need to change.
>
>>>   {
>>> -	return; /* XXX: Not implemented yet */
>>> +	vmemmap_free(page, nr_pages);
>>>   }
>>>   static void free_map_bootmem(struct page *page, unsigned long nr_pages)
>>>   {
>>> +	vmemmap_free(page, nr_pages);
>>>   }
>>>   #else
>>>   static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
>>
>>
>> .
>>
>
>
>

Patch

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5657670..1f26af5 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1642,6 +1642,7 @@  int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
 void vmemmap_populate_print_last(void);
 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
 				  unsigned long size);
+void vmemmap_free(struct page *memmap, unsigned long nr_pages);
 
 enum mf_flags {
 	MF_COUNT_INCREASED = 1 << 0,
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 1b7e22a..242cb28 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -29,6 +29,10 @@ 
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 
+#ifdef CONFIG_MEMORY_HOTREMOVE
+#include <asm/tlbflush.h>
+#endif
+
 /*
  * Allocate a block of memory to be used to back the virtual memory map
  * or to back the page tables that are used to create the mapping.
@@ -224,3 +228,213 @@  void __init sparse_mem_maps_populate_node(struct page **map_map,
 		vmemmap_buf_end = NULL;
 	}
 }
+
+#ifdef CONFIG_MEMORY_HOTREMOVE
+static void vmemmap_free_pages(struct page *page, int order)
+{
+	struct zone *zone;
+	unsigned long magic;
+
+	magic = (unsigned long) page->lru.next;
+	if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) {
+		put_page_bootmem(page);
+
+		zone = page_zone(page);
+		zone_span_writelock(zone);
+		zone->present_pages++;
+		zone_span_writeunlock(zone);
+		totalram_pages++;
+	} else {
+		if (is_vmalloc_addr(page_address(page)))
+			vfree(page_address(page));
+		else
+			free_pages((unsigned long)page_address(page), order);
+	}
+}
+
+static void free_pte_table(pmd_t *pmd)
+{
+	pte_t *pte, *pte_start;
+	int i;
+
+	pte_start = (pte_t *)pmd_page_vaddr(*pmd);
+	for (i = 0; i < PTRS_PER_PTE; i++) {
+		pte = pte_start + i;
+		if (pte_val(*pte))
+			return;
+	}
+
+	/* free a pte talbe */
+	vmemmap_free_pages(pmd_page(*pmd), 0);
+	spin_lock(&init_mm.page_table_lock);
+	pmd_clear(pmd);
+	spin_unlock(&init_mm.page_table_lock);
+}
+
+static void free_pmd_table(pud_t *pud)
+{
+	pmd_t *pmd, *pmd_start;
+	int i;
+
+	pmd_start = (pmd_t *)pud_page_vaddr(*pud);
+	for (i = 0; i < PTRS_PER_PMD; i++) {
+		pmd = pmd_start + i;
+		if (pmd_val(*pmd))
+			return;
+	}
+
+	/* free a pmd talbe */
+	vmemmap_free_pages(pud_page(*pud), 0);
+	spin_lock(&init_mm.page_table_lock);
+	pud_clear(pud);
+	spin_unlock(&init_mm.page_table_lock);
+}
+
+static void free_pud_table(pgd_t *pgd)
+{
+	pud_t *pud, *pud_start;
+	int i;
+
+	pud_start = (pud_t *)pgd_page_vaddr(*pgd);
+	for (i = 0; i < PTRS_PER_PUD; i++) {
+		pud = pud_start + i;
+		if (pud_val(*pud))
+			return;
+	}
+
+	/* free a pud table */
+	vmemmap_free_pages(pgd_page(*pgd), 0);
+	spin_lock(&init_mm.page_table_lock);
+	pgd_clear(pgd);
+	spin_unlock(&init_mm.page_table_lock);
+}
+
+static int split_large_page(pte_t *kpte, unsigned long address, pte_t *pbase)
+{
+	struct page *page = pmd_page(*(pmd_t *)kpte);
+	int i = 0;
+	unsigned long magic;
+	unsigned long section_nr;
+
+	__split_large_page(kpte, address, pbase);
+	__flush_tlb_all();
+
+	magic = (unsigned long) page->lru.next;
+	if (magic == SECTION_INFO) {
+		section_nr = pfn_to_section_nr(page_to_pfn(page));
+		while (i < PTRS_PER_PMD) {
+			page++;
+			i++;
+			get_page_bootmem(section_nr, page, SECTION_INFO);
+		}
+	}
+
+	return 0;
+}
+
+static void vmemmap_pte_remove(pmd_t *pmd, unsigned long addr, unsigned long end)
+{
+	pte_t *pte;
+	unsigned long next;
+
+	pte = pte_offset_kernel(pmd, addr);
+	for (; addr < end; pte++, addr += PAGE_SIZE) {
+		next = (addr + PAGE_SIZE) & PAGE_MASK;
+		if (next > end)
+			next = end;
+
+		if (pte_none(*pte))
+			continue;
+		if (IS_ALIGNED(addr, PAGE_SIZE) &&
+		    IS_ALIGNED(end, PAGE_SIZE)) {
+			vmemmap_free_pages(pte_page(*pte), 0);
+			spin_lock(&init_mm.page_table_lock);
+			pte_clear(&init_mm, addr, pte);
+			spin_unlock(&init_mm.page_table_lock);
+		}
+	}
+
+	free_pte_table(pmd);
+	__flush_tlb_all();
+}
+
+static void vmemmap_pmd_remove(pud_t *pud, unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pmd_t *pmd;
+
+	pmd = pmd_offset(pud, addr);
+	for (; addr < end; addr = next, pmd++) {
+		next = pmd_addr_end(addr, end);
+		if (pmd_none(*pmd))
+			continue;
+
+		if (cpu_has_pse) {
+			unsigned long pte_base;
+
+			if (IS_ALIGNED(addr, PMD_SIZE) &&
+			    IS_ALIGNED(next, PMD_SIZE)) {
+				vmemmap_free_pages(pmd_page(*pmd),
+						   get_order(PMD_SIZE));
+				spin_lock(&init_mm.page_table_lock);
+				pmd_clear(pmd);
+				spin_unlock(&init_mm.page_table_lock);
+				continue;
+			}
+
+			/*
+			 * We use 2M page, but we need to remove part of them,
+			 * so split 2M page to 4K page.
+			 */
+			pte_base = get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
+			split_large_page((pte_t *)pmd, addr, (pte_t *)pte_base);
+			__flush_tlb_all();
+
+			spin_lock(&init_mm.page_table_lock);
+			pmd_populate_kernel(&init_mm, pmd, (pte_t *)pte_base);
+			spin_unlock(&init_mm.page_table_lock);
+		}
+
+		vmemmap_pte_remove(pmd, addr, next);
+	}
+
+	free_pmd_table(pud);
+	__flush_tlb_all();
+}
+
+static void vmemmap_pud_remove(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+	unsigned long next;
+	pud_t *pud;
+
+	pud = pud_offset(pgd, addr);
+	for (; addr < end; addr = next, pud++) {
+		next = pud_addr_end(addr, end);
+		if (pud_none(*pud))
+			continue;
+
+		vmemmap_pmd_remove(pud, addr, next);
+	}
+
+	free_pud_table(pgd);
+	__flush_tlb_all();
+}
+
+void vmemmap_free(struct page *memmap, unsigned long nr_pages)
+{
+	unsigned long addr = (unsigned long)memmap;
+	unsigned long end = (unsigned long)(memmap + nr_pages);
+	unsigned long next;
+
+	for (; addr < end; addr = next) {
+		pgd_t *pgd = pgd_offset_k(addr);
+
+		next = pgd_addr_end(addr, end);
+		if (!pgd_present(*pgd))
+			continue;
+
+		vmemmap_pud_remove(pgd, addr, next);
+		sync_global_pgds(addr, next);
+	}
+}
+#endif
diff --git a/mm/sparse.c b/mm/sparse.c
index fac95f2..3a16d68 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -613,12 +613,13 @@  static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
 	/* This will make the necessary allocations eventually. */
 	return sparse_mem_map_populate(pnum, nid);
 }
-static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
+static void __kfree_section_memmap(struct page *page, unsigned long nr_pages)
 {
-	return; /* XXX: Not implemented yet */
+	vmemmap_free(page, nr_pages);
 }
 static void free_map_bootmem(struct page *page, unsigned long nr_pages)
 {
+	vmemmap_free(page, nr_pages);
 }
 #else
 static struct page *__kmalloc_section_memmap(unsigned long nr_pages)