diff options
| author | Steve Capper <steve.capper@linaro.org> | 2014-03-21 10:55:36 +0000 |
|---|---|---|
| committer | Steve Capper <steve.capper@linaro.org> | 2014-03-26 09:33:49 +0000 |
| commit | ce076a3b56c2e011bf860124e2dd43eba34afa65 (patch) | |
| tree | 49f96986255eb09c9771e897d49feae67e9859c5 | |
| parent | 4a31becf6d669e6d71f72442eb5f64f834aa483a (diff) | |
| download | linux-fast_gup/general.tar.gz | |
arm64: mm: Enable RCU fast_gupfast_gup/general
Activate the RCU fast_gup for ARM64. We also need to force THP splits
to broadcast an IPI s.t. we block in the fast_gup page walker. As THP
splits are comparatively rare, this should not lead to a noticeable
performance degradation.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
| -rw-r--r-- | arch/arm64/Kconfig | 3 | ||||
| -rw-r--r-- | arch/arm64/include/asm/pgtable.h | 4 | ||||
| -rw-r--r-- | arch/arm64/mm/flush.c | 19 |
3 files changed, 26 insertions, 0 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6185f95108f..9f5a81a6ea1 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -86,6 +86,9 @@ config GENERIC_CSUM config GENERIC_CALIBRATE_DELAY def_bool y +config HAVE_RCU_GUP + def_bool y + config ZONE_DMA32 def_bool y diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index aa3917c8b62..0e148ae4249 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -245,6 +245,10 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT)) #define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING) +#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH +struct vm_area_struct; +void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp); #endif #define PMD_BIT_FUNC(fn,op) \ diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index e4193e3adc7..ddf96c1feaa 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -103,3 +103,22 @@ EXPORT_SYMBOL(flush_dcache_page); */ EXPORT_SYMBOL(flush_cache_all); EXPORT_SYMBOL(flush_icache_range); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifdef CONFIG_HAVE_RCU_TABLE_FREE +static void thp_splitting_flush_sync(void *arg) +{ +} + +void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp) +{ + pmd_t pmd = pmd_mksplitting(*pmdp); + VM_BUG_ON(address & ~PMD_MASK); + set_pmd_at(vma->vm_mm, address, pmdp, pmd); + + /* dummy IPI to serialise against fast_gup */ + smp_call_function(thp_splitting_flush_sync, NULL, 1); +} +#endif /* CONFIG_HAVE_RCU_TABLE_FREE */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |