[PATCH v2 0/2] Support Armv8.9/v9.4 FEAT_HAFT

[PATCH v2 0/2] Support Armv8.9/v9.4 FEAT_HAFT

[Yicong Yang]

From: Yicong Yang <yangyicong@hisilicon.com>

This series adds basic support for FEAT_HAFT introduced in Armv8.9/v9.4
and enable ARCH_HAS_NONLEAF_PMD_YOUNG. The latter will be used in
lru-gen aging. Tested with lru-gen in below steps:
1. Generate a 1GiB workingset by `stress-ng --vm 1`. Then hang the task to
   stop accessing the memory. (AF bit won't be updated)
2. try to age the memory by /sys/kernel/debug/lru_gen

Run above steps with LRU_GEN_NONLEAF_YOUNG(0x4) and not respectively
(switching by /sys/kernel/mm/lru_gen/enabled). LRU_GEN_NONLEAF_YOUNG
will clear and test the PMD AF bit on page walking for aging,
otherwise will clear and test the PTE AF bit for aging. In this case
LRU_GEN_NONLEAF_YOUNG will improve the efficiency of page scanning
since pages won't be accessed and we don't need to scan each PTE.
Observed ~40% time saved for 1GiB memory on our emulated platform
with LRU_GEN_NONLEAF_YOUNG.

For lru-gen aging:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/admin-guide/mm/multigen_lru.rst?h=v6.11-rc1#n94

Change since v1:
- Address comments from Marc, improve comments/Kconfig, clean code. Thanks for
  the comments.
Link: https://lore.kernel.org/linux-arm-kernel/20240802093458.32683-1-yangyicong@huawei.com/

Yicong Yang (2):
  arm64: Add support for FEAT_HAFT
  arm64: Enable ARCH_HAS_NONLEAF_PMD_YOUNG

 arch/arm64/Kconfig               | 20 ++++++++++++++++++++
 arch/arm64/include/asm/pgtable.h | 14 ++++++++++++--
 arch/arm64/kernel/cpufeature.c   | 26 ++++++++++++++++++++++++++
 arch/arm64/tools/cpucaps         |  1 +
 arch/arm64/tools/sysreg          |  1 +
 5 files changed, 60 insertions(+), 2 deletions(-)

-- 
2.24.0

[PATCH v2 1/2] arm64: Add support for FEAT_HAFT

[Yicong Yang]

From: Yicong Yang <yangyicong@hisilicon.com>

Armv8.9/v9.4 introduces the feature Hardware managed Access Flag
for Table descriptors (FEAT_HAFT). The feature is indicated by
ID_AA64MMFR1_EL1.HAFDBS == 0b0011 and can be enabled by
TCR2_EL1.HAFT so it has a dependency on FEAT_TCR2.

This patch adds the Kconfig for FEAT_HAFT and support detecting
and enabling the feature.

Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
---
 arch/arm64/Kconfig             | 19 +++++++++++++++++++
 arch/arm64/kernel/cpufeature.c | 26 ++++++++++++++++++++++++++
 arch/arm64/tools/cpucaps       |  1 +
 arch/arm64/tools/sysreg        |  1 +
 4 files changed, 47 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a2f8ff354ca6..869792458a23 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2137,6 +2137,25 @@ config ARM64_EPAN
 	  if the cpu does not implement the feature.
 endmenu # "ARMv8.7 architectural features"
 
+menu "ARMv8.9 architectural features"
+
+config ARM64_HAFT
+	bool "Support for Hardware managed Access Flag for Table Descriptor"
+	depends on ARM64_HW_AFDBM
+	default y
+	help
+	  The ARMv8.9/ARMv9.5 introduces the feature Hardware managed Access
+	  Flag for Table descriptors. When enabled an architectural executed
+	  memory access will update the Access Flag in each Table descriptor
+	  which is accessed during the translation table walk and for which
+	  the Access Flag is 0. The Access Flag of the Table descriptor use
+	  the same bit of PTE_AF.
+
+	  The feature will only be enabled if all the CPUs in the system
+	  support this feature. If unsure, say Y.
+
+endmenu # "ARMv8.9 architectural features"
+
 config ARM64_SVE
 	bool "ARM Scalable Vector Extension support"
 	default y
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 646ecd3069fd..ed4c968be935 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2044,6 +2044,17 @@ static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
 
 #endif
 
+#if CONFIG_ARM64_HAFT
+
+static void cpu_enable_haft(struct arm64_cpu_capabilities const *cap)
+{
+	sysreg_clear_set_s(SYS_TCR2_EL1, 0, TCR2_EL1x_HAFT);
+	isb();
+	local_flush_tlb_all();
+}
+
+#endif
+
 #ifdef CONFIG_ARM64_AMU_EXTN
 
 /*
@@ -2580,6 +2591,21 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.cpus = &dbm_cpus,
 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
 	},
+#endif
+#ifdef CONFIG_ARM64_HAFT
+	{
+		.desc = "Hardware managed Access Flag for Table Descriptor",
+		/*
+		 * Contrary to the page/block access flag, the table access flag
+		 * cannot be emulated in software (no access fault will occur).
+		 * Therefore mandate that all CPUs have FEAT_HAFT.
+		 */
+		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
+		.capability = ARM64_HAFT,
+		.matches = has_cpuid_feature,
+		.cpu_enable = cpu_enable_haft,
+		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, HAFT)
+	},
 #endif
 	{
 		.desc = "CRC32 instructions",
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index ac3429d892b9..0b7a3a237e5d 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -55,6 +55,7 @@ HAS_TLB_RANGE
 HAS_VA52
 HAS_VIRT_HOST_EXTN
 HAS_WFXT
+HAFT
 HW_DBM
 KVM_HVHE
 KVM_PROTECTED_MODE
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 7ceaa1e0b4bc..9b3d15ea8a63 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -1688,6 +1688,7 @@ UnsignedEnum	3:0	HAFDBS
 	0b0000	NI
 	0b0001	AF
 	0b0010	DBM
+	0b0011	HAFT
 EndEnum
 EndSysreg
 
-- 
2.24.0

Re: [PATCH v2 1/2] arm64: Add support for FEAT_HAFT

[Mark Brown]

[-- Attachment #1: Type: text/plain, Size: 484 bytes --]

On Wed, Aug 14, 2024 at 05:23:32PM +0800, Yicong Yang wrote:

> --- a/arch/arm64/tools/sysreg
> +++ b/arch/arm64/tools/sysreg
> @@ -1688,6 +1688,7 @@ UnsignedEnum	3:0	HAFDBS
>  	0b0000	NI
>  	0b0001	AF
>  	0b0010	DBM
> +	0b0011	HAFT
>  EndEnum
>  EndSysreg

Please if you're doing this update the entire sysreg to whatever the
current version of the XML is, ideally as a separate commit to avoid
duplication of work.  It makes it easier to track any issues in the
sysreg conversions.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH v2 1/2] arm64: Add support for FEAT_HAFT

[Yicong Yang]

On 2024/8/14 20:07, Mark Brown wrote:
> On Wed, Aug 14, 2024 at 05:23:32PM +0800, Yicong Yang wrote:
> 
>> --- a/arch/arm64/tools/sysreg
>> +++ b/arch/arm64/tools/sysreg
>> @@ -1688,6 +1688,7 @@ UnsignedEnum	3:0	HAFDBS
>>  	0b0000	NI
>>  	0b0001	AF
>>  	0b0010	DBM
>> +	0b0011	HAFT
>>  EndEnum
>>  EndSysreg
> 
> Please if you're doing this update the entire sysreg to whatever the
> current version of the XML is, ideally as a separate commit to avoid
> duplication of work.  It makes it easier to track any issues in the
> sysreg conversions.
> 

Got it. Currently I only update the it according to the feature to support.
Will make it into a separate patch and update the definition of both
HAFT and HDBSS according to the lastest register XML.

Thanks.

Re: [PATCH v2 1/2] arm64: Add support for FEAT_HAFT

[Will Deacon]

On Wed, Aug 14, 2024 at 05:23:32PM +0800, Yicong Yang wrote:
> From: Yicong Yang <yangyicong@hisilicon.com>
> 
> Armv8.9/v9.4 introduces the feature Hardware managed Access Flag
> for Table descriptors (FEAT_HAFT). The feature is indicated by
> ID_AA64MMFR1_EL1.HAFDBS == 0b0011 and can be enabled by
> TCR2_EL1.HAFT so it has a dependency on FEAT_TCR2.
> 
> This patch adds the Kconfig for FEAT_HAFT and support detecting
> and enabling the feature.
> 
> Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
> ---
>  arch/arm64/Kconfig             | 19 +++++++++++++++++++
>  arch/arm64/kernel/cpufeature.c | 26 ++++++++++++++++++++++++++
>  arch/arm64/tools/cpucaps       |  1 +
>  arch/arm64/tools/sysreg        |  1 +
>  4 files changed, 47 insertions(+)
> 
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index a2f8ff354ca6..869792458a23 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -2137,6 +2137,25 @@ config ARM64_EPAN
>  	  if the cpu does not implement the feature.
>  endmenu # "ARMv8.7 architectural features"
>  
> +menu "ARMv8.9 architectural features"
> +
> +config ARM64_HAFT
> +	bool "Support for Hardware managed Access Flag for Table Descriptor"
> +	depends on ARM64_HW_AFDBM
> +	default y
> +	help
> +	  The ARMv8.9/ARMv9.5 introduces the feature Hardware managed Access
> +	  Flag for Table descriptors. When enabled an architectural executed
> +	  memory access will update the Access Flag in each Table descriptor
> +	  which is accessed during the translation table walk and for which
> +	  the Access Flag is 0. The Access Flag of the Table descriptor use
> +	  the same bit of PTE_AF.
> +
> +	  The feature will only be enabled if all the CPUs in the system
> +	  support this feature. If unsure, say Y.
> +
> +endmenu # "ARMv8.9 architectural features"
> +
>  config ARM64_SVE
>  	bool "ARM Scalable Vector Extension support"
>  	default y
> diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
> index 646ecd3069fd..ed4c968be935 100644
> --- a/arch/arm64/kernel/cpufeature.c
> +++ b/arch/arm64/kernel/cpufeature.c
> @@ -2044,6 +2044,17 @@ static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
>  
>  #endif
>  
> +#if CONFIG_ARM64_HAFT
> +
> +static void cpu_enable_haft(struct arm64_cpu_capabilities const *cap)
> +{
> +	sysreg_clear_set_s(SYS_TCR2_EL1, 0, TCR2_EL1x_HAFT);
> +	isb();
> +	local_flush_tlb_all();
> +}

As this isn't a per-TTBR enable, should we be initialising the kernel
table entries in TTBR1 as YOUNG to avoid potential races with the
hardware update? It looks like the bit is ignored on CPUs without HAFT,
so we can just do this unconditionally.

At the very least, we should be able to enable HAFT in __cpu_setup(),
like we do for HA.

> +#endif
> +
>  #ifdef CONFIG_ARM64_AMU_EXTN
>  
>  /*
> @@ -2580,6 +2591,21 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
>  		.cpus = &dbm_cpus,
>  		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
>  	},
> +#endif
> +#ifdef CONFIG_ARM64_HAFT
> +	{
> +		.desc = "Hardware managed Access Flag for Table Descriptor",
> +		/*
> +		 * Contrary to the page/block access flag, the table access flag
> +		 * cannot be emulated in software (no access fault will occur).
> +		 * Therefore mandate that all CPUs have FEAT_HAFT.
> +		 */

It's a bit of a pity that we can't handle this mismatch. After all,
access flag data is imprecise (unlike the dirty bit) and so you could
envisage a mechanism for falling back to leaf-level AF at runtime rather
than refusing to online a CPU.

Of course, it's hard to tell whether this really matters until we see
what people try to glue together.

Will

Re: [PATCH v2 1/2] arm64: Add support for FEAT_HAFT

[Yicong Yang]

On 2024/8/21 0:18, Will Deacon wrote:
> On Wed, Aug 14, 2024 at 05:23:32PM +0800, Yicong Yang wrote:
>> From: Yicong Yang <yangyicong@hisilicon.com>
>>
>> Armv8.9/v9.4 introduces the feature Hardware managed Access Flag
>> for Table descriptors (FEAT_HAFT). The feature is indicated by
>> ID_AA64MMFR1_EL1.HAFDBS == 0b0011 and can be enabled by
>> TCR2_EL1.HAFT so it has a dependency on FEAT_TCR2.
>>
>> This patch adds the Kconfig for FEAT_HAFT and support detecting
>> and enabling the feature.
>>
>> Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
>> ---
>>  arch/arm64/Kconfig             | 19 +++++++++++++++++++
>>  arch/arm64/kernel/cpufeature.c | 26 ++++++++++++++++++++++++++
>>  arch/arm64/tools/cpucaps       |  1 +
>>  arch/arm64/tools/sysreg        |  1 +
>>  4 files changed, 47 insertions(+)
>>
>> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
>> index a2f8ff354ca6..869792458a23 100644
>> --- a/arch/arm64/Kconfig
>> +++ b/arch/arm64/Kconfig
>> @@ -2137,6 +2137,25 @@ config ARM64_EPAN
>>  	  if the cpu does not implement the feature.
>>  endmenu # "ARMv8.7 architectural features"
>>  
>> +menu "ARMv8.9 architectural features"
>> +
>> +config ARM64_HAFT
>> +	bool "Support for Hardware managed Access Flag for Table Descriptor"
>> +	depends on ARM64_HW_AFDBM
>> +	default y
>> +	help
>> +	  The ARMv8.9/ARMv9.5 introduces the feature Hardware managed Access
>> +	  Flag for Table descriptors. When enabled an architectural executed
>> +	  memory access will update the Access Flag in each Table descriptor
>> +	  which is accessed during the translation table walk and for which
>> +	  the Access Flag is 0. The Access Flag of the Table descriptor use
>> +	  the same bit of PTE_AF.
>> +
>> +	  The feature will only be enabled if all the CPUs in the system
>> +	  support this feature. If unsure, say Y.
>> +
>> +endmenu # "ARMv8.9 architectural features"
>> +
>>  config ARM64_SVE
>>  	bool "ARM Scalable Vector Extension support"
>>  	default y
>> diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
>> index 646ecd3069fd..ed4c968be935 100644
>> --- a/arch/arm64/kernel/cpufeature.c
>> +++ b/arch/arm64/kernel/cpufeature.c
>> @@ -2044,6 +2044,17 @@ static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
>>  
>>  #endif
>>  
>> +#if CONFIG_ARM64_HAFT
>> +
>> +static void cpu_enable_haft(struct arm64_cpu_capabilities const *cap)
>> +{
>> +	sysreg_clear_set_s(SYS_TCR2_EL1, 0, TCR2_EL1x_HAFT);
>> +	isb();
>> +	local_flush_tlb_all();
>> +}
> 
> As this isn't a per-TTBR enable, should we be initialising the kernel
> table entries in TTBR1 as YOUNG to avoid potential races with the
> hardware update? It looks like the bit is ignored on CPUs without HAFT,
> so we can just do this unconditionally.
> 

I'm a bit uncertain about the race here, is it because of the access fault
or the inconsistent observation of table entries' AF status when enable the
HAFT? and...

> At the very least, we should be able to enable HAFT in __cpu_setup(),
> like we do for HA.
> 

...if we enable this in __cpu_setup() as suggested we should have no race problem,
since this is enabled before MMU on?

>> +#endif
>> +
>>  #ifdef CONFIG_ARM64_AMU_EXTN
>>  
>>  /*
>> @@ -2580,6 +2591,21 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
>>  		.cpus = &dbm_cpus,
>>  		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
>>  	},
>> +#endif
>> +#ifdef CONFIG_ARM64_HAFT
>> +	{
>> +		.desc = "Hardware managed Access Flag for Table Descriptor",
>> +		/*
>> +		 * Contrary to the page/block access flag, the table access flag
>> +		 * cannot be emulated in software (no access fault will occur).
>> +		 * Therefore mandate that all CPUs have FEAT_HAFT.
>> +		 */
> 
> It's a bit of a pity that we can't handle this mismatch. After all,
> access flag data is imprecise (unlike the dirty bit) and so you could
> envisage a mechanism for falling back to leaf-level AF at runtime rather
> than refusing to online a CPU.
> 

ok. I suppose enable this should be ok if no users. So it's possible to just
try to enable this at each CPU's startup, but don't advertise this feature to
the upper users if not all the CPUs in the system support this. This won't
affect leaf-level AF since leaf-level AF has its own detection and enabling
and doesn't depend on HAFT.

> Of course, it's hard to tell whether this really matters until we see
> what people try to glue together.
> 
> Will
> .
> 

Re: [PATCH v2 1/2] arm64: Add support for FEAT_HAFT

[Catalin Marinas]

On Thu, Aug 29, 2024 at 02:29:41PM +0800, Yicong Yang wrote:
> On 2024/8/21 0:18, Will Deacon wrote:
> > On Wed, Aug 14, 2024 at 05:23:32PM +0800, Yicong Yang wrote:
> >> +static void cpu_enable_haft(struct arm64_cpu_capabilities const *cap)
> >> +{
> >> +	sysreg_clear_set_s(SYS_TCR2_EL1, 0, TCR2_EL1x_HAFT);
> >> +	isb();
> >> +	local_flush_tlb_all();
> >> +}
> > 
> > As this isn't a per-TTBR enable, should we be initialising the kernel
> > table entries in TTBR1 as YOUNG to avoid potential races with the
> > hardware update? It looks like the bit is ignored on CPUs without HAFT,
> > so we can just do this unconditionally.
> 
> I'm a bit uncertain about the race here, is it because of the access fault
> or the inconsistent observation of table entries' AF status when enable the
> HAFT? and...

Linux doesn't use this mechanism on kernel page tables. The hardware
update of the access bit is atomic and, while it could race with a
non-atomic entry update, it shouldn't matter since no-one checks this
bit. However, I'd still set the AF bit when creating the kernel page
table, it saves the hardware from having to update them at run-time.

> > At the very least, we should be able to enable HAFT in __cpu_setup(),
> > like we do for HA.
> > 
> 
> ...if we enable this in __cpu_setup() as suggested we should have no race problem,
> since this is enabled before MMU on?

I suspect Will was referring to the actual page tables. But it makes
sense to enable this in __cpu_setup(), it saves us from having to do the
TLBI.

> > It's a bit of a pity that we can't handle this mismatch. After all,
> > access flag data is imprecise (unlike the dirty bit) and so you could
> > envisage a mechanism for falling back to leaf-level AF at runtime rather
> > than refusing to online a CPU.
> 
> ok. I suppose enable this should be ok if no users. So it's possible to just
> try to enable this at each CPU's startup, but don't advertise this feature to
> the upper users if not all the CPUs in the system support this. This won't
> affect leaf-level AF since leaf-level AF has its own detection and enabling
> and doesn't depend on HAFT.

Yes, I think this should work as long as arch_has_hw_pte_young() returns
false if one of the CPUs doesn't support it. I had a quick grep through
the kernel and all (most?) cases where the non-leaf pmd is checked or
the AF bit cleared are conditional on should_clear_pmd_young().
Personally I'd have introduced pmd_nonleaf_young() etc. functions in the
core code to make it more explicit. I guess, if we really want, we could
add a VM_WARN_ON if pmd_young() or pmdp_test_and_clear_young() is called
on a non-leaf pmd and FEAT_HAFT is disabled.

-- 
Catalin

[PATCH v2 2/2] arm64: Enable ARCH_HAS_NONLEAF_PMD_YOUNG

[Yicong Yang]

From: Yicong Yang <yangyicong@hisilicon.com>

With the support of FEAT_HAFT, the NONLEAF_PMD_YOUNG can be enabled
on arm64 since the hardware is capable of updating the AF flag for
PMD table descriptor. Since the AF bit of the table descriptor
shares the same bit position in block descriptors, we only need
to implement arch_has_hw_nonleaf_pmd_young() and select related
configs. The related pmd_young test/update operations keeps the
same with and already implemented for transparent page support.

Currently ARCH_HAS_NONLEAF_PMD_YOUNG is used to improve the
efficiency of lru-gen aging.

Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
---
 arch/arm64/Kconfig               |  1 +
 arch/arm64/include/asm/pgtable.h | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 869792458a23..74451a7ccc4f 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -36,6 +36,7 @@ config ARM64
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_NMI_SAFE_THIS_CPU_OPS
 	select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
+	select ARCH_HAS_NONLEAF_PMD_YOUNG if ARM64_HAFT
 	select ARCH_HAS_PTE_DEVMAP
 	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_HW_PTE_YOUNG
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 7a4f5604be3f..077bea37867e 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -1205,7 +1205,7 @@ static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
 	return young;
 }
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 					    unsigned long address,
@@ -1213,7 +1213,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 {
 	return __ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
 }
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
 
 static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
 				       unsigned long address, pte_t *ptep)
@@ -1448,6 +1448,16 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
  */
 #define arch_has_hw_pte_young		cpu_has_hw_af
 
+#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG
+
+#define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young
+static inline bool arch_has_hw_nonleaf_pmd_young(void)
+{
+	return cpus_have_final_cap(ARM64_HAFT);
+}
+
+#endif
+
 /*
  * Experimentally, it's cheap to set the access flag in hardware and we
  * benefit from prefaulting mappings as 'old' to start with.
-- 
2.24.0