From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id 044B5C54F51 for ; Wed, 29 Jul 2026 12:26:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender:List-Subscribe:List-Help :List-Post:List-Archive:List-Unsubscribe:List-Id:Content-Transfer-Encoding: MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc:To:From: Reply-To:Content-Type:Content-ID:Content-Description:Resent-Date:Resent-From: Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID:List-Owner; bh=n6glQyinDiG+knWCf5YuYqgT1r5WAXpU+YxPMWMH1Aw=; b=IyKMNduQHgcgRNifFeRm6Buvrp 2OT9Z0xp+ZI+IXl6pUmXFIdANAYJftHPWb96l0BsW8r7zNXqEzwaSxxzLBTYAT6xkVuJIPluciofq 2dJuy7lJ40pbekiDwoIWD7WzywG4g+Mj4NT8lr+dU5xmZNfXyc+zdC0cYEfP7OU8zLmsGTCqa4fOh 0zl7lVSljViUmc6hvq1XRTXDxyNh8jxF8YrOAqKZC6a9EpAotSQ6yfYpRMhOdpjaVGmkbNV08RSTW /KHlQqG76BRA6EmAAYGDpXHOTAOzfX9GB3wBnoiRGFtzE0ExoOJ9sbzi68IaiD8uIyjks6cM+ghz3 FFi6NVQw==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.99.1 #2 (Red Hat Linux)) id 1wp3Mo-00000007u52-2364; Wed, 29 Jul 2026 12:26:34 +0000 Received: from foss.arm.com ([217.140.110.172]) by bombadil.infradead.org with esmtp (Exim 4.99.1 #2 (Red Hat Linux)) id 1wp3MW-00000007tpB-1MxT for linux-arm-kernel@lists.infradead.org; Wed, 29 Jul 2026 12:26:18 +0000 Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 630D11688; Wed, 29 Jul 2026 05:26:11 -0700 (PDT) Received: from a085714.blr.arm.com (a085714.arm.com [10.164.19.28]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id DCEBE3F66F; Wed, 29 Jul 2026 05:26:11 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=arm.com; s=foss; t=1785327975; bh=g9NV7nRdXqfFL3CF8X7Uv58zGdVCBUS7EOV62bO/WS4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=DItmoeNADsjRN4kC/vUk9pTatICCrxY30uhea8jvosirq9eKyBdxl46nh5vk+JUeK NrIgHuMNJdP2tKiFSAdB4gMWzkdqaUVnEgwQpvOITIxt67Qgeslf1bg6cZLeXUwvCd 4FEx7FaYj9qc2uYRIStTSNKgNPRhbVHuFONaVtGo= From: Anshuman Khandual To: linux-mm@kvack.org, linux-arm-kernel@lists.infradead.org Cc: Anshuman Khandual , Catalin Marinas , Will Deacon , Ryan Roberts , Mark Rutland , Lorenzo Stoakes , Andrew Morton , David Hildenbrand , Mike Rapoport , Linu Cherian , linux-kernel@vger.kernel.org Subject: [PATCH 16/17] arm64/mm: Enable TLBIP instruction based TLB flush Date: Wed, 29 Jul 2026 17:54:51 +0530 Message-ID: <20260729122452.3797443-17-anshuman.khandual@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260729122452.3797443-1-anshuman.khandual@arm.com> References: <20260729122452.3797443-1-anshuman.khandual@arm.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.9.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20260729_052616_835873_B1D7F65A X-CRM114-Status: GOOD ( 10.91 ) X-BeenThere: linux-arm-kernel@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: "linux-arm-kernel" Errors-To: linux-arm-kernel-bounces+linux-arm-kernel=archiver.kernel.org@lists.infradead.org From: Linu Cherian Add TLBIP instruction support for TLB flush operation updating all relevant macros as necessary. This enables back pgtable level hint and address range support during TLB flush operation for D128 translation. Signed-off-by: Linu Cherian Signed-off-by: Anshuman Khandual --- arch/arm64/include/asm/tlbflush.h | 68 +++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c103380ec87e..3b6384b9c200 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -41,6 +41,25 @@ #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) +#ifdef CONFIG_ARM64_D128 +#define __tlbip(op, arg) do { \ + asm (ARM64_ASM_PREAMBLE \ + ".arch_extension d128\n\t" \ + "tlbip " #op ", %0, %H0\n" \ + : : "r" (arg.full)); \ +} while (0) + +#define __tlbip_user(op, arg) do { \ + if (arm64_kernel_unmapped_at_el0()) { \ + arg.low |= USER_ASID_FLAG; \ + __tlbip(op, (arg)); \ + } \ +} while (0) + +#endif + +#define TLBI_ASID_MASK GENMASK_ULL(63, 48) + #define __tlbi_user(op, arg) do { \ if (arm64_kernel_unmapped_at_el0()) \ __tlbi(op, (arg) | USER_ASID_FLAG); \ @@ -130,9 +149,15 @@ static inline void sme_dvmsync_batch(void) #define TLBI_TTL_UNKNOWN INT_MAX +#ifdef CONFIG_ARM64_D128 +typedef union __u128_halves tlbi_arg_t; +#define __tlbi_wrapper(op, arg) __tlbip(op, arg) +#define __tlbi_user_wrapper(op, arg) __tlbip_user(op, arg) +#else typedef u64 tlbi_arg_t; #define __tlbi_wrapper(op, arg) __tlbi(op, arg) #define __tlbi_user_wrapper(op, arg) __tlbi_user(op, arg) +#endif typedef void (*tlbi_op)(tlbi_arg_t arg); @@ -179,17 +204,28 @@ static __always_inline void ipas2e1is(tlbi_arg_t arg) __tlbi_wrapper(ipas2e1is, arg); } -static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, - u16 asid) +static __always_inline void __tlbi_update_level(u32 level, u64 *arg) { - u64 arg = __TLBI_VADDR(addr, asid); - if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) { u64 ttl = level | (get_trans_granule() << 2); - FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); + FIELD_MODIFY(TLBI_TTL_MASK, arg, ttl); } +} + +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, u16 asid) +{ +#ifdef CONFIG_ARM64_D128 + union __u128_halves arg; + + arg.low = FIELD_PREP(TLBI_ASID_MASK, asid); + __tlbi_update_level(level, &arg.low); + arg.high = addr >> 12; +#else + u64 arg = __TLBI_VADDR(addr, asid); + __tlbi_update_level(level, &arg); +#endif op(arg); } @@ -475,19 +511,33 @@ static __always_inline void ripas2e1is(tlbi_arg_t arg) __tlbi_wrapper(ripas2e1is, arg); } -static __always_inline void __tlbi_range(tlbi_op op, u64 addr, - u16 asid, int scale, int num, - u32 level, bool lpa2) +static __always_inline u64 __tlbi_range_args_encode_comm(u16 asid, int scale, int num, u32 level) { u64 arg = 0; - arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); arg |= FIELD_PREP(TLBIR_NUM_MASK, num); arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale); arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); + return arg; +} + +static __always_inline void __tlbi_range(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2) +{ +#ifdef CONFIG_ARM64_D128 + union __u128_halves arg; + + arg.low = __tlbi_range_args_encode_comm(asid, scale, num, level); + arg.high = addr >> 12; +#else + u64 arg = __tlbi_range_args_encode_comm(asid, scale, num, level); + + arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); +#endif op(arg); } -- 2.43.0