From nobody Fri Jul 26 06:25:23 2024 X-Original-To: dev-commits-src-all@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 4WVd6C3pWhz5RRxf; Fri, 26 Jul 2024 06:25:23 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R11" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4WVd6C1kWtz48tp; Fri, 26 Jul 2024 06:25:23 +0000 (UTC) (envelope-from git@FreeBSD.org) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1721975123; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=axfg9vnaLQ0Jnyfj7MN3pQATa9MwRwOvG2/G9MMBZLY=; b=aJkaXBAKaW7o5LeSnPFHPu8EqDP1rdic/KMt9EjgdNAG79qxAmdH01rVbmuo1GmbO9/1Kl N0vboDMmlkni8NMur63MbQa/fUdj+DaAQScqZRa/S+louX8gq09yHYBsPNG00hvdI6KN3j P0ILEpkNpQRwoA1SP6VDz332IciTy/gmUmvlX0aCeivOnlvpmIbcO6xtnqciPj9UpjJaVN XxVukwzBCucOsUm9ihEPdGB1hJkbgjBAxNOIxzX2a4BanKKGdM+9PLn7XMTHu+LoX66OpI AO06mEwzt8vkn3maJVVGlZiAUtPqM+6tJmaV6k68MNsgBW7hML4wPf4HR8Nk8w== ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1721975123; a=rsa-sha256; cv=none; b=wREH6xqeXTfE1vjI1ug+qWk2rPXST/FIBaZ0ReYYJFl76SRQ12znY4XIH3DZof8GX/N6rI sDkfhrzCBiQsP7+O0W7p+E3pRQUB2owj9blThLEixQZ8amPtYyVwEON2ZBK9E8wqqGDkGW V5nych08i3uRJf2oNQvPMuJf9dxW3/DHg6cC4SPGp/xojr6zZ/WaSN7M6tAV1V7b4BMUFb HwWxL9sxaQb/i9kmqXptlmnVilhIAYQ/3g47lh3fP12796e/ikf9DPUFMBu7zlJxj/vwAH 6trgAUoF5nQWmcly9qZe/1Y8BLxG1CDOIS4xm2U67I3FaaPSSGGGcigBJDannQ== ARC-Authentication-Results: i=1; mx1.freebsd.org; none ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1721975123; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=axfg9vnaLQ0Jnyfj7MN3pQATa9MwRwOvG2/G9MMBZLY=; b=v6EEJivITB7320muPklcFwWK6YXOXX9Ku/5lcS5D3DMylu9Pn99t+g73hzWnPSmp4wg4gi vUo4QMAorEjkWZTctyd4/4jU2+j6PAd5ySXBXEcSo9Ju1g7YLHdtsA05qUApf6mFQpZGYK ZChM8e8fx7GAXrT1C0EShe33tT3kHftT/B+m3molRmg+LcPA2AVk2Gdbm8gu4wYChBxt+n yqjsxMZcPZAW3NmcSSCwvouy8oEZfUkKZ+S1fV+xSnEKzXxVhKs+/OnSeR7e8+u0DLxs/9 TUUjlJXf9de6vBpkLZoSvm/8HMYv+CUE0769KqPo9+iDLpW/ZNZ92R5KbLNcxg== Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 4WVd6C1KW0z1RLC; Fri, 26 Jul 2024 06:25:23 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.18.1/8.18.1) with ESMTP id 46Q6PNu1044400; Fri, 26 Jul 2024 06:25:23 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.18.1/8.18.1/Submit) id 46Q6PNff044397; Fri, 26 Jul 2024 06:25:23 GMT (envelope-from git) Date: Fri, 26 Jul 2024 06:25:23 GMT Message-Id: <202407260625.46Q6PNff044397@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Alan Cox Subject: git: 5b8c01d13a09 - main - amd64 pmap: Optimize PKU lookups when creating superpage mappings List-Id: Commit messages for all branches of the src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-all List-Help: List-Post: List-Subscribe: List-Unsubscribe: X-BeenThere: dev-commits-src-all@freebsd.org Sender: owner-dev-commits-src-all@FreeBSD.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: alc X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: 5b8c01d13a0970b11f47503fcd627d249a6e638a Auto-Submitted: auto-generated The branch main has been updated by alc: URL: https://cgit.FreeBSD.org/src/commit/?id=5b8c01d13a0970b11f47503fcd627d249a6e638a commit 5b8c01d13a0970b11f47503fcd627d249a6e638a Author: Alan Cox AuthorDate: 2024-07-25 06:57:53 +0000 Commit: Alan Cox CommitDate: 2024-07-26 05:38:46 +0000 amd64 pmap: Optimize PKU lookups when creating superpage mappings Modify pmap_pkru_same() to update the prototype PTE at the same time as checking the address range. This eliminates the need for calling pmap_pkru_get() in addition to pmap_pkru_same(). pmap_pkru_same() was already doing most of the work of pmap_pkru_get(). Reviewed by: kib Differential Revision: https://reviews.freebsd.org/D46135 --- sys/amd64/amd64/pmap.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index dcf9b4f5a4f3..778d07689ff0 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -573,7 +573,8 @@ struct pmap_pkru_range { }; static uma_zone_t pmap_pkru_ranges_zone; -static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); +static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, + pt_entry_t *pte); static pt_entry_t pmap_pkru_get(pmap_t pmap, vm_offset_t va); static void pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); static void *pkru_dup_range(void *ctx, void *data); @@ -7071,11 +7072,9 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, PG_V = pmap_valid_bit(pmap); restart: - if (!pmap_pkru_same(pmap, va, va + pagesizes[psind])) - return (KERN_PROTECTION_FAILURE); pten = newpte; - if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) - pten |= pmap_pkru_get(pmap, va); + if (!pmap_pkru_same(pmap, va, va + pagesizes[psind], &pten)) + return (KERN_PROTECTION_FAILURE); if (psind == 2) { /* 1G */ pml4e = pmap_pml4e(pmap, va); @@ -7529,14 +7528,10 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags, * and let vm_fault() cope. Check after pde allocation, since * it could sleep. */ - if (!pmap_pkru_same(pmap, va, va + NBPDR)) { + if (!pmap_pkru_same(pmap, va, va + NBPDR, &newpde)) { pmap_abort_ptp(pmap, va, pdpg); return (KERN_PROTECTION_FAILURE); } - if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) { - newpde &= ~X86_PG_PKU_MASK; - newpde |= pmap_pkru_get(pmap, va); - } /* * If there are existing mappings, either abort or remove them. @@ -11460,13 +11455,21 @@ pmap_pkru_deassign_all(pmap_t pmap) rangeset_remove_all(&pmap->pm_pkru); } +/* + * Returns true if the PKU setting is the same across the specified address + * range, and false otherwise. When returning true, updates the referenced PTE + * to reflect the PKU setting. + */ static bool -pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) +pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte) { struct pmap_pkru_range *next_ppr, *ppr; vm_offset_t va; + u_int keyidx; PMAP_LOCK_ASSERT(pmap, MA_OWNED); + KASSERT(pmap->pm_type != PT_X86 || (*pte & X86_PG_PKU_MASK) == 0, + ("pte %p has unexpected PKU %ld", pte, *pte & X86_PG_PKU_MASK)); if (pmap->pm_type != PT_X86 || (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0 || sva >= VM_MAXUSER_ADDRESS) @@ -11478,14 +11481,16 @@ pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) return (ppr == NULL || ppr->pkru_rs_el.re_start >= eva); } + keyidx = ppr->pkru_keyidx; while ((va = ppr->pkru_rs_el.re_end) < eva) { next_ppr = rangeset_next(&pmap->pm_pkru, va); if (next_ppr == NULL || va != next_ppr->pkru_rs_el.re_start || - ppr->pkru_keyidx != next_ppr->pkru_keyidx) + keyidx != next_ppr->pkru_keyidx) return (false); ppr = next_ppr; } + *pte |= X86_PG_PKU(keyidx); return (true); }