From nobody Mon Mar 14 14:46:16 2022 X-Original-To: dev-commits-src-all@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 213091A27519; Mon, 14 Mar 2022 14:46:17 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4KHK9N5CFFz3hXr; Mon, 14 Mar 2022 14:46:16 +0000 (UTC) (envelope-from git@FreeBSD.org) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1647269176; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=78yl8sZiVHGzysGHirJSzylb7r9hwhasZXgcrBDUn4U=; b=OCywQAdB5fagInUiFosAICRFlKCKSSmWTV6PshGsw8IFpHKUwQUswRqMAQvt9vC6gwGHO4 smZGHX5fyCUDPcfTNRVkhjHzy2NaOO/1+s3v6IxBKbNBEWarqKJcePPr5UpNCRu2YqaDRx rkptWZQPzyvPA9kZDa85s5ZZh1jOFQHBUEvVANeniBmVtn5V4Ysf70SSmjfpJqfQstKUPG uoVGYpgK2c7jQbjdUoHd7gEOCkj2QJc/ePrgN/OTn5cODOAOEnjOwZWPdI3L2driKZIy/X wiKDXRq+fdq8IJ77rhyxfpyKhoU3OnWZMPV+SXrMZ+OT4vlMFTCs61el+cVfaQ== Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 52E277696; Mon, 14 Mar 2022 14:46:16 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 22EEkGa7052104; Mon, 14 Mar 2022 14:46:16 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 22EEkGE4052103; Mon, 14 Mar 2022 14:46:16 GMT (envelope-from git) Date: Mon, 14 Mar 2022 14:46:16 GMT Message-Id: <202203141446.22EEkGE4052103@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Mark Johnston Subject: git: 25b18f0a12c9 - stable/13 - riscv: Add support for dynamically allocating L1 page table pages List-Id: Commit messages for all branches of the src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-all List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-all@freebsd.org X-BeenThere: dev-commits-src-all@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: markj X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 25b18f0a12c9acc23baae4efde53a0311c1e43da Auto-Submitted: auto-generated ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1647269176; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=78yl8sZiVHGzysGHirJSzylb7r9hwhasZXgcrBDUn4U=; b=qvyJuM2jGLbBAo5ied0kpcSSwlGkQS4uoYoc+sryz7X0l/jSlPjeWenzUPRIqRsf75hN9L ONkwOSRSVSUeY803zk8btJ0XlDK6Fvd0Mr3SVzrxjAVeYQ6oWcTLtvo63YgIQbQ83TZBte XkR5belSLGktFsz7Sr1r517r3oyxHgteu9/9Tb0tRTEf9vTWqiaG6yZ9FVM77TFyx9tsba iWr/O0jfdkso+Dity1NTCyO7VJjZVOCmXyPL3tCRm+HohcTMUCmPuAx/+d0BS5Zs8OCvKp TRluF63cmIrLc/Dm3Sk+8kaiQDC+8h+HUKym/hfGdNKAp7T0UbkNkrP7r3LMHw== ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1647269176; a=rsa-sha256; cv=none; b=p1Ttcxj2pwWxm/y65uip/YGcF9XBUlbzXezI2ZSIF5VA7tdEsnFzwDGIAKO359aVXYQSEl rnNNbHS6nclx670RIkzHT8+6cswWJzBl7GxtAUu2aEIsYb220CKfvYMlpmYefJQyUmpNii Oq+M790WEhrReoi6AbvurxvutIwG9O7S8JKDlxLsyMZBF0EXZSyXIWejsFKL1wUbFM00Jd T9JUaKT/kt2/QQ/+ibk+r2YTmGUKFLAC/u1WkaVm180RgBEbqVCePPSTBanbiR7PSrzU+2 MaHVm20AJMUCK8mFRicfTnlNdC6w2XEDY54orFbH81FT2TGZpbuVdEYMPSXseA== ARC-Authentication-Results: i=1; mx1.freebsd.org; none X-ThisMailContainsUnwantedMimeParts: N The branch stable/13 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=25b18f0a12c9acc23baae4efde53a0311c1e43da commit 25b18f0a12c9acc23baae4efde53a0311c1e43da Author: Mark Johnston AuthorDate: 2022-03-01 14:06:57 +0000 Commit: Mark Johnston CommitDate: 2022-03-14 14:45:47 +0000 riscv: Add support for dynamically allocating L1 page table pages This is required in SV48 mode. Reviewed by: jhb Sponsored by: The FreeBSD Foundation (cherry picked from commit 6ce716f7c31a0a1b5ca8224cbb627632f44e4e6c) --- sys/riscv/riscv/pmap.c | 127 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 101 insertions(+), 26 deletions(-) diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c index 8aa99d52f2c9..a6b604d43984 100644 --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -1190,6 +1190,8 @@ pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) static inline boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { + KASSERT(m->ref_count > 0, + ("%s: page %p ref count underflow", __func__, m)); --m->ref_count; if (m->ref_count == 0) { @@ -1206,7 +1208,11 @@ _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) vm_paddr_t phys; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - if (m->pindex >= NUL2E) { + if (m->pindex >= NUL2E + NUL1E) { + pd_entry_t *l0; + l0 = pmap_l0(pmap, va); + pmap_clear(l0); + } else if (m->pindex >= NUL2E) { pd_entry_t *l1; l1 = pmap_l1(pmap, va); pmap_clear(l1); @@ -1225,6 +1231,15 @@ _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) phys = PTE_TO_PHYS(pmap_load(l1)); pdpg = PHYS_TO_VM_PAGE(phys); pmap_unwire_ptp(pmap, va, pdpg, free); + } else if (m->pindex < NUL2E + NUL1E && pmap_mode != PMAP_MODE_SV39) { + pd_entry_t *l0; + vm_page_t pdpg; + + MPASS(pmap_mode != PMAP_MODE_SV39); + l0 = pmap_l0(pmap, va); + phys = PTE_TO_PHYS(pmap_load(l0)); + pdpg = PHYS_TO_VM_PAGE(phys); + pmap_unwire_ptp(pmap, va, pdpg, free); } pmap_invalidate_page(pmap, va); @@ -1323,7 +1338,7 @@ pmap_pinit(pmap_t pmap) static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) { - vm_page_t m, /*pdppg, */pdpg; + vm_page_t m, pdpg; pt_entry_t entry; vm_paddr_t phys; pn_t pn; @@ -1356,39 +1371,96 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) * Map the pagetable page into the process address space, if * it isn't already there. */ - - if (ptepindex >= NUL2E) { - pd_entry_t *l1; - vm_pindex_t l1index; + pn = VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT; + if (ptepindex >= NUL2E + NUL1E) { + pd_entry_t *l0; + vm_pindex_t l0index; + + KASSERT(pmap_mode != PMAP_MODE_SV39, + ("%s: pindex %#lx in SV39 mode", __func__, ptepindex)); + KASSERT(ptepindex < NUL2E + NUL1E + NUL0E, + ("%s: pindex %#lx out of range", __func__, ptepindex)); + + l0index = ptepindex - (NUL2E + NUL1E); + l0 = &pmap->pm_top[l0index]; + KASSERT((pmap_load(l0) & PTE_V) == 0, + ("%s: L0 entry %#lx is valid", __func__, pmap_load(l0))); + + entry = PTE_V | (pn << PTE_PPN0_S); + pmap_store(l0, entry); + } else if (ptepindex >= NUL2E) { + pd_entry_t *l0, *l1; + vm_pindex_t l0index, l1index; l1index = ptepindex - NUL2E; - l1 = &pmap->pm_top[l1index]; + if (pmap_mode == PMAP_MODE_SV39) { + l1 = &pmap->pm_top[l1index]; + } else { + l0index = l1index >> Ln_ENTRIES_SHIFT; + l0 = &pmap->pm_top[l0index]; + if (pmap_load(l0) == 0) { + /* Recurse to allocate the L1 page. */ + if (_pmap_alloc_l3(pmap, + NUL2E + NUL1E + l0index, lockp) == NULL) + goto fail; + phys = PTE_TO_PHYS(pmap_load(l0)); + } else { + phys = PTE_TO_PHYS(pmap_load(l0)); + pdpg = PHYS_TO_VM_PAGE(phys); + pdpg->ref_count++; + } + l1 = (pd_entry_t *)PHYS_TO_DMAP(phys); + l1 = &l1[ptepindex & Ln_ADDR_MASK]; + } KASSERT((pmap_load(l1) & PTE_V) == 0, ("%s: L1 entry %#lx is valid", __func__, pmap_load(l1))); - pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE); - entry = (PTE_V); - entry |= (pn << PTE_PPN0_S); + entry = PTE_V | (pn << PTE_PPN0_S); pmap_store(l1, entry); pmap_distribute_l1(pmap, l1index, entry); } else { - vm_pindex_t l1index; - pd_entry_t *l1, *l2; + vm_pindex_t l0index, l1index; + pd_entry_t *l0, *l1, *l2; l1index = ptepindex >> (L1_SHIFT - L2_SHIFT); - l1 = &pmap->pm_top[l1index]; - if (pmap_load(l1) == 0) { - /* recurse for allocating page dir */ - if (_pmap_alloc_l3(pmap, NUL2E + l1index, - lockp) == NULL) { - vm_page_unwire_noq(m); - vm_page_free_zero(m); - return (NULL); + if (pmap_mode == PMAP_MODE_SV39) { + l1 = &pmap->pm_top[l1index]; + if (pmap_load(l1) == 0) { + /* recurse for allocating page dir */ + if (_pmap_alloc_l3(pmap, NUL2E + l1index, + lockp) == NULL) + goto fail; + } else { + phys = PTE_TO_PHYS(pmap_load(l1)); + pdpg = PHYS_TO_VM_PAGE(phys); + pdpg->ref_count++; } } else { - phys = PTE_TO_PHYS(pmap_load(l1)); - pdpg = PHYS_TO_VM_PAGE(phys); - pdpg->ref_count++; + l0index = l1index >> Ln_ENTRIES_SHIFT; + l0 = &pmap->pm_top[l0index]; + if (pmap_load(l0) == 0) { + /* Recurse to allocate the L1 entry. */ + if (_pmap_alloc_l3(pmap, NUL2E + l1index, + lockp) == NULL) + goto fail; + phys = PTE_TO_PHYS(pmap_load(l0)); + l1 = (pd_entry_t *)PHYS_TO_DMAP(phys); + l1 = &l1[l1index & Ln_ADDR_MASK]; + } else { + phys = PTE_TO_PHYS(pmap_load(l0)); + l1 = (pd_entry_t *)PHYS_TO_DMAP(phys); + l1 = &l1[l1index & Ln_ADDR_MASK]; + if (pmap_load(l1) == 0) { + /* Recurse to allocate the L2 page. */ + if (_pmap_alloc_l3(pmap, + NUL2E + l1index, lockp) == NULL) + goto fail; + } else { + phys = PTE_TO_PHYS(pmap_load(l1)); + pdpg = PHYS_TO_VM_PAGE(phys); + pdpg->ref_count++; + } + } } phys = PTE_TO_PHYS(pmap_load(l1)); @@ -1397,15 +1469,18 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) KASSERT((pmap_load(l2) & PTE_V) == 0, ("%s: L2 entry %#lx is valid", __func__, pmap_load(l2))); - pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE); - entry = (PTE_V); - entry |= (pn << PTE_PPN0_S); + entry = PTE_V | (pn << PTE_PPN0_S); pmap_store(l2, entry); } pmap_resident_count_inc(pmap, 1); return (m); + +fail: + vm_page_unwire_noq(m); + vm_page_free_zero(m); + return (NULL); } static vm_page_t