git: e85eaa930862 - main - Have rtld query the page size from the kernel
Date: Thu, 07 Apr 2022 14:38:49 UTC
The branch main has been updated by andrew:
URL: https://cgit.FreeBSD.org/src/commit/?id=e85eaa930862d5b4dc917bc31e8d7254a693635d
commit e85eaa930862d5b4dc917bc31e8d7254a693635d
Author: Andrew Turner <andrew@FreeBSD.org>
AuthorDate: 2022-04-04 15:05:40 +0000
Commit: Andrew Turner <andrew@FreeBSD.org>
CommitDate: 2022-04-07 14:37:37 +0000
Have rtld query the page size from the kernel
To allow for a dynamic page size on arm64 have the runtime linker
query the kernel for the currentl page size.
Reviewed by: kib
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D34765
---
libexec/rtld-elf/map_object.c | 54 +++++++++++++++++-----------------
libexec/rtld-elf/rtld-libc/rtld_libc.c | 24 ++-------------
libexec/rtld-elf/rtld.c | 35 +++++++++++++++-------
libexec/rtld-elf/rtld.h | 3 ++
4 files changed, 58 insertions(+), 58 deletions(-)
diff --git a/libexec/rtld-elf/map_object.c b/libexec/rtld-elf/map_object.c
index 2da323c115cc..7c637fe1d6ba 100644
--- a/libexec/rtld-elf/map_object.c
+++ b/libexec/rtld-elf/map_object.c
@@ -49,8 +49,7 @@ int __getosreldate(void);
static bool
phdr_in_zero_page(const Elf_Ehdr *hdr)
{
- return (hdr->e_phoff + hdr->e_phnum * sizeof(Elf_Phdr) <=
- (size_t)PAGE_SIZE);
+ return (hdr->e_phoff + hdr->e_phnum * sizeof(Elf_Phdr) <= page_size);
}
/*
@@ -134,14 +133,15 @@ map_object(int fd, const char *path, const struct stat *sb)
case PT_LOAD:
segs[++nsegs] = phdr;
- if ((segs[nsegs]->p_align & (PAGE_SIZE - 1)) != 0) {
+ if ((segs[nsegs]->p_align & (page_size - 1)) != 0) {
_rtld_error("%s: PT_LOAD segment %d not page-aligned",
path, nsegs);
goto error;
}
if ((segs[nsegs]->p_flags & PF_X) == PF_X) {
text_end = MAX(text_end,
- round_page(segs[nsegs]->p_vaddr + segs[nsegs]->p_memsz));
+ rtld_round_page(segs[nsegs]->p_vaddr +
+ segs[nsegs]->p_memsz));
}
break;
@@ -168,18 +168,18 @@ map_object(int fd, const char *path, const struct stat *sb)
break;
case PT_NOTE:
- if (phdr->p_offset > PAGE_SIZE ||
- phdr->p_offset + phdr->p_filesz > PAGE_SIZE) {
- note_map_len = round_page(phdr->p_offset +
- phdr->p_filesz) - trunc_page(phdr->p_offset);
+ if (phdr->p_offset > page_size ||
+ phdr->p_offset + phdr->p_filesz > page_size) {
+ note_map_len = rtld_round_page(phdr->p_offset +
+ phdr->p_filesz) - rtld_trunc_page(phdr->p_offset);
note_map = mmap(NULL, note_map_len, PROT_READ,
- MAP_PRIVATE, fd, trunc_page(phdr->p_offset));
+ MAP_PRIVATE, fd, rtld_trunc_page(phdr->p_offset));
if (note_map == MAP_FAILED) {
_rtld_error("%s: error mapping PT_NOTE (%d)", path, errno);
goto error;
}
note_start = (Elf_Addr)(note_map + phdr->p_offset -
- trunc_page(phdr->p_offset));
+ rtld_trunc_page(phdr->p_offset));
} else {
note_start = (Elf_Addr)(char *)hdr + phdr->p_offset;
}
@@ -203,13 +203,13 @@ map_object(int fd, const char *path, const struct stat *sb)
* Map the entire address space of the object, to stake out our
* contiguous region, and to establish the base address for relocation.
*/
- base_vaddr = trunc_page(segs[0]->p_vaddr);
- base_vlimit = round_page(segs[nsegs]->p_vaddr + segs[nsegs]->p_memsz);
+ base_vaddr = rtld_trunc_page(segs[0]->p_vaddr);
+ base_vlimit = rtld_round_page(segs[nsegs]->p_vaddr + segs[nsegs]->p_memsz);
mapsize = base_vlimit - base_vaddr;
base_addr = (caddr_t) base_vaddr;
base_flags = __getosreldate() >= P_OSREL_MAP_GUARD ? MAP_GUARD :
MAP_PRIVATE | MAP_ANON | MAP_NOCORE;
- if (npagesizes > 1 && round_page(segs[0]->p_filesz) >= pagesizes[1])
+ if (npagesizes > 1 && rtld_round_page(segs[0]->p_filesz) >= pagesizes[1])
base_flags |= MAP_ALIGNED_SUPER;
if (base_vaddr != 0)
base_flags |= MAP_FIXED | MAP_EXCL;
@@ -228,9 +228,9 @@ map_object(int fd, const char *path, const struct stat *sb)
for (i = 0; i <= nsegs; i++) {
/* Overlay the segment onto the proper region. */
- data_offset = trunc_page(segs[i]->p_offset);
- data_vaddr = trunc_page(segs[i]->p_vaddr);
- data_vlimit = round_page(segs[i]->p_vaddr + segs[i]->p_filesz);
+ data_offset = rtld_trunc_page(segs[i]->p_offset);
+ data_vaddr = rtld_trunc_page(segs[i]->p_vaddr);
+ data_vlimit = rtld_round_page(segs[i]->p_vaddr + segs[i]->p_filesz);
data_addr = mapbase + (data_vaddr - base_vaddr);
data_prot = convert_prot(segs[i]->p_flags);
data_flags = convert_flags(segs[i]->p_flags) | MAP_FIXED;
@@ -248,12 +248,12 @@ map_object(int fd, const char *path, const struct stat *sb)
/* Clear any BSS in the last page of the segment. */
clear_vaddr = segs[i]->p_vaddr + segs[i]->p_filesz;
clear_addr = mapbase + (clear_vaddr - base_vaddr);
- clear_page = mapbase + (trunc_page(clear_vaddr) - base_vaddr);
+ clear_page = mapbase + (rtld_trunc_page(clear_vaddr) - base_vaddr);
if ((nclear = data_vlimit - clear_vaddr) > 0) {
/* Make sure the end of the segment is writable */
if ((data_prot & PROT_WRITE) == 0 && -1 ==
- mprotect(clear_page, PAGE_SIZE, data_prot|PROT_WRITE)) {
+ mprotect(clear_page, page_size, data_prot|PROT_WRITE)) {
_rtld_error("%s: mprotect failed: %s", path,
rtld_strerror(errno));
goto error1;
@@ -263,12 +263,12 @@ map_object(int fd, const char *path, const struct stat *sb)
/* Reset the data protection back */
if ((data_prot & PROT_WRITE) == 0)
- mprotect(clear_page, PAGE_SIZE, data_prot);
+ mprotect(clear_page, page_size, data_prot);
}
/* Overlay the BSS segment onto the proper region. */
bss_vaddr = data_vlimit;
- bss_vlimit = round_page(segs[i]->p_vaddr + segs[i]->p_memsz);
+ bss_vlimit = rtld_round_page(segs[i]->p_vaddr + segs[i]->p_memsz);
bss_addr = mapbase + (bss_vaddr - base_vaddr);
if (bss_vlimit > bss_vaddr) { /* There is something to do */
if (mmap(bss_addr, bss_vlimit - bss_vaddr, data_prot,
@@ -324,14 +324,14 @@ map_object(int fd, const char *path, const struct stat *sb)
obj->tlsinit = mapbase + phtls->p_vaddr;
}
obj->stack_flags = stack_flags;
- obj->relro_page = obj->relocbase + trunc_page(relro_page);
- obj->relro_size = trunc_page(relro_page + relro_size) -
- trunc_page(relro_page);
+ obj->relro_page = obj->relocbase + rtld_trunc_page(relro_page);
+ obj->relro_size = rtld_trunc_page(relro_page + relro_size) -
+ rtld_trunc_page(relro_page);
if (note_start < note_end)
digest_notes(obj, note_start, note_end);
if (note_map != NULL)
munmap(note_map, note_map_len);
- munmap(hdr, PAGE_SIZE);
+ munmap(hdr, page_size);
return (obj);
error1:
@@ -341,7 +341,7 @@ error:
munmap(note_map, note_map_len);
if (!phdr_in_zero_page(hdr))
munmap(phdr, hdr->e_phnum * sizeof(phdr[0]));
- munmap(hdr, PAGE_SIZE);
+ munmap(hdr, page_size);
return (NULL);
}
@@ -391,7 +391,7 @@ get_elf_header(int fd, const char *path, const struct stat *sbp,
return (NULL);
}
- hdr = mmap(NULL, PAGE_SIZE, PROT_READ, MAP_PRIVATE | MAP_PREFAULT_READ,
+ hdr = mmap(NULL, page_size, PROT_READ, MAP_PRIVATE | MAP_PREFAULT_READ,
fd, 0);
if (hdr == MAP_FAILED) {
_rtld_error("%s: read error: %s", path, rtld_strerror(errno));
@@ -423,7 +423,7 @@ get_elf_header(int fd, const char *path, const struct stat *sbp,
return (hdr);
error:
- munmap(hdr, PAGE_SIZE);
+ munmap(hdr, page_size);
return (NULL);
}
diff --git a/libexec/rtld-elf/rtld-libc/rtld_libc.c b/libexec/rtld-elf/rtld-libc/rtld_libc.c
index dfad9465f4e8..69e801191db4 100644
--- a/libexec/rtld-elf/rtld-libc/rtld_libc.c
+++ b/libexec/rtld-elf/rtld-libc/rtld_libc.c
@@ -98,31 +98,13 @@ __assert(const char *func, const char *file, int line, const char *failedexpr)
/*
* Avoid pulling in all of pthreads from getpagesize().
* It normally uses libc/gen/auxv.c which pulls in pthread_once().
+ * This relies on init_pagesizes setting page_size so must not be called
+ * before that.
*/
int
getpagesize(void)
{
- int mib[2], value;
- size_t size;
- static int pagesize;
-
- if (pagesize != 0)
- return (pagesize);
-
- if (npagesizes > 0)
- pagesize = pagesizes[0];
-
- if (pagesize == 0) {
- mib[0] = CTL_HW;
- mib[1] = HW_PAGESIZE;
- size = sizeof(value);
- if (sysctl(mib, nitems(mib), &value, &size, NULL, 0) == -1)
- pagesize = PAGE_SIZE;
- else
- pagesize = value;
- }
-
- return (pagesize);
+ return (page_size);
}
extern int __sys___sysctl(const int *name, u_int namelen, void *oldp,
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 6e499ebc5208..aa5400d29fc2 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -268,6 +268,7 @@ Elf_Addr _rtld_bind(Obj_Entry *obj, Elf_Size reloff);
int npagesizes;
static int osreldate;
size_t *pagesizes;
+size_t page_size;
static int stack_prot = PROT_READ | PROT_WRITE | RTLD_DEFAULT_STACK_EXEC;
static int max_stack_flags;
@@ -477,6 +478,18 @@ set_ld_elf_hints_path(void)
ld_elf_hints_path = ld_elf_hints_default;
}
+uintptr_t
+rtld_round_page(uintptr_t x)
+{
+ return (roundup2(x, page_size));
+}
+
+uintptr_t
+rtld_trunc_page(uintptr_t x)
+{
+ return (rounddown2(x, page_size));
+}
+
/*
* Main entry point for dynamic linking. The first argument is the
* stack pointer. The stack is expected to be laid out as described
@@ -1647,10 +1660,10 @@ digest_phdr(const Elf_Phdr *phdr, int phnum, caddr_t entry, const char *path)
case PT_LOAD:
if (nsegs == 0) { /* First load segment */
- obj->vaddrbase = trunc_page(ph->p_vaddr);
+ obj->vaddrbase = rtld_trunc_page(ph->p_vaddr);
obj->mapbase = obj->vaddrbase + obj->relocbase;
} else { /* Last load segment */
- obj->mapsize = round_page(ph->p_vaddr + ph->p_memsz) -
+ obj->mapsize = rtld_round_page(ph->p_vaddr + ph->p_memsz) -
obj->vaddrbase;
}
nsegs++;
@@ -1674,9 +1687,9 @@ digest_phdr(const Elf_Phdr *phdr, int phnum, caddr_t entry, const char *path)
break;
case PT_GNU_RELRO:
- obj->relro_page = obj->relocbase + trunc_page(ph->p_vaddr);
- obj->relro_size = trunc_page(ph->p_vaddr + ph->p_memsz) -
- trunc_page(ph->p_vaddr);
+ obj->relro_page = obj->relocbase + rtld_trunc_page(ph->p_vaddr);
+ obj->relro_size = rtld_trunc_page(ph->p_vaddr + ph->p_memsz) -
+ rtld_trunc_page(ph->p_vaddr);
break;
case PT_NOTE:
@@ -2325,8 +2338,8 @@ parse_rtld_phdr(Obj_Entry *obj)
break;
case PT_GNU_RELRO:
obj->relro_page = obj->relocbase +
- trunc_page(ph->p_vaddr);
- obj->relro_size = round_page(ph->p_memsz);
+ rtld_trunc_page(ph->p_vaddr);
+ obj->relro_size = rtld_round_page(ph->p_memsz);
break;
case PT_NOTE:
note_start = (Elf_Addr)obj->relocbase + ph->p_vaddr;
@@ -2455,6 +2468,8 @@ psa_filled:
/* Discard any invalid entries at the end of the array. */
while (npagesizes > 0 && pagesizes[npagesizes - 1] == 0)
npagesizes--;
+
+ page_size = pagesizes[0];
}
/*
@@ -3208,9 +3223,9 @@ reloc_textrel_prot(Obj_Entry *obj, bool before)
l--, ph++) {
if (ph->p_type != PT_LOAD || (ph->p_flags & PF_W) != 0)
continue;
- base = obj->relocbase + trunc_page(ph->p_vaddr);
- sz = round_page(ph->p_vaddr + ph->p_filesz) -
- trunc_page(ph->p_vaddr);
+ base = obj->relocbase + rtld_trunc_page(ph->p_vaddr);
+ sz = rtld_round_page(ph->p_vaddr + ph->p_filesz) -
+ rtld_trunc_page(ph->p_vaddr);
prot = before ? (PROT_READ | PROT_WRITE) :
convert_prot(ph->p_flags);
if (mprotect(base, sz, prot) == -1) {
diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h
index cbdbd29f2aa9..807437af5ff3 100644
--- a/libexec/rtld-elf/rtld.h
+++ b/libexec/rtld-elf/rtld.h
@@ -55,6 +55,7 @@ extern int tls_max_index;
extern int npagesizes;
extern size_t *pagesizes;
+extern size_t page_size;
extern int main_argc;
extern char **main_argv;
@@ -376,6 +377,8 @@ void dump_Elf_Rela(Obj_Entry *, const Elf_Rela *, u_long);
/*
* Function declarations.
*/
+uintptr_t rtld_round_page(uintptr_t);
+uintptr_t rtld_trunc_page(uintptr_t);
unsigned long elf_hash(const char *);
const Elf_Sym *find_symdef(unsigned long, const Obj_Entry *,
const Obj_Entry **, int, SymCache *, struct Struct_RtldLockState *);