svn commit: r328469 - stable/11/sys/vm
Konstantin Belousov
kib at FreeBSD.org
Sat Jan 27 11:40:47 UTC 2018
Author: kib
Date: Sat Jan 27 11:40:46 2018
New Revision: 328469
URL: https://svnweb.freebsd.org/changeset/base/328469
Log:
MFC r328192:
Assign map->header values to avoid boundary checks.
Modified:
stable/11/sys/vm/vm_map.c
stable/11/sys/vm/vm_map.h
stable/11/sys/vm/vm_mmap.c
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/vm/vm_map.c
==============================================================================
--- stable/11/sys/vm/vm_map.c Sat Jan 27 11:33:21 2018 (r328468)
+++ stable/11/sys/vm/vm_map.c Sat Jan 27 11:40:46 2018 (r328469)
@@ -998,12 +998,10 @@ vm_map_entry_link(vm_map_t map,
"vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map,
map->nentries, entry, after_where);
VM_MAP_ASSERT_LOCKED(map);
- KASSERT(after_where == &map->header ||
- after_where->end <= entry->start,
+ KASSERT(after_where->end <= entry->start,
("vm_map_entry_link: prev end %jx new start %jx overlap",
(uintmax_t)after_where->end, (uintmax_t)entry->start));
- KASSERT(after_where->next == &map->header ||
- entry->end <= after_where->next->start,
+ KASSERT(entry->end <= after_where->next->start,
("vm_map_entry_link: new end %jx next start %jx overlap",
(uintmax_t)entry->end, (uintmax_t)after_where->next->start));
@@ -1025,8 +1023,7 @@ vm_map_entry_link(vm_map_t map,
entry->right = map->root;
entry->left = NULL;
}
- entry->adj_free = (entry->next == &map->header ? map->max_offset :
- entry->next->start) - entry->end;
+ entry->adj_free = entry->next->start - entry->end;
vm_map_entry_set_max_free(entry);
map->root = entry;
}
@@ -1045,8 +1042,7 @@ vm_map_entry_unlink(vm_map_t map,
else {
root = vm_map_entry_splay(entry->start, entry->left);
root->right = entry->right;
- root->adj_free = (entry->next == &map->header ? map->max_offset :
- entry->next->start) - root->end;
+ root->adj_free = entry->next->start - root->end;
vm_map_entry_set_max_free(root);
}
map->root = root;
@@ -1082,8 +1078,7 @@ vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t
if (entry != map->root)
map->root = vm_map_entry_splay(entry->start, map->root);
- entry->adj_free = (entry->next == &map->header ? map->max_offset :
- entry->next->start) - entry->end;
+ entry->adj_free = entry->next->start - entry->end;
vm_map_entry_set_max_free(entry);
}
@@ -1213,7 +1208,7 @@ vm_map_insert(vm_map_t map, vm_object_t object, vm_oof
/*
* Assert that the next entry doesn't overlap the end point.
*/
- if (prev_entry->next != &map->header && prev_entry->next->start < end)
+ if (prev_entry->next->start < end)
return (KERN_NO_SPACE);
if ((cow & MAP_CREATE_GUARD) != 0 && (object != NULL ||
@@ -2032,8 +2027,7 @@ vm_map_protect(vm_map_t map, vm_offset_t start, vm_off
/*
* Make a first pass to check for protection violations.
*/
- for (current = entry; current != &map->header && current->start < end;
- current = current->next) {
+ for (current = entry; current->start < end; current = current->next) {
if ((current->eflags & MAP_ENTRY_GUARD) != 0)
continue;
if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
@@ -2051,8 +2045,7 @@ vm_map_protect(vm_map_t map, vm_offset_t start, vm_off
* now will do cow due to allowed write (e.g. debugger sets
* breakpoint on text segment)
*/
- for (current = entry; current != &map->header && current->start < end;
- current = current->next) {
+ for (current = entry; current->start < end; current = current->next) {
vm_map_clip_end(map, current, end);
@@ -2106,8 +2099,7 @@ vm_map_protect(vm_map_t map, vm_offset_t start, vm_off
* Go back and fix up protections. [Note that clipping is not
* necessary the second time.]
*/
- for (current = entry; current != &map->header && current->start < end;
- current = current->next) {
+ for (current = entry; current->start < end; current = current->next) {
if ((current->eflags & MAP_ENTRY_GUARD) != 0)
continue;
@@ -2216,10 +2208,8 @@ vm_map_madvise(
* We clip the vm_map_entry so that behavioral changes are
* limited to the specified address range.
*/
- for (current = entry;
- (current != &map->header) && (current->start < end);
- current = current->next
- ) {
+ for (current = entry; current->start < end;
+ current = current->next) {
if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
continue;
@@ -2263,10 +2253,8 @@ vm_map_madvise(
* Since we don't clip the vm_map_entry, we have to clip
* the vm_object pindex and count.
*/
- for (current = entry;
- (current != &map->header) && (current->start < end);
- current = current->next
- ) {
+ for (current = entry; current->start < end;
+ current = current->next) {
vm_offset_t useEnd, useStart;
if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
@@ -2362,7 +2350,7 @@ vm_map_inherit(vm_map_t map, vm_offset_t start, vm_off
vm_map_clip_start(map, entry, start);
} else
entry = temp_entry->next;
- while ((entry != &map->header) && (entry->start < end)) {
+ while (entry->start < end) {
vm_map_clip_end(map, entry, end);
if ((entry->eflags & MAP_ENTRY_GUARD) == 0 ||
new_inheritance != VM_INHERIT_ZERO)
@@ -2404,7 +2392,7 @@ vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offs
}
last_timestamp = map->timestamp;
entry = first_entry;
- while (entry != &map->header && entry->start < end) {
+ while (entry->start < end) {
if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
/*
* We have not yet clipped the entry.
@@ -2467,8 +2455,7 @@ vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offs
* If VM_MAP_WIRE_HOLESOK was specified, skip this check.
*/
if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
- (entry->end < end && (entry->next == &map->header ||
- entry->next->start > entry->end))) {
+ (entry->end < end && entry->next->start > entry->end)) {
end = entry->end;
rv = KERN_INVALID_ADDRESS;
goto done;
@@ -2494,8 +2481,7 @@ done:
else
KASSERT(result, ("vm_map_unwire: lookup failed"));
}
- for (entry = first_entry; entry != &map->header && entry->start < end;
- entry = entry->next) {
+ for (entry = first_entry; entry->start < end; entry = entry->next) {
/*
* If VM_MAP_WIRE_HOLESOK was specified, an empty
* space in the unwired region could have been mapped
@@ -2609,7 +2595,7 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset
}
last_timestamp = map->timestamp;
entry = first_entry;
- while (entry != &map->header && entry->start < end) {
+ while (entry->start < end) {
if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
/*
* We have not yet clipped the entry.
@@ -2746,8 +2732,7 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset
*/
next_entry:
if ((flags & VM_MAP_WIRE_HOLESOK) == 0 &&
- entry->end < end && (entry->next == &map->header ||
- entry->next->start > entry->end)) {
+ entry->end < end && entry->next->start > entry->end) {
end = entry->end;
rv = KERN_INVALID_ADDRESS;
goto done;
@@ -2764,8 +2749,7 @@ done:
else
KASSERT(result, ("vm_map_wire: lookup failed"));
}
- for (entry = first_entry; entry != &map->header && entry->start < end;
- entry = entry->next) {
+ for (entry = first_entry; entry->start < end; entry = entry->next) {
/*
* If VM_MAP_WIRE_HOLESOK was specified, an empty
* space in the unwired region could have been mapped
@@ -2869,15 +2853,13 @@ vm_map_sync(
/*
* Make a first pass to check for user-wired memory and holes.
*/
- for (current = entry; current != &map->header && current->start < end;
- current = current->next) {
+ for (current = entry; current->start < end; current = current->next) {
if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) {
vm_map_unlock_read(map);
return (KERN_INVALID_ARGUMENT);
}
if (end > current->end &&
- (current->next == &map->header ||
- current->end != current->next->start)) {
+ current->end != current->next->start) {
vm_map_unlock_read(map);
return (KERN_INVALID_ADDRESS);
}
@@ -2891,7 +2873,7 @@ vm_map_sync(
* Make a second pass, cleaning/uncaching pages from the indicated
* objects as we go.
*/
- for (current = entry; current != &map->header && current->start < end;) {
+ for (current = entry; current->start < end;) {
offset = current->offset + (start - current->start);
size = (end <= current->end ? end : current->end) - start;
if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
@@ -3068,7 +3050,7 @@ vm_map_delete(vm_map_t map, vm_offset_t start, vm_offs
/*
* Step through all entries in this region
*/
- while ((entry != &map->header) && (entry->start < end)) {
+ while (entry->start < end) {
vm_map_entry_t next;
/*
@@ -3176,8 +3158,6 @@ vm_map_check_protection(vm_map_t map, vm_offset_t star
entry = tmp_entry;
while (start < end) {
- if (entry == &map->header)
- return (FALSE);
/*
* No holes allowed!
*/
@@ -3641,8 +3621,7 @@ vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
/*
* If we can't accommodate max_ssize in the current mapping, no go.
*/
- if ((prev_entry->next != &map->header) &&
- (prev_entry->next->start < addrbos + max_ssize))
+ if (prev_entry->next->start < addrbos + max_ssize)
return (KERN_NO_SPACE);
/*
Modified: stable/11/sys/vm/vm_map.h
==============================================================================
--- stable/11/sys/vm/vm_map.h Sat Jan 27 11:33:21 2018 (r328468)
+++ stable/11/sys/vm/vm_map.h Sat Jan 27 11:40:46 2018 (r328469)
@@ -172,15 +172,19 @@ vm_map_entry_system_wired_count(vm_map_entry_t entry)
* A map is a set of map entries. These map entries are
* organized both as a binary search tree and as a doubly-linked
* list. Both structures are ordered based upon the start and
- * end addresses contained within each map entry. Sleator and
- * Tarjan's top-down splay algorithm is employed to control
- * height imbalance in the binary search tree.
+ * end addresses contained within each map entry. The list
+ * header has max start value and min end value to act as
+ * sentinels for sequential search of the doubly-linked list.
+ * Sleator and Tarjan's top-down splay algorithm is employed to
+ * control height imbalance in the binary search tree.
*
* List of locks
* (c) const until freed
*/
struct vm_map {
struct vm_map_entry header; /* List of entries */
+#define min_offset header.end /* (c) */
+#define max_offset header.start /* (c) */
struct sx lock; /* Lock for map data */
struct mtx system_mtx;
int nentries; /* Number of entries */
@@ -191,8 +195,6 @@ struct vm_map {
vm_flags_t flags; /* flags for this vm_map */
vm_map_entry_t root; /* Root of a binary search tree */
pmap_t pmap; /* (c) Physical map */
-#define min_offset header.start /* (c) */
-#define max_offset header.end /* (c) */
int busy;
};
Modified: stable/11/sys/vm/vm_mmap.c
==============================================================================
--- stable/11/sys/vm/vm_mmap.c Sat Jan 27 11:33:21 2018 (r328468)
+++ stable/11/sys/vm/vm_mmap.c Sat Jan 27 11:40:46 2018 (r328469)
@@ -541,8 +541,7 @@ kern_munmap(struct thread *td, uintptr_t addr0, size_t
*/
pkm.pm_address = (uintptr_t) NULL;
if (vm_map_lookup_entry(map, addr, &entry)) {
- for (;
- entry != &map->header && entry->start < addr + size;
+ for (; entry->start < addr + size;
entry = entry->next) {
if (vm_map_check_protection(map, entry->start,
entry->end, VM_PROT_EXECUTE) == TRUE) {
@@ -768,16 +767,12 @@ RestartScan:
* up the pages elsewhere.
*/
lastvecindex = -1;
- for (current = entry;
- (current != &map->header) && (current->start < end);
- current = current->next) {
+ for (current = entry; current->start < end; current = current->next) {
/*
* check for contiguity
*/
- if (current->end < end &&
- (entry->next == &map->header ||
- current->next->start > current->end)) {
+ if (current->end < end && current->next->start > current->end) {
vm_map_unlock_read(map);
return (ENOMEM);
}
More information about the svn-src-stable
mailing list