From a36bbdd351085826241144001bd0bdea2c862fd8 Mon Sep 17 00:00:00 2001 From: mlarkin Date: Thu, 4 Jun 2026 05:22:04 +0000 Subject: [PATCH] amd64: Place the direct map at a random location Places the direct map at a 512GB-aligned random location selected from a 16TB VA window. With help and suggestions from deraadt@. Tested by many. --- sys/arch/amd64/amd64/bus_dma.c | 4 ++-- sys/arch/amd64/amd64/bus_space.c | 4 ++-- sys/arch/amd64/amd64/genassym.cf | 3 ++- sys/arch/amd64/amd64/locore0.S | 7 ++++-- sys/arch/amd64/amd64/machdep.c | 15 ++++++++---- sys/arch/amd64/amd64/mem.c | 4 ++-- sys/arch/amd64/amd64/pmap.c | 29 +++++++++++++--------- sys/arch/amd64/include/pmap.h | 41 ++++++++++++++++++-------------- 8 files changed, 65 insertions(+), 42 deletions(-) diff --git a/sys/arch/amd64/amd64/bus_dma.c b/sys/arch/amd64/amd64/bus_dma.c index 80a45e032ee..f4efff95a18 100644 --- a/sys/arch/amd64/amd64/bus_dma.c +++ b/sys/arch/amd64/amd64/bus_dma.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bus_dma.c,v 1.61 2026/04/19 09:59:22 kettenis Exp $ */ +/* $OpenBSD: bus_dma.c,v 1.62 2026/06/04 05:22:04 mlarkin Exp $ */ /* $NetBSD: bus_dma.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $ */ /*- @@ -748,7 +748,7 @@ _bus_dmamem_unmap(bus_dma_tag_t t, caddr_t kva, size_t size) if ((u_long)kva & PGOFSET) panic("_bus_dmamem_unmap"); #endif - if (kva >= (caddr_t)PMAP_DIRECT_BASE && kva <= (caddr_t)PMAP_DIRECT_END) + if (kva >= (caddr_t)pmap_direct_base && kva <= (caddr_t)pmap_direct_end) return; km_free(kva, round_page(size), &kv_any, &kp_none); diff --git a/sys/arch/amd64/amd64/bus_space.c b/sys/arch/amd64/amd64/bus_space.c index b9aca179b0b..9439ca6b954 100644 --- a/sys/arch/amd64/amd64/bus_space.c +++ b/sys/arch/amd64/amd64/bus_space.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bus_space.c,v 1.30 2025/09/17 18:39:50 sf Exp $ */ +/* $OpenBSD: bus_space.c,v 1.31 2026/06/04 05:22:04 mlarkin Exp $ */ /* $NetBSD: bus_space.c,v 1.2 2003/03/14 18:47:53 christos Exp $ */ /*- @@ -612,7 +612,7 @@ bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size) if (IOM_BEGIN <= bpa && bpa <= IOM_END) goto ok; - if (bsh >= PMAP_DIRECT_BASE && bsh < PMAP_DIRECT_END) { + if (bsh >= pmap_direct_base && bsh < pmap_direct_end) { bpa = PMAP_DIRECT_UNMAP(bsh); goto ok; } diff --git a/sys/arch/amd64/amd64/genassym.cf b/sys/arch/amd64/amd64/genassym.cf index c9d90f13284..922699f7668 100644 --- a/sys/arch/amd64/amd64/genassym.cf +++ b/sys/arch/amd64/amd64/genassym.cf @@ -1,4 +1,4 @@ -# $OpenBSD: genassym.cf,v 1.50 2026/01/07 03:25:44 deraadt Exp $ +# $OpenBSD: genassym.cf,v 1.51 2026/06/04 05:22:04 mlarkin Exp $ # Written by Artur Grabowski art@openbsd.org, Public Domain include @@ -162,6 +162,7 @@ export NDML2_ENTRIES export NBPD_L2 export NPDPG +export DIRECT_MAP_START_MASK export PDIR_SLOT_DIRECT export PCID_PROC export PCID_PROC_INTEL diff --git a/sys/arch/amd64/amd64/locore0.S b/sys/arch/amd64/amd64/locore0.S index 705a4d52b81..f8ba97805ce 100644 --- a/sys/arch/amd64/amd64/locore0.S +++ b/sys/arch/amd64/amd64/locore0.S @@ -1,4 +1,4 @@ -/* $OpenBSD: locore0.S,v 1.34 2026/01/15 12:09:49 hshoexer Exp $ */ +/* $OpenBSD: locore0.S,v 1.35 2026/06/04 05:22:04 mlarkin Exp $ */ /* $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $ */ /* @@ -655,6 +655,10 @@ store_pte: fillkpt_nx leal (PROC0_PML4_OFF + PDIR_SLOT_DIRECT * 8)(%esi), %ebx + movb RELOC((pmap_direct_rand)), %al + andl $DIRECT_MAP_START_MASK, %eax + shll $0x3, %eax + addl %eax, %ebx leal (PROC0_DMP3_OFF)(%esi), %eax orl $(PG_V|PG_KW), %eax movl $NDML3_ENTRIES, %ecx @@ -921,4 +925,3 @@ farjmp64: .align 8, 0xcc .space 512 tmpstk: - diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c index 3f1ac1d5b4b..00d9b97cfc0 100644 --- a/sys/arch/amd64/amd64/machdep.c +++ b/sys/arch/amd64/amd64/machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: machdep.c,v 1.309 2026/04/03 14:20:23 kettenis Exp $ */ +/* $OpenBSD: machdep.c,v 1.310 2026/06/04 05:22:04 mlarkin Exp $ */ /* $NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $ */ /*- @@ -1483,7 +1483,14 @@ init_x86_64(paddr_t first_avail) struct region_descriptor region; bios_memmap_t *bmp; int x, ist; - uint64_t max_dm_size = ((uint64_t)512 * NUM_L4_SLOT_DIRECT) << 30; + uint64_t max_dm_size = DIRECT_MAP_SIZE; + extern vaddr_t pmap_direct_base, pmap_direct_end; + extern char pmap_direct_rand; + + pmap_direct_base = (VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))); + pmap_direct_base = (VA_SIGN_NEG((pmap_direct_base + + ((pmap_direct_rand & DIRECT_MAP_START_MASK) * NBPD_L4)))); + pmap_direct_end = pmap_direct_base + DIRECT_MAP_SIZE; /* * locore0 mapped 2 pages for use as GHCB before pmap is initialized. @@ -1639,8 +1646,8 @@ init_x86_64(paddr_t first_avail) } /* - * The direct map is limited to 512GB * NUM_L4_SLOT_DIRECT of - * memory, so discard anything above that. + * The direct map is limited to DIRECT_MAP_SIZE of memory, so + * discard anything above that. */ if (e1 >= max_dm_size) { e1 = max_dm_size; diff --git a/sys/arch/amd64/amd64/mem.c b/sys/arch/amd64/amd64/mem.c index 43ae1c0e3ef..bf92147d71d 100644 --- a/sys/arch/amd64/amd64/mem.c +++ b/sys/arch/amd64/amd64/mem.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mem.c,v 1.39 2024/12/30 02:46:00 guenther Exp $ */ +/* $OpenBSD: mem.c,v 1.40 2026/06/04 05:22:04 mlarkin Exp $ */ /* * Copyright (c) 1988 University of Utah. * Copyright (c) 1982, 1986, 1990, 1993 @@ -156,7 +156,7 @@ mmrw(dev_t dev, struct uio *uio, int flags) return EFAULT; } else if ((!uvm_kernacc((caddr_t)v, c, uio->uio_rw == UIO_READ ? B_READ : B_WRITE)) && - (v < PMAP_DIRECT_BASE || v > PMAP_DIRECT_END - c)) + (v < pmap_direct_base || v > pmap_direct_end - c)) return (EFAULT); error = uiomove((caddr_t)v, c, uio); continue; diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c index 44552332d90..845970b90e2 100644 --- a/sys/arch/amd64/amd64/pmap.c +++ b/sys/arch/amd64/amd64/pmap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.c,v 1.190 2026/04/06 18:27:33 mlarkin Exp $ */ +/* $OpenBSD: pmap.c,v 1.191 2026/06/04 05:22:04 mlarkin Exp $ */ /* $NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $ */ /* @@ -218,6 +218,10 @@ pd_entry_t *const normal_pdes[] = PDES_INITIALIZER; #define pmap_pte_clearbits(p, b) x86_atomic_clearbits_u64(p, b) #define pmap_pte_setbits(p, b) x86_atomic_setbits_u64(p, b) +vaddr_t pmap_direct_base; +vaddr_t pmap_direct_end; +char pmap_direct_rand __attribute((section(".openbsd.randomdata"))); + /* * global data structures */ @@ -661,7 +665,7 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa) { vaddr_t kva_start = VM_MIN_KERNEL_ADDRESS; struct pmap *kpm; - int curslot, i, j, p; + int curslot, i, j, p, pdir_rand_slot; long ndmpdp; paddr_t dmpd, dmpdp, start_cur, cur_pa; vaddr_t kva, kva_end; @@ -669,6 +673,9 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa) KASSERT(((0x1000ULL | pg_crypt) & pg_frame) == 0x1000ULL); + pdir_rand_slot = PDIR_SLOT_DIRECT + + (pmap_direct_rand & DIRECT_MAP_START_MASK); + /* * define the boundaries of the managed kernel virtual address * space. @@ -795,7 +802,7 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa) if (ndmpdp > 512) ndmpdp = 512; /* At most 512GB */ - dmpdp = kpm->pm_pdir[PDIR_SLOT_DIRECT] & pg_frame; + dmpdp = kpm->pm_pdir[pdir_rand_slot] & pg_frame; dmpd = first_avail; first_avail += ndmpdp * PAGE_SIZE; memset((void *)PMAP_DIRECT_MAP(dmpd), 0, ndmpdp * PAGE_SIZE); @@ -824,11 +831,11 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa) pg_crypt; } - kpm->pm_pdir[PDIR_SLOT_DIRECT] = dmpdp | PG_V | PG_KW | PG_U | - PG_M | pg_nx | pg_crypt; + kpm->pm_pdir[pdir_rand_slot] = + dmpdp | PG_V | PG_KW | PG_U | PG_M | pg_nx | pg_crypt; /* Map any remaining physical memory > 512GB */ - for (curslot = 1 ; curslot < NUM_L4_SLOT_DIRECT ; curslot++) { + for (curslot = 1 ; curslot < DIRECT_MAP_PML4_SLOTS ; curslot++) { /* * Start of current range starts at PA (curslot) * 512GB */ @@ -838,7 +845,7 @@ pmap_bootstrap(paddr_t first_avail, paddr_t max_pa) dmpd = first_avail; first_avail += PAGE_SIZE; pml3 = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd); memset(pml3, 0, PAGE_SIZE); - kpm->pm_pdir[PDIR_SLOT_DIRECT + curslot] = dmpd | + kpm->pm_pdir[pdir_rand_slot + curslot] = dmpd | PG_KW | PG_V | PG_U | PG_M | pg_nx | pg_crypt; /* Calculate full 1GB pages in this 512GB region */ @@ -1329,7 +1336,7 @@ pmap_pdp_ctor(pd_entry_t *pdir) memset(&pdir[PDIR_SLOT_KERN + npde], 0, (NTOPLEVEL_PDES - (PDIR_SLOT_KERN + npde)) * sizeof(pd_entry_t)); - for (i = 0; i < NUM_L4_SLOT_DIRECT; i++) + for (i = 0; i < DIRECT_MAP_RESERVED_PML4_SLOTS; i++) pdir[PDIR_SLOT_DIRECT + i] = kpm->pm_pdir[PDIR_SLOT_DIRECT + i]; #if VM_MIN_KERNEL_ADDRESS != KERNBASE @@ -1565,9 +1572,9 @@ pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) pt_entry_t *ptes, pte; int level, offs; - if (pmap == pmap_kernel() && va >= PMAP_DIRECT_BASE && - va < PMAP_DIRECT_END) { - *pap = va - PMAP_DIRECT_BASE; + if (pmap == pmap_kernel() && va >= pmap_direct_base && + va < pmap_direct_end) { + *pap = va - pmap_direct_base; return 1; } diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h index c45bc85385a..227e080e052 100644 --- a/sys/arch/amd64/include/pmap.h +++ b/sys/arch/amd64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.94 2025/07/07 00:55:15 jsg Exp $ */ +/* $OpenBSD: pmap.h,v 1.95 2026/06/04 05:22:04 mlarkin Exp $ */ /* $NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $ */ /* @@ -92,11 +92,8 @@ * The other levels are kept as physical pages in 3 UVM objects and are * temporarily mapped for virtual access when needed. * - * The other obvious difference from i386 is that it has a direct map of all - * physical memory in the VA range: - * - * 0xfffffd8000000000 - 0xffffff7fffffffff - * + * The other obvious difference from i386 is that it has a direct map of + * physical memory in a randomized VA subrange of the direct-map window. * The direct map is used in some cases to access PTEs of non-current pmaps. * * Note that address space is signed, so the layout for 48 bits is: @@ -104,8 +101,8 @@ * +---------------------------------+ 0xffffffffffffffff * | Kernel Image | * +---------------------------------+ 0xffffff8000000000 - * | Direct Map | - * +---------------------------------+ 0xfffffd8000000000 + * | Direct Map Window | + * +---------------------------------+ 0xffffee0000000000 * ~ ~ * | | * | Kernel Space | @@ -139,13 +136,23 @@ */ #define VA_SIGN_POS(va) ((va) & ~VA_SIGN_MASK) -#define L4_SLOT_PTE 255 -#define L4_SLOT_KERN 256 -#define L4_SLOT_KERNBASE 511 -#define NUM_L4_SLOT_DIRECT 4 -#define L4_SLOT_DIRECT (L4_SLOT_KERNBASE - NUM_L4_SLOT_DIRECT) +#define L4_SLOT_PTE 255 +#define L4_SLOT_KERN 256 +#define L4_SLOT_KERNBASE 511 +#define DIRECT_MAP_PML4_SLOTS 4 +#define DIRECT_MAP_START_CHOICES 32 +#define DIRECT_MAP_START_MASK (DIRECT_MAP_START_CHOICES - 1) +#define DIRECT_MAP_RESERVED_PML4_SLOTS (DIRECT_MAP_PML4_SLOTS + \ + DIRECT_MAP_START_CHOICES - 1) +#define DIRECT_MAP_SIZE ((vaddr_t)DIRECT_MAP_PML4_SLOTS * NBPD_L4) +#define L4_SLOT_DIRECT (L4_SLOT_KERNBASE - \ + DIRECT_MAP_RESERVED_PML4_SLOTS) #define L4_SLOT_EARLY (L4_SLOT_DIRECT - 1) +#if (DIRECT_MAP_START_CHOICES & DIRECT_MAP_START_MASK) != 0 +#error DIRECT_MAP_START_CHOICES must be a power of two +#endif + #define PDIR_SLOT_KERN L4_SLOT_KERN #define PDIR_SLOT_PTE L4_SLOT_PTE #define PDIR_SLOT_DIRECT L4_SLOT_DIRECT @@ -160,9 +167,7 @@ */ #define PTE_BASE ((pt_entry_t *) (L4_SLOT_PTE * NBPD_L4)) -#define PMAP_DIRECT_BASE (VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))) -#define PMAP_DIRECT_END (VA_SIGN_NEG(((L4_SLOT_DIRECT + \ - NUM_L4_SLOT_DIRECT) * NBPD_L4))) +extern vaddr_t pmap_direct_base, pmap_direct_end; #define L1_BASE PTE_BASE @@ -498,8 +503,8 @@ kvtopte(vaddr_t va) return (PTE_BASE + pl1_i(va)); } -#define PMAP_DIRECT_MAP(pa) ((vaddr_t)PMAP_DIRECT_BASE + (pa)) -#define PMAP_DIRECT_UNMAP(va) ((paddr_t)(va) - PMAP_DIRECT_BASE) +#define PMAP_DIRECT_MAP(pa) ((vaddr_t)pmap_direct_base + (pa)) +#define PMAP_DIRECT_UNMAP(va) ((paddr_t)(va) - pmap_direct_base) #define pmap_map_direct(pg) PMAP_DIRECT_MAP(VM_PAGE_TO_PHYS(pg)) #define pmap_unmap_direct(va) PHYS_TO_VM_PAGE(PMAP_DIRECT_UNMAP(va))