From 17f3efc862ee1035ca33e342ae7cf9c7e7f9d345 Mon Sep 17 00:00:00 2001 From: xalestar Date: Mon, 9 Nov 2026 01:07:49 +0800 Subject: [PATCH] Map and detach SysV shm attaches as Linux does A SysV shm attach was private anonymous memory, so it counted toward RLIMIT_DATA, a fork child got a copy, and shmdt left the mapping in place. A Linux attach is VM_SHARED: it counts toward RLIMIT_AS only, a fork child shares it, and shmdt unmaps it. The attach is now mapped shared anonymous (a fork child shares it when it starts on a host page), named /SYSV (deleted) as Linux names it, and its regions carry a serial unique to the attach, kept across fork, so the attach never merges with another mapping, and its own pieces merge only where their offsets into it are contiguous. The region struct grew, so the fork IPC magic moves to ELFT. shmdt finds the attach from the mappings, as Linux does: a region with the serial whose offset into the attach matches its distance from the address. It writes back and unmaps only the parts found there: the first one whole, even when mremap grew it past the segment, and the later ones only when they end within the segment size. So an attach moved with mremap detaches at its new address, and the record stays live while another part is mapped. A record whose serial no region carries is retired at the next shmat or shmdt, so unmapping attaches does not exhaust the 8 host attaches macOS allows a process. --- src/core/guest.c | 45 +++- src/core/guest.h | 31 ++- src/runtime/fork-state.h | 2 +- src/syscall/internal.h | 50 ++++- src/syscall/mem.c | 142 ++++++++++--- src/syscall/sysvipc.c | 150 ++++++++++--- tests/test-fork-ipc-protocol-host.c | 7 +- tests/test-rlimit-mem.c | 316 ++++++++++++++++++++++++++++ 8 files changed, 654 insertions(+), 89 deletions(-) diff --git a/src/core/guest.c b/src/core/guest.c index 0d2d4cebf..934d5727c 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1957,12 +1957,12 @@ int guest_get_used_regions(const guest_t *g, * Check whether two adjacent regions have merge-compatible layouts. An actual * merge additionally requires a shared vma_id; a new same-generation mapping * may adopt its compatible neighbor's ID below. Regions must be contiguous in - * address space, have identical protection/flags/name, and have contiguous file - * offsets (so the merged region still represents valid mapping). For anonymous - * regions the offset is meaningless (always 0, but may become non-zero after - * split/trim), so the contiguity check is skipped. Without this, adjacent - * anonymous mmaps (common in megablock-style allocators) each create separate - * entries that exhaust the region table. + * address space, have identical protection/flags/name/shm_attach, and have + * contiguous file offsets (so the merged region still represents valid + * mapping). For anonymous regions the offset is meaningless (always 0, but may + * become non-zero after split/trim), so the contiguity check is skipped. + * Without this, adjacent anonymous mmaps (common in megablock-style allocators) + * each create separate entries that exhaust the region table. */ static bool regions_mergeable_layout(const guest_region_t *a, const guest_region_t *b) @@ -1993,6 +1993,15 @@ static bool regions_mergeable_layout(const guest_region_t *a, return false; if (strcmp(a->name, b->name) != 0) return false; + if (a->shm_attach != b->shm_attach) + return false; + + /* Pieces of one shm attach merge only where their offsets into the attach + * run on: a page mremap moved past another of the same attach is its own + * VMA on Linux, and a merged region would carry the wrong offset for it. + */ + if (a->shm_attach) + return a->offset + (a->end - a->start) == b->offset; /* Anonymous regions have no file offset to preserve. The flags field * reliably distinguishes anonymous from file-backed because every @@ -2163,12 +2172,15 @@ static uint64_t allocate_vma_id(guest_t *g) void guest_reseed_next_vma_id(guest_t *g) { - uint64_t max_id = g->next_vma_id; + uint64_t max_id = g->next_vma_id, max_attach = g->next_shm_attach; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].vma_id > max_id) max_id = g->regions[i].vma_id; + if (g->regions[i].shm_attach > max_attach) + max_attach = g->regions[i].shm_attach; } g->next_vma_id = max_id; + g->next_shm_attach = max_attach; } int guest_region_add_ex_owned(guest_t *g, @@ -2198,6 +2210,24 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int owned_backing_fd, bool inherited_at_fork, uint64_t vma_id) +{ + return guest_region_add_attach_gpa(g, start, end, gpa_base, prot, flags, + offset, name, owned_backing_fd, + inherited_at_fork, vma_id, 0); +} + +int guest_region_add_attach_gpa(guest_t *g, + uint64_t start, + uint64_t end, + uint64_t gpa_base, + int prot, + int flags, + uint64_t offset, + const char *name, + int owned_backing_fd, + bool inherited_at_fork, + uint64_t vma_id, + uint64_t shm_attach) { if (g->nregions >= GUEST_MAX_REGIONS) { log_error( @@ -2234,6 +2264,7 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0; r->backing_ro = false; r->inherited_at_fork = inherited_at_fork; + r->shm_attach = shm_attach; guest_region_clear_overlay(r); if (name) { str_copy_trunc(r->name, name, sizeof(r->name)); diff --git a/src/core/guest.h b/src/core/guest.h index 1aa42e4ae..7a8db9f05 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -292,6 +292,10 @@ typedef struct { */ uint64_t overlay_end; /* Host-page-aligned overlay end (exclusive). */ char name[64]; /* Label: "[heap]", "[stack]", ELF path, etc. */ + uint64_t shm_attach; /* Nonzero for a SysV shm attach: a serial unique + * to that attach, so the region never merges with + * or lends its vma_id to any other mapping. + */ } guest_region_t; /* TLB invalidation request kinds. After every page-table modification, the shim @@ -540,8 +544,9 @@ typedef struct { /* Semantic region tracking for munmap/mprotect/proc-self-maps */ guest_region_t regions[GUEST_MAX_REGIONS]; - int nregions; /* Number of active regions */ - uint64_t next_vma_id; /* Last logical-VMA lineage ID allocated. */ + int nregions; /* Number of active regions */ + uint64_t next_vma_id; /* Last logical-VMA lineage ID allocated. */ + uint64_t next_shm_attach; /* Last SysV shm attach serial handed out. */ /* Set when a full region table stopped a mutation from being recorded: * guest_region_set_prot unable to split (leaving a region's old prot, or * applying the new one to a tail no PTE work covered), a munmap split @@ -1318,10 +1323,24 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int owned_backing_fd, bool inherited_at_fork, uint64_t vma_id); - -/* Re-seed the logical-VMA allocator from a restored region snapshot. Fork IPC - * restores regions by value, so next_vma_id must be advanced past every - * serialized lineage before child-private mappings are added. +/* guest_region_add_ex_owned_gpa, for a region that carries @shm_attach. */ +int guest_region_add_attach_gpa(guest_t *g, + uint64_t start, + uint64_t end, + uint64_t gpa_base, + int prot, + int flags, + uint64_t offset, + const char *name, + int owned_backing_fd, + bool inherited_at_fork, + uint64_t vma_id, + uint64_t shm_attach); + +/* Re-seed the logical-VMA allocator and the shm attach serial from a restored + * region snapshot. Fork IPC restores regions by value, so next_vma_id and + * next_shm_attach must be advanced past every serialized lineage and attach + * before child-private mappings are added. */ void guest_reseed_next_vma_id(guest_t *g); diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index 1df816791..1afb7edad 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -20,7 +20,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C4653U /* "ELFS" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C4654U /* "ELFT" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 66a0c4eb5..5c25116d6 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -150,9 +150,8 @@ * log_mutex (debug/log.c) rosettad_path_lock (rosetta.c) * nl_lock (netlink.c) session_lock (proc-identity.c) * overlay_lock (chown-overlay.c) shm_dir_lock (procemu.c) - * proc_scratch_lock (procemu.c) shm_lock (sysvipc.c) - * removed_overlay_lock (fs.c) syscpu_dir_lock (procemu.c) - * sysinfo_lock (sys.c) + * proc_scratch_lock (procemu.c) syscpu_dir_lock (procemu.c) + * removed_overlay_lock (fs.c) sysinfo_lock (sys.c) * sysroot_lock (proc-state.c) * usb_lock (runtime/usb-sysfs.c) * usbdev_loop_lock (usbdev.c) @@ -964,12 +963,45 @@ int mac_to_linux_status_flags(int mac_flags); /* Translate Linux status flags (F_SETFL arg) to macOS equivalents. */ int linux_to_mac_status_flags(int linux_flags); -/* Anonymous mmap for other modules. */ - -/* Allocate anonymous guest memory. Wraps the static sys_mmap with - * MAP_PRIVATE|MAP_ANONYMOUS. Caller must hold mmap_lock. - */ -int64_t sys_mmap_anon(guest_t *g, uint64_t addr, uint64_t length, int prot); +/* Guest memory for a SysV shm attach of the segment with @key: shared + * anonymous, as a Linux attach is VM_SHARED. Its region carries a new serial, + * which is stored in *@attach. Caller must hold mmap_lock. + */ +int64_t sys_mmap_shm(guest_t *g, + uint64_t length, + int prot, + int32_t key, + uint64_t *attach); + +/* The first part starting in [@gva, @end) whose region carries the shm attach + * serial @attach at the offset into the attach that an attach at @base would + * put there, as [*@lo, *@hi) in guest addresses. *@hi is the end of that region + * and can lie past @end. This is the part Linux shmdt at @base finds: a VMA of + * the segment whose page offset matches its distance from @base. When @host is + * not NULL, *@host is set to the host memory behind *@lo, which is readable + * whatever the guest page protection. + * + * Returns false when no part of the range carries it. Caller must hold + * mmap_lock. + */ +bool mem_attach_next(guest_t *g, + uint64_t base, + uint64_t gva, + uint64_t end, + uint64_t attach, + uint64_t *lo, + uint64_t *hi, + void **host); + +/* Set @mapped[k] to whether any region carries the shm attach serial + * @attach[k], wherever the guest has moved it, for each of the @n serials. One + * walk over the regions answers all of them, and a region without a serial + * costs one test. Caller must hold mmap_lock. + */ +void mem_attach_mapped(const guest_t *g, + const uint64_t *attach, + bool *mapped, + int n); /* Borrowed-or-pinned host fd reference. * diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 2e5d825d4..cd8302684 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -115,6 +115,7 @@ typedef struct { bool backing_ro; bool inherited_at_fork; char name[sizeof(((guest_region_t *) 0)->name)]; + uint64_t shm_attach; } region_snapshot_t; /* Undo record for one boundary split. A split rewrites exactly one entry and @@ -463,6 +464,7 @@ typedef struct { int backing_fd; int tail_backing_fd; char name[sizeof(((guest_region_t *) 0)->name)]; + uint64_t shm_attach; } mremap_track_t; /* Release whatever descriptors the tracker still owns. Idempotent, so a failure @@ -511,6 +513,7 @@ static bool mremap_track_init(mremap_track_t *track, if (src_reg->backing_fd >= 0 && track->backing_fd < 0) return false; str_copy_trunc(track->name, src_reg->name, sizeof(track->name)); + track->shm_attach = src_reg->shm_attach; if (mremap_track_splits(track, new_size) && track->backing_fd >= 0) { track->tail_backing_fd = dup(track->backing_fd); @@ -552,17 +555,17 @@ static int add_mremap_region(guest_t *g, close(backing_fd); return -1; } - if (guest_region_add_ex_owned(g, start, start + inherited_size, prot, - flags, offset, name, backing_fd, true, - vma_id) < 0) { + if (guest_region_add_attach_gpa(g, start, start + inherited_size, start, + prot, flags, offset, name, backing_fd, + true, vma_id, track->shm_attach) < 0) { if (tail_backing_fd >= 0) close(tail_backing_fd); return -1; } - if (guest_region_add_ex_owned(g, start + inherited_size, - start + new_size, prot, flags, - offset + inherited_size, name, - tail_backing_fd, false, vma_id) < 0) { + if (guest_region_add_attach_gpa( + g, start + inherited_size, start + new_size, + start + inherited_size, prot, flags, offset + inherited_size, + name, tail_backing_fd, false, vma_id, track->shm_attach) < 0) { guest_region_remove(g, start, start + inherited_size); return -1; } @@ -571,9 +574,10 @@ static int add_mremap_region(guest_t *g, if (tail_backing_fd >= 0) close(tail_backing_fd); - return guest_region_add_ex_owned( - g, start, start + new_size, prot, flags, offset, name, backing_fd, - inherited_size == new_size && inherited_size > 0, vma_id); + return guest_region_add_attach_gpa( + g, start, start + new_size, start, prot, flags, offset, name, + backing_fd, inherited_size == new_size && inherited_size > 0, vma_id, + track->shm_attach); } static void region_clip_overlay(guest_region_t *r) @@ -1901,6 +1905,7 @@ static int capture_region_snapshots(guest_t *g, snap->backing_ro = r->backing_ro; snap->inherited_at_fork = r->inherited_at_fork; str_copy_trunc(snap->name, r->name, sizeof(snap->name)); + snap->shm_attach = r->shm_attach; } if (splits_out) @@ -2016,10 +2021,11 @@ static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n) { for (int i = 0; i < n; i++) { region_snapshot_t *snap = &snaps[i]; - if (guest_region_add_ex_owned_gpa( + if (guest_region_add_attach_gpa( g, snap->start, snap->end, snap->gpa_base, snap->prot, snap->flags, snap->offset, snap->name[0] ? snap->name : NULL, - snap->backing_fd, snap->inherited_at_fork, snap->vma_id) < 0) { + snap->backing_fd, snap->inherited_at_fork, snap->vma_id, + snap->shm_attach) < 0) { snap->backing_fd = -1; close_region_snapshots(snaps, n); return -LINUX_ENOMEM; @@ -3697,19 +3703,23 @@ static int mmap_extend_fresh(guest_t *g, return 0; } -/* Why this stays one function. +/* sys_mmap, but the region it records gets the name @name and the shm attach + * serial @shm_attach. Only a non-fixed request with no high-VA hint passes + * them, and that request records its region at one place below. * - * Flag combinations, not steps: MAP_FIXED, MAP_SHARED, the file-backed overlay - * and the anonymous path each qualify the others. Splitting by flag duplicates - * the interactions between them. + * Why this stays one function: flag combinations, not steps. MAP_FIXED, + * MAP_SHARED, the file-backed overlay and the anonymous path each qualify the + * others. Splitting by flag duplicates the interactions between them. */ -int64_t sys_mmap(guest_t *g, - uint64_t addr, - uint64_t length, - int prot, - int flags, - int fd, - int64_t offset) +static int64_t mmap_tagged(guest_t *g, + uint64_t addr, + uint64_t length, + int prot, + int flags, + int fd, + int64_t offset, + const char *name, + uint64_t shm_attach) { bool is_anon = (flags & LINUX_MAP_ANONYMOUS) != 0; bool needs_exec = (prot & LINUX_PROT_EXEC) != 0; @@ -3990,9 +4000,10 @@ int64_t sys_mmap(guest_t *g, */ int owned_backing_fd = fresh.track_backing_fd; fresh.track_backing_fd = -1; - if (guest_region_add_ex_owned(g, result_off, result_off + length, prot, - track_flags, is_anon ? 0 : (uint64_t) offset, - NULL, owned_backing_fd, false, 0) < 0) { + if (guest_region_add_attach_gpa( + g, result_off, result_off + length, result_off, prot, track_flags, + is_anon ? 0 : (uint64_t) offset, name, owned_backing_fd, false, 0, + shm_attach) < 0) { /* Region table was full: undo any host overlay we just installed so the * file is not left mmap'd at host_base+ipa with no tracking. Without * this, a later operation in that range would memset zeros directly @@ -4950,11 +4961,82 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) } } -/* Anonymous mmap wrapper for other modules. */ -int64_t sys_mmap_anon(guest_t *g, uint64_t addr, uint64_t length, int prot) +int64_t sys_mmap(guest_t *g, + uint64_t addr, + uint64_t length, + int prot, + int flags, + int fd, + int64_t offset) +{ + return mmap_tagged(g, addr, length, prot, flags, fd, offset, NULL, 0); +} + +/* Guest memory for a SysV shm attach, which sys_shmat fills by copying from the + * host segment. A Linux attach is VM_SHARED: it counts toward RLIMIT_AS only, + * /proc/self/maps shows it as shared, and a fork child shares it with the + * parent. Mapping it MAP_SHARED | MAP_ANONYMOUS gives all three. The region is + * named as Linux names it in maps, and tagged with a serial no other region + * has, so it never merges with another mapping or another attach. + */ +int64_t sys_mmap_shm(guest_t *g, + uint64_t length, + int prot, + int32_t key, + uint64_t *attach) +{ + char name[sizeof(((guest_region_t *) 0)->name)]; + snprintf(name, sizeof(name), "/SYSV%08x (deleted)", (uint32_t) key); + *attach = ++g->next_shm_attach; + return mmap_tagged(g, 0, length, prot, + LINUX_MAP_SHARED | LINUX_MAP_ANONYMOUS, -1, 0, name, + *attach); +} + +bool mem_attach_next(guest_t *g, + uint64_t base, + uint64_t gva, + uint64_t end, + uint64_t attach, + uint64_t *lo, + uint64_t *hi, + void **host) +{ + if (gva >= end) + return false; + uint64_t off = gva >= g->guest_size ? gva : gva - g->ipa_base; + uint64_t off_end = off + (end - gva); + for (int i = guest_region_first_end_above(g, off); i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= off_end) + break; + uint64_t piece = r->start > off ? r->start : off; + if (r->shm_attach == attach && + r->offset + (piece - r->start) == gva + (piece - off) - base) { + *lo = gva + (piece - off); + *hi = gva + (r->end - off); + if (host) + *host = host_ptr_for_gpa(g, r->gpa_base + (piece - r->start)); + return true; + } + } + return false; +} + +void mem_attach_mapped(const guest_t *g, + const uint64_t *attach, + bool *mapped, + int n) { - return sys_mmap(g, addr, length, prot, - LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, -1, 0); + for (int k = 0; k < n; k++) + mapped[k] = false; + for (int i = 0; i < g->nregions; i++) { + uint64_t serial = g->regions[i].shm_attach; + for (int k = 0; serial && k < n; k++) { + if (attach[k] == serial) + mapped[k] = true; + } + } } static int compare_range_pair(const void *a, const void *b) diff --git a/src/syscall/sysvipc.c b/src/syscall/sysvipc.c index 4c6c75560..823f01c39 100644 --- a/src/syscall/sysvipc.c +++ b/src/syscall/sysvipc.c @@ -128,25 +128,76 @@ typedef struct { #define LINUX_MSG_EXCEPT 020000 #define LINUX_MSG_COPY 040000 -/* shmat tracking table. */ +/* shmat tracking table. mmap_lock protects it: shmat and shmdt run under it, + * and shmctl does not touch the table. + */ #define MAX_SHM_ATTACH 64 typedef struct { bool active; int shmid; - void *host_addr; /* shmat() return on host */ - uint64_t guest_gva; /* guest virtual address */ - uint64_t size; /* segment size */ - bool rdonly; /* attached read-only */ + void *host_addr; /* shmat() return on host */ + uint64_t size; /* segment size */ + uint64_t attach; /* serial the guest mapping's regions carry */ + bool rdonly; /* attached read-only */ } shm_attach_t; static shm_attach_t shm_table[MAX_SHM_ATTACH]; -static pthread_mutex_t shm_lock = PTHREAD_MUTEX_INITIALIZER; #define SHM_FOR_EACH(e) \ for (shm_attach_t *e = shm_table; e < shm_table + MAX_SHM_ATTACH; e++) +/* Retire every record whose attach no region carries any more, wherever the + * guest moved it, and release its host attach. Linux detaches an attach when + * its last VMA is unmapped, by munmap or by a mapping placed over it, so such a + * record names nothing, and keeping it would hold one of the host's few + * attaches (kern.sysv.shmseg, 8 by default on macOS) and a table slot. The + * caller holds mmap_lock. + */ +static void shm_retire_unmapped(guest_t *g) +{ + shm_attach_t *rec[MAX_SHM_ATTACH]; + uint64_t attach[MAX_SHM_ATTACH]; + bool mapped[MAX_SHM_ATTACH]; + void *host[MAX_SHM_ATTACH]; + int n = 0, nhost = 0; + SHM_FOR_EACH (e) { + if (e->active) { + rec[n] = e; + attach[n++] = e->attach; + } + } + mem_attach_mapped(g, attach, mapped, n); + for (int k = 0; k < n; k++) { + if (!mapped[k]) { + rec[k]->active = false; + host[nhost++] = rec[k]->host_addr; + } + } + for (int i = 0; i < nhost; i++) + shmdt(host[i]); +} + +/* The next part of the attach at @base that shmdt detaches, from @cur on, with + * @cur equal to @base for the first call. Linux unmaps the first VMA of the + * segment it finds whole, even one that mremap grew past the segment, then only + * the later VMAs that end within the segment size from @base, and stops at the + * first one that reaches past it. Caller must hold mmap_lock. + */ +static bool shm_next_part(guest_t *g, + uint64_t base, + uint64_t cur, + uint64_t end, + uint64_t attach, + uint64_t *lo, + uint64_t *hi, + void **host) +{ + return mem_attach_next(g, base, cur, end, attach, lo, hi, host) && + (cur == base || *hi <= end); +} + /* Helper: macOS ipc_perm -> Linux ipc64_perm. */ static void mac_to_linux_ipc_perm(const struct ipc_perm *mac, @@ -235,6 +286,8 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) bool rdonly = (shmflg & LINUX_SHM_RDONLY) != 0; + shm_retire_unmapped(g); + /* Attach on host side */ void *host_addr = shmat(shmid, NULL, rdonly ? SHM_RDONLY : 0); if (host_addr == (void *) -1) @@ -249,8 +302,9 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) /* Always allocate RW initially so the code can copy shm content in. For * SHM_RDONLY, downgrade to read-only via mprotect after the copy. */ - int64_t gva = - sys_mmap_anon(g, 0, seg_size, LINUX_PROT_READ | LINUX_PROT_WRITE); + uint64_t attach; + int64_t gva = sys_mmap_shm(g, seg_size, LINUX_PROT_READ | LINUX_PROT_WRITE, + (int32_t) info.shm_perm._key, &attach); if (gva < 0) { shmdt(host_addr); return gva; /* propagate mmap error */ @@ -258,6 +312,7 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) /* Copy host shm content into guest memory */ if (guest_write(g, (uint64_t) gva, host_addr, seg_size) < 0) { + sys_munmap(g, (uint64_t) gva, PAGE_ALIGN_UP(seg_size)); shmdt(host_addr); return -LINUX_EFAULT; } @@ -270,7 +325,6 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) LINUX_PROT_READ); /* Track the attachment */ - pthread_mutex_lock(&shm_lock); shm_attach_t *entry = NULL; SHM_FOR_EACH (e) { if (!e->active) { @@ -279,55 +333,83 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) } } if (!entry) { - pthread_mutex_unlock(&shm_lock); + sys_munmap(g, (uint64_t) gva, PAGE_ALIGN_UP(seg_size)); shmdt(host_addr); return -LINUX_ENOMEM; } entry->active = true; entry->shmid = shmid; entry->host_addr = host_addr; - entry->guest_gva = (uint64_t) gva; entry->size = seg_size; + entry->attach = attach; entry->rdonly = rdonly; - pthread_mutex_unlock(&shm_lock); return gva; } int64_t sys_shmdt(guest_t *g, uint64_t shmaddr_gva) { - pthread_mutex_lock(&shm_lock); + /* Linux finds the attach to detach from the mappings, not from where it was + * made: the first VMA of a segment at or above the address whose page + * offset matches its distance from the address, so an attach the guest + * moved with mremap is detached at its new address and not at its old one. + * The record taken is likewise the one with such a part in its range from + * the address, the lowest part winning. Records whose attach is mapped + * nowhere are retired first. When no record has such a part, Linux finds no + * attach and fails with EINVAL, and what is mapped there now is neither + * copied nor unmapped. Only the parts found are detached: Linux keeps the + * segment attached while another part is mapped elsewhere, so the record + * stays live until no region carries its serial. The caller holds + * mmap_lock, which keeps the record from being taken meanwhile. + */ + shm_retire_unmapped(g); + + uint64_t lo, hi, best = UINT64_MAX; shm_attach_t *match = NULL; SHM_FOR_EACH (e) { - if (e->active && e->guest_gva == shmaddr_gva) { + uint64_t end = shmaddr_gva + PAGE_ALIGN_UP(e->size); + if (e->active && end > shmaddr_gva && + mem_attach_next(g, shmaddr_gva, shmaddr_gva, end, e->attach, &lo, + &hi, NULL) && + lo < best) { match = e; - break; + best = lo; } } - if (!match) { - pthread_mutex_unlock(&shm_lock); + if (!match) return -LINUX_EINVAL; - } - - shm_attach_t entry = *match; - match->active = false; - pthread_mutex_unlock(&shm_lock); - /* Write back guest modifications to host shm (unless read-only) */ - if (!entry.rdonly) { - /* Read guest memory back to host shm buffer */ - guest_read(g, entry.guest_gva, entry.host_addr, entry.size); + /* Write back each part found to host shm (unless read-only), then release + * it as Linux does at detach, so it stops counting toward RLIMIT_AS. The + * copy reads the memory behind the part directly, so a part the guest made + * PROT_NONE is written back too. Each step looks the next part up from the + * end of this one, so unmapping a part does not change which comes next. A + * part that fails to unmap does not stop the others, and the first error is + * returned. + */ + uint64_t gva = shmaddr_gva; + uint64_t end = gva + PAGE_ALIGN_UP(match->size); + void *src; + int64_t err = 0; + for (uint64_t cur = gva; + shm_next_part(g, gva, cur, end, match->attach, &lo, &hi, &src); + cur = hi) { + uint64_t copy_end = hi < gva + match->size ? hi : gva + match->size; + if (!match->rdonly && src) + memcpy((char *) match->host_addr + (lo - gva), src, copy_end - lo); + int64_t rc = sys_munmap(g, lo, hi - lo); + if (rc < 0 && err == 0) + err = rc; } - /* Detach host shm */ - shmdt(entry.host_addr); - - /* Guest memory remains allocated (no munmap). Linux shmdt does not - * guarantee immediate unmap either; the pages become undefined. A real - * implementation would munmap the guest region here. - */ + bool mapped; + mem_attach_mapped(g, &match->attach, &mapped, 1); + if (!mapped) { + match->active = false; + shmdt(match->host_addr); + } - return 0; + return err; } int64_t sys_shmctl(guest_t *g, int shmid, int cmd, uint64_t buf_gva) diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 075a14cc5..8923d3944 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -24,9 +24,10 @@ #define PREVIOUS_ELFP_MAGIC 0x454C4650U #define PREVIOUS_ELFQ_MAGIC 0x454C4651U #define PREVIOUS_ELFR_MAGIC 0x454C4652U +#define PREVIOUS_ELFS_MAGIC 0x454C4653U -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4653U, - "fork IPC protocol magic must remain ELFS until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4654U, + "fork IPC protocol magic must remain ELFT until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -46,6 +47,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFQ_MAGIC, "inherited pidfd state requires rejecting ELFQ peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFR_MAGIC, "the rlimit table in the header requires rejecting ELFR peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFS_MAGIC, + "the region shm_attach field requires rejecting ELFS peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-rlimit-mem.c b/tests/test-rlimit-mem.c index 9e5bcb42b..c2ce95e8c 100644 --- a/tests/test-rlimit-mem.c +++ b/tests/test-rlimit-mem.c @@ -16,7 +16,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -439,6 +441,320 @@ static int data_child(void) if (msync(dst, 4096, MS_ASYNC) < 0) return 12; + /* 13: a SysV shm attach is shared, so RLIMIT_DATA does not count it. */ + int id = shmget(IPC_PRIVATE, 2 * MIB, IPC_CREAT | 0600); + data = maps_size(1); + if (id < 0 || data == 0 || set_soft(RLIMIT_DATA, data + MIB / 2) < 0) { + if (id >= 0) + shmctl(id, IPC_RMID, NULL); + return 100; + } + char *seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (seg == (void *) -1) + return 13; + + /* 14: and a fork child shares it, so its write reaches the parent. */ + seg[0] = 'P'; + pid_t pid = fork(); + if (pid == 0) { + seg[0] = 'C'; + _exit(0); + } + int status; + if (pid < 0 || waitpid(pid, &status, 0) < 0 || !WIFEXITED(status)) + return 100; + if (seg[0] != 'C') + return 14; + shmdt(seg); + + /* 15: a detach releases the attach, so attaching and detaching more than + * RLIMIT_AS leaves room for in all does not run out. + */ + id = shmget(IPC_PRIVATE, 2 * MIB, IPC_CREAT | 0600); + size_t vm = vm_size(); + if (id < 0 || vm == 0 || set_soft(RLIMIT_AS, vm + 8 * MIB) < 0) { + if (id >= 0) + shmctl(id, IPC_RMID, NULL); + return 100; + } + for (int i = 0; i < 16; i++) { + seg = shmat(id, NULL, 0); + if (seg == (void *) -1 || shmdt(seg) < 0) { + shmctl(id, IPC_RMID, NULL); + return 15; + } + } + + /* 16: a detach after the attach was mapped over fails with EINVAL and + * leaves the new mapping in place. + */ + seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (seg == (void *) -1 || + map_anon(seg, 2 * MIB, rw, MAP_PRIVATE | MAP_FIXED) != seg) + return 100; + seg[0] = 'N'; + errno = 0; + if (shmdt(seg) != -1 || errno != EINVAL || msync(seg, 4096, MS_ASYNC) < 0 || + seg[0] != 'N') + return 16; + + /* 17: when only part of the attach was mapped over, the detach writes back + * and unmaps the part that remains and leaves the new mapping alone. The + * new mapping is data, so the soft RLIMIT_DATA from step 13 is raised. + */ + struct rlimit dl; + if (getrlimit(RLIMIT_DATA, &dl) < 0 || + set_soft(RLIMIT_DATA, dl.rlim_max) < 0) + return 100; + id = shmget(IPC_PRIVATE, 2 * MIB, IPC_CREAT | 0600); + seg = id < 0 ? (void *) -1 : shmat(id, NULL, 0); + if (seg == (void *) -1 || + map_anon(seg + MIB, MIB, rw, MAP_PRIVATE | MAP_FIXED) != seg + MIB) { + if (id >= 0) + shmctl(id, IPC_RMID, NULL); + return 100; + } + seg[0] = 'A'; + seg[MIB] = 'X'; + char *again = shmdt(seg) < 0 ? (void *) -1 : shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (again == (void *) -1 || again[0] != 'A' || again[MIB] != 0 || + (again != seg + MIB && seg[MIB] != 'X')) + return 17; + + /* 18: so is a shared anonymous one over the first page, which Linux keeps + * apart from the attach. + */ + id = shmget(IPC_PRIVATE, 64 * 1024, IPC_CREAT | 0600); + seg = id < 0 ? (void *) -1 : shmat(id, NULL, 0); + if (seg == (void *) -1 || munmap(seg, 4096) < 0 || + map_anon(seg, 4096, rw, MAP_SHARED | MAP_FIXED) != seg) { + if (id >= 0) + shmctl(id, IPC_RMID, NULL); + return 100; + } + seg[0] = 'S'; + seg[4096] = 'B'; + again = shmdt(seg) < 0 ? (void *) -1 : shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (again == (void *) -1 || again[0] != 0 || again[4096] != 'B' || + (again != seg && seg[0] != 'S')) + return 18; + + /* 19: an attach that was unmapped does not hide one made at its address + * since, which the detach then takes. + */ + int ida = shmget(IPC_PRIVATE, 64 * 1024, IPC_CREAT | 0600); + int idb = shmget(IPC_PRIVATE, 64 * 1024, IPC_CREAT | 0600); + char *sa = ida < 0 ? (void *) -1 : shmat(ida, NULL, 0); + char *sb = sa == (void *) -1 || munmap(sa, 64 * 1024) < 0 || idb < 0 + ? (void *) -1 + : shmat(idb, sa, 0); + if (ida >= 0) + shmctl(ida, IPC_RMID, NULL); + if (idb >= 0) + shmctl(idb, IPC_RMID, NULL); + if (sb == (void *) -1) + return 100; + if (sb == sa && (shmdt(sa) < 0 || msync(sa, 4096, MS_ASYNC) == 0)) + return 19; + + /* 20: a detach writes back a part the guest made PROT_NONE. */ + id = shmget(IPC_PRIVATE, 64 * 1024, IPC_CREAT | 0600); + seg = id < 0 ? (void *) -1 : shmat(id, NULL, 0); + if (seg == (void *) -1) { + if (id >= 0) + shmctl(id, IPC_RMID, NULL); + return 100; + } + seg[100] = 'W'; + if (mprotect(seg, 64 * 1024, PROT_NONE) < 0 || shmdt(seg) < 0) { + shmctl(id, IPC_RMID, NULL); + return 20; + } + again = shmat(id, NULL, 0); + if (again == (void *) -1 || again[100] != 'W') { + shmctl(id, IPC_RMID, NULL); + return 20; + } + + /* 21: so is an attach that mremap shrank and grew back in place. */ + if (mremap(again, 64 * 1024, 32 * 1024, 0) != again || + mremap(again, 32 * 1024, 64 * 1024, 0) != again) { + shmctl(id, IPC_RMID, NULL); + return 100; + } + again[0] = 'Q'; + if (shmdt(again) < 0 || msync(again, 4096, MS_ASYNC) == 0) { + shmctl(id, IPC_RMID, NULL); + return 21; + } + seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (seg == (void *) -1 || seg[0] != 'Q') + return 21; + + /* 22: an attach the guest unmapped is detached, so unmapping attaches + * without shmdt does not run out of them. elfuse keeps 64 records, and the + * macOS host allows 8 attaches per process by default. + */ + id = shmget(IPC_PRIVATE, 64 * 1024, IPC_CREAT | 0600); + if (id < 0) + return 100; + for (int i = 0; i < 80; i++) { + char *a = shmat(id, NULL, 0); + if (a == (void *) -1 || munmap(a, 64 * 1024) < 0) { + shmctl(id, IPC_RMID, NULL); + return 22; + } + } + shmctl(id, IPC_RMID, NULL); + + /* 23: an attach moved with mremap is detached at its new address, with the + * writes made there, and not at its old one. + */ + char *to = map_anon(NULL, 4096, rw, MAP_PRIVATE); + if (to == MAP_FAILED) + return 100; + id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0600); + if (id < 0) + return 100; + seg = shmat(id, NULL, 0); + if (seg == (void *) -1 || + mremap(seg, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, to) != to) { + shmctl(id, IPC_RMID, NULL); + return 23; + } + to[0] = 'M'; + errno = 0; + int old_rc = shmdt(seg); + int old_errno = errno; + int new_rc = shmdt(to); + seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (old_rc != -1 || old_errno != EINVAL || new_rc != 0 || + seg == (void *) -1 || seg[0] != 'M') + return 23; + shmdt(seg); + + /* 24: a fork child's attaches do not take the serials of the attaches it + * inherited. In a child holding 8 attaches, a grandchild attaches and + * unmaps 20 times; an attach that shared an inherited one's serial would + * look still mapped and never be detached. + */ + id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0600); + if (id < 0) + return 100; + pid_t holder = fork(); + if (holder == 0) { + for (int i = 0; i < 8; i++) { + if (shmat(id, NULL, 0) == (void *) -1) + _exit(100); + } + pid_t gc = fork(); + if (gc == 0) { + for (int i = 0; i < 20; i++) { + char *a = shmat(id, NULL, 0); + if (a == (void *) -1 || munmap(a, 4096) < 0) + _exit(24); + } + _exit(0); + } + int gst; + _exit(gc < 0 || waitpid(gc, &gst, 0) < 0 || !WIFEXITED(gst) + ? 24 + : WEXITSTATUS(gst)); + } + int hst; + int hrc = holder < 0 || waitpid(holder, &hst, 0) < 0 || !WIFEXITED(hst) + ? 24 + : WEXITSTATUS(hst); + shmctl(id, IPC_RMID, NULL); + if (hrc != 0) + return hrc; + + /* 25: a detach at the address of a part moved away detaches only that part, + * and the part left behind still detaches at the old address, with the + * writes made there. + */ + to = map_anon(NULL, 4096, rw, MAP_PRIVATE); + if (to == MAP_FAILED) + return 100; + id = shmget(IPC_PRIVATE, 2 * 4096, IPC_CREAT | 0600); + if (id < 0) + return 100; + seg = shmat(id, NULL, 0); + if (seg == (void *) -1 || + mremap(seg, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, to) != to) { + shmctl(id, IPC_RMID, NULL); + return 25; + } + seg[4096] = 'S'; + new_rc = shmdt(to); + old_rc = shmdt(seg); + seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (new_rc != 0 || old_rc != 0 || seg == (void *) -1 || seg[4096] != 'S') + return 25; + shmdt(seg); + + /* 26: an attach mremap grew in place past its segment is detached whole, + * with the writes made in the segment. + */ + char *at = map_anon(NULL, 2 * 4096, PROT_NONE, MAP_PRIVATE); + if (at == MAP_FAILED || munmap(at, 2 * 4096) < 0) + return 100; + id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0600); + if (id < 0) + return 100; + seg = shmat(id, at, 0); + if (seg != at || mremap(seg, 4096, 2 * 4096, 0) != seg) { + shmctl(id, IPC_RMID, NULL); + return 26; + } + seg[0] = 'G'; + if (shmdt(seg) < 0 || msync(seg, 4096, MS_ASYNC) == 0 || + msync(seg + 4096, 4096, MS_ASYNC) == 0) { + shmctl(id, IPC_RMID, NULL); + return 26; + } + seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (seg == (void *) -1 || seg[0] != 'G') + return 26; + shmdt(seg); + + /* 27: a page moved past the next page of its attach does not merge with it. + * Detaching at the old address leaves the moved page attached, and each + * page's write reaches its own segment offset. + */ + at = map_anon(NULL, 3 * 4096, PROT_NONE, MAP_PRIVATE); + if (at == MAP_FAILED || munmap(at, 3 * 4096) < 0) + return 100; + id = shmget(IPC_PRIVATE, 2 * 4096, IPC_CREAT | 0600); + if (id < 0) + return 100; + seg = shmat(id, at, 0); + to = at + 2 * 4096; + if (seg != at || + mremap(seg, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, to) != to) { + shmctl(id, IPC_RMID, NULL); + return 27; + } + to[0] = 'L'; + seg[4096] = 'H'; + old_rc = shmdt(seg); + int moved_mapped = msync(to, 4096, MS_ASYNC) == 0; + new_rc = shmdt(to); + seg = shmat(id, NULL, 0); + shmctl(id, IPC_RMID, NULL); + if (old_rc != 0 || !moved_mapped || new_rc != 0 || seg == (void *) -1 || + seg[0] != 'L' || seg[4096] != 'H') + return 27; + shmdt(seg); + return 0; }