Enforce guest RLIMIT_AS and RLIMIT_DATA

src/core/guest.c

@@ -1957,12 +1957,12 @@ int guest_get_used_regions(const guest_t *g,
  * Check whether two adjacent regions have merge-compatible layouts. An actual
  * merge additionally requires a shared vma_id; a new same-generation mapping
  * may adopt its compatible neighbor's ID below. Regions must be contiguous in
- * address space, have identical protection/flags/name, and have contiguous file
- * offsets (so the merged region still represents valid mapping). For anonymous
- * regions the offset is meaningless (always 0, but may become non-zero after
- * split/trim), so the contiguity check is skipped. Without this, adjacent
- * anonymous mmaps (common in megablock-style allocators) each create separate
- * entries that exhaust the region table.
+ * address space, have identical protection/flags/name/shm_attach, and have
+ * contiguous file offsets (so the merged region still represents valid
+ * mapping). For anonymous regions the offset is meaningless (always 0, but may
+ * become non-zero after split/trim), so the contiguity check is skipped.
+ * Without this, adjacent anonymous mmaps (common in megablock-style allocators)
+ * each create separate entries that exhaust the region table.
  */
 static bool regions_mergeable_layout(const guest_region_t *a,
                                      const guest_region_t *b)
@@ -1993,6 +1993,8 @@ static bool regions_mergeable_layout(const guest_region_t *a,
         return false;
     if (strcmp(a->name, b->name) != 0)
         return false;
+    if (a->shm_attach != b->shm_attach)
+        return false;
 
     /* Anonymous regions have no file offset to preserve. The flags field
      * reliably distinguishes anonymous from file-backed because every
@@ -2163,12 +2165,15 @@ static uint64_t allocate_vma_id(guest_t *g)
 
 void guest_reseed_next_vma_id(guest_t *g)
 {
-    uint64_t max_id = g->next_vma_id;
+    uint64_t max_id = g->next_vma_id, max_attach = g->next_shm_attach;
     for (int i = 0; i < g->nregions; i++) {
         if (g->regions[i].vma_id > max_id)
             max_id = g->regions[i].vma_id;
+        if (g->regions[i].shm_attach > max_attach)
+            max_attach = g->regions[i].shm_attach;
     }
     g->next_vma_id = max_id;
+    g->next_shm_attach = max_attach;
 }
 
 int guest_region_add_ex_owned(guest_t *g,
@@ -2198,6 +2203,24 @@ int guest_region_add_ex_owned_gpa(guest_t *g,
                                   int owned_backing_fd,
                                   bool inherited_at_fork,
                                   uint64_t vma_id)
+{
+    return guest_region_add_attach_gpa(g, start, end, gpa_base, prot, flags,
+                                       offset, name, owned_backing_fd,
+                                       inherited_at_fork, vma_id, 0);
+}
+
+int guest_region_add_attach_gpa(guest_t *g,
+                                uint64_t start,
+                                uint64_t end,
+                                uint64_t gpa_base,
+                                int prot,
+                                int flags,
+                                uint64_t offset,
+                                const char *name,
+                                int owned_backing_fd,
+                                bool inherited_at_fork,
+                                uint64_t vma_id,
+                                uint64_t shm_attach)
 {
     if (g->nregions >= GUEST_MAX_REGIONS) {
         log_error(
@@ -2234,6 +2257,7 @@ int guest_region_add_ex_owned_gpa(guest_t *g,
     r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0;
     r->backing_ro = false;
     r->inherited_at_fork = inherited_at_fork;
+    r->shm_attach = shm_attach;
     guest_region_clear_overlay(r);
     if (name) {
         str_copy_trunc(r->name, name, sizeof(r->name));

src/core/guest.h

@@ -292,6 +292,10 @@ typedef struct {
                              */
     uint64_t overlay_end;   /* Host-page-aligned overlay end (exclusive). */
     char name[64];          /* Label: "[heap]", "[stack]", ELF path, etc. */
+    uint64_t shm_attach;    /* Nonzero for a SysV shm attach: a serial unique
+                             * to that attach, so the region never merges with
+                             * or lends its vma_id to any other mapping.
+                             */
 } guest_region_t;
 
 /* TLB invalidation request kinds. After every page-table modification, the shim
@@ -540,8 +544,9 @@ typedef struct {
 
     /* Semantic region tracking for munmap/mprotect/proc-self-maps */
     guest_region_t regions[GUEST_MAX_REGIONS];
-    int nregions;         /* Number of active regions */
-    uint64_t next_vma_id; /* Last logical-VMA lineage ID allocated. */
+    int nregions;             /* Number of active regions */
+    uint64_t next_vma_id;     /* Last logical-VMA lineage ID allocated. */
+    uint64_t next_shm_attach; /* Last SysV shm attach serial handed out. */
     /* Set when a full region table stopped a mutation from being recorded:
      * guest_region_set_prot unable to split (leaving a region's old prot, or
      * applying the new one to a tail no PTE work covered), a munmap split
@@ -844,7 +849,7 @@ static inline bool guest_range_hits_infra(const guest_t *g,
 
 /* Whether a tracked region is address space a Linux process would have. The
  * shim and its data sit in the infra reserve, which Linux has no mapping for,
- * so they do not count toward VmSize.
+ * so they count toward neither VmSize nor RLIMIT_AS and RLIMIT_DATA.
  */
 static inline bool guest_region_in_process_vm(const guest_t *g,
                                               const guest_region_t *r)
@@ -1318,10 +1323,24 @@ int guest_region_add_ex_owned_gpa(guest_t *g,
                                   int owned_backing_fd,
                                   bool inherited_at_fork,
                                   uint64_t vma_id);
-
-/* Re-seed the logical-VMA allocator from a restored region snapshot. Fork IPC
- * restores regions by value, so next_vma_id must be advanced past every
- * serialized lineage before child-private mappings are added.
+/* guest_region_add_ex_owned_gpa, for a region that carries @shm_attach. */
+int guest_region_add_attach_gpa(guest_t *g,
+                                uint64_t start,
+                                uint64_t end,
+                                uint64_t gpa_base,
+                                int prot,
+                                int flags,
+                                uint64_t offset,
+                                const char *name,
+                                int owned_backing_fd,
+                                bool inherited_at_fork,
+                                uint64_t vma_id,
+                                uint64_t shm_attach);
+
+/* Re-seed the logical-VMA allocator and the shm attach serial from a restored
+ * region snapshot. Fork IPC restores regions by value, so next_vma_id and
+ * next_shm_attach must be advanced past every serialized lineage and attach
+ * before child-private mappings are added.
  */
 void guest_reseed_next_vma_id(guest_t *g);
 

src/runtime/fork-state.h

@@ -20,7 +20,7 @@
 /* Fork IPC protocol identity. Bump this whenever the header layout or ordered
  * fork payload changes incompatibly.
  */
-#define FORK_IPC_PROTOCOL_MAGIC 0x454C4653U /* "ELFS" */
+#define FORK_IPC_PROTOCOL_MAGIC 0x454C4654U /* "ELFT" */
 
 #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC
 #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */

src/syscall/internal.h

@@ -964,12 +964,45 @@ int mac_to_linux_status_flags(int mac_flags);
 /* Translate Linux status flags (F_SETFL arg) to macOS equivalents. */
 int linux_to_mac_status_flags(int linux_flags);
 
-/* Anonymous mmap for other modules. */
-
-/* Allocate anonymous guest memory. Wraps the static sys_mmap with
- * MAP_PRIVATE|MAP_ANONYMOUS. Caller must hold mmap_lock.
- */
-int64_t sys_mmap_anon(guest_t *g, uint64_t addr, uint64_t length, int prot);
+/* Guest memory for a SysV shm attach of the segment with @key: shared
+ * anonymous, as a Linux attach is VM_SHARED. Its region carries a new serial,
+ * which is stored in *@attach. Caller must hold mmap_lock.
+ */
+int64_t sys_mmap_shm(guest_t *g,
+                     uint64_t length,
+                     int prot,
+                     int32_t key,
+                     uint64_t *attach);
+
+/* The first part starting in [@gva, @end) whose region carries the shm attach
+ * serial @attach at the offset into the attach that an attach at @base would
+ * put there, as [*@lo, *@hi) in guest addresses. *@hi is the end of that region
+ * and can lie past @end. This is the part Linux shmdt at @base finds: a VMA of
+ * the segment whose page offset matches its distance from @base. When @host is
+ * not NULL, *@host is set to the host memory behind *@lo, which is readable
+ * whatever the guest page protection.
+ *
+ * Returns false when no part of the range carries it. Caller must hold
+ * mmap_lock.
+ */
+bool mem_attach_next(guest_t *g,
+                     uint64_t base,
+                     uint64_t gva,
+                     uint64_t end,
+                     uint64_t attach,
+                     uint64_t *lo,
+                     uint64_t *hi,
+                     void **host);
+
+/* Set @mapped[k] to whether any region carries the shm attach serial
+ * @attach[k], wherever the guest has moved it, for each of the @n serials. One
+ * walk over the regions answers all of them, and a region without a serial
+ * costs one test. Caller must hold mmap_lock.
+ */
+void mem_attach_mapped(const guest_t *g,
+                       const uint64_t *attach,
+                       bool *mapped,
+                       int n);
 
 /* Borrowed-or-pinned host fd reference.
  *