From 4e60f321d3949312b3ab8ecc9649b05b9a8a7b25 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 18 Oct 2026 09:05:01 -0700 Subject: [PATCH] feat(qemu-acpi): model guest NUMA nodes without PXB expanders A QEMU wrapper can give a CVM guest NUMA nodes (SRAT and CPU _PXM) while leaving every device on the root bus. Such a layout depends only on the vCPU count and memory size, so the KMS can reproduce it, unlike the multi-PXB layout whose expander _CRS comes from firmware enumeration. NumaNodeConfig.pxb_bus becomes optional and is omitted when absent, so existing vm_configs deserialize and serialize unchanged and their MRs and verifier cache keys stay the same. Mixed layouts are still rejected. An older KMS fails to parse a node without pxb_bus and refuses the CVM. dstack-mr measure gains --numa-nodes, and both differential harnesses cover the new layout. Signed-off-by: Kevin Wang --- dstack/crates/qemu-acpi/examples/dump.rs | 8 +++- dstack/crates/qemu-acpi/fixtures/README.md | 4 ++ ...-11.1-q35-hotplug-off-numa2-plain-base.bin | Bin 0 -> 9536 bytes ...1.1-q35-hotplug-off-numa2-plain-loader.bin | Bin 0 -> 4096 bytes ...-11.1-q35-hotplug-off-numa2-plain-rsdp.bin | Bin 0 -> 20 bytes .../qemu-acpi/scripts/differential-qemu.sh | 20 ++++++--- .../qemu-acpi/scripts/differential-random.py | 40 +++++++++++++++--- dstack/crates/qemu-acpi/src/golden_tests.rs | 26 ++++++++++++ dstack/crates/qemu-acpi/src/topology.rs | 13 ++++-- dstack/dstack-mr/cli/src/main.rs | 18 +++++--- dstack/dstack-mr/src/acpi.rs | 2 +- dstack/dstack-types/src/lib.rs | 26 +++++++++++- dstack/vmm/src/app.rs | 2 +- 13 files changed, 130 insertions(+), 29 deletions(-) create mode 100644 dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-base.bin create mode 100644 dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-loader.bin create mode 100644 dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-rsdp.bin diff --git a/dstack/crates/qemu-acpi/examples/dump.rs b/dstack/crates/qemu-acpi/examples/dump.rs index 312ab4fec..eb531c3a6 100644 --- a/dstack/crates/qemu-acpi/examples/dump.rs +++ b/dstack/crates/qemu-acpi/examples/dump.rs @@ -29,11 +29,15 @@ fn main() -> Result<(), Box> { let memory_size = optional(&mut args, 2u64 << 30)?; let volumes = optional(&mut args, 0)?; let pic = optional::(&mut args, 0)? == 1; - // Comma-separated PXB bus of each NUMA node; empty keeps the legacy layout. + // Comma-separated PXB bus of each NUMA node, `-` for none; empty keeps the + // legacy layout. let numa_nodes = optional::(&mut args, String::new())? .split(',') .filter(|bus| !bus.is_empty()) - .map(|bus| bus.parse().map(|bus| NumaNode { pxb_bus: Some(bus) })) + .map(|bus| match bus { + "-" => Ok(NumaNode { pxb_bus: None }), + bus => bus.parse().map(|bus| NumaNode { pxb_bus: Some(bus) }), + }) .collect::>()?; // Devices appended after the root ports: `e` endpoint, `r` root port. let extra_root_devices = optional::(&mut args, String::new())? diff --git a/dstack/crates/qemu-acpi/fixtures/README.md b/dstack/crates/qemu-acpi/fixtures/README.md index e6fb7aa49..2b2d11942 100644 --- a/dstack/crates/qemu-acpi/fixtures/README.md +++ b/dstack/crates/qemu-acpi/fixtures/README.md @@ -55,6 +55,10 @@ for each compatibility family cover the Cartesian product of: whose PXBs sit on buses 5, 6 and 8 (12 vCPUs, 6 GiB, two NICs), together with their loader and RSDP blobs. +`qemu-11.1-q35-hotplug-off-numa2-plain-*.bin` capture two guest NUMA nodes +without PXBs (8 vCPUs, 2 GiB, one NIC, two GPU root ports and one extra +endpoint on the root bus). + `qemu-11.1-q35-one-nic.bin` is an untrimmed byte-for-byte regression fixture. Fixture names identify the emulated compatibility version, not necessarily the source tree's release number. diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-base.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-base.bin new file mode 100644 index 0000000000000000000000000000000000000000..9e3f1f418c55cba227e59a06fcb8572e8c339189 GIT binary patch literal 9536 zcmbVSO>7&-8J*$pYDr3zL|gt*BE?D36fF|UvYnv6B9}j6B+4{bj#JP8Da&dlCqRfK zI=DdqTP_;MrzV1f9u-5m=hSNh^we8!4bVe#=_Qv2Xo0wwvhSO3hn!hbMpOdg&Nuts z_syH#owwXorQrEX;{X4Y{L*^&86l)7UiOy#`QmF0Zyq0sl+>newidfCT6aptYFnlv zG1__`M@;#!feG3QEAMLB% z+?S<2-46A9lIAW-wQ%@)ZY5qW`Sr}frTtr?I~kRtb8zM(9>5nb{pBD2#oyFFocX$b z`p;hduyc@nxZ_3+J6-qg{f-dmnBBkSCc9Jdh{%WzX5kGXHo6jjil%QI}UQev9t&|0~CE67+m8O0-f}pB{D1v1%m5HT$Jyd2~5D^Q&ao(qR zfa;K#$gi*w#RN8ph^e6wp_qUwCPHFjGzuG4Og)0AAf|>!nF*+3A|$4+5>re)f|z1z zXpEV9@CZ$vaZ6`hF*P*KOg(snrp|<=GohFony_^uG<7B|ok>e)($qKbk9J6$eSvtpTod`{xuBFqpbh@@qgr?4vr88yeOxZdSnmW^#&a|a7ZRdaU= zGnUSbtrMZCbKKH7Zs{Dibs{u%PFOl8ES(d!PK2h;tfez+>CDEr)(xdllin_ z>NWmp#nfy3)6CQhH$q}+Cz-Qq&RI3*?3xI6WC_=+P4JHiKV3L6vOfpb|2?I5lFi?X@ z25K*k3=|;_ zgAR$Q=N1NvNY5<{6p@}=GEjtiZplCq;%EDrHY`*er1#a2)kFG0`bmN|h1_)QioY%o z#@==BY>Pv8?9ff+F4V4g%`w<+Ge0@*ranlBG4<>Bws=5&ZXDmW?z`LWgrF7@HS?{# z)g*krpdTKjC-{+pgVO%wNq72#aWNK!LkXzSawA0~!VJIE?R2=a7)r-|Bhw9CH#s{U zZa0R~ao@;vL)XCxJ00#phSG80$aF*J^+`J&ZcK*Kao@;vL+9y?oep;^L+QBBPM6KK zs|}vHOu=i^^7;HFk*(k2$Ji@l*I$fMlODe|nq^;~o-VFRJjc#>jjQv^Z@$xgWB1qd zZ*9H(=I&dYTW^b&^ct%x!D*Ut-KzXD{bgFFZ{sOD7SeT@`ZDdNTX&zQpdjYGMtMzo zwKZRQ&HAg-E5DMbuZk2=&#RZ5*4-&WMm@j0IuCfMus+|~Tkc$cfyRb0@VEj23wf%{ zkK5z9a>84${4#Rj?r)3Vd62h5O-ap#orBc=wi}`Ld2UCky~yntwJ&fxPVI}_PEh+2 zx0BRl3G=A)&P`!{7*-0aFu7cTSo#Bn~Dc^K}+k>33DVEn`V4Y#+y<@QkT)9xtH!E-}X zN#|2{^F9rFL+ox~cq2e`z7YtDw#4ohK)ajtf~OqgH$23?AvWn8+61t*Fj~&g6r(ua z3L#MuiMH^PgevGnZuC25!RU5xj6PKLG4Ha!o<8DGjXvXsMu|!9?u?kut{VyW67*KM z6NhKy=vw!Fj_V?iH3%QP(!sUi9MTnnE3;l$pG>)qz*7sl7tW32Mc8_Z#E9*kTUf9$V(%cG?J}Rvh#QZMfEa+(a?QmKU*ZCvZedk z+z8BDkIjvQ^DW}dPB(RMUGhZ~E?*blIh4bOgub(d#3+IcO`C>Ys zY~53OFVathkY3!%_4CE!;e2s3Up$=e5*}@Z^T7rB`4Wk6zJ!@C5zhA#9(aZG;byC! zFPRMIOPcwT{d~Ez@Bx@uyPQ8S0>2_pprtHZm}HaS*Rk{(@|BeP$NL!(!P6vsdh{i$ z@^SicS~}>LX?0w`#qP)Hew@MUMj`&C-e2-@$FHwmkA8j8CnBsjn|JVL^Y6SzbgVt! z^m>2#FYoz${J{_LHZ;fvU<5%fLXHsN7gpC@#vN;aO2lh0?D=dP0vW|!t@=39FyKc83;^>(57YB$0I;@h|JAfy{dnf!01FnO;)k(iHz|YZrORae& z;&$A0){7}=t-C*dF-qbHNqZ&ze#US8D@FS9=&R(2d2GtmuFzMNzLuTV-i4K72~)uT z9}va5K{JV8`gDe#a!*ucM|6b5iFs4p#P(c79slR*m#_9&6S!SASdJS%NAJ;FdyC%X z`hw0K$BKzNHGCV`Np~EleDrZtIL~IqDI$HZx$gB-r*LR#Bw068g|&Bubj}G$unE6X z2o7CDU|y)B7<5F@QNc~*zj5+JpYZ<{baE}>PYOJL)*oJ7L@d}+w}bY)?vIEV{rKrk z^zdMnKeVEM`dj3|rVj> zgFZv>WX)gkbsyNCaM-isN!CwHF38j`Do!Z?@`(ikK!r@iD#oFX5hxC~za%j!C$$)dN=#t~pb$tu zC4d>InOyrlfqET)xIhs|kO@HHN3Q+k>bnATFALPY17ZJ-0k+>piGhIw>VHb&e<#o% z8-Vx=nQ;%&Mv47vf&N$m#2W_4{#a!Oh7ceQU?krG6!~8T=zk!#A>Vq6!v7`E?gv1u zL4o}Pk^gj52R8p346yws18l#=0NZbaWIrYOPY`Gh2M}9OEB%w${vfyh1GQUNpl(L- fKZ*9k>JSkM{6DI7Gz3ONU^E0qLtr!n1|kFi&GxPS literal 0 HcmV?d00001 diff --git a/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-rsdp.bin b/dstack/crates/qemu-acpi/fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-rsdp.bin new file mode 100644 index 0000000000000000000000000000000000000000..6128229d5f6b03fa69406ffecb3d06f5ff6808e3 GIT binary patch literal 20 bcmWFvc2Nij2~uEi@^|(KR$$;zWncgRF|Y&N literal 0 HcmV?d00001 diff --git a/dstack/crates/qemu-acpi/scripts/differential-qemu.sh b/dstack/crates/qemu-acpi/scripts/differential-qemu.sh index 07bce4053..1b34e0c34 100755 --- a/dstack/crates/qemu-acpi/scripts/differential-qemu.sh +++ b/dstack/crates/qemu-acpi/scripts/differential-qemu.sh @@ -24,17 +24,19 @@ rust_dump="$workspace/target/debug/examples/dump" run_case() { local version=$1 topology=$2 - # pxb_buses: one guest NUMA node per bus, GPUs split evenly across them. - local hugepages=0 gpus=0 pxb_buses=() + # pxb_buses: one guest NUMA node per bus, GPUs split evenly across them; + # plain_nodes: guest NUMA nodes without expanders, GPUs on the root bus. + local hugepages=0 gpus=0 pxb_buses=() plain_nodes=0 hotplug_off=0 case "$topology" in normal) ;; numa) hugepages=1 ;; numa-pxb) hugepages=1; gpus=8; pxb_buses=(5) ;; numa2-pxb) hugepages=1; gpus=8; pxb_buses=(5 10) ;; numa4-pxb-adjacent) hugepages=1; gpus=4; pxb_buses=(5 6 8 9) ;; + numa2-plain) gpus=4; plain_nodes=2; hotplug_off=1 ;; *) echo "invalid internal topology: $topology" >&2; exit 2 ;; esac - local nodes=${#pxb_buses[@]} + local nodes=$((${#pxb_buses[@]} + plain_nodes)) ((hugepages && nodes == 0)) && nodes=1 local output="$tmp/$version-$topology" @@ -58,26 +60,30 @@ run_case() { -numa "node,nodeid=$node,cpus=$((node * 8 / nodes))-$(((node + 1) * 8 / nodes - 1)),memdev=mem$node" -object "memory-backend-file,id=mem$node,size=$((2048 / nodes))M,mem-path=/dev/hugepages,share=on,prealloc=no" ) - if ((gpus)); then + if ((gpus && !plain_nodes)); then args+=(-device "pxb-pcie,id=pcie.node$node,bus=pcie.0,addr=$((10 + node)),numa_node=$node,bus_nr=${pxb_buses[$node]}") fi done if ((gpus)); then args+=(-object "iommufd,id=iommufd0") for ((index = 0; index < gpus; index++)); do + local bus=pcie.0 + ((plain_nodes)) || bus=pcie.node$((index * nodes / gpus)) args+=( - -device "pcie-root-port,id=pci.$index,bus=pcie.node$((index * nodes / gpus)),chassis=$index" + -device "pcie-root-port,id=pci.$index,bus=$bus,chassis=$index" -device "vfio-pci,host=00:00.0,bus=pci.$index,iommufd=iommufd0" ) done fi + ((hotplug_off)) && args+=(-global ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off) QEMU_ACPI_COMPAT_VER="$version" QEMU_ACPI_DUMP_DIR="$output" \ "$qemu" "${args[@]}" -machine "$machine" local numa="" ((nodes > 1)) && numa=$(IFS=,; echo "${pxb_buses[*]}") + ((plain_nodes)) && numa=$(printf -- '-,%.0s' $(seq "$plain_nodes")) QEMU_ACPI_OUTPUT_DIR="$output/rust" \ - "$rust_dump" 2 8 "$version" "$gpus" 0 "$hugepages" 1 0 0 0 $((2048 << 20)) 0 0 "$numa" + "$rust_dump" 2 8 "$version" "$gpus" 0 "$hugepages" 1 "$hotplug_off" 0 0 $((2048 << 20)) 0 0 "$numa" for blob in tables loader rsdp; do cmp "$output/$blob.bin" "$output/rust/$blob.bin" done @@ -85,7 +91,7 @@ run_case() { } for version in 8.0.0 8.2.0 9.0.0 9.1.0 9.2.0 10.0.0 10.2.0 11.0.0 11.1.0 11.2.0; do - for topology in normal numa numa-pxb numa2-pxb numa4-pxb-adjacent; do + for topology in normal numa numa-pxb numa2-pxb numa4-pxb-adjacent numa2-plain; do run_case "$version" "$topology" done done diff --git a/dstack/crates/qemu-acpi/scripts/differential-random.py b/dstack/crates/qemu-acpi/scripts/differential-random.py index 9f688936b..405248c2a 100755 --- a/dstack/crates/qemu-acpi/scripts/differential-random.py +++ b/dstack/crates/qemu-acpi/scripts/differential-random.py @@ -39,7 +39,8 @@ class Case: smm: bool = False pic: bool = False pci_hole64_size: int = 0 - # PXB bus of each guest NUMA node; empty keeps the layout `hugepages` implies. + # PXB bus of each guest NUMA node, None for a node without one; empty + # keeps the layout `hugepages` implies. pxb_buses: tuple = () # Devices a QEMU wrapper appends to the root bus: "e" endpoint, "r" root port. extra: str = "" @@ -51,7 +52,7 @@ def numa_nodes(self): def pxbs(self): """PXB bus of each node that has one, explicit or implied.""" if self.pxb_buses: - return self.pxb_buses + return tuple(bus for bus in self.pxb_buses if bus is not None) return (5,) if self.hugepages and self.gpus else () @@ -106,6 +107,28 @@ def fixed_cases(): pxb_buses=(5, 7), extra="eree", ), + # Guest NUMA nodes without expanders: SRAT and _PXM only. + Case(cpus=2, pxb_buses=(None, None)), + Case(cpus=12, memory_mib=6144, nics=2, pxb_buses=(None,) * 3), + Case( + cpus=192, + memory_mib=8192, + gpus=4, + hotplug_off=True, + pci_hole64_size=1 << 43, + pxb_buses=(None, None), + extra="eeeee", + ), + Case( + version="9.2.0", + cpus=4, + hugepages=True, + gpus=2, + switches=1, + hotplug_off=True, + pxb_buses=(None, None), + extra="er", + ), ] return cases @@ -114,11 +137,14 @@ def random_case(rng): hugepages = rng.choice([False, True]) gpus = rng.randint(0, 8) switches = rng.randint(0, 4) if gpus else 0 - nodes = rng.choice([0, 0, 2, 3, 4]) if hugepages else 0 + nodes = rng.choice([0, 0, 2, 3, 4]) pxb_buses = [] - for _ in range(nodes): - pxb_buses.append(pxb_buses[-1] + rng.randint(1, 6) if pxb_buses else 5) - pxbs = len(pxb_buses) or int(hugepages and gpus > 0) + if hugepages: + for _ in range(nodes): + pxb_buses.append(pxb_buses[-1] + rng.randint(1, 6) if pxb_buses else 5) + else: + pxb_buses = [None] * nodes + pxbs = sum(bus is not None for bus in pxb_buses) or int(hugepages and gpus > 0) pxb = pxbs > 0 available = 26 - pxbs passthrough = switches if pxb else gpus + switches @@ -365,7 +391,7 @@ def run_case(case, image, rust_dump, output): str(case.memory_mib * 1024 * 1024), str(case.volumes), str(int(case.pic)), - ",".join(map(str, case.pxb_buses)), + ",".join("-" if bus is None else str(bus) for bus in case.pxb_buses), case.extra, ], check=True, diff --git a/dstack/crates/qemu-acpi/src/golden_tests.rs b/dstack/crates/qemu-acpi/src/golden_tests.rs index 2fb64fb16..0fe2f6448 100644 --- a/dstack/crates/qemu-acpi/src/golden_tests.rs +++ b/dstack/crates/qemu-acpi/src/golden_tests.rs @@ -89,6 +89,31 @@ mod tests { Ok(()) } + /// Two nodes without expanders: GPUs and a wrapper's endpoint stay on + /// the root bus, and only the SRAT and the CPUs' `_PXM` carry the nodes. + #[test] + fn numa_nodes_without_pxb_match_qemu_byte_for_byte() -> Result<(), Error> { + let mut c = config(1, 0); + c.cpu_count = 8; + c.numa_nodes = vec![NumaNode { pxb_bus: None }; 2]; + c.num_gpus = 2; + c.extra_root_devices = vec![RootBusDevice::Endpoint]; + c.hotplug_off = true; + let actual = build(&c)?; + let expected = include_bytes!("../fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-base.bin"); + assert_eq!(&actual.tables[..expected.len()], expected); + assert!(actual.tables[expected.len()..].iter().all(|b| *b == 0)); + assert_eq!( + actual.loader, + include_bytes!("../fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-loader.bin") + ); + assert_eq!( + actual.rsdp, + include_bytes!("../fixtures/qemu-11.1-q35-hotplug-off-numa2-plain-rsdp.bin") + ); + Ok(()) + } + #[test] fn numa_layouts_qemu_cannot_build_are_rejected() { let nodes = |buses: &[u8]| { @@ -107,6 +132,7 @@ mod tests { 2, vec![NumaNode { pxb_bus: None }, NumaNode { pxb_bus: Some(5) }], ), + (3, vec![NumaNode { pxb_bus: None }; 2]), ] { c.cpu_count = cpus; c.numa_nodes = buses; diff --git a/dstack/crates/qemu-acpi/src/topology.rs b/dstack/crates/qemu-acpi/src/topology.rs index 8e2a66f63..40f74ca63 100644 --- a/dstack/crates/qemu-acpi/src/topology.rs +++ b/dstack/crates/qemu-acpi/src/topology.rs @@ -89,8 +89,8 @@ pub enum TopologyError { }, #[error("{0} NUMA nodes exceed QEMU's limit of 128")] TooManyNumaNodes(usize), - #[error("every explicit NUMA node needs a PXB expander")] - NumaNodeWithoutPxb, + #[error("NUMA nodes must either all have a PXB expander or none")] + MixedPxbNumaNodes, #[error("{0} PXB expanders do not fit on the root bus")] TooManyPxbs(usize), #[error("PXB bus numbers must be nonzero and distinct: {0:?}")] @@ -125,8 +125,13 @@ impl MachineConfig { return Err(TopologyError::TooManyNumaNodes(nodes)); } // Node `i`'s expander sits on slot 0x10 + i only if every node has one. - if self.numa_nodes.iter().any(|node| node.pxb_bus.is_none()) { - return Err(TopologyError::NumaNodeWithoutPxb); + let pxbs = self + .numa_nodes + .iter() + .filter(|n| n.pxb_bus.is_some()) + .count(); + if pxbs != 0 && pxbs != nodes { + return Err(TopologyError::MixedPxbNumaNodes); } for (what, count) in [ ("cpu_count", u64::from(self.cpu_count)), diff --git a/dstack/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs index 104d6cf21..4f67c5182 100644 --- a/dstack/dstack-mr/cli/src/main.rs +++ b/dstack/dstack-mr/cli/src/main.rs @@ -66,6 +66,11 @@ struct MachineConfig { #[arg(long, value_delimiter = ',')] numa_pxb_buses: Vec, + /// Number of guest NUMA nodes without PXB expanders (every device on the + /// root bus) + #[arg(long, conflicts_with = "numa_pxb_buses")] + numa_nodes: Option, + /// Number of GPUs #[arg(long, default_value = "0")] num_gpus: u32, @@ -132,11 +137,14 @@ fn main() -> Result<()> { // The image declares its OVMF layout. Older metadata.json files // predate the field, so fall back to the only layout that existed. let ovmf_variant = image_info.ovmf_variant.unwrap_or_default(); - let numa_nodes: Vec<_> = config - .numa_pxb_buses - .iter() - .map(|&bus| dstack_types::NumaNodeConfig { pxb_bus: bus }) - .collect(); + let numa_nodes = match config.numa_nodes { + Some(nodes) => vec![dstack_types::NumaNodeConfig { pxb_bus: None }; nodes], + None => config + .numa_pxb_buses + .iter() + .map(|&bus| dstack_types::NumaNodeConfig { pxb_bus: Some(bus) }) + .collect(), + }; let machine = Machine::builder() .cpu_count(config.cpu) diff --git a/dstack/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs index facbdefc8..15d026ea2 100644 --- a/dstack/dstack-mr/src/acpi.rs +++ b/dstack/dstack-mr/src/acpi.rs @@ -40,7 +40,7 @@ impl Machine<'_> { .numa_nodes .iter() .map(|node| NumaNode { - pxb_bus: Some(node.pxb_bus), + pxb_bus: node.pxb_bus, }) .collect(), num_gpus: self.num_gpus, diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index a513d6401..0e52e878e 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -1427,8 +1427,10 @@ fn is_false(value: &bool) -> bool { /// node order. #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] pub struct NumaNodeConfig { - /// Bus number of the node's `pxb-pcie` expander (its `bus_nr`). - pub pxb_bus: u8, + /// Bus number of the node's `pxb-pcie` expander (its `bus_nr`). Either + /// every node has one or none does, leaving all devices on the root bus. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pxb_bus: Option, } /// A device a host QEMU wrapper appends to the root PCIe bus. @@ -2656,6 +2658,26 @@ mod vm_config_device_count_tests { ); } + #[test] + fn numa_nodes_keep_their_legacy_serialization() { + let mut json = legacy_json(); + json["numa_nodes"] = serde_json::json!([{ "pxb_bus": 5 }, { "pxb_bus": 10 }]); + let cfg: VmConfig = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(cfg.numa_nodes[1].pxb_bus, Some(10)); + assert_eq!( + serde_json::to_value(&cfg).unwrap()["numa_nodes"], + json["numa_nodes"] + ); + + json["numa_nodes"] = serde_json::json!([{}, {}]); + let cfg: VmConfig = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(cfg.numa_nodes[0].pxb_bus, None); + assert_eq!( + serde_json::to_value(&cfg).unwrap()["numa_nodes"], + json["numa_nodes"] + ); + } + #[test] fn swtpm_is_serialized_only_when_enabled() { let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 5baf6aeae..b43710e7b 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -2213,7 +2213,7 @@ fn make_vm_config( } else { pxb_buses .into_iter() - .map(|pxb_bus| dstack_types::NumaNodeConfig { pxb_bus }) + .map(|bus| dstack_types::NumaNodeConfig { pxb_bus: Some(bus) }) .collect() }; // Each resolved network interface becomes one virtio-net-pci device in the