From ae399703469b07eae379a5f9c26f718c2dafa877 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:07:04 +0000 Subject: [PATCH 01/33] macOS guests 6/6: experimental native image builds Opt-in native macOS image builds, off by default. POST /builds takes an installed, digest-pinned darwin/arm64 base and a source tarball containing build.sh. The script runs as root in an isolated VZ guest; the guest is strictly stopped, its closed storage exported, and the result published to this server's registry as an OCI machine image. Derivatives of a base in the same registry publish a delta layer of changed 64 KiB disk chunks. Mac hosts pull the image on instance create. Linux builds are unchanged. Requires macos_only and macos_builds, with a loopback or system-trusted registry URL. Base publication tooling is a separate change. --- cmd/api/api/builds.go | 17 +- cmd/api/api/capabilities.go | 36 +- cmd/api/api/capabilities_machine_test.go | 34 + cmd/api/api/instances.go | 8 +- cmd/api/api/macos.go | 18 +- cmd/api/api/macos_builds.go | 125 +++ cmd/api/api/macos_builds_test.go | 226 +++++ cmd/api/api/macos_test.go | 38 + cmd/api/config/config.go | 24 +- cmd/api/config/macos_builds_test.go | 26 + cmd/api/main.go | 13 +- config.example.darwin.yaml | 8 + docs/macos-builds.md | 125 +++ docs/macos-support.md | 2 +- lib/builds/macos_build.go | 300 +++++++ lib/builds/macos_build_cancel.go | 70 ++ lib/builds/macos_build_failure_test.go | 198 +++++ lib/builds/macos_build_guest.go | 185 ++++ lib/builds/macos_build_job_test.go | 37 + lib/builds/macos_build_layer_test.go | 70 ++ lib/builds/macos_build_lifecycle_test.go | 135 +++ lib/builds/macos_build_manager.go | 199 +++++ lib/builds/macos_build_manager_test.go | 228 +++++ lib/builds/macos_build_policy_test.go | 44 + lib/builds/macos_build_publisher.go | 194 +++++ lib/builds/macos_build_publisher_test.go | 96 +++ lib/builds/macos_build_recovery_test.go | 88 ++ lib/builds/macos_build_source.go | 119 +++ lib/builds/macos_build_source_test.go | 67 ++ lib/builds/macos_build_test.go | 226 +++++ lib/builds/macos_delta_publish.go | 172 ++++ lib/builds/macos_delta_publish_test.go | 202 +++++ lib/builds/macos_registry.go | 105 +++ lib/builds/macos_registry_test.go | 54 ++ lib/builds/manager.go | 105 ++- lib/builds/source.go | 75 ++ lib/builds/source_test.go | 36 + lib/builds/storage.go | 8 +- lib/builds/types.go | 7 + lib/images/macos_bundle.go | 15 + lib/images/macos_delta.go | 261 ++++++ lib/images/macos_delta_test.go | 203 +++++ lib/images/macos_import_darwin.go | 8 +- lib/images/macos_unpack.go | 23 +- lib/instances/create.go | 3 + lib/instances/create_image.go | 16 +- lib/instances/create_image_test.go | 50 ++ lib/instances/delete.go | 2 +- lib/instances/fork.go | 2 +- lib/instances/macos.go | 15 +- lib/instances/macos_agent_startup.go | 2 +- lib/instances/macos_build_export.go | 5 + lib/instances/macos_build_export_darwin.go | 133 +++ .../macos_build_export_darwin_test.go | 53 ++ lib/instances/macos_identity.go | 2 +- lib/instances/manager.go | 2 +- lib/instances/process_identity_linux_test.go | 8 +- lib/instances/restart_policy.go | 2 +- lib/instances/stop.go | 32 +- lib/middleware/oapi_auth.go | 24 + lib/oapi/oapi.go | 812 +++++++++--------- lib/providers/providers.go | 23 +- lib/registry/registry.go | 27 + lib/registrypush/macos_test.go | 48 ++ openapi.yaml | 18 + 65 files changed, 5043 insertions(+), 466 deletions(-) create mode 100644 cmd/api/api/capabilities_machine_test.go create mode 100644 cmd/api/api/macos_builds.go create mode 100644 cmd/api/api/macos_builds_test.go create mode 100644 cmd/api/config/macos_builds_test.go create mode 100644 docs/macos-builds.md create mode 100644 lib/builds/macos_build.go create mode 100644 lib/builds/macos_build_cancel.go create mode 100644 lib/builds/macos_build_failure_test.go create mode 100644 lib/builds/macos_build_guest.go create mode 100644 lib/builds/macos_build_job_test.go create mode 100644 lib/builds/macos_build_layer_test.go create mode 100644 lib/builds/macos_build_lifecycle_test.go create mode 100644 lib/builds/macos_build_manager.go create mode 100644 lib/builds/macos_build_manager_test.go create mode 100644 lib/builds/macos_build_policy_test.go create mode 100644 lib/builds/macos_build_publisher.go create mode 100644 lib/builds/macos_build_publisher_test.go create mode 100644 lib/builds/macos_build_recovery_test.go create mode 100644 lib/builds/macos_build_source.go create mode 100644 lib/builds/macos_build_source_test.go create mode 100644 lib/builds/macos_build_test.go create mode 100644 lib/builds/macos_delta_publish.go create mode 100644 lib/builds/macos_delta_publish_test.go create mode 100644 lib/builds/macos_registry.go create mode 100644 lib/builds/macos_registry_test.go create mode 100644 lib/builds/source.go create mode 100644 lib/builds/source_test.go create mode 100644 lib/images/macos_delta.go create mode 100644 lib/images/macos_delta_test.go create mode 100644 lib/instances/macos_build_export.go create mode 100644 lib/instances/macos_build_export_darwin.go create mode 100644 lib/instances/macos_build_export_darwin_test.go create mode 100644 lib/registrypush/macos_test.go diff --git a/cmd/api/api/builds.go b/cmd/api/api/builds.go index ae8e74c27..2f4174b3b 100644 --- a/cmd/api/api/builds.go +++ b/cmd/api/api/builds.go @@ -44,9 +44,15 @@ func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequ // CreateBuild creates a new build job func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) { if s.Config != nil && s.Config.MacOSOnly { - return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + if !s.Config.MacOSBuilds { + return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + } + req, source, err := parseMachineBuildMultipart(request.Body) + if err != nil { + return oapi.CreateBuild400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil + } + return s.submitBuild(ctx, req, source) } - log := logger.FromContext(ctx) // Parse multipart form fields var sourceData []byte @@ -69,6 +75,8 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } switch part.FormName() { + case "machine_base_image", "machine_network_mode": + return oapi.CreateBuild400JSONResponse{Code: "unsupported", Message: "machine builds require macos_only and macos_builds"}, nil case "source": sourceData, err = io.ReadAll(part) if err != nil { @@ -256,6 +264,11 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } } + return s.submitBuild(ctx, domainReq, sourceData) +} + +func (s *ApiService) submitBuild(ctx context.Context, domainReq builds.CreateBuildRequest, sourceData []byte) (oapi.CreateBuildResponseObject, error) { + log := logger.FromContext(ctx) build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData) if err != nil { switch { diff --git a/cmd/api/api/capabilities.go b/cmd/api/api/capabilities.go index e19ca8035..63c82e31f 100644 --- a/cmd/api/api/capabilities.go +++ b/cmd/api/api/capabilities.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" @@ -26,6 +25,8 @@ const ( featureLogs = "logs" featureDevices = "devices" featureRosettaEmulation = "rosetta-emulation" + featureMacOSImages = "macos-machine-images" + featureMacOSBuilds = "macos-machine-builds-v1" ) // apiVersion is the API contract version from the embedded OpenAPI document. @@ -133,13 +134,11 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities }, Runtimes: runtimes, Network: *networkCaps, - Images: oapi.CapabilitiesImages{ - Platforms: imagePlatforms(runtime.GOARCH, emulation), - DefaultPlatform: images.HostPlatformString(), - }, + Images: hostImageCapabilities(runtime.GOOS, runtime.GOARCH, emulation, s.Config.MacOSOnly), Features: serverFeatures(runtime.GOOS, emulation), } + resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH, s.Config.MacOSBuilds)...) return oapi.GetCapabilities200JSONResponse(resp), nil } @@ -210,6 +209,33 @@ func imagePlatforms(goarch string, emulation bool) []string { return platforms } +func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi.CapabilitiesImages { + caps := oapi.CapabilitiesImages{ + Platforms: imagePlatforms(goarch, emulation), + DefaultPlatform: "linux/" + goarch, + } + if goos == "darwin" && goarch == "arm64" { + if macOSOnly { + caps.Platforms = []string{"darwin/arm64"} + caps.DefaultPlatform = "darwin/arm64" + } else { + caps.Platforms = append(caps.Platforms, "darwin/arm64") + } + } + return caps +} + +func machineImageFeatures(goos, goarch string, builds bool) []string { + if goos != "darwin" || goarch != "arm64" { + return nil + } + features := []string{featureMacOSImages} + if builds { + features = append(features, featureMacOSBuilds) + } + return features +} + // serverFeatures builds the server-level feature ID list: API surfaces that // are always present plus host-platform conditionals. func serverFeatures(goos string, emulation bool) []string { diff --git a/cmd/api/api/capabilities_machine_test.go b/cmd/api/api/capabilities_machine_test.go new file mode 100644 index 000000000..705eabf86 --- /dev/null +++ b/cmd/api/api/capabilities_machine_test.go @@ -0,0 +1,34 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHostMachineImageCapabilities(t *testing.T) { + for _, tc := range []struct { + name, os, arch, defaultPlatform string + emulation, macOSOnly bool + platforms []string + }{ + {"Linux unchanged", "linux", "amd64", "linux/amd64", false, false, []string{"linux/amd64"}}, + {"Linux arm unchanged", "linux", "arm64", "linux/arm64", false, false, []string{"linux/arm64"}}, + {"Mac mixed", "darwin", "arm64", "linux/arm64", false, false, []string{"linux/arm64", "darwin/arm64"}}, + {"Mac mixed Rosetta", "darwin", "arm64", "linux/arm64", true, false, []string{"linux/arm64", "linux/amd64", "darwin/arm64"}}, + {"Mac only", "darwin", "arm64", "darwin/arm64", true, true, []string{"darwin/arm64"}}, + } { + t.Run(tc.name, func(t *testing.T) { + caps := hostImageCapabilities(tc.os, tc.arch, tc.emulation, tc.macOSOnly) + require.Equal(t, tc.defaultPlatform, caps.DefaultPlatform) + require.Equal(t, tc.platforms, caps.Platforms) + }) + } +} + +func TestMachineImageFeatureActivation(t *testing.T) { + require.Empty(t, machineImageFeatures("linux", "arm64", true)) + require.Empty(t, machineImageFeatures("darwin", "amd64", true)) + require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64", false)) + require.Equal(t, []string{"macos-machine-images", "macos-machine-builds-v1"}, machineImageFeatures("darwin", "arm64", true)) +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d5aa47fcf..ced753723 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -63,7 +63,11 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image) + platform := derefString(request.Body.Platform) + if s.Config != nil && s.Config.MacOSOnly && platform == "" { + platform = "darwin/arm64" + } + macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform) if err != nil { log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{Code: "internal_error", Message: "failed to look up image"}, nil @@ -337,7 +341,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Name: request.Body.Name, Image: request.Body.Image, Size: size, - Platform: derefString(request.Body.Platform), + Platform: platform, HotplugSize: hotplugSize, OverlaySize: overlaySize, Vcpus: vcpus, diff --git a/cmd/api/api/macos.go b/cmd/api/api/macos.go index 5dc42cbdb..62e6652ce 100644 --- a/cmd/api/api/macos.go +++ b/cmd/api/api/macos.go @@ -16,11 +16,19 @@ import ( // macOSHost is true where macOS guests can exist. Tests override it. var macOSHost = runtime.GOOS == "darwin" -// isMacOSImage reports whether a create names an imported macOS image. Other -// hosts skip the lookup, so Linux creates are unchanged. An image that is not -// cached yet is not a macOS image; any other lookup error is returned. -func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error) { - if !macOSHost || s.ImageManager == nil { +// isMacOSImage identifies macOS creates before Linux defaults are applied. An +// explicit Darwin platform and macOS-only servers can resolve uncached images. +func (s *ApiService) isMacOSImage(ctx context.Context, name, platform string) (bool, error) { + if !macOSHost { + return false, nil + } + if s.Config != nil && s.Config.MacOSOnly { + return true, nil + } + if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { + return true, nil + } + if s.ImageManager == nil { return false, nil } img, err := s.ImageManager.GetImage(ctx, name) diff --git a/cmd/api/api/macos_builds.go b/cmd/api/api/macos_builds.go new file mode 100644 index 000000000..9ec28c825 --- /dev/null +++ b/cmd/api/api/macos_builds.go @@ -0,0 +1,125 @@ +package api + +import ( + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/scopes" +) + +// maxConcurrentMachineBuildUploads bounds how many machine build bodies, each up +// to 65 MiB, are read into memory at once. +const maxConcurrentMachineBuildUploads = 4 + +// LimitMachineBuildBody must wrap the validator, which also consumes request bodies. +// A build upload is authenticated before it takes an upload slot, so unauthenticated +// requests cannot hold the slots. +func LimitMachineBuildBody(jwtSecret string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + uploads := make(chan struct{}, maxConcurrentMachineBuildUploads) + upload := mw.JwtAuth(jwtSecret)(scopes.RequireScope(scopes.BuildWrite)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case uploads <- struct{}{}: + defer func() { <-uploads }() + default: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"code":"too_many_requests","message":"too many machine build uploads in progress"}`)) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 65<<20) + next.ServeHTTP(w, r) + }))) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/builds" { + upload.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// Machine-only HTTP admission is bounded regardless of multipart field order. +// Linux retains its existing parser and source contract. +func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildRequest, []byte, error) { + req := builds.CreateBuildRequest{BuildPolicy: &builds.BuildPolicy{}} + policy := req.BuildPolicy + numbers := map[string]*int{"timeout_seconds": &policy.TimeoutSeconds, "memory_mb": &policy.MemoryMB, "cpus": &policy.CPUs} + var source []byte + seen := map[string]bool{} + if body == nil { + return req, nil, fmt.Errorf("multipart source is required") + } + for { + part, err := body.NextPart() + if err == io.EOF { + break + } + if err != nil { + return req, nil, fmt.Errorf("invalid multipart form") + } + name := part.FormName() + if seen[name] { + return req, nil, fmt.Errorf("duplicate multipart field") + } + seen[name] = true + limit := int64(64 << 10) + switch name { + case "source": + limit = 64 << 20 + case "machine_base_image", "machine_network_mode", "timeout_seconds", "memory_mb", "cpus", "tags": + default: + return req, nil, fmt.Errorf("unsupported machine build field: %s", name) + } + data, err := io.ReadAll(io.LimitReader(part, limit+1)) + if err != nil || int64(len(data)) > limit { + return req, nil, fmt.Errorf("invalid or oversized %s field", name) + } + if err = part.Close(); err != nil { + return req, nil, fmt.Errorf("invalid multipart field") + } + switch name { + case "source": + source = data + case "machine_base_image": + req.MachineBaseImage = string(data) + case "machine_network_mode": + if string(data) != "isolated" && string(data) != "egress" { + return req, nil, fmt.Errorf("machine_network_mode must be isolated or egress") + } + req.BuildPolicy.NetworkMode = string(data) + case "tags": + req.Tags, err = parseTagsJSON(string(data)) + if err != nil { + return req, nil, fmt.Errorf("tags must be a JSON object of strings") + } + default: + value, err := strconv.Atoi(string(data)) + if err != nil || value < 0 { + return req, nil, fmt.Errorf("%s must be a nonnegative integer", name) + } + *numbers[name] = value + } + } + if len(source) == 0 { + return req, nil, fmt.Errorf("source is required") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !builds.ValidMachineDigest(ref.Digest()) { + return req, nil, fmt.Errorf("machine_base_image must be pinned by sha256 digest") + } + if req.BuildPolicy.TimeoutSeconds > 86400 { + return req, nil, fmt.Errorf("timeout_seconds exceeds 86400") + } + if err = req.BuildPolicy.Validate(); err != nil { + return req, nil, err + } + return req, source, nil +} diff --git a/cmd/api/api/macos_builds_test.go b/cmd/api/api/macos_builds_test.go new file mode 100644 index 000000000..2fe07942f --- /dev/null +++ b/cmd/api/api/macos_builds_test.go @@ -0,0 +1,226 @@ +package api + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +func machineMultipart(t *testing.T, fields ...[2]string) *multipart.Reader { + t.Helper() + var data bytes.Buffer + writer := multipart.NewWriter(&data) + for _, field := range fields { + require.NoError(t, writer.WriteField(field[0], field[1])) + } + require.NoError(t, writer.Close()) + return multipart.NewReader(&data, writer.Boundary()) +} + +type machineZeroReader struct{} + +func (machineZeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil } + +func TestMachineBuildBodyBoundBeforeValidation(t *testing.T) { + for _, path := range []string{"/builds", "/images"} { + r := authorizedMachineBuildRequest(t, path, io.LimitReader(machineZeroReader{}, (65<<20)+1)) + LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n, err := io.Copy(io.Discard, r.Body) + if path == "/builds" { + require.Error(t, err) + require.EqualValues(t, 65<<20, n) + } else { + require.NoError(t, err) + require.EqualValues(t, (65<<20)+1, n) + } + })).ServeHTTP(httptest.NewRecorder(), r) + } +} + +func TestMachineBuildUploadsAreBounded(t *testing.T) { + release := make(chan struct{}) + entered := make(chan struct{}, maxConcurrentMachineBuildUploads) + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entered <- struct{}{} + <-release + })) + done := make(chan struct{}) + for range maxConcurrentMachineBuildUploads { + go func() { + handler.ServeHTTP(httptest.NewRecorder(), authorizedMachineBuildRequest(t, "/builds", nil)) + done <- struct{}{} + }() + <-entered + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusTooManyRequests, w.Code) + + close(release) + for range maxConcurrentMachineBuildUploads { + <-done + } + w = httptest.NewRecorder() + go func() { <-entered }() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "a slot frees once an upload finishes") +} + +func TestMachineBuildMultipartAdmission(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + valid := [][2]string{{"source", "synthetic"}, {"machine_base_image", base}} + for _, tc := range []struct { + name string + extra [][2]string + ok bool + }{ + {name: "inherit", ok: true}, + {name: "policy", extra: [][2]string{{"cpus", "4"}, {"memory_mb", "8192"}, {"timeout_seconds", "1200"}}, ok: true}, + {name: "isolated network", extra: [][2]string{{"machine_network_mode", "isolated"}}, ok: true}, + {name: "egress network", extra: [][2]string{{"machine_network_mode", "egress"}}, ok: true}, + {name: "invalid network", extra: [][2]string{{"machine_network_mode", "bridge"}}}, + {name: "empty network", extra: [][2]string{{"machine_network_mode", ""}}}, + {name: "duplicate", extra: [][2]string{{"source", "second"}}}, + {name: "linux", extra: [][2]string{{"dockerfile", "FROM scratch"}}}, + {name: "secret", extra: [][2]string{{"secrets", "[]"}}}, + {name: "invalid number", extra: [][2]string{{"cpus", "nan"}}}, + {name: "negative", extra: [][2]string{{"memory_mb", "-1"}}}, + {name: "timeout", extra: [][2]string{{"timeout_seconds", "86401"}}}, + {name: "metadata bound", extra: [][2]string{{"tags", strings.Repeat("x", (64<<10)+1)}}}, + } { + t.Run(tc.name, func(t *testing.T) { + req, source, err := parseMachineBuildMultipart(machineMultipart(t, append(append([][2]string{}, valid...), tc.extra...)...)) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, base, req.MachineBaseImage) + require.Equal(t, []byte("synthetic"), source) + if tc.name == "egress network" { + require.Equal(t, "egress", req.BuildPolicy.NetworkMode) + } + }) + } + _, _, err := parseMachineBuildMultipart(machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", "example.test/base:latest"})) + require.Error(t, err) +} + +type machineHTTPManager struct { + builds.Manager + req builds.CreateBuildRequest + calls int +} + +func (m *machineHTTPManager) CreateBuild(_ context.Context, req builds.CreateBuildRequest, source []byte) (*builds.Build, error) { + m.calls++ + m.req = req + return &builds.Build{ID: "synthetic-build", Status: builds.StatusQueued}, nil +} + +func TestMachineBuildHTTPGateAndDispatch(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + for _, tc := range []struct { + name string + cfg *config.Config + accepted bool + }{ + {"disabled", &config.Config{MacOSOnly: true}, false}, + {"linux", &config.Config{}, false}, + {"enabled", &config.Config{MacOSOnly: true, MacOSBuilds: true}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := &machineHTTPManager{} + service := &ApiService{Config: tc.cfg, BuildManager: manager} + response, err := service.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{Body: machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", base})}) + require.NoError(t, err) + if tc.accepted { + require.IsType(t, oapi.CreateBuild202JSONResponse{}, response) + require.Equal(t, 1, manager.calls) + require.Equal(t, base, manager.req.MachineBaseImage) + } else { + // Disabled machine builds are unsupported (501) on macOS-only servers, and + // a Linux server rejects the machine selector as a bad request (400). + if tc.cfg.MacOSOnly { + require.IsType(t, oapi.CreateBuild501JSONResponse{}, response) + } else { + require.IsType(t, oapi.CreateBuild400JSONResponse{}, response) + } + require.Zero(t, manager.calls) + } + }) + } +} + +const machineBuildTestSecret = "machine-build-test-secret" + +func signedMachineBuildToken(t *testing.T, claims jwt.MapClaims) string { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(machineBuildTestSecret)) + require.NoError(t, err) + return token +} + +func authorizedMachineBuildRequest(t *testing.T, path string, body io.Reader) *http.Request { + t.Helper() + token := signedMachineBuildToken(t, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + }) + r := httptest.NewRequest(http.MethodPost, path, body) + r.Header.Set("Authorization", "Bearer "+token) + return r +} + +func TestMachineBuildUploadsRequireAuthBeforeTakingASlot(t *testing.T) { + readOnly := signedMachineBuildToken(t, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + "permissions": []string{"build:read"}, + }) + for _, tc := range []struct { + name string + header string + status int + }{ + {"unauthenticated", "", http.StatusUnauthorized}, + {"read-only token", "Bearer " + readOnly, http.StatusForbidden}, + } { + t.Run(tc.name, func(t *testing.T) { + reached := false + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + reached = true + })) + for range maxConcurrentMachineBuildUploads + 1 { + r := httptest.NewRequest(http.MethodPost, "/builds", nil) + if tc.header != "" { + r.Header.Set("Authorization", tc.header) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + require.Equal(t, tc.status, w.Code) + } + require.False(t, reached, "rejected uploads must not reach processing") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "rejected requests hold no upload slot") + require.True(t, reached) + }) + } +} diff --git a/cmd/api/api/macos_test.go b/cmd/api/api/macos_test.go index ce19cce6b..fc19ca81f 100644 --- a/cmd/api/api/macos_test.go +++ b/cmd/api/api/macos_test.go @@ -84,6 +84,38 @@ func TestLinuxCreateDefaultsMatchSchema(t *testing.T) { require.EqualValues(t, linuxDefaultVcpus, props["vcpus"].Value.Default) } +func TestMacOSCreateDefaultsToDarwinPlatform(t *testing.T) { + useMacOSHost(t) + darwin := "darwin/arm64" + for _, tc := range []struct { + name string + cfg *config.Config + images images.Manager + body string + platform *string + }{ + {"macOS-only server", &config.Config{MacOSOnly: true}, nil, `{"name":"macos","image":"registry.example/macos/app:latest"}`, nil}, + {"uncached darwin image", &config.Config{}, missingMacOSImageManager{}, `{"name":"macos","image":"registry.example/macos/app:latest","platform":"darwin/arm64"}`, &darwin}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := newCaptureCreateManager(nil) + service := &ApiService{InstanceManager: manager, ImageManager: tc.images, Config: tc.cfg} + ctx, status := recordCreateBody(t, tc.body) + require.Equal(t, http.StatusOK, status) + size, overlay, vcpus := linuxDefaultSize, linuxDefaultOverlaySize, linuxDefaultVcpus + _, err := service.CreateInstance(ctx, oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{ + Name: "macos", Image: "registry.example/macos/app:latest", Platform: tc.platform, + Size: &size, OverlaySize: &overlay, Vcpus: &vcpus, + }}) + require.NoError(t, err) + require.Equal(t, "darwin/arm64", manager.lastReq.Platform) + require.Zero(t, manager.lastReq.Size) + require.Zero(t, manager.lastReq.OverlaySize) + require.Zero(t, manager.lastReq.Vcpus) + }) + } +} + func TestMacOSCreateIgnoresInjectedLinuxDefaults(t *testing.T) { useMacOSHost(t) m := newCaptureCreateManager(nil) @@ -172,3 +204,9 @@ type macOSImageManager struct{ images.Manager } func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) { return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil } + +type missingMacOSImageManager struct{ images.Manager } + +func (missingMacOSImageManager) GetImage(context.Context, string) (*images.Image, error) { + return nil, images.ErrNotFound +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 0bffb655b..544a5d494 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -285,7 +285,8 @@ type Config struct { JwtSecret string `koanf:"jwt_secret"` Env string `koanf:"env"` Version string `koanf:"version"` - MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. + MacOSBuilds bool `koanf:"macos_builds"` // Experimental: enable machine builds on the macOS-only server. + MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. Network NetworkConfig `koanf:"network"` Caddy CaddyConfig `koanf:"caddy"` @@ -606,6 +607,9 @@ func (c *Config) Validate() error { if err := c.validateAPIListenPorts(); err != nil { return err } + if err := c.validateMacOSBuilds(); err != nil { + return err + } if strings.TrimSpace(c.Metrics.ListenAddress) == "" { return fmt.Errorf("metrics.listen_address must not be empty") } @@ -852,3 +856,21 @@ func (c *Config) validateFirecrackerUFFDGraduation() error { func intPtr(v int) *int { return &v } + +// validateMacOSBuilds checks the experimental machine-build gate. The publisher +// validates the registry URL itself, since the base publication tool shares it. +func (c *Config) validateMacOSBuilds() error { + if !c.MacOSBuilds { + return nil + } + if !c.MacOSOnly { + return fmt.Errorf("macos_builds requires macos_only") + } + if c.Registry.URL == "" { + return fmt.Errorf("macos_builds requires an explicit registry URL") + } + if c.Registry.CACertFile != "" || c.Registry.Insecure { + return fmt.Errorf("macos_builds requires system-trusted registry TLS or loopback HTTP") + } + return nil +} diff --git a/cmd/api/config/macos_builds_test.go b/cmd/api/config/macos_builds_test.go new file mode 100644 index 000000000..6a4cd08d9 --- /dev/null +++ b/cmd/api/config/macos_builds_test.go @@ -0,0 +1,26 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildsFailClosedConfiguration(t *testing.T) { + require.False(t, (&Config{}).MacOSBuilds) + c := *defaultConfig() + c.MacOSBuilds = true + require.ErrorContains(t, c.validateMacOSBuilds(), "requires macos_only") + c.MacOSOnly = true + c.Hypervisor.Default = "vz" + c.Registry.URL = "" + require.ErrorContains(t, c.validateMacOSBuilds(), "explicit registry URL") + c.Registry.URL = "http://localhost:4973" + c.Registry.CACertFile = "custom.pem" + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.CACertFile = "" + c.Registry.Insecure = true + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.Insecure = false + require.NoError(t, c.validateMacOSBuilds()) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 347b395ff..1c8241323 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -535,6 +535,11 @@ func run() error { r.Use(api.RecordCreateFields) } + // Bound machine multipart input before OpenAPI validation reads the body. + if app.Config.MacOSBuilds { + r.Use(api.LimitMachineBuildBody(app.Config.JwtSecret)) + } + // OpenAPI request validation with authentication validatorOptions := &nethttpmiddleware.Options{ Options: openapi3filter.Options{ @@ -632,10 +637,10 @@ func run() error { logger.Error("failed to start builders manager", "error", err) return err } - if err := app.BuildManager.Start(gctx); err != nil { - logger.Error("failed to start build manager", "error", err) - return err - } + } + if err := app.BuildManager.Start(gctx); err != nil { + logger.Error("failed to start build manager", "error", err) + return err } grp.Go(func() error { diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index e7745fb1a..d838b9887 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -27,6 +27,14 @@ data_dir: ~/Library/Application Support/hypeman # Server configuration port: "4973" +# Native macOS guests/builds are opt-in; these defaults retain Linux guests on VZ. +macos_only: false +macos_builds: false +# For a dedicated native Mac builder, set both true after the release gates in +# docs/macos-builds.md pass, and replace registry.url below with an explicit +# system-trusted HTTPS address reachable by the build and destination hosts. +# Machine publication runs on the host, not at the Linux guest NAT gateway. + # ============================================================================= # Hypervisor Configuration (IMPORTANT FOR MACOS) # ============================================================================= diff --git a/docs/macos-builds.md b/docs/macos-builds.md new file mode 100644 index 000000000..81f2eb217 --- /dev/null +++ b/docs/macos-builds.md @@ -0,0 +1,125 @@ +# macOS image builds (experimental, opt-in) + +A build job can provision a digest-pinned installed macOS base in an isolated VZ +VM, run the source's `build.sh` in it, and publish the result as an OCI machine +image. This is not macOS installation, Linux rootfs conversion, memory forking or +identity rekeying. It depends on the shared system GuestService. + +The normal build manager owns the queue, persisted request, deadline, source +staging, status, provenance and the image-readiness gate. `GuestMachineBuildDriver` +runs the guest and `StreamingMachinePublisher` writes the registry. + +## Enabling + +Off by default. Set `macos_only: true` and `macos_builds: true` on an Apple-silicon +VZ server, with `registry.url` set to an address the host can reach (not a guest +gateway), including its scheme: `http://localhost:4973` for loopback, or a +system-trusted `https://` endpoint. Credentials, paths, queries, fragments, custom +CAs and insecure HTTPS are rejected. Linux servers reject the machine selector. + +```sh +curl -H "Authorization: Bearer $TOKEN" \ + -F "machine_base_image=registry.example/base@sha256:$BASE_DIGEST" \ + -F "source=@source.tar.gz;type=application/gzip" \ + -F "timeout_seconds=1200" "$API/builds" +``` + +The base must already be Ready. A 202 queues an ordinary job; status, cancellation +and provenance use the shared `/builds` APIs. Only `source`, `machine_base_image`, +`machine_network_mode`, `timeout_seconds`, `memory_mb`, `cpus` and `tags` are +accepted. Duplicate or unknown fields, malformed numbers and mutable bases return +400. Limits: 64 KiB per metadata field, 64 MiB source, 65 MiB total body. Linux +source parsing is unchanged. + +## Using the result + +`GET /builds/{id}` returns a digest-pinned `image_ref` for instance create. Other +Mac hosts need `registry.url` reachable from them; loopback only works on the build +host. A host that hasn't cached the image pulls on create, which can exceed the +readiness wait for large images, so retry or pre-pull. Distribute by digest with +`POST /pushes`. Consumers should pin the digest, not a tag. + +## Base and policy + +The base must be pinned by canonical SHA256, be `darwin/arm64`, and declare a +provisioned system agent on port 2222. The driver also checks guest UID 0 and never +installs an agent or falls back to host SSH. CPU and memory match the base exactly; +zero request values inherit them. Timeout defaults to 600 s, capped at 24 h. + +Networking defaults to isolated. `machine_network_mode=egress` gives unrestricted +VZ NAT for checksum-verified downloads; it is not a domain allowlist. Linux builder, +cache, build-arg, secret and admin options are rejected. + +Jobs on the same base digest are serialized. Identity is preserved, so concurrent +guests with the same identity are never allowed. + +## Source + +The source tar.gz is staged privately and SHA256-verified, including on recovery, +and `provenance.source_hash` pins exactly what ran. Input is capped at 64 MiB +compressed, 128 MiB uncompressed, 1024 entries and 64 MiB per file. Only regular +files and directories are accepted; links, devices, duplicates and path escapes fail +before transfer. The archive must contain a non-empty `build.sh` at its root. It is +copied to `/var/root/hypeman-build-{id}` in the guest and run as root with +`/bin/sh build.sh`. + +Before stopping, the guest's build directory is removed and its absence verified, +and the host copy is removed. No registry token, host credential or secret is +injected. Guest stdout and stderr are discarded, so build logs never contain guest +output. Failures report the phase and, for the script, its exit code. + +## Stop, export and cleanup + +Publication is two instance operations. `StopMacOSBuildInstance` does a strict +graceful stop and rejects forced fallback, an instance already Stopped, non-macOS +guests, and any stop without shutdown acknowledgement and confirmed VMM exit. +`ExportStoppedMacOSBuildInstance` then copies the closed storage of a still-Stopped +instance under the instance lock, so a concurrent Start makes export fail. The output +is a private 0700 directory holding only `disk.img`, `aux.img` and `config.json`. + +Failure cleanup uses a forced stop and exports nothing. Storage is deleted only after +confirmed VMM exit; a VM with no confirmed exit stays quarantined for an operator. +Cleanup has its own 2-minute budget. A re-enqueued attempt first reaps the builder +left by the interrupted attempt, and fails rather than start a second one if that +delete fails. Builders orphaned outside this path have no general reaper. + +The builder is destroyed **before** publication. Source stays staged until a terminal +status is persisted, so a job interrupted while pushing is rebuilt on recovery. +Startup on a macOS-only server recovers machine jobs without preparing Linux builder +state, reaps cancelled builders and removes terminal-job source. Cancellation +interrupts the running job and keeps Cancelled terminal. Guest errors that may carry +secrets are masked in strings and logs. + +## Publication + +The publisher uses a server-configured registry and a token scoped to `builds/{id}`. +Plaintext is allowed only on loopback; redirects are never followed and proxies are +disabled. The layer is never written to disk: a first pass computes its digest, size +and DiffID, the upload regenerates the same bytes, and the registry rejects any +mismatch. Blobs use monolithic POST uploads, every persisted blob is re-read and +hashed, the manifest is committed last, and the digest content is verified. Ambiguous +errors never return success. + +A derivative built from a base in the same registry publishes a delta: the base's +layers are mounted by digest and only a layer of changed 64 KiB disk chunks is +uploaded. See `lib/images/macos_delta.go`. Bases outside the registry publish +standalone. There is no garbage-collection rule yet; deleting a base that a derivative +references breaks a later re-pull of that derivative. + +## Status and gates + +Race tests cover admission, the shared queue, cancellation without publication, phase +failures and quarantine, export validation, and authenticated publication with +manifest-last ordering. Test disks are synthetic and prove nothing about bootability. +A development base has been run end to end through the HTTP API on Apple silicon, +including a cold boot of the result; this trimmed branch has not been re-run natively. + +Before production use: a reviewed fresh base with approved artifact pins (base +publication tooling is a separate PR), cross-host publish and pull with sequential +cold boot, disk-space monitoring (this change has no disk admission, so run machine +builds serially on a dedicated host), SDK and CLI release, and Apple licence review. +Keep `macos_builds` off until then. + +**Rollback:** stop new submissions, let or cancel running jobs through the normal +APIs, repoint workloads to the prior validated digest, then set `macos_builds: false`. +Disabling does not delete persisted jobs or images. diff --git a/docs/macos-support.md b/docs/macos-support.md index 2ee42b466..13afe299e 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -14,7 +14,7 @@ owns the behavior. | Stats | Partial | CPU and memory from the VM process. Network counters are 0 under NAT. | PR 1 | | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 | -| Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 6 | +| Builds | Partial | Opt-in `macos_builds`. Runs the source's build.sh as root in the guest. Linux Dockerfile builds unsupported. | PR 6 | | Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | | Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | | Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go new file mode 100644 index 000000000..6eb8e9ef9 --- /dev/null +++ b/lib/builds/macos_build.go @@ -0,0 +1,300 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// MachineBuildRequest is the internal backend contract for one machine build. +// The normal build manager owns ID, source staging and publish authorization. A concrete driver must require a provisioned system agent. +type MachineBuildRequest struct { + ID string + BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. + SourceHash string // Optional expected lowercase SHA256 of the source archive. + Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. + onStarted func(string) error // Normal manager records the owned builder before provisioning. + onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. + deltaBase *MachineDeltaBase // Set when the base is in the publisher's registry; publishes a derivative. +} + +type MachineStopReceipt = instances.MacOSBuildExportReceipt +type MachinePublication struct{ Reference, Digest string } +type MachineBuildResult struct { + Publication MachinePublication + Provenance BuildProvenance +} + +type MachineBuildDriver interface { + // Start owns an isolated instance and identity lease, named name. A partially + // created session must be returned even with an error so cleanup can stop it. + Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) +} + +type MachineBuildSession interface { + ID() string + Provision(context.Context, string) error // Build-owned staged source, never a client host path. + Sanitize(context.Context) error // Remove build credentials, source and secret injection artifacts. + // Stop must await actual VMM exit. Only a strict graceful stop may produce a + // receipt that permits export. With force, a failed strict stop falls back to + // a forced stop, which is failure-only cleanup: nothing is exported after it. + Stop(ctx context.Context, force bool) (MachineStopReceipt, error) + Export(context.Context, string) error // Stopped disk.img, aux.img, config.json only. + // Destroy removes instance storage only after proven VMM exit, not the export. + Destroy(context.Context) error +} + +type MachineBuildPublisher interface { + // Publisher owns registry address, scoped token and builds/{id} destination. + // It must stream blobs and commit/verify the manifest last. An ambiguous + // network error is not a verified successful publication. + Publish(context.Context, string, string) (MachinePublication, error) +} + +type MachineBuildBackend struct { + Driver MachineBuildDriver + Publisher MachineBuildPublisher +} + +// machinePhaseError keeps potentially secret-bearing guest/command errors out of +// ordinary log/error text, while retaining errors.Is/As for internal handling. +type machinePhaseError struct { + phase string + cause error +} + +func (e *machinePhaseError) Error() string { + if exit := (*machineExitError)(nil); errors.As(e.cause, &exit) { + return "machine build " + e.phase + " failed: " + exit.Error() + } + // Host admission errors, such as a template identity already in use, carry + // no guest output. + if errors.Is(e.cause, instances.ErrInsufficientResources) || errors.Is(e.cause, instances.ErrInvalidState) { + return "machine build " + e.phase + " failed: " + e.cause.Error() + } + return "machine build " + e.phase + " failed" +} +func (e *machinePhaseError) Unwrap() error { return e.cause } +func machineFailure(phase string, err error) error { return &machinePhaseError{phase, err} } + +// machineExitError is the entrypoint's nonzero exit code. It holds no guest +// output, so it is safe to show. +type machineExitError struct{ code int } + +func (e *machineExitError) Error() string { + return fmt.Sprintf("%s exited with code %d", machineEntrypoint, e.code) +} + +// normalizeMachinePolicy applies machine build policy defaults and rejects what +// machine builds do not support. Admission, the runner and the guest driver share it. +func normalizeMachinePolicy(policy *BuildPolicy) error { + if policy.TimeoutSeconds == 0 { + policy.TimeoutSeconds = 600 + } + if policy.TimeoutSeconds < 1 || policy.TimeoutSeconds > 86400 { + return fmt.Errorf("invalid machine build timeout") + } + if policy.NetworkMode == "" { + policy.NetworkMode = "isolated" + } + if len(policy.AllowedDomains) != 0 || (policy.NetworkMode != "isolated" && policy.NetworkMode != "egress") { + return fmt.Errorf("unsupported machine network policy") + } + return nil +} + +// runPrepared executes only machine-specific phases. The shared build manager +// owns resolution, source staging/hash verification, deadline, queue and status. +func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath, workDir string) (result *MachineBuildResult, err error) { + if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(workDir) || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { + return nil, fmt.Errorf("machine driver, publisher, private workspace and verified source required") + } + if _, bounded := ctx.Deadline(); !bounded { + return nil, fmt.Errorf("prepared machine job requires a deadline") + } + if len(req.ID) == 0 || len(req.ID) > 128 || strings.Trim(req.ID, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" { + return nil, fmt.Errorf("invalid machine build ID") + } + ref, e := images.ParseNormalizedRef(req.BaseImage) + if e != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := req.Policy + if e = normalizeMachinePolicy(&policy); e != nil { + return nil, e + } + if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || !base.MacOS.GuestAgent || base.Digest != ref.Digest() || base.MacOS.CPUs < 2 || base.MacOS.CPUs > MaxBuildCPUs || base.MacOS.Memory < 4<<30 || base.MacOS.Memory > uint64(MaxBuildMemoryMB)<<20 || base.MacOS.Memory%(1<<20) != 0 { + return nil, fmt.Errorf("base is not the pinned ready macOS machine") + } + if base.MacOS.Validate() != nil { + return nil, fmt.Errorf("base has invalid machine identity") + } + memory := int(base.MacOS.Memory >> 20) + cpus := int(base.MacOS.CPUs) + if (policy.MemoryMB != 0 && policy.MemoryMB != memory) || (policy.CPUs != 0 && policy.CPUs != cpus) { + return nil, fmt.Errorf("machine resources must match installed base") + } + policy.MemoryMB = memory + policy.CPUs = cpus + if e = policy.Validate(); e != nil { + return nil, e + } + workspace, e := os.MkdirTemp(workDir, "machine-build-") + if e != nil { + return nil, machineFailure("workspace", e) + } + defer func() { + if cleanupErr := os.RemoveAll(workspace); cleanupErr != nil { + err = errors.Join(err, machineFailure("cleanup_workspace", cleanupErr)) + result = nil + } + }() + if e = verifyBuildSource(ctx, sourcePath, req.SourceHash); e != nil { + return nil, machineFailure("verify_source", e) + } + if e = ctx.Err(); e != nil { + return nil, e + } + session, e := r.Driver.Start(ctx, machineBuilderName(req.ID), base, policy) + if session == nil { + if e == nil { + e = fmt.Errorf("driver returned no session") + } + return nil, machineFailure("start", e) + } + stopped, destroyed := false, false + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), machineCleanupTimeout) + defer cancel() + if !stopped { + receipt, stopErr := session.Stop(cleanup, true) + stopped = receipt.VMMExited + if !stopped { + err = errors.Join(err, machineFailure("quarantine_unconfirmed_vmm_exit", stopErr)) + result = nil + return + } + } + if !destroyed { + if destroyErr := session.Destroy(cleanup); destroyErr != nil { + err = errors.Join(err, machineFailure("cleanup", destroyErr)) + result = nil + } + } + }() + instanceID := session.ID() + if req.onStarted != nil { + if recordErr := req.onStarted(instanceID); recordErr != nil { + return nil, machineFailure("record_builder", recordErr) + } + } + if e != nil { + return nil, machineFailure("start", e) + } + if e = session.Provision(ctx, sourcePath); e != nil { + return nil, machineFailure("provision", e) + } + if e = session.Sanitize(ctx); e != nil { + return nil, machineFailure("sanitize", e) + } + receipt, e := session.Stop(ctx, false) + stopped = receipt.VMMExited + if e != nil || !receipt.Graceful || !stopped { + return nil, machineFailure("graceful_stop", e) + } + bundle := filepath.Join(workspace, "bundle") + if e = os.Mkdir(bundle, 0700); e != nil { + return nil, machineFailure("export", e) + } + if e = session.Export(ctx, bundle); e != nil { + return nil, machineFailure("export", e) + } + if e = validateMachineExport(bundle, base.MacOS); e != nil { + return nil, machineFailure("validate_export", e) + } + if e = session.Destroy(ctx); e != nil { + return nil, machineFailure("cleanup", e) + } + destroyed = true + if e = ctx.Err(); e != nil { + return nil, e + } + publication, e := r.publishBuild(ctx, req, bundle) + if e != nil { + return nil, machineFailure("publish", e) + } + output, e := images.ParseNormalizedRef(publication.Reference) + if e != nil || !output.IsDigest() || !ValidMachineDigest(publication.Digest) || output.Digest() != publication.Digest { + return nil, fmt.Errorf("machine publication receipt is unverified") + } + if req.onPublished != nil { + if e = req.onPublished(ctx, publication); e != nil { + return nil, machineFailure("image_ready", e) + } + } + provenance := BuildProvenance{BaseImageDigest: base.Digest, SourceHash: req.SourceHash, Timestamp: time.Now().UTC()} + return &MachineBuildResult{Publication: publication, Provenance: provenance}, nil +} + +// machineCleanupTimeout covers a strict graceful stop attempt (30s power-off +// budget) plus forced fallback and storage deletion. Export is outside cleanup. +const machineCleanupTimeout = 2 * time.Minute + +// ValidMachineDigest reports whether s is a canonical lowercase sha256 digest. +// It is the single validation rule for machine base and output digests. +func ValidMachineDigest(s string) bool { + if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { + return false + } + return strings.Trim(s[7:], "0123456789abcdef") == "" +} + +func validateMachineExport(root string, base *images.MacOSImage) error { + directory, err := os.Lstat(root) + if err != nil || !directory.IsDir() { + return fmt.Errorf("export root must be a real directory") + } + entries, err := os.ReadDir(root) + if err != nil { + return err + } + if len(entries) != 3 { + return fmt.Errorf("export must contain only machine payload files") + } + for _, entry := range entries { + if entry.Name() != "disk.img" && entry.Name() != "aux.img" && entry.Name() != "config.json" { + return fmt.Errorf("unexpected export file") + } + info, err := os.Lstat(filepath.Join(root, entry.Name())) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("export files must be regular") + } + } + platform, err := images.ValidateMacOSBundle(root) + if err != nil { + return err + } + if base != nil && (!bytes.Equal(platform.HardwareModel, base.HardwareModel) || !bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || !strings.EqualFold(platform.MAC, base.MAC) || platform.CPUs != base.CPUs || platform.Memory != base.Memory) { + return fmt.Errorf("machine export changed preserved identity/resources") + } + for _, entry := range entries { + if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { + return err + } + } + return nil +} + +// machineBuilderName names a build's builder instance after the build, so a retry +// can find and reap a builder that an interrupted attempt created but never recorded. +func machineBuilderName(buildID string) string { + return "macos-build-" + strings.ToLower(buildID) +} diff --git a/lib/builds/macos_build_cancel.go b/lib/builds/macos_build_cancel.go new file mode 100644 index 000000000..09b984000 --- /dev/null +++ b/lib/builds/macos_build_cancel.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "fmt" + "sync" + "time" +) + +type machineBuildControl struct { + mu sync.Mutex + cancel context.CancelFunc +} + +func (m *manager) recordMachineBuilder(id, instanceID string) error { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status == StatusCancelled { + return context.Canceled + } + meta.BuilderInstance = &instanceID + return writeMetadata(m.paths, meta) +} + +func (m *manager) cancelQueuedMachineBuild(meta *buildMetadata) error { + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + m.removeMachineSource(meta.ID) + m.notifyStatusChange(meta.ID, StatusCancelled) + return nil +} + +func (m *manager) cancelMachineBuild(id string) error { + active, ok := m.machineBuilds.Load(id) + if !ok { + return ErrBuildInProgress + } + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status != StatusBuilding && meta.Status != StatusPushing { + return fmt.Errorf("machine build no longer running") + } + // Persist cancellation before interrupting phases. The backend performs bounded + // graceful cleanup or quarantine; Cancel must not force-delete mutable storage. + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + control.cancel() + m.notifyStatusChange(id, StatusCancelled) + return nil +} diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go new file mode 100644 index 000000000..c0e4e6e0f --- /dev/null +++ b/lib/builds/macos_build_failure_test.go @@ -0,0 +1,198 @@ +package builds + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func runMachineBuildFailure(t *testing.T, f *machineFixture, ctx context.Context, req MachineBuildRequest, source io.Reader) (*machineRunnerFixture, error) { + t.Helper() + runner := machineTestRunner(t, f) + result, err := runner.Run(ctx, req, source) + if err == nil || result != nil { + t.Fatal("failed build returned a result") + } + return runner, err +} + +type failingMachineSource struct{} + +func (failingMachineSource) Read(p []byte) (int, error) { + return copy(p, "partial"), errors.New("secret-source-error") +} + +type cancelMachineSource struct{ cancel context.CancelFunc } + +func (s cancelMachineSource) Read(p []byte) (int, error) { + s.cancel() + return copy(p, "source"), io.EOF +} + +func TestMachineBuildSourceFailureDoesNotStart(t *testing.T) { + for _, tc := range []struct { + name string + source func(context.CancelFunc) io.Reader + wantCancel bool + }{ + {name: "read failure", source: func(context.CancelFunc) io.Reader { return failingMachineSource{} }}, + {name: "canceled", source: func(cancel context.CancelFunc) io.Reader { return cancelMachineSource{cancel} }, wantCancel: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + r, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), tc.source(cancel)) + if tc.wantCancel { + require.ErrorIs(t, err, context.Canceled) + } else { + require.NotContains(t, err.Error(), "secret-source-error", "source failure was not masked") + } + require.Empty(t, f.calls, "source failure started guest") + entries, err := os.ReadDir(r.WorkDir) + require.NoError(t, err) + require.Empty(t, entries, "partial source retained") + }) + } +} + +// Cancellation and timeout both interrupt a running build. Cleanup must still +// run to completion on a live context, and nothing may publish. +func TestMachineBuildInterruptedCleanup(t *testing.T) { + assertCleanup := func(t *testing.T, f *machineFixture, err, want error) { + t.Helper() + require.ErrorIs(t, err, want) + require.False(t, f.publisherCalled, "interrupted build published") + require.True(t, f.cleanupContextLive, "cleanup ran on a cancelled context") + require.Equal(t, []string{"start", "abort", "destroy"}, f.calls) + } + t.Run("cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + f.cancel = cancel + _, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), strings.NewReader("source")) + assertCleanup(t, f, err, context.Canceled) + }) + t.Run("timeout", func(t *testing.T) { + f := newMachineFixture() + f.waitForCancel = true + req := machineTestRequest() + req.Policy.TimeoutSeconds = 1 + parent, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := runMachineBuildFailure(t, f, parent, req, strings.NewReader("source")) + assertCleanup(t, f, err, context.DeadlineExceeded) + }) +} + +func TestMachineBuildPublicationReceipt(t *testing.T) { + for _, kind := range []string{"ambiguous error", "tag receipt", "digest mismatch", "invalid digest"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + switch kind { + case "ambiguous error": + f.fail = "publish" + case "tag receipt": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test:latest", Digest: machineTestDigest} + case "digest mismatch": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "sha256:" + strings.Repeat("b", 64)} + case "invalid digest": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "not-a-digest"} + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if !f.publisherCalled { + t.Fatal("unverified receipt became ready") + } + if strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("publisher error leaked") + } + count := 0 + for _, phase := range f.calls { + if phase == "publish" { + count++ + } + } + if count != 1 { + t.Fatal("ambiguous publication was retried") + } + }) + } +} + +func TestMachineBuildMalformedExport(t *testing.T) { + for _, kind := range []string{"empty disk", "empty aux", "symlink disk", "hardlink payloads", "truncated config", "oversized config", "resource change", "bad MAC", "identity", "extra file", "unknown metadata"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + f.mutateExport = func(root string) error { + disk, aux, config := filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"), filepath.Join(root, "config.json") + switch kind { + case "empty disk": + return os.Truncate(disk, 0) + case "empty aux": + return os.Truncate(aux, 0) + case "symlink disk": + if err := os.Remove(disk); err != nil { + return err + } + return os.Symlink(aux, disk) + case "hardlink payloads": + if err := os.Remove(aux); err != nil { + return err + } + return os.Link(disk, aux) + case "truncated config": + return os.WriteFile(config, []byte(`{"hardware_model":`), 0600) + case "oversized config": + return os.WriteFile(config, []byte(strings.Repeat(" ", 65<<10)), 0600) + case "resource change": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), `"cpus":4`, `"cpus":2`, 1)), 0600) + case "bad MAC": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:01", "02:00:00:00:00:01:02:03", 1)), 0600) + case "identity", "unknown metadata": + var raw map[string]any + data, err := os.ReadFile(config) + if err == nil { + err = json.Unmarshal(data, &raw) + } + if err != nil { + return err + } + if kind == "identity" { + raw["machine_identifier"] = []byte{9} + } else { + raw["password"] = "must-not-publish" + } + data, _ = json.Marshal(raw) + return os.WriteFile(config, data, 0600) + case "extra file": + return os.WriteFile(filepath.Join(root, "secret.txt"), []byte("must-not-publish"), 0600) + } + return nil + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("malformed export published") + } + if strings.Contains(err.Error(), "must-not-publish") { + t.Fatal("export secret leaked in failure") + } + }) + } +} diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go new file mode 100644 index 000000000..ebf341318 --- /dev/null +++ b/lib/builds/macos_build_guest.go @@ -0,0 +1,185 @@ +package builds + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +type machineExportManager interface { + StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) + ExportStoppedMacOSBuildInstance(context.Context, string, string) error +} + +// GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system +// GuestService. It never installs an agent or silently upgrades an unmanaged base. +// The backend validates the base and policy before Start. +type GuestMachineBuildDriver struct { + Instances instances.Manager + WorkDir string +} + +func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) { + exporter, ok := d.Instances.(machineExportManager) + if !ok || !filepath.IsAbs(d.WorkDir) { + return nil, fmt.Errorf("strict machine export and private workspace required") + } + if _, ok = ctx.Deadline(); !ok { + return nil, fmt.Errorf("machine start requires a deadline") + } + ref, err := images.ParseNormalizedRef(base.Name) + if err != nil { + return nil, err + } + // Workspaces are named per builder, so one left by an interrupted attempt + // of this job is removed here. + prefix := "guest-machine-" + name + "-" + stale, _ := filepath.Glob(filepath.Join(d.WorkDir, prefix+"*")) + for _, dir := range stale { + _ = os.RemoveAll(dir) + } + dir, err := os.MkdirTemp(d.WorkDir, prefix) + if err != nil { + return nil, err + } + inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) + if inst == nil { + _ = os.RemoveAll(dir) + if err == nil { + err = fmt.Errorf("instance creation returned no instance") + } + return nil, err + } + session := &guestMachineSession{manager: d.Instances, exporter: exporter, instance: inst, workspace: dir, guestDir: "/var/root/hypeman-build-" + inst.Id} + return session, err +} + +type guestMachineSession struct { + manager instances.Manager + exporter machineExportManager + instance *instances.Instance + workspace, guestDir string + receipt MachineStopReceipt + sanitized bool +} + +func (s *guestMachineSession) ID() string { return s.instance.Id } +func (s *guestMachineSession) dialer() (hypervisor.VsockDialer, error) { + return hypervisor.NewVsockDialer(s.instance.HypervisorType, s.instance.VsockSocket, s.instance.VsockCID) +} +func (s *guestMachineSession) exec(ctx context.Context, command []string, cwd string, env map[string]string, wait time.Duration) error { + dialer, err := s.dialer() + if err != nil { + return err + } + // Drain output without retaining or publishing it. Redaction cannot reliably + // make arbitrary guest output safe: even successful commands may print secrets. + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{Command: command, Cwd: cwd, Env: env, WaitForAgent: wait, Stdout: io.Discard, Stderr: io.Discard}) + if err != nil { + return err + } + if exit == nil { + return fmt.Errorf("machine guest command did not succeed") + } + if exit.Code != 0 { + return &machineExitError{code: exit.Code} + } + return nil +} + +func (s *guestMachineSession) Provision(ctx context.Context, source string) error { + root := filepath.Join(s.workspace, "source") + if err := extractMachineSource(ctx, source, root); err != nil { + return err + } + err := s.exec(ctx, []string{"/bin/sh", "-c", `test "$(/usr/bin/id -u)" -eq 0`}, "", nil, 30*time.Second) + if err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 30*time.Second); err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/mkdir", "-m", "700", s.guestDir}, "", nil, 0); err != nil { + return err + } + dialer, err := s.dialer() + if err != nil { + return err + } + if err = guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{SrcPath: root, DstPath: s.guestDir}); err != nil { + return err + } + return s.exec(ctx, []string{"/bin/sh", machineEntrypoint}, s.guestDir, nil, 0) +} + +func (s *guestMachineSession) Sanitize(ctx context.Context) error { + if err := s.exec(ctx, []string{"/bin/rm", "-rf", "--", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(s.workspace, "source")); err != nil { + return err + } + s.sanitized = true + return nil +} + +// Stop is the publication path: only an acknowledged graceful power-off with +// confirmed VMM exit may produce a receipt that permits export. With force, a +// failed strict stop falls back to the ordinary forced hypervisor stop. Nothing +// from a failed build is exported, and Destroy still requires confirmed VMM exit, +// so a non-exited instance remains quarantined. +func (s *guestMachineSession) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { + if s.receipt.VMMExited { + return s.receipt, nil + } + receipt, err := s.exporter.StopMacOSBuildInstance(ctx, s.ID()) + if !force { + s.receipt = receipt + return s.receipt, err + } + if err == nil && receipt.VMMExited { + s.receipt = receipt + return s.receipt, nil + } + if inst, err := s.manager.GetInstance(ctx, s.ID()); err != nil || inst.State != instances.StateStopped { + inst, err = s.manager.StopInstance(ctx, s.ID()) + if err != nil { + return s.receipt, err + } + if inst == nil || inst.State != instances.StateStopped { + return s.receipt, fmt.Errorf("forced stop did not confirm VMM exit") + } + } + s.receipt = MachineStopReceipt{VMMExited: true} + return s.receipt, nil +} + +// Export copies only from closed storage of a Stopped instance, under the +// instance lock, after a strict graceful receipt and sanitation. +func (s *guestMachineSession) Export(ctx context.Context, destination string) error { + if !s.receipt.Graceful || !s.receipt.VMMExited || !s.sanitized { + return fmt.Errorf("sanitized strict stop receipt required") + } + return s.exporter.ExportStoppedMacOSBuildInstance(ctx, s.ID(), destination) +} + +func (s *guestMachineSession) Destroy(ctx context.Context) error { + if !s.receipt.VMMExited { + return fmt.Errorf("unconfirmed VMM exit: instance quarantined") + } + if err := s.manager.DeleteInstance(ctx, s.ID()); err != nil { + return err + } + return os.RemoveAll(s.workspace) +} diff --git a/lib/builds/macos_build_job_test.go b/lib/builds/macos_build_job_test.go new file mode 100644 index 000000000..12394b213 --- /dev/null +++ b/lib/builds/macos_build_job_test.go @@ -0,0 +1,37 @@ +package builds + +import ( + "context" + "io" + "os" + "path/filepath" + "time" +) + +type machineRunnerFixture struct { + MachineBuildBackend + WorkDir string +} + +func (r *machineRunnerFixture) Run(ctx context.Context, req MachineBuildRequest, source io.Reader) (*MachineBuildResult, error) { + timeout := req.Policy.TimeoutSeconds + if timeout == 0 { + timeout = 600 + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + job, err := os.MkdirTemp(r.WorkDir, "prepared-job-") + if err != nil { + return nil, err + } + defer os.RemoveAll(job) + sourcePath := filepath.Join(job, "source.tar.gz") + hash, err := stageBuildSource(ctx, source, sourcePath, maxBuildSourceBytes) + if err != nil { + return nil, machineFailure("stage_source", err) + } + if req.SourceHash == "" { + req.SourceHash = hash + } + return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath, r.WorkDir) +} diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_build_layer_test.go new file mode 100644 index 000000000..6798c5582 --- /dev/null +++ b/lib/builds/macos_build_layer_test.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { + parent := t.TempDir() + bundle := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + disk := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk + copy(disk[1<<20:], "data") + for name, data := range map[string][]byte{"disk.img": disk, "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.NotEmpty(t, diffID.Hex) + digest, err := layer.Digest() + require.NoError(t, err) + size, err := layer.Size() + require.NoError(t, err) + for range 2 { + body, err := layer.Compressed() + require.NoError(t, err) + sum := sha256.New() + n, err := io.Copy(sum, body) + require.NoError(t, err) + require.NoError(t, body.Close()) + require.Equal(t, size, n) + require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) + } + entries, err := os.ReadDir(parent) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing is spooled beside the bundle") +} + +func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { + bundle := t.TempDir() + for name, data := range map[string][]byte{"disk.img": make([]byte, 8<<20), "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + before := runtime.NumGoroutine() + body, err := layer.Compressed() + require.NoError(t, err) + _, err = io.CopyN(io.Discard, body, 16) + require.NoError(t, err) + require.NoError(t, body.Close()) + // Polled directly: require.Eventually runs its condition in another goroutine. + deadline := time.Now().Add(5 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), before, "the layer generator exits when the upload is abandoned") +} diff --git a/lib/builds/macos_build_lifecycle_test.go b/lib/builds/macos_build_lifecycle_test.go new file mode 100644 index 000000000..f7359e95c --- /dev/null +++ b/lib/builds/macos_build_lifecycle_test.go @@ -0,0 +1,135 @@ +package builds + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineLifecycleInstances struct { + instances.Manager + lookedUp string + state, stoppedState instances.State + missing, stopped bool + deleted bool + stopErr, deleteErr error +} + +func (m *machineLifecycleInstances) GetInstance(_ context.Context, id string) (*instances.Instance, error) { + m.lookedUp = id + if m.missing { + return nil, instances.ErrNotFound + } + return &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "prior"}, State: m.state}, nil +} +func (m *machineLifecycleInstances) StopInstance(context.Context, string) (*instances.Instance, error) { + m.stopped = true + if m.stopErr != nil { + return nil, m.stopErr + } + if m.stoppedState != "" { + m.state = m.stoppedState + } + return &instances.Instance{State: m.state}, nil +} +func (m *machineLifecycleInstances) DeleteInstance(context.Context, string) error { + m.deleted = true + return m.deleteErr +} + +type abortExporter struct { + receipt instances.MacOSBuildExportReceipt + err error +} + +func (e abortExporter) StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) { + return e.receipt, e.err +} +func (abortExporter) ExportStoppedMacOSBuildInstance(context.Context, string, string) error { + return errors.New("abort must not export") +} + +func TestMachineForcedStopCleanup(t *testing.T) { + for _, tc := range []struct { + name string + state, stoppedState instances.State + graceful, forceError, wantExit bool + }{ + {"graceful exit", instances.StateRunning, "", true, false, true}, + {"already stopped", instances.StateStopped, "", false, false, true}, + {"force running VM", instances.StateRunning, instances.StateStopped, false, false, true}, + {"unconfirmed stop", instances.StateRunning, "", false, false, false}, + {"failed force stop", instances.StateRunning, "", false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + m := &machineLifecycleInstances{state: tc.state, stoppedState: tc.stoppedState} + stop := MachineStopReceipt{Graceful: tc.graceful, VMMExited: tc.graceful} + var stopErr error + if !tc.graceful { + stopErr = errors.New("agent never became ready") + } + if tc.forceError { + m.stopErr = errors.New("hypervisor still alive") + } + s := &guestMachineSession{manager: m, exporter: abortExporter{receipt: stop, err: stopErr}, instance: &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "builder"}}} + receipt, err := s.Stop(context.Background(), true) + if tc.wantExit { + require.NoError(t, err) + require.Equal(t, MachineStopReceipt{Graceful: tc.graceful, VMMExited: true}, receipt) + } else { + wantErr := "forced stop did not confirm VMM exit" + if tc.forceError { + wantErr = "hypervisor still alive" + } + require.EqualError(t, err, wantErr) + require.False(t, receipt.VMMExited) + require.Error(t, s.Destroy(context.Background()), "unconfirmed exit must remain quarantined") + } + require.Equal(t, !tc.graceful && tc.state != instances.StateStopped, m.stopped) + }) + } +} + +func TestReapPriorMachineBuilder(t *testing.T) { + for _, tc := range []struct { + name string + state instances.State + missing bool + unrecorded bool + deleteErr error + wantStop bool + wantDelete bool + wantError bool + }{ + {"stop and delete", instances.StateRunning, false, false, nil, true, true, false}, + {"delete failure blocks retry", instances.StateStopped, false, false, errors.New("storage still open"), false, true, true}, + {"missing builder", "", true, false, nil, false, false, false}, + {"find unrecorded builder", instances.StateRunning, false, true, nil, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := paths.New(t.TempDir()) + meta := &buildMetadata{ID: "job", Status: StatusQueued} + if !tc.unrecorded { + prior := "prior" + meta.BuilderInstance = &prior + } + require.NoError(t, writeMetadata(p, meta)) + fake := &machineLifecycleInstances{state: tc.state, missing: tc.missing, deleteErr: tc.deleteErr} + m := &manager{paths: p, instanceManager: fake, logger: slog.Default()} + err := m.reapPriorMachineBuilder(context.Background(), "job") + if tc.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.unrecorded, fake.lookedUp == machineBuilderName("job")) + require.Equal(t, tc.wantStop, fake.stopped) + require.Equal(t, tc.wantDelete, fake.deleted) + }) + } +} diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go new file mode 100644 index 000000000..dcd5be191 --- /dev/null +++ b/lib/builds/macos_build_manager.go @@ -0,0 +1,199 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// machineBuild reports whether a request selects the native macOS build path. +func machineBuild(req *CreateBuildRequest) bool { + return req != nil && req.MachineBaseImage != "" +} + +func (m *manager) skipMachineBuildRecovery(meta *buildMetadata) bool { + if !m.config.MachineBuildOnly { + return false + } + if machineBuild(meta.Request) && m.config.MachineBuild != nil { + return false + } + m.logger.Warn("leaving unsupported build pending in machine-build-only mode", "id", meta.ID) + return true +} + +// executeMachineBuild is a backend of the same queued build job. runBuild owns +// deadline/status/log completion, and CreateBuild already staged verified source. +func (m *manager) executeMachineBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + backend := m.config.MachineBuild + if backend == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if err := m.reapPriorMachineBuilder(ctx, id); err != nil { + return nil, machineFailure("reap_prior_builder", err) + } + base, err := m.imageManager.GetImage(ctx, req.MachineBaseImage) + if err != nil { + return nil, machineFailure("resolve_base", err) + } + sourcePath, err := filepath.Abs(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")) + if err != nil { + return nil, err + } + // An interrupted attempt skips its own cleanup; drop its workspace first. + removeMachineWorkspaces(m.paths.BuildDir(id)) + defer removeMachineWorkspaces(m.paths.BuildDir(id)) + job := MachineBuildRequest{ + ID: id, BaseImage: req.MachineBaseImage, SourceHash: req.SourceHash, Policy: *policy, + deltaBase: m.machineDeltaBase(backend.Publisher, req.MachineBaseImage, base), + onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, + onPublished: func(ctx context.Context, publication MachinePublication) error { + return m.waitMachinePublication(ctx, id, publication.Digest) + }, + } + result, err := backend.runPrepared(ctx, job, base, sourcePath, filepath.Dir(filepath.Dir(sourcePath))) + if err != nil { + return nil, err + } + ref, err := images.ParseNormalizedRef(result.Publication.Reference) + if err != nil || ref.Repository() != strings.TrimSuffix(stripRegistryScheme(m.config.RegistryURL), "/")+"/builds/"+id { + return nil, fmt.Errorf("machine publication is outside the authorized job repository") + } + return &BuildResult{Success: true, ImageDigest: result.Publication.Digest, ImageRef: result.Publication.Reference, Provenance: result.Provenance}, nil +} + +func (m *manager) waitMachinePublication(ctx context.Context, id, digest string) error { + if !ValidMachineDigest(digest) { + return fmt.Errorf("canonical machine publication digest required") + } + if err := m.imageManager.WaitForReady(ctx, "builds/"+id+"@"+digest); err != nil { + return fmt.Errorf("machine image import: %w", err) + } + return nil +} + +// reapPriorMachineBuilder removes the builder recorded by an interrupted attempt of +// this same job before a re-enqueued attempt starts. Nothing from that builder is +// published, so a forced stop is acceptable; a failed delete fails this attempt +// rather than leaving two builders with storage in play. +func (m *manager) reapPriorMachineBuilder(ctx context.Context, id string) error { + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + // An attempt interrupted before it recorded its builder still named it after the build. + prior := machineBuilderName(id) + if meta.BuilderInstance != nil && *meta.BuilderInstance != "" { + prior = *meta.BuilderInstance + } + inst, err := m.instanceManager.GetInstance(ctx, prior) + if errors.Is(err, instances.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if inst.State != instances.StateStopped { + if _, err := m.instanceManager.StopInstance(ctx, inst.Id); err != nil { + return fmt.Errorf("stop prior builder: %w", err) + } + } + if err := m.instanceManager.DeleteInstance(ctx, inst.Id); err != nil && !errors.Is(err, instances.ErrNotFound) { + return fmt.Errorf("delete prior builder: %w", err) + } + m.logger.Info("reaped prior machine builder", "build_id", id, "instance_id", inst.Id) + return nil +} + +// storeMachineSource stages machine source privately and records its SHA256 +// provenance. A caller-supplied source hash must match. +func (m *manager) storeMachineSource(ctx context.Context, meta *buildMetadata, data []byte) error { + sourceDir := m.paths.BuildSourceDir(meta.ID) + if err := os.MkdirAll(sourceDir, 0700); err != nil { + return fmt.Errorf("store source: %w", err) + } + hash, err := stageBuildSource(ctx, bytes.NewReader(data), filepath.Join(sourceDir, "source.tar.gz"), int64(len(data))) + if err != nil { + return fmt.Errorf("store source: %w", err) + } + if meta.Request.SourceHash != "" && meta.Request.SourceHash != hash { + return ErrSourceHashMismatch + } + meta.Request.SourceHash = hash + if err := writeMetadata(m.paths, meta); err != nil { + return fmt.Errorf("write source provenance: %w", err) + } + return nil +} + +// admitMachineBuild validates a machine build request and returns its policy. +// Machine resources inherit the installed base, not Linux builder defaults. +func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) (*BuildPolicy, error) { + if m.config.MachineBuild == nil || m.config.MachineBuild.Driver == nil || m.config.MachineBuild.Publisher == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if req.Dockerfile != "" || req.BuilderID != "" || len(req.BuildArgs) != 0 || len(req.Secrets) != 0 || req.CacheScope != "" || req.GlobalCacheKey != "" || req.IsAdminBuild { + return nil, fmt.Errorf("machine builds do not support Linux builder/cache/secret options") + } + if len(sourceData) > maxBuildSourceBytes { + return nil, fmt.Errorf("machine build source too large") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := &BuildPolicy{} + if req.BuildPolicy != nil { + requested := *req.BuildPolicy + policy = &requested + } + if err := normalizeMachinePolicy(policy); err != nil { + return nil, err + } + req.BuildPolicy = policy + return policy, nil +} + +// enqueueMachineBuild queues a machine build whose source is already staged. +func machineBuildSerialKey(req CreateBuildRequest) string { + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() { + return "machine:" + req.MachineBaseImage + } + return "machine:" + ref.Digest() +} + +func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { + queuePos := m.queue.EnqueueSerial(meta.ID, req, machineBuildSerialKey(req), func() { + m.runBuild(context.Background(), meta.ID, req, policy) + }) + build := meta.toBuild() + if queuePos > 0 { + build.QueuePosition = &queuePos + } + m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) + return build +} + +// removeMachineWorkspaces deletes export workspaces under a build directory. Each +// holds a full exported disk. +func removeMachineWorkspaces(buildDir string) { + matches, _ := filepath.Glob(filepath.Join(buildDir, "machine-build-*")) + for _, dir := range matches { + _ = os.RemoveAll(dir) + } +} + +// removeMachineSource deletes a machine build's staged source after the job is terminal. +func (m *manager) removeMachineSource(id string) { + if err := os.Remove(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")); err != nil && !errors.Is(err, os.ErrNotExist) { + m.logger.Warn("failed to remove machine build source", "id", id, "error", err) + } +} diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go new file mode 100644 index 000000000..fdc2cab7d --- /dev/null +++ b/lib/builds/macos_build_manager_test.go @@ -0,0 +1,228 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineJobImages struct { + images.Manager + fixture *machineFixture + resolveErr error +} + +func (m machineJobImages) GetImage(ctx context.Context, ref string) (*images.Image, error) { + if m.resolveErr != nil { + return nil, m.resolveErr + } + if ref == "registry/base@"+machineTestDigest { + return m.fixture.ResolveBase(ctx, ref) + } + return &images.Image{Status: images.StatusReady, Digest: machineTestDigest}, nil +} + +func (m machineJobImages) WaitForReady(context.Context, string) error { return nil } + +func newMachineTestManager(t *testing.T, f *machineFixture, publisher MachineBuildPublisher, imageManager images.Manager, machineOnly bool) *manager { + t.Helper() + config := DefaultConfig() + config.RegistrySecret = "synthetic-test-secret" + config.MachineBuildOnly = machineOnly + if f != nil && publisher != nil { + config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: publisher} + } + if imageManager == nil && f != nil { + imageManager = machineJobImages{fixture: f} + } + api, err := NewManager(paths.New(t.TempDir()), config, noPriorBuilder{}, nil, nil, imageManager, nil, nil, nil) + require.NoError(t, err) + return api.(*manager) +} + +type machineJobPublisher struct { + fixture *machineFixture + publishedID string + // sourcePath, when set, must still exist at publish time. + sourcePath func(id string) string +} + +func (p *machineJobPublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + p.publishedID = id + if p.sourcePath != nil { + if _, err := os.Stat(p.sourcePath(id)); err != nil { + return MachinePublication{}, errors.New("source removed before publish") + } + } + // Reuse the fixture's export assertions with its fixed ID. + receipt, err := p.fixture.Publish(ctx, "build-test", bundle) + if err == nil { + receipt.Reference = "localhost:4973/builds/" + id + "@" + receipt.Digest + } + return receipt, err +} +func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { + f := newMachineFixture() + publisher := &machineJobPublisher{fixture: f} + m := newMachineTestManager(t, f, publisher, nil, false) + publisher.sourcePath = func(id string) string { return filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz") } + source := []byte("synthetic compressed-source bytes") + sum := sha256.Sum256(source) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: hex.EncodeToString(sum[:])} + build, err := m.CreateBuild(context.Background(), req, source) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && (b.Status == StatusReady || b.Status == StatusFailed) + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, done.Status, "failure: %v", done.Error) + require.NotNil(t, done.ImageRef) + require.Equal(t, "localhost:4973/builds/"+build.ID+"@"+machineTestDigest, *done.ImageRef) + require.Equal(t, build.ID, publisher.publishedID) + require.Equal(t, req.SourceHash, done.Provenance.SourceHash) + require.Equal(t, machineTestDigest, done.Provenance.BaseImageDigest) + require.Equal(t, f.ID(), *done.BuilderInstanceID) + meta, err := readMetadata(m.paths, build.ID) + require.NoError(t, err) + require.Equal(t, req.MachineBaseImage, meta.Request.MachineBaseImage) + require.Equal(t, "isolated", meta.Request.BuildPolicy.NetworkMode) + require.Equal(t, 0, meta.Request.BuildPolicy.CPUs, "machine defaults must not use Linux CPU defaults") + _, err = readBuildConfig(m.paths, build.ID) + require.Error(t, err, "machine builds must not persist a builder config or registry token") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) +} +func TestMachineBuildSerializesByBaseDigest(t *testing.T) { + first := CreateBuildRequest{MachineBaseImage: "registry-a/base@" + machineTestDigest} + second := CreateBuildRequest{MachineBaseImage: "registry-b/another-name@" + machineTestDigest} + require.Equal(t, machineBuildSerialKey(first), machineBuildSerialKey(second)) +} + +func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, false) + _, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.ErrorContains(t, err, "not configured") + f := newMachineFixture() + m.config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: f} + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base:mutable"}, []byte("source")) + require.ErrorContains(t, err, "pinned") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, Secrets: []SecretRef{{ID: "unsupported"}}}, []byte("source")) + require.ErrorContains(t, err, "secret options") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: "wrong"}, []byte("source")) + require.True(t, errors.Is(err, ErrSourceHashMismatch)) + _, err = m.admitMachineBuild(&CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, make([]byte, maxBuildSourceBytes+1)) + require.ErrorContains(t, err, "source too large") + list, err := m.ListBuilds(context.Background()) + require.NoError(t, err) + require.Empty(t, list) +} + +func TestQueuedMachineCancellationRemovesSource(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) + require.NoError(t, err) + t.Cleanup(func() { + _ = m.CancelBuild(context.Background(), first.ID) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) + }) + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, first.ID) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) + require.NoError(t, err) + source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") + require.FileExists(t, source) + require.NoError(t, m.CancelBuild(context.Background(), queued.ID)) + done, err := m.GetBuild(context.Background(), queued.ID) + require.NoError(t, err) + require.Equal(t, StatusCancelled, done.Status) + require.NotNil(t, done.CompletedAt) + require.NoFileExists(t, source) +} + +func TestMachineCancellationInterruptsPhasesWithoutPublishing(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + build, e := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("synthetic source")) + require.NoError(t, e) + require.Eventually(t, func() bool { + meta, e := readMetadata(m.paths, build.ID) + return e == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), build.ID)) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(build.ID); return !active }, 5*time.Second, 10*time.Millisecond) + done, e := m.GetBuild(context.Background(), build.ID) + require.NoError(t, e) + require.Equal(t, StatusCancelled, done.Status) + require.False(t, fixture.publisherCalled) + require.Contains(t, fixture.calls, "abort") + require.Contains(t, fixture.calls, "destroy") +} + +func TestMachineFailureUsesSharedFailedStatusWithoutPublication(t *testing.T) { + for _, tc := range []struct { + name string + phase string + resolveErr error + wantError string + }{ + {name: "base resolution", resolveErr: errors.New("sensitive-resolution-error"), wantError: "machine build resolve_base failed"}, + {name: "provision", phase: "provision", wantError: "machine build provision failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + m := newMachineTestManager(t, f, f, machineJobImages{fixture: f, resolveErr: tc.resolveErr}, false) + build, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && b.Status == StatusFailed + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusFailed, done.Status) + require.Nil(t, done.ImageDigest) + require.NotContains(t, f.calls, "publish") + require.NotContains(t, *done.Error, "sensitive-resolution-error") + require.Equal(t, tc.wantError, *done.Error) + if tc.phase == "provision" { + require.Contains(t, f.calls, "destroy") + } else { + require.Empty(t, f.calls) + } + }) + } +} + +func TestMachineFailureShowsOnlyExitCode(t *testing.T) { + err := machineFailure("provision", fmt.Errorf("guest said secret-token: %w", &machineExitError{code: 7})) + require.Equal(t, "machine build provision failed: build.sh exited with code 7", err.Error()) + require.Equal(t, "machine build provision failed", machineFailure("provision", fmt.Errorf("secret-token")).Error()) + admission := fmt.Errorf("%w: macOS template identity is already in use", instances.ErrInsufficientResources) + require.Contains(t, machineFailure("start", admission).Error(), "machine build start failed: insufficient resources") +} + +// noPriorBuilder reports that no earlier attempt left a builder instance behind. +type noPriorBuilder struct{ instances.Manager } + +func (noPriorBuilder) GetInstance(context.Context, string) (*instances.Instance, error) { + return nil, instances.ErrNotFound +} diff --git a/lib/builds/macos_build_policy_test.go b/lib/builds/macos_build_policy_test.go new file mode 100644 index 000000000..32c1592f6 --- /dev/null +++ b/lib/builds/macos_build_policy_test.go @@ -0,0 +1,44 @@ +package builds + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachinePolicyNormalization(t *testing.T) { + for _, tc := range []struct { + name string + policy BuildPolicy + err string + }{ + {name: "defaults"}, + {name: "minimum timeout", policy: BuildPolicy{TimeoutSeconds: 1}}, + {name: "maximum timeout", policy: BuildPolicy{TimeoutSeconds: 86400}}, + {name: "egress", policy: BuildPolicy{NetworkMode: "egress"}}, + {name: "negative timeout", policy: BuildPolicy{TimeoutSeconds: -1}, err: "invalid machine build timeout"}, + {name: "excessive timeout", policy: BuildPolicy{TimeoutSeconds: 86401}, err: "invalid machine build timeout"}, + {name: "unsupported mode", policy: BuildPolicy{NetworkMode: "host"}, err: "unsupported machine network policy"}, + {name: "domains", policy: BuildPolicy{NetworkMode: "egress", AllowedDomains: []string{"example.test"}}, err: "unsupported machine network policy"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := tc.policy + err := normalizeMachinePolicy(&policy) + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + require.NoError(t, err) + want := tc.policy + if want.TimeoutSeconds == 0 { + want.TimeoutSeconds = 600 + } + if want.NetworkMode == "" { + want.NetworkMode = "isolated" + } + require.Equal(t, want, policy) + require.NoError(t, normalizeMachinePolicy(&policy)) + require.Equal(t, want, policy) + }) + } +} diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go new file mode 100644 index 000000000..4945020d3 --- /dev/null +++ b/lib/builds/macos_build_publisher.go @@ -0,0 +1,194 @@ +package builds + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// StreamingMachinePublisher targets only the configured internal registry and a +// build-scoped repository. go-containerregistry hashes and compresses the layer +// from a reopenable tar stream, so it is never written to disk. Blobs use the +// registry's disk-streaming monolithic upload and are independently verified before +// the manifest is committed last. +// No credentials, redirect destinations or arbitrary output repositories come from the source. +type StreamingMachinePublisher struct { + RegistryURL string + Tokens *RegistryTokenGenerator +} + +type machineBlob struct { + open func() (io.ReadCloser, error) + descriptor v1.Descriptor + mountFrom string // set for a base layer mounted from this repository instead of uploaded +} + +// Publish commits bundle as a standalone machine image in builds/{id}. +func (p *StreamingMachinePublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + return p.publish(ctx, id, bundle, nil) +} + +// PublishDerived commits bundle as a delta over base. The base layers are mounted +// from base.Repo rather than uploaded; only the delta layer and config are uploaded. +func (p *StreamingMachinePublisher) PublishDerived(ctx context.Context, id, bundle string, base MachineDeltaBase) (MachinePublication, error) { + if err := base.validate(); err != nil { + return MachinePublication{}, err + } + return p.publish(ctx, id, bundle, &base) +} + +func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle string, source *MachineDeltaBase) (MachinePublication, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return MachinePublication{}, fmt.Errorf("bounded publication required") + } + if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil { + return MachinePublication{}, fmt.Errorf("scoped machine publisher required") + } + address := p.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { + return MachinePublication{}, fmt.Errorf("invalid internal registry address") + } + if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { + return MachinePublication{}, fmt.Errorf("plaintext registry must be loopback") + } + if err := validateMachineExport(bundle, nil); err != nil { + return MachinePublication{}, err + } + repo := "builds/" + id + access := []RepoPermission{{Repo: repo, Scope: "push"}} + if source != nil { + access = append(access, RepoPermission{Repo: source.Repo, Scope: "pull"}) + } + token, err := p.Tokens.GenerateToken(id, access, time.Until(deadline)) + if err != nil { + return MachinePublication{}, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + base := strings.TrimRight(address, "/") + "/v2/" + repo + layers, diffIDs, err := machineLayers(ctx, client, token, address, bundle, source) + if err != nil { + return MachinePublication{}, err + } + config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, Config: v1.Config{Labels: map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + }}} + configBytes, err := json.Marshal(config) + if err != nil { + return MachinePublication{}, err + } + configBlob := machineBlob{open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(configBytes)), nil }, descriptor: v1.Descriptor{MediaType: types.OCIConfigJSON, Digest: machineHash(configBytes), Size: int64(len(configBytes))}} + descriptors := make([]v1.Descriptor, 0, len(layers)) + for _, layer := range layers { + descriptors = append(descriptors, layer.descriptor) + } + manifest := v1.Manifest{SchemaVersion: 2, MediaType: types.OCIManifestSchema1, Config: configBlob.descriptor, Layers: descriptors} + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return MachinePublication{}, err + } + digest := machineHash(manifestBytes).String() + for _, blob := range append([]machineBlob{configBlob}, layers...) { + if blob.mountFrom != "" { + if err := mountMachineBlob(ctx, client, token, base, blob); err != nil { + return MachinePublication{}, err + } + continue + } + body, e := blob.open() + if e != nil { + return MachinePublication{}, e + } + response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, base+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) + closeErr := body.Close() + if e != nil { + return MachinePublication{}, e + } + if closeErr != nil { + response.Body.Close() + return MachinePublication{}, closeErr + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") + } + if _, e = machineFetch(ctx, client, token, base+"/blobs/"+blob.descriptor.Digest.String(), blob.descriptor.Digest.String(), blob.descriptor.Size); e != nil { + return MachinePublication{}, fmt.Errorf("machine blob verification failed: %w", e) + } + } + target := base + "/manifests/latest" + response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target, string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) + if err != nil { + return MachinePublication{}, err + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") + } + // The digest covers manifestBytes exactly, so a matching fetch is the same bytes. + if _, err = machineFetch(ctx, client, token, base+"/manifests/"+digest, digest, int64(len(manifestBytes))); err != nil { + return MachinePublication{}, fmt.Errorf("machine manifest verification failed: %w", err) + } + return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil +} + +func machineTarballLayer(ctx context.Context, bundle string) (v1.Layer, error) { + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(writeMachineTar(ctx, bundle, writer)) }() + return reader, nil + }, tarball.WithMediaType(types.OCILayer), tarball.WithCompressionLevel(gzip.DefaultCompression)) +} + +// writeMachineTar writes a deterministic tar of the exported bundle. +func writeMachineTar(ctx context.Context, bundle string, w io.Writer) error { + archive := tar.NewWriter(w) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + source, e := os.Open(filepath.Join(bundle, name)) + if e != nil { + return e + } + info, e := source.Stat() + if e != nil { + source.Close() + return e + } + e = archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) + if e == nil { + _, e = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) + } + closeErr := source.Close() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + } + return archive.Close() +} diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go new file mode 100644 index 000000000..c710a49e7 --- /dev/null +++ b/lib/builds/macos_build_publisher_test.go @@ -0,0 +1,96 @@ +package builds_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +type machineImportStub struct { + images.Manager + imported chan string +} + +func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, digest string) (*images.Image, error) { + m.imported <- repo + return &images.Image{Status: images.StatusReady, Digest: digest}, nil +} + +func machinePublisherBundle(t *testing.T) string { + t.Helper() + root := t.TempDir() + platform := images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} + data, e := json.Marshal(platform) + require.NoError(t, e) + for name, data := range map[string][]byte{"config.json": data, "disk.img": []byte("synthetic disk"), "aux.img": []byte("synthetic aux")} { + require.NoError(t, os.WriteFile(filepath.Join(root, name), data, 0600)) + } + return root +} + +func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { + // Tiny storage is intentional: this proves auth/streaming/commit ordering, not bootability. + imports := &machineImportStub{imported: make(chan string, 1)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads, verified atomic.Int32 + handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NotEqual(t, http.MethodPatch, r.Method, "uploads must use the disk-streaming monolithic endpoint") + if r.Method == http.MethodPost { + require.NotEmpty(t, r.URL.Query().Get("digest")) + require.Greater(t, r.ContentLength, int64(0)) + uploads.Add(1) + } + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { + verified.Add(1) + } + if r.Method == http.MethodPut { + require.Equal(t, int32(2), uploads.Load(), "manifest must be last") + } + reg.Handler().ServeHTTP(w, r) + })) + server := httptest.NewServer(handler) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + result, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.NoError(t, e) + require.Contains(t, result.Reference, "/builds/qa-build@sha256:") + select { + case repo := <-imports.imported: + require.Equal(t, "builds/qa-build", repo) + case <-ctx.Done(): + t.Fatal("manifest did not reach production conversion boundary") + } +} + +func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing.T) { + var visited atomic.Bool + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { visited.Store(true) })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic secret")} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.Error(t, e) + require.False(t, visited.Load()) +} diff --git a/lib/builds/macos_build_recovery_test.go b/lib/builds/macos_build_recovery_test.go new file mode 100644 index 000000000..411c12102 --- /dev/null +++ b/lib/builds/macos_build_recovery_test.go @@ -0,0 +1,88 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { + for _, status := range []string{StatusQueued, StatusBuilding, StatusPushing} { + t.Run(status, func(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, true) + m.config.BuilderImage = "linux-builder-must-not-start" + policy := &BuildPolicy{} + require.NoError(t, normalizeMachinePolicy(policy)) + meta := &buildMetadata{ID: "build-test", Status: status, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: policy}} + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("synthetic compressed-source bytes"))) + // A workspace left by the interrupted attempt. + stale := filepath.Join(m.paths.BuildDir(meta.ID), "machine-build-interrupted") + require.NoError(t, os.MkdirAll(filepath.Join(stale, "bundle"), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(stale, "bundle", "disk.img"), []byte("disk"), 0600)) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "linux-job", Status: StatusQueued, Request: &CreateBuildRequest{BuilderID: "linux-builder"}})) + t.Cleanup(func() { + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) + }) + + // Hold the recovered build active so the second recovery sees it running. + f.startGate = make(chan struct{}) + require.NoError(t, m.Start(context.Background())) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 1 }, 5*time.Second, 10*time.Millisecond) + m.RecoverPendingBuilds() + close(f.startGate) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 && m.queue.PendingCount() == 0 }, 5*time.Second, 10*time.Millisecond) + build, err := m.GetBuild(context.Background(), meta.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, build.Status, "failure: %v", build.Error) + require.NoDirExists(t, stale, "the interrupted attempt's workspace is removed") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz"), "source is removed once the image is ready") + require.False(t, m.ReadyForBuilds(), "Linux builder image must not be prepared") + var starts int + for _, call := range f.calls { + if call == "start" { + starts++ + } + } + require.Equal(t, 1, starts, "recovery must not duplicate an active build") + linux, err := readMetadata(m.paths, "linux-job") + require.NoError(t, err) + require.Equal(t, StatusQueued, linux.Status, "Linux jobs are preserved for Linux-mode recovery") + require.True(t, m.BuilderHasBuilds("linux-builder"), "unrecovered disk jobs still protect their builders") + }) + } +} + +func TestRecoveryCleansCancelledMachineBuildSource(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, false) + meta := &buildMetadata{ + ID: "cancelled-build", + Status: StatusCancelled, + Request: &CreateBuildRequest{ + MachineBaseImage: "registry/base@" + machineTestDigest, + }, + } + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("source"))) + meta.Status = StatusCancelled + require.NoError(t, writeMetadata(m.paths, meta)) + + m.RecoverPendingBuilds() + + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz")) +} + +func TestMachineOnlyStartPreservesBuildsWhenBackendDisabled(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, true) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "disabled", Status: StatusBuilding, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}})) + require.NoError(t, m.Start(context.Background())) + require.Equal(t, 0, m.queue.ActiveCount()) + meta, err := readMetadata(m.paths, "disabled") + require.NoError(t, err) + require.Equal(t, StatusBuilding, meta.Status) + require.False(t, m.ReadyForBuilds()) +} diff --git a/lib/builds/macos_build_source.go b/lib/builds/macos_build_source.go new file mode 100644 index 000000000..37b7ee2f5 --- /dev/null +++ b/lib/builds/macos_build_source.go @@ -0,0 +1,119 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "unicode/utf8" +) + +// machineEntrypoint is the script a machine build source must contain at its +// root. It runs as root with /bin/sh in the guest's private build directory. +const machineEntrypoint = "build.sh" +const maxMachineExtractBytes = 128 << 20 + +// extractMachineSource unpacks a build-owned tar.gz on the host into an exclusive +// directory, never with guest tar or a caller-supplied destination. Links, +// devices, duplicates and decompression bombs fail before anything is copied to +// the VM. The archive must contain the entrypoint as a regular file. +func extractMachineSource(ctx context.Context, source, destination string) error { + file, err := os.Open(source) + if err != nil { + return err + } + defer file.Close() + compressed, err := gzip.NewReader(file) + if err != nil { + return err + } + defer compressed.Close() + compressed.Multistream(false) + if err = os.Mkdir(destination, 0700); err != nil { + return err + } + complete := false + defer func() { + if !complete { + _ = os.RemoveAll(destination) + } + }() + limited := &io.LimitedReader{R: compressed, N: maxMachineExtractBytes + 1} + reader := tar.NewReader(limited) + seen := map[string]bool{} + var total int64 + for count := 0; ; count++ { + if err = ctx.Err(); err != nil { + return err + } + header, e := reader.Next() + if e == io.EOF { + break + } + if e != nil { + return e + } + if count >= 1024 { + return fmt.Errorf("too many machine source entries") + } + name := path.Clean(header.Name) + if name == "." && header.Typeflag == tar.TypeDir { + continue + } + if !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || !filepath.IsLocal(name) || seen[name] { + return fmt.Errorf("invalid or duplicate machine source path") + } + seen[name] = true + if header.Size < 0 || header.Size > 64<<20 || total+header.Size > maxMachineExtractBytes { + return fmt.Errorf("machine source expands beyond limit") + } + total += header.Size + target := filepath.Join(destination, name) + switch header.Typeflag { + case tar.TypeDir: + if header.Size != 0 { + return fmt.Errorf("directory has data") + } + if err = os.MkdirAll(target, 0700); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + output, e := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600|os.FileMode(header.Mode)&0100) + if e != nil { + return e + } + _, copyErr := io.Copy(output, reader) + closeErr := output.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("machine source must contain only regular files and directories") + } + } + // Consume bounded trailing tar padding to verify gzip integrity without allowing + // an unbounded post-tar decompression stream. + if _, err = io.Copy(io.Discard, limited); err != nil { + return err + } + if limited.N <= 0 { + return fmt.Errorf("machine source expands beyond limit") + } + info, err := os.Lstat(filepath.Join(destination, machineEntrypoint)) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine source must contain %s", machineEntrypoint) + } + complete = true + return nil +} diff --git a/lib/builds/macos_build_source_test.go b/lib/builds/macos_build_source_test.go new file mode 100644 index 000000000..00f0d5f4d --- /dev/null +++ b/lib/builds/macos_build_source_test.go @@ -0,0 +1,67 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeMachineSource(t *testing.T, headers []*tar.Header, bodies []string) string { + t.Helper() + name := filepath.Join(t.TempDir(), "source.tar.gz") + f, e := os.Create(name) + require.NoError(t, e) + gz := gzip.NewWriter(f) + writer := tar.NewWriter(gz) + for i, h := range headers { + require.NoError(t, writer.WriteHeader(h)) + if h.Typeflag == tar.TypeReg { + _, e = writer.Write([]byte(bodies[i])) + require.NoError(t, e) + } + } + require.NoError(t, writer.Close()) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) + return name +} + +func TestMachineSourceExtraction(t *testing.T) { + source := writeMachineSource(t, []*tar.Header{{Name: machineEntrypoint, Size: 4, Typeflag: tar.TypeReg, Mode: 0755}, {Name: "lib/data", Size: 1, Typeflag: tar.TypeReg}}, []string{"true", "x"}) + root := filepath.Join(t.TempDir(), "source") + require.NoError(t, extractMachineSource(context.Background(), source, root)) + info, e := os.Stat(filepath.Join(root, machineEntrypoint)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0700), info.Mode().Perm()) + info, e = os.Stat(filepath.Join(root, "lib", "data")) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +func TestMachineSourceRejectsUnsafeArchives(t *testing.T) { + entry := &tar.Header{Name: machineEntrypoint, Typeflag: tar.TypeReg, Size: 4} + for _, tc := range []struct { + name string + headers []*tar.Header + bodies []string + }{ + {"missing entrypoint", []*tar.Header{{Name: "other", Typeflag: tar.TypeReg, Size: 1}}, []string{"x"}}, + {"empty entrypoint", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeReg}}, []string{""}}, + {"entrypoint directory", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeDir}}, []string{""}}, + {"traversal", []*tar.Header{entry, {Name: "../outside", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x"}}, + {"symlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc"}}, []string{"true", ""}}, + {"hardlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeLink, Linkname: "other"}}, []string{"true", ""}}, + {"duplicate", []*tar.Header{entry, {Name: "a", Typeflag: tar.TypeReg, Size: 1}, {Name: "a", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x", "y"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := filepath.Join(t.TempDir(), "source") + require.Error(t, extractMachineSource(context.Background(), writeMachineSource(t, tc.headers, tc.bodies), root)) + require.NoDirExists(t, root) + }) + } +} diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go new file mode 100644 index 000000000..da848ce78 --- /dev/null +++ b/lib/builds/macos_build_test.go @@ -0,0 +1,226 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +type machineFixture struct { + image *images.Image + calls []string + fail string + stop MachineStopReceipt + cancel context.CancelFunc + waitForCancel bool + mutateExport func(string) error + publicationOverride *MachinePublication + publisherCalled bool + cleanupContextLive bool + startGate chan struct{} +} + +func newMachineFixture() *machineFixture { + return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} +} +func (f *machineFixture) record(phase string) error { + f.calls = append(f.calls, phase) + if f.fail == phase { + return errors.New("sensitive-command-argument") + } + return nil +} +func (f *machineFixture) ResolveBase(context.Context, string) (*images.Image, error) { + if err := f.record("resolve"); err != nil { + return nil, err + } + return f.image, nil +} +func (f *machineFixture) Start(_ context.Context, _ string, _ *images.Image, p BuildPolicy) (MachineBuildSession, error) { + if p.CPUs != 4 || p.MemoryMB != 8192 { + return nil, errors.New("wrong resource defaults") + } + if f.startGate != nil { + <-f.startGate + } + return f, f.record("start") +} +func (f *machineFixture) ID() string { return "isolated-test-instance" } +func (f *machineFixture) Provision(ctx context.Context, source string) error { + if _, err := os.ReadFile(source); err != nil { + return err + } + if f.cancel != nil { + f.cancel() + return ctx.Err() + } + if f.waitForCancel { + <-ctx.Done() + return ctx.Err() + } + return f.record("provision") +} +func (f *machineFixture) Sanitize(context.Context) error { return f.record("sanitize") } +func (f *machineFixture) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { + f.cleanupContextLive = ctx.Err() == nil + phase := "stop" + if force { + phase = "abort" + } + return f.stop, f.record(phase) +} +func (f *machineFixture) Export(_ context.Context, root string) error { + if err := f.record("export"); err != nil { + return err + } + data, _ := json.Marshal(*f.image.MacOS) + for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { + if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { + return err + } + } + if f.mutateExport != nil { + return f.mutateExport(root) + } + return nil +} +func (f *machineFixture) Destroy(ctx context.Context) error { + f.cleanupContextLive = ctx.Err() == nil + return f.record("destroy") +} +func (f *machineFixture) Publish(ctx context.Context, id, root string) (MachinePublication, error) { + f.publisherCalled = true + if err := ctx.Err(); err != nil { + return MachinePublication{}, err + } + if id != "build-test" { + return MachinePublication{}, errors.New("wrong job namespace") + } + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, err := os.Stat(filepath.Join(root, name)) + if err != nil || info.Mode().Perm() != 0600 { + return MachinePublication{}, errors.New("unsafe export mode") + } + } + publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} + if f.publicationOverride != nil { + publication = *f.publicationOverride + } + return publication, f.record("publish") +} +func machineTestRunner(t *testing.T, f *machineFixture) *machineRunnerFixture { + t.Helper() + return &machineRunnerFixture{MachineBuildBackend: MachineBuildBackend{Driver: f, Publisher: f}, WorkDir: t.TempDir()} +} +func machineTestRequest() MachineBuildRequest { + return MachineBuildRequest{ID: "build-test", BaseImage: "localhost/macos@" + machineTestDigest} +} + +func TestMachineBuildPublicationOrder(t *testing.T) { + f := newMachineFixture() + r := machineTestRunner(t, f) + result, err := r.Run(context.Background(), machineTestRequest(), strings.NewReader("source")) + if err != nil { + t.Fatal(err) + } + expected := []string{"start", "provision", "sanitize", "stop", "export", "destroy", "publish"} + if !reflect.DeepEqual(f.calls, expected) { + t.Fatalf("wrong phase order: %v", f.calls) + } + hash := sha256.Sum256([]byte("source")) + if result.Provenance.SourceHash != hex.EncodeToString(hash[:]) || result.Provenance.BaseImageDigest != machineTestDigest { + t.Fatal("unverified provenance") + } + entries, err := os.ReadDir(r.WorkDir) + if err != nil || len(entries) != 0 { + t.Fatal("workspace not cleaned") + } +} + +func TestMachineBuildFailuresDoNotPublish(t *testing.T) { + for _, tc := range []struct { + name string + phase string + receipt *MachineStopReceipt + }{ + {name: "start", phase: "start"}, + {name: "provision", phase: "provision"}, + {name: "sanitize", phase: "sanitize"}, + {name: "stop", phase: "stop"}, + {name: "export", phase: "export"}, + {name: "destroy", phase: "destroy"}, + {name: "unconfirmed stop", receipt: &MachineStopReceipt{}}, + {name: "forced stop", receipt: &MachineStopReceipt{VMMExited: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + if tc.receipt != nil { + f.stop = *tc.receipt + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("failed build reached publication") + } + if tc.phase != "" && strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("raw guest error leaked") + } + for _, phase := range f.calls { + if phase == "export" && tc.receipt != nil { + t.Fatal("unconfirmed stop exported") + } + if phase == "destroy" && tc.receipt != nil && !tc.receipt.VMMExited { + t.Fatal("destroyed unconfirmed live storage") + } + } + }) + } +} + +func TestMachineBuildInputAdmission(t *testing.T) { + for _, kind := range []string{"floating base", "wrong digest", "linux", "pending", "shaping", "source hash", "CPU overflow", "memory precision"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + req := machineTestRequest() + switch kind { + case "floating base": + req.BaseImage = "localhost/macos:latest" + case "wrong digest": + f.image.Digest = "sha256:" + strings.Repeat("b", 64) + case "linux": + f.image.Platform = "linux/arm64" + case "pending": + f.image.Status = images.StatusPending + case "shaping": + req.Policy.MemoryMB = 4096 + case "source hash": + req.SourceHash = strings.Repeat("0", 64) + case "CPU overflow": + f.image.MacOS.CPUs = ^uint(0) + case "memory precision": + f.image.MacOS.Memory++ + } + runMachineBuildFailure(t, f, context.Background(), req, strings.NewReader("source")) + for _, phase := range f.calls { + if phase == "start" { + t.Fatal("bad input started a VM") + } + } + }) + } +} + +var _ MachineBuildDriver = (*machineFixture)(nil) +var _ MachineBuildSession = (*machineFixture)(nil) +var _ MachineBuildPublisher = (*machineFixture)(nil) diff --git a/lib/builds/macos_delta_publish.go b/lib/builds/macos_delta_publish.go new file mode 100644 index 000000000..f0590eb47 --- /dev/null +++ b/lib/builds/macos_delta_publish.go @@ -0,0 +1,172 @@ +package builds + +import ( + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// maxMachineMetadataBytes bounds a base manifest or config fetched for a delta. +const maxMachineMetadataBytes = 4 << 20 + +// MachineDeltaBase is a pinned base that a derivative is published against. Its +// layers are mounted from Repo, so the base must already be in the registry the +// publisher writes to. +type MachineDeltaBase struct { + Repo string // builds/{id} repository holding the base manifest and layers + Digest string // canonical sha256 digest of the base manifest + DiskPath string // absolute path of the base boot disk on this host +} + +func (m MachineDeltaBase) validate() error { + if !strings.HasPrefix(m.Repo, "builds/") || strings.Contains(m.Repo, "..") || len(m.Repo) > 256 { + return fmt.Errorf("delta base repository must be under builds/") + } + if !ValidMachineDigest(m.Digest) { + return fmt.Errorf("delta base digest must be a canonical sha256 digest") + } + if !filepath.IsAbs(m.DiskPath) { + return fmt.Errorf("delta base disk path must be absolute") + } + return nil +} + +// machineLayers returns a machine image's layer blobs and their diff IDs. A +// standalone image has one uploaded layer. A derivative mounts the base's layers +// and uploads one delta layer that patches the base disk. +func machineLayers(ctx context.Context, client *http.Client, token, registry, bundle string, source *MachineDeltaBase) ([]machineBlob, []v1.Hash, error) { + if source == nil { + layer, err := machineTarballLayer(ctx, bundle) + if err != nil { + return nil, nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, nil, err + } + digest, err := layer.Digest() + if err != nil { + return nil, nil, err + } + size, err := layer.Size() + if err != nil { + return nil, nil, err + } + blob := machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: digest, Size: size}} + return []machineBlob{blob}, []v1.Hash{diffID}, nil + } + baseLayers, baseDiffIDs, err := fetchMachineSource(ctx, client, token, registry, source) + if err != nil { + return nil, nil, err + } + blobs := make([]machineBlob, 0, len(baseLayers)+1) + for _, descriptor := range baseLayers { + blobs = append(blobs, machineBlob{descriptor: descriptor, mountFrom: source.Repo}) + } + diskPath := filepath.Join(bundle, "disk.img") + delta, err := images.ComputeMacOSDelta(source.DiskPath, diskPath) + if err != nil { + return nil, nil, fmt.Errorf("compute disk delta: %w", err) + } + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(images.WriteMacOSDeltaTar(writer, delta, diskPath, bundle)) }() + return reader, nil + }, tarball.WithMediaType(images.MacOSDeltaMediaType), tarball.WithCompressionLevel(gzip.DefaultCompression)) + if err != nil { + return nil, nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, nil, err + } + digest, err := layer.Digest() + if err != nil { + return nil, nil, err + } + size, err := layer.Size() + if err != nil { + return nil, nil, err + } + blobs = append(blobs, machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: images.MacOSDeltaMediaType, Digest: digest, Size: size}}) + return blobs, append(baseDiffIDs, diffID), nil +} + +// fetchMachineSource reads the base manifest and config from the registry, checking +// each against its digest. It returns the base's layer descriptors and diff IDs. +func fetchMachineSource(ctx context.Context, client *http.Client, token, registry string, source *MachineDeltaBase) ([]v1.Descriptor, []v1.Hash, error) { + root := strings.TrimRight(registry, "/") + "/v2/" + source.Repo + manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+source.Digest, source.Digest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + var manifest v1.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + configDigest := manifest.Config.Digest.String() + configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + var config v1.ConfigFile + if err := json.Unmarshal(configBytes, &config); err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + if config.OS != "darwin" || config.Architecture != "arm64" || len(manifest.Layers) == 0 || len(config.RootFS.DiffIDs) != len(manifest.Layers) { + return nil, nil, fmt.Errorf("base is not a darwin/arm64 machine image with matching layers") + } + return manifest.Layers, config.RootFS.DiffIDs, nil +} + +// machineDeltaBase returns the base as a delta source when the publisher writes to +// the registry that stores the base. Any other base publishes standalone. +func (m *manager) machineDeltaBase(publisher MachineBuildPublisher, baseImage string, base *images.Image) *MachineDeltaBase { + publisherWithRegistry, ok := publisher.(*StreamingMachinePublisher) + if !ok || base == nil { + return nil + } + address := publisherWithRegistry.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" { + return nil + } + ref, err := images.ParseNormalizedRef(baseImage) + if err != nil || !ref.IsDigest() { + return nil + } + prefix := registry.Host + "/" + if !strings.HasPrefix(ref.Repository(), prefix) { + return nil + } + disk, err := images.GetDiskPath(m.paths, baseImage, base.Digest) + if err != nil { + return nil + } + return &MachineDeltaBase{Repo: strings.TrimPrefix(ref.Repository(), prefix), Digest: base.Digest, DiskPath: disk} +} + +// publishBuild publishes bundle as a derivative when the base is a registry source, +// and as a standalone image otherwise. +func (r *MachineBuildBackend) publishBuild(ctx context.Context, req MachineBuildRequest, bundle string) (MachinePublication, error) { + if derived, ok := r.Publisher.(interface { + PublishDerived(context.Context, string, string, MachineDeltaBase) (MachinePublication, error) + }); ok && req.deltaBase != nil { + return derived.PublishDerived(ctx, req.ID, bundle, *req.deltaBase) + } + return r.Publisher.Publish(ctx, req.ID, bundle) +} diff --git a/lib/builds/macos_delta_publish_test.go b/lib/builds/macos_delta_publish_test.go new file mode 100644 index 000000000..034cff270 --- /dev/null +++ b/lib/builds/macos_delta_publish_test.go @@ -0,0 +1,202 @@ +package builds_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +const deltaTestSecret = "synthetic registry secret" + +type deltaTestRegistry struct { + server *httptest.Server + tokens *builds.RegistryTokenGenerator + mounts atomic.Int32 + uploads atomic.Int32 + puts atomic.Int32 +} + +func newDeltaTestRegistry(t *testing.T) *deltaTestRegistry { + t.Helper() + imports := &machineImportStub{imported: make(chan string, 16)} + reg, err := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, err) + dr := &deltaTestRegistry{tokens: builds.NewRegistryTokenGenerator(deltaTestSecret)} + handler := middleware.JwtAuth(deltaTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Query().Get("mount") != "" { + dr.mounts.Add(1) + } else if r.Method == http.MethodPost && r.URL.Query().Get("digest") != "" { + dr.uploads.Add(1) + } + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + dr.puts.Add(1) + } + reg.Handler().ServeHTTP(w, r) + })) + dr.server = httptest.NewServer(handler) + t.Cleanup(dr.server.Close) + // Each manifest PUT starts a conversion goroutine that writes the OCI layout + // after the response. Its import call is the last step, so wait for every one + // before the TempDir cleanup removes the registry's data. + t.Cleanup(func() { + for range dr.puts.Load() { + select { + case <-imports.imported: + case <-time.After(10 * time.Second): + return + } + } + }) + return dr +} + +// deltaTestDisk is a disk of eight 64 KiB chunks, each a distinct byte. +func deltaTestDisk() []byte { + disk := make([]byte, 8*64<<10) + for i := range 8 { + for j := range 64 << 10 { + disk[i*64<<10+j] = byte(i + 1) + } + } + return disk +} + +func deltaTestBundle(t *testing.T, disk []byte, aux string) string { + t.Helper() + bundle := machinePublisherBundle(t) + require.NoError(t, os.WriteFile(filepath.Join(bundle, "disk.img"), disk, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(bundle, "aux.img"), []byte(aux), 0600)) + return bundle +} + +// pullManifest fetches a manifest from the registry with a pull-scoped token. +func (dr *deltaTestRegistry) pullManifest(t *testing.T, repo, digest string) v1.Manifest { + t.Helper() + token, err := dr.tokens.GenerateToken("reader", []builds.RepoPermission{{Repo: repo, Scope: "pull"}}, time.Minute) + require.NoError(t, err) + request, err := http.NewRequest(http.MethodGet, dr.server.URL+"/v2/"+repo+"/manifests/"+digest, nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + var manifest v1.Manifest + require.NoError(t, json.NewDecoder(response.Body).Decode(&manifest)) + return manifest +} + +func TestStreamingMachinePublisherDerivedMountsBaseLayers(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + baseBundle := deltaTestBundle(t, deltaTestDisk(), "base aux") + base, err := publisher.Publish(ctx, "base-x", baseBundle) + require.NoError(t, err) + baseManifest := dr.pullManifest(t, "builds/base-x", base.Digest) + + derived := deltaTestDisk() + for j := range 64 << 10 { + derived[3*64<<10+j] = 0xAA + } + derivedBundle := deltaTestBundle(t, derived, "derived aux") + dr.mounts.Store(0) + dr.uploads.Store(0) + result, err := publisher.PublishDerived(ctx, "derived-x", derivedBundle, builds.MachineDeltaBase{ + Repo: "builds/base-x", + Digest: base.Digest, + DiskPath: filepath.Join(baseBundle, "disk.img"), + }) + require.NoError(t, err) + require.Equal(t, int32(1), dr.mounts.Load(), "the base layer is mounted, not uploaded") + require.Equal(t, int32(2), dr.uploads.Load(), "only the config and the delta layer are uploaded") + + derivedManifest := dr.pullManifest(t, "builds/derived-x", result.Digest) + require.Len(t, derivedManifest.Layers, 2) + require.Equal(t, baseManifest.Layers[0].Digest, derivedManifest.Layers[0].Digest) + require.Equal(t, images.MacOSDeltaMediaType, derivedManifest.Layers[1].MediaType) +} + +func TestStreamingMachinePublisherDerivedRejectsMissingBase(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + bundle := deltaTestBundle(t, deltaTestDisk(), "aux") + missing := fmt.Sprintf("sha256:%064x", 1) + _, err := publisher.PublishDerived(ctx, "derived-y", bundle, builds.MachineDeltaBase{ + Repo: "builds/never-published", + Digest: missing, + DiskPath: filepath.Join(bundle, "disk.img"), + }) + require.Error(t, err) +} + +func TestStreamingMachinePublisherDerivedRejectsNonBaseRepo(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + bundle := deltaTestBundle(t, deltaTestDisk(), "aux") + _, err := publisher.PublishDerived(context.Background(), "derived-z", bundle, builds.MachineDeltaBase{ + Repo: "library/alpine", + Digest: fmt.Sprintf("sha256:%064x", 2), + DiskPath: filepath.Join(bundle, "disk.img"), + }) + require.ErrorContains(t, err, "under builds/") +} + +// mountRequest posts a blob mount with the given token and reports the status. +func (dr *deltaTestRegistry) mountRequest(t *testing.T, token, repo, digest, from string) int { + t.Helper() + target := dr.server.URL + "/v2/" + repo + "/blobs/uploads/?mount=" + digest + "&from=" + from + request, err := http.NewRequest(http.MethodPost, target, bytes.NewReader(nil)) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+token) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + _, _ = io.Copy(io.Discard, response.Body) + return response.StatusCode +} + +func TestBlobMountFallsThroughToUploadForMissingBlob(t *testing.T) { + dr := newDeltaTestRegistry(t) + token, err := dr.tokens.GenerateToken("mounter", []builds.RepoPermission{ + {Repo: "builds/target-x", Scope: "push"}, + {Repo: "builds/source-x", Scope: "pull"}, + }, time.Minute) + require.NoError(t, err) + status := dr.mountRequest(t, token, "builds/target-x", fmt.Sprintf("sha256:%064x", 3), "builds/source-x") + require.Equal(t, http.StatusAccepted, status, "a missing blob opens an upload session") +} + +func TestBlobMountRefusedFromUnauthorizedSource(t *testing.T) { + dr := newDeltaTestRegistry(t) + token, err := dr.tokens.GenerateToken("mounter", []builds.RepoPermission{ + {Repo: "builds/target-x", Scope: "push"}, + }, time.Minute) + require.NoError(t, err) + status := dr.mountRequest(t, token, "builds/target-x", fmt.Sprintf("sha256:%064x", 4), "builds/other-private") + require.Equal(t, http.StatusForbidden, status, "a token without pull on the source cannot mount from it") +} diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go new file mode 100644 index 000000000..a9d2f35cd --- /dev/null +++ b/lib/builds/macos_registry.go @@ -0,0 +1,105 @@ +package builds + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// machineRegistryRequest sends one authenticated registry request. The caller +// closes the response body. +func machineRegistryRequest(ctx context.Context, client *http.Client, token, method, target, media string, body io.Reader, size int64) (*http.Response, error) { + request, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return nil, err + } + request.ContentLength = size + request.Header.Set("Authorization", "Bearer "+token) + if media != "" { + request.Header.Set("Content-Type", media) + } + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("machine registry request failed") + } + return response, nil +} + +func machineHash(data []byte) v1.Hash { + sum := sha256.Sum256(data) + return v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum[:])} +} + +// machineFetch GETs a registry object and checks it against digest. A size of 0 +// marks metadata: the body is buffered up to maxMachineMetadataBytes and returned. +// Any other size must match exactly and is only hashed, never kept, so layers of +// tens of gigabytes are not held in memory. +func machineFetch(ctx context.Context, client *http.Client, token, target, digest string, size int64) ([]byte, error) { + response, err := machineRegistryRequest(ctx, client, token, http.MethodGet, target, "", nil, 0) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned status %d", response.StatusCode) + } + metadata := size == 0 + limit := size + 1 + if metadata { + limit = maxMachineMetadataBytes + 1 + } + sum := sha256.New() + var kept bytes.Buffer + out := io.Writer(sum) + if metadata { + out = io.MultiWriter(sum, &kept) + } + n, err := io.Copy(out, io.LimitReader(response.Body, limit)) + if err != nil { + return nil, err + } + if n >= limit || (!metadata && n != size) { + return nil, fmt.Errorf("object does not match digest %s", digest) + } + if (v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum.Sum(nil))}).String() != digest { + return nil, fmt.Errorf("object does not match digest %s", digest) + } + if !metadata { + return nil, nil + } + return kept.Bytes(), nil +} + +// mountMachineBlob links a base layer into the build repository from its source. The +// registry answers 201 only when it already stores the blob; the size is then checked +// with a HEAD request. The bytes are not downloaded or hashed again. +func mountMachineBlob(ctx context.Context, client *http.Client, token, base string, blob machineBlob) error { + digest := blob.descriptor.Digest.String() + target := base + "/blobs/uploads/?mount=" + digest + "&from=" + url.QueryEscape(blob.mountFrom) + response, err := machineRegistryRequest(ctx, client, token, http.MethodPost, target, "", nil, 0) + if err != nil { + return err + } + response.Body.Close() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("base layer %s was not mounted", digest) + } + head, err := machineRegistryRequest(ctx, client, token, http.MethodHead, base+"/blobs/"+digest, "", nil, 0) + if err != nil { + return err + } + head.Body.Close() + if head.StatusCode != http.StatusOK || head.ContentLength != blob.descriptor.Size { + return fmt.Errorf("mounted base layer %s is not verified", digest) + } + return nil +} diff --git a/lib/builds/macos_registry_test.go b/lib/builds/macos_registry_test.go new file mode 100644 index 000000000..77ccbac79 --- /dev/null +++ b/lib/builds/macos_registry_test.go @@ -0,0 +1,54 @@ +package builds + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachineFetchVerifiesSizeAndDigest(t *testing.T) { + body := []byte("synthetic object") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(body) + })) + defer server.Close() + digest := machineHash(body).String() + oversize := strings.Repeat("x", maxMachineMetadataBytes+1) + bigServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(oversize)) + })) + defer bigServer.Close() + + for _, tc := range []struct { + name string + target string + digest string + size int64 + ok bool + }{ + {"metadata matches digest", server.URL, digest, 0, true}, + {"sized object matches", server.URL, digest, int64(len(body)), true}, + {"wrong digest", server.URL, machineHash([]byte("other")).String(), 0, false}, + {"size shorter than body", server.URL, digest, int64(len(body)) - 1, false}, + {"size longer than body", server.URL, digest, int64(len(body)) + 1, false}, + {"metadata over cap", bigServer.URL, machineHash([]byte(oversize)).String(), 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := machineFetch(context.Background(), http.DefaultClient, "token", tc.target, tc.digest, tc.size) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.size == 0 { + require.Equal(t, body, got, "metadata is returned") + } else { + require.Nil(t, got, "sized objects are verified, never buffered") + } + }) + } +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 806ae5622..a4327e358 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -87,6 +87,12 @@ type Manager interface { // Config holds configuration for the build manager type Config struct { + // MachineBuild is the opt-in native macOS build backend. + MachineBuild *MachineBuildBackend + + // MachineBuildOnly disables Linux builder preparation and recovery. + MachineBuildOnly bool + // MaxConcurrentBuilds is the maximum number of concurrent builds MaxConcurrentBuilds int @@ -149,6 +155,7 @@ type manager struct { logger *slog.Logger metrics *Metrics createMu sync.Mutex + machineBuilds sync.Map // build ID -> *machineBuildControl builderReady atomic.Bool // pendingRecovered is set once RecoverPendingBuilds has re-enqueued // persisted pending builds; until then BuilderHasBuilds also scans @@ -204,6 +211,10 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { + if m.config.MachineBuildOnly { + m.RecoverPendingBuilds() + return nil + } go func() { for { m.ensureBuilderImage(ctx) @@ -444,7 +455,12 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc // Apply defaults to build policy policy := req.BuildPolicy - if policy == nil { + if machineBuild(&req) { + var err error + if policy, err = m.admitMachineBuild(&req, sourceData); err != nil { + return nil, err + } + } else if policy == nil { defaultPolicy := DefaultBuildPolicy() policy = &defaultPolicy } else { @@ -481,6 +497,15 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc CreatedAt: time.Now(), } + if machineBuild(&req) { + // Stage verified input before making the job visible to recovery. + if err := m.storeMachineSource(ctx, meta, sourceData); err != nil { + deleteBuild(m.paths, id) + return nil, err + } + return m.enqueueMachineBuild(meta, req, policy), nil + } + // Write initial metadata if err := writeMetadata(m.paths, meta); err != nil { return nil, fmt.Errorf("write metadata: %w", err) @@ -678,18 +703,29 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques start := time.Now() m.logger.Info("starting build", "id", id) - // Update status to building - m.updateStatus(id, StatusBuilding, nil) - - // Create timeout context + // Register machine cancellation before exposing the running status. Ordinary + // Linux builder cancellation retains its existing instance-delete behavior. buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + if machineBuild(&req) { + m.machineBuilds.Store(id, &machineBuildControl{cancel: cancel}) + defer m.machineBuilds.Delete(id) + defer func() { + meta, err := readMetadata(m.paths, id) + if err == nil && (meta.Status == StatusReady || meta.Status == StatusFailed || meta.Status == StatusCancelled) { + m.removeMachineSource(id) + } + }() + } + m.updateStatus(id, StatusBuilding, nil) // Mirror base images to the local registry before launching the VM. // BuildKit is configured with our registry as a mirror for docker.io, // so pre-cached images will be served locally without pulling from Docker Hub. - if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { - m.logger.Warn("failed to mirror base images", "id", id, "error", err) + if !machineBuild(&req) { + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } } // Run the build in a builder VM @@ -754,6 +790,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // If image_name is set, re-tag the image so it's accessible by that name. // The builder pushed to builds/{id} but the user wants it as image_name. imageRef := buildRepo + if machineBuild(&req) && result.ImageRef != "" { + imageRef = result.ImageRef + } if req.ImageName != "" { ref, err := images.ParseNormalizedRef(req.ImageName) if err != nil { @@ -798,6 +837,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // executeBuild runs the build in a builder VM func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + if machineBuild(&req) { + return m.executeMachineBuild(ctx, id, req, policy) + } if !m.builderReady.Load() { return nil, fmt.Errorf("builder image is being prepared, please retry shortly") } @@ -1106,6 +1148,11 @@ func (m *manager) updateStatus(id string, status string, err error) { // updateBuildComplete updates the build with final results func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } meta, readErr := readMetadata(m.paths, id) if readErr != nil { m.logger.Error("read metadata for completion", "id", id, "error", readErr) @@ -1246,12 +1293,18 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { case StatusQueued: // Remove from queue if m.queue.Cancel(id) { + if machineBuild(meta.Request) { + return m.cancelQueuedMachineBuild(meta) + } m.updateStatus(id, StatusCancelled, nil) return nil } return ErrBuildInProgress // Was already picked up case StatusBuilding, StatusPushing: + if machineBuild(meta.Request) { + return m.cancelMachineBuild(id) + } // Can't cancel a running build easily // Would need to terminate the builder instance if meta.BuilderInstance != nil { @@ -1449,15 +1502,36 @@ func (m *manager) StreamBuildEvents(ctx context.Context, id string, follow bool) // RecoverPendingBuilds recovers builds that were interrupted on restart func (m *manager) RecoverPendingBuilds() { - pending, err := listPendingBuilds(m.paths) + all, err := listAllBuilds(m.paths) if err != nil { - m.logger.Error("list pending builds for recovery", "error", err) + m.logger.Error("list builds for machine cleanup", "error", err) return } + for _, meta := range all { + if !machineBuild(meta.Request) { + continue + } + switch meta.Status { + case StatusReady, StatusFailed: + m.removeMachineSource(meta.ID) + case StatusCancelled: + if m.config.MachineBuild != nil { + if err := m.reapPriorMachineBuilder(context.Background(), meta.ID); err != nil { + m.logger.Error("failed to reap cancelled machine build", "id", meta.ID, "error", err) + continue + } + } + m.removeMachineSource(meta.ID) + } + } + pending := pendingBuilds(all) for _, meta := range pending { meta := meta // Shadow loop variable for closure capture m.logger.Info("recovering build", "id", meta.ID, "status", meta.Status) + if m.skipMachineBuildRecovery(meta) { + continue + } // Re-enqueue the build if meta.Request != nil { @@ -1472,7 +1546,11 @@ func (m *manager) RecoverPendingBuilds() { continue } - m.queue.EnqueueSerial(meta.ID, *meta.Request, meta.Request.BuilderID, func() { + serialKey := meta.Request.BuilderID + if machineBuild(meta.Request) { + serialKey = machineBuildSerialKey(*meta.Request) + } + m.queue.EnqueueSerial(meta.ID, *meta.Request, serialKey, func() { policy := DefaultBuildPolicy() if meta.Request.BuildPolicy != nil { policy = *meta.Request.BuildPolicy @@ -1485,11 +1563,16 @@ func (m *manager) RecoverPendingBuilds() { if len(pending) > 0 { m.logger.Info("recovered pending builds", "count", len(pending)) } - m.pendingRecovered.Store(true) + if !m.config.MachineBuildOnly { + m.pendingRecovered.Store(true) + } } // refreshBuildToken regenerates the registry token for a build and updates the config file func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) error { + if machineBuild(req) { + return nil // Machine builds have no builder config or registry token. + } // Read existing build config config, err := readBuildConfig(m.paths, buildID) if err != nil { diff --git a/lib/builds/source.go b/lib/builds/source.go new file mode 100644 index 000000000..b2075607f --- /dev/null +++ b/lib/builds/source.go @@ -0,0 +1,75 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" +) + +const maxBuildSourceBytes = 64 << 20 + +type buildContextReader struct { + ctx context.Context + r io.Reader +} + +func (r buildContextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +// stageBuildSource streams compressed source once into a private, exclusive job +// file. Both backends consume this file and use its hash for input verification. +func stageBuildSource(ctx context.Context, source io.Reader, path string, limit int64) (hash string, err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return "", err + } + defer func() { + if closeErr := file.Close(); err == nil { + err = closeErr + } + }() + h := sha256.New() + n, err := io.Copy(io.MultiWriter(file, h), io.LimitReader(buildContextReader{ctx, source}, limit+1)) + if err != nil { + return "", err + } + if n > limit { + return "", fmt.Errorf("build source too large") + } + if err = file.Sync(); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// verifyBuildSource rejects missing/replaced source on recovery before a machine +// is launched. It never writes a second source copy. +func verifyBuildSource(ctx context.Context, path, expected string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxBuildSourceBytes { + return fmt.Errorf("invalid staged source") + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + h := sha256.New() + if _, err = io.Copy(h, io.LimitReader(buildContextReader{ctx, file}, maxBuildSourceBytes+1)); err != nil { + return err + } + if hex.EncodeToString(h.Sum(nil)) != expected { + return ErrSourceHashMismatch + } + return nil +} diff --git a/lib/builds/source_test.go b/lib/builds/source_test.go new file mode 100644 index 000000000..1bb7e8a84 --- /dev/null +++ b/lib/builds/source_test.go @@ -0,0 +1,36 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSharedBuildSourceStagingAndRecoveryVerification(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.tar.gz") + hash, err := stageBuildSource(ctx, strings.NewReader("source"), path, maxBuildSourceBytes) + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.NoError(t, verifyBuildSource(ctx, path, hash)) + _, err = stageBuildSource(ctx, strings.NewReader("replacement"), path, maxBuildSourceBytes) + require.Error(t, err, "exclusive staging must preserve existing source") + require.NoError(t, verifyBuildSource(ctx, path, hash)) + canceled, cancel := context.WithCancel(ctx) + cancel() + require.ErrorIs(t, verifyBuildSource(canceled, path, hash), context.Canceled) + require.NoError(t, os.WriteFile(path, []byte("changed"), 0600)) + require.ErrorIs(t, verifyBuildSource(ctx, path, hash), ErrSourceHashMismatch) + require.NoError(t, os.Remove(path)) + require.Error(t, verifyBuildSource(ctx, path, hash)) + target := filepath.Join(filepath.Dir(path), "target") + require.NoError(t, os.WriteFile(target, []byte("source"), 0600)) + require.NoError(t, os.Symlink(target, path)) + require.ErrorContains(t, verifyBuildSource(ctx, path, hash), "invalid staged source") +} diff --git a/lib/builds/storage.go b/lib/builds/storage.go index ce555aac9..1add7a3c1 100644 --- a/lib/builds/storage.go +++ b/lib/builds/storage.go @@ -144,7 +144,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { if err != nil { return nil, err } + return pendingBuilds(all), nil +} +func pendingBuilds(all []*buildMetadata) []*buildMetadata { var pending []*buildMetadata for _, meta := range all { switch meta.Status { @@ -152,13 +155,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { pending = append(pending, meta) } } - - // Sort by created_at ascending (oldest first for FIFO recovery) sort.Slice(pending, func(i, j int) bool { return pending[i].CreatedAt.Before(pending[j].CreatedAt) }) - - return pending, nil + return pending } // deleteBuild removes a build's data from disk diff --git a/lib/builds/types.go b/lib/builds/types.go index 4ad6008e9..571259328 100644 --- a/lib/builds/types.go +++ b/lib/builds/types.go @@ -43,6 +43,10 @@ type CreateBuildRequest struct { // The Dockerfile specifies the runtime (e.g., FROM node:20-alpine). Dockerfile string `json:"dockerfile,omitempty"` + // MachineBaseImage selects the installed-machine backend. The HTTP multipart + // field machine_base_image maps here; it is enabled only with macos_builds. + MachineBaseImage string `json:"machine_base_image,omitempty"` + // BaseImageDigest optionally pins the base image by digest for reproducibility BaseImageDigest string `json:"base_image_digest,omitempty"` @@ -219,6 +223,9 @@ type BuildResult struct { // ImageDigest is the digest of the pushed image (only on success) ImageDigest string `json:"image_digest,omitempty"` + // ImageRef is the pullable output reference for machine builds. + ImageRef string `json:"image_ref,omitempty"` + // Error is the error message (only on failure) Error string `json:"error,omitempty"` diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 5e41bb8e7..73b8a4e0a 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -2,10 +2,12 @@ package images import ( "bytes" + "context" "encoding/json" "fmt" "io" "os" + "os/exec" "path/filepath" ) @@ -19,6 +21,19 @@ type macOSMachinePayload struct { Platform *MacOSImage } +// RequireClosedFile fails unless no process holds path open. lsof exits 1 when +// nothing holds the file; any other failure leaves the storage unverified. +func RequireClosedFile(ctx context.Context, path string) error { + err := exec.CommandContext(ctx, "/usr/sbin/lsof", "-t", path).Run() + if err == nil { + return fmt.Errorf("storage is open: stop its VM first") + } + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + return fmt.Errorf("cannot verify storage is closed: %w", err) + } + return nil +} + // ValidateMacOSBundle validates a fixed-layout export, rejecting unknown metadata // fields so an accidental credential field cannot become part of an image. func ValidateMacOSBundle(root string) (*MacOSImage, error) { diff --git a/lib/images/macos_delta.go b/lib/images/macos_delta.go new file mode 100644 index 000000000..17255ac55 --- /dev/null +++ b/lib/images/macos_delta.go @@ -0,0 +1,261 @@ +package images + +import ( + "archive/tar" + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// MacOSDeltaMediaType marks a machine layer that patches the boot disk unpacked +// from the layers beneath it. Its tar holds, in this order, delta.json, the changed +// chunks of disk.img, and aux.img and config.json in full. +const MacOSDeltaMediaType types.MediaType = "application/vnd.hypeman.machine.disk-delta.v1.tar+gzip" + +// MacOSDeltaChunkSize is the granularity at which a delta records changed bytes. +const MacOSDeltaChunkSize = 64 << 10 + +const ( + macOSDeltaVersion = 1 + macOSDeltaManifestEntry = "delta.json" + macOSDeltaPatchEntry = "disk.img.patch" + macOSDeltaDisk = "disk.img" + macOSDeltaAux = "aux.img" + macOSDeltaConfig = "config.json" + maxMacOSDeltaManifestBytes = 1 << 20 + macOSDeltaEntries = 4 +) + +// MacOSDelta lists the chunks of a boot disk that differ from the disk beneath it. +// Size is the disk's logical size, which the base disk must match. +type MacOSDelta struct { + Version int `json:"version"` + ChunkSize int64 `json:"chunk_size"` + Size int64 `json:"size"` + Chunks []int64 `json:"chunks"` +} + +// Validate checks a delta before any chunk is applied: a supported version and +// chunk size, a whole number of chunks, and changed chunks in ascending order. +func (d *MacOSDelta) Validate() error { + if d.Version != macOSDeltaVersion || d.ChunkSize != MacOSDeltaChunkSize { + return fmt.Errorf("unsupported machine disk delta version %d with chunk size %d", d.Version, d.ChunkSize) + } + if d.Size <= 0 || d.Size%d.ChunkSize != 0 { + return fmt.Errorf("machine disk delta size %d is not a positive multiple of %d", d.Size, d.ChunkSize) + } + count := d.Size / d.ChunkSize + previous := int64(-1) + for _, index := range d.Chunks { + if index <= previous || index >= count { + return fmt.Errorf("machine disk delta chunk %d is out of order or out of range", index) + } + previous = index + } + return nil +} + +// ComputeMacOSDelta compares a derived boot disk with its base chunk by chunk. Both +// files must be the same size and a whole number of chunks. +func ComputeMacOSDelta(basePath, derivedPath string) (MacOSDelta, error) { + base, err := os.Open(basePath) + if err != nil { + return MacOSDelta{}, err + } + defer base.Close() + derived, err := os.Open(derivedPath) + if err != nil { + return MacOSDelta{}, err + } + defer derived.Close() + baseInfo, err := base.Stat() + if err != nil { + return MacOSDelta{}, err + } + derivedInfo, err := derived.Stat() + if err != nil { + return MacOSDelta{}, err + } + if baseInfo.Size() != derivedInfo.Size() { + return MacOSDelta{}, fmt.Errorf("derived disk is %d bytes, base disk is %d", derivedInfo.Size(), baseInfo.Size()) + } + delta := MacOSDelta{Version: macOSDeltaVersion, ChunkSize: MacOSDeltaChunkSize, Size: derivedInfo.Size(), Chunks: []int64{}} + if err := delta.Validate(); err != nil { + return MacOSDelta{}, err + } + baseReader := bufio.NewReaderSize(base, 4<<20) + derivedReader := bufio.NewReaderSize(derived, 4<<20) + baseChunk := make([]byte, MacOSDeltaChunkSize) + derivedChunk := make([]byte, MacOSDeltaChunkSize) + for index := int64(0); index*MacOSDeltaChunkSize < delta.Size; index++ { + if _, err := io.ReadFull(baseReader, baseChunk); err != nil { + return MacOSDelta{}, fmt.Errorf("read base disk: %w", err) + } + if _, err := io.ReadFull(derivedReader, derivedChunk); err != nil { + return MacOSDelta{}, fmt.Errorf("read derived disk: %w", err) + } + if !bytes.Equal(baseChunk, derivedChunk) { + delta.Chunks = append(delta.Chunks, index) + } + } + return delta, nil +} + +// WriteMacOSDeltaTar writes a delta layer's tar. The changed chunks are read from +// derived, which must be the disk the delta was computed against. aux.img and +// config.json are copied in full from bundle. +func WriteMacOSDeltaTar(w io.Writer, delta MacOSDelta, derived, bundle string) error { + if err := delta.Validate(); err != nil { + return err + } + manifest, err := json.Marshal(delta) + if err != nil { + return err + } + disk, err := os.Open(derived) + if err != nil { + return err + } + defer disk.Close() + archive := tar.NewWriter(w) + if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaManifestEntry, Mode: 0600, Size: int64(len(manifest)), Typeflag: tar.TypeReg}); err != nil { + return err + } + if _, err := archive.Write(manifest); err != nil { + return err + } + patchSize := int64(len(delta.Chunks)) * delta.ChunkSize + if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaPatchEntry, Mode: 0600, Size: patchSize, Typeflag: tar.TypeReg}); err != nil { + return err + } + chunk := make([]byte, delta.ChunkSize) + for _, index := range delta.Chunks { + if _, err := disk.ReadAt(chunk, index*delta.ChunkSize); err != nil { + return fmt.Errorf("read changed chunk %d: %w", index, err) + } + if _, err := archive.Write(chunk); err != nil { + return err + } + } + for _, name := range []string{macOSDeltaAux, macOSDeltaConfig} { + if err := writeMacOSDeltaFile(archive, filepath.Join(bundle, name), name); err != nil { + return err + } + } + return archive.Close() +} + +func writeMacOSDeltaFile(archive *tar.Writer, path, name string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + if err := archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}); err != nil { + return err + } + _, err = io.Copy(archive, file) + return err +} + +// macOSDeltaApplier applies the entries of a delta layer to the boot disk already +// unpacked into targetDir. The entries must arrive in the order WriteMacOSDeltaTar +// writes them, so a layer cannot name files outside the format. +type macOSDeltaApplier struct { + targetDir string + step int + manifest *MacOSDelta +} + +func (a *macOSDeltaApplier) entry(name string, size int64, archive io.Reader) error { + switch a.step { + case 0: + if name != macOSDeltaManifestEntry { + return fmt.Errorf("delta layer must start with %s, got %q", macOSDeltaManifestEntry, name) + } + if size > maxMacOSDeltaManifestBytes { + return fmt.Errorf("machine disk delta manifest is %d bytes", size) + } + decoder := json.NewDecoder(io.LimitReader(archive, size)) + decoder.DisallowUnknownFields() + var delta MacOSDelta + if err := decoder.Decode(&delta); err != nil { + return fmt.Errorf("decode machine disk delta: %w", err) + } + if err := delta.Validate(); err != nil { + return err + } + a.manifest = &delta + case 1: + if name != macOSDeltaPatchEntry { + return fmt.Errorf("delta layer entry %q must be %s", name, macOSDeltaPatchEntry) + } + if size != int64(len(a.manifest.Chunks))*a.manifest.ChunkSize { + return fmt.Errorf("machine disk patch is %d bytes, want %d", size, int64(len(a.manifest.Chunks))*a.manifest.ChunkSize) + } + if err := a.applyChunks(archive); err != nil { + return err + } + case 2, 3: + want := macOSDeltaAux + if a.step == 3 { + want = macOSDeltaConfig + } + if name != want { + return fmt.Errorf("delta layer entry %q must be %s", name, want) + } + if err := writeSparseFile(filepath.Join(a.targetDir, name), archive, size); err != nil { + return err + } + default: + return fmt.Errorf("delta layer has more than %d entries", macOSDeltaEntries) + } + a.step++ + return nil +} + +// applyChunks writes each changed chunk over the base disk in place. The base must +// be a regular file of the delta's size, so the offsets land where the diff computed. +func (a *macOSDeltaApplier) applyChunks(r io.Reader) error { + path := filepath.Join(a.targetDir, macOSDeltaDisk) + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("delta has no base disk: %w", err) + } + if !info.Mode().IsRegular() || info.Size() != a.manifest.Size { + return fmt.Errorf("base disk does not match the delta's %d-byte size", a.manifest.Size) + } + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return err + } + buf := make([]byte, a.manifest.ChunkSize) + for _, index := range a.manifest.Chunks { + if _, err = io.ReadFull(r, buf); err != nil { + break + } + if _, err = file.WriteAt(buf, index*a.manifest.ChunkSize); err != nil { + break + } + } + return errors.Join(err, file.Close()) +} + +// complete reports an error if the layer ended before its last entry. +func (a *macOSDeltaApplier) complete() error { + if a.step != macOSDeltaEntries { + return fmt.Errorf("delta layer has %d of %d entries", a.step, macOSDeltaEntries) + } + return nil +} diff --git a/lib/images/macos_delta_test.go b/lib/images/macos_delta_test.go new file mode 100644 index 000000000..4e0227e85 --- /dev/null +++ b/lib/images/macos_delta_test.go @@ -0,0 +1,203 @@ +package images + +import ( + "archive/tar" + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/stretchr/testify/require" +) + +type machineTestFile struct { + name string + data []byte +} + +// machineFilesLayer builds a layer of regular files with the given media type. +func machineFilesLayer(t *testing.T, mediaType types.MediaType, files ...machineTestFile) gcr.Layer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, f := range files { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0600, Size: int64(len(f.data)), Typeflag: tar.TypeReg})) + _, err := tw.Write(f.data) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(mediaType)) + require.NoError(t, err) + return layer +} + +// deltaTestDisk returns a disk of n chunks, each filled with a distinct byte. +func deltaTestDisk(n int) []byte { + disk := make([]byte, n*MacOSDeltaChunkSize) + for i := range n { + for j := range MacOSDeltaChunkSize { + disk[i*MacOSDeltaChunkSize+j] = byte(i + 1) + } + } + return disk +} + +func writeDeltaTestFile(t *testing.T, path string, data []byte) { + t.Helper() + require.NoError(t, os.WriteFile(path, data, 0600)) +} + +// deltaTestLayer builds a delta layer from a manifest and a bundle directory. +func deltaTestLayer(t *testing.T, delta MacOSDelta, derived, bundle string) gcr.Layer { + t.Helper() + var buf bytes.Buffer + require.NoError(t, WriteMacOSDeltaTar(&buf, delta, derived, bundle)) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(MacOSDeltaMediaType)) + require.NoError(t, err) + return layer +} + +func TestMacOSDeltaRoundTripAppliesOnlyChangedChunks(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + base := deltaTestDisk(8) + derived := append([]byte(nil), base...) + for j := range MacOSDeltaChunkSize { + derived[2*MacOSDeltaChunkSize+j] = 0xAA + derived[5*MacOSDeltaChunkSize+j] = 0 + } + basePath := filepath.Join(dir, "base-disk.img") + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, basePath, base) + writeDeltaTestFile(t, derivedPath, derived) + bundle := filepath.Join(dir, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("derived aux")) + writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte(`{"derived":true}`)) + + delta, err := ComputeMacOSDelta(basePath, derivedPath) + require.NoError(t, err) + require.Equal(t, []int64{2, 5}, delta.Chunks) + + baseLayer := machineFilesLayer(t, types.OCILayer, + machineTestFile{name: "disk.img", data: base}, + machineTestFile{name: "aux.img", data: []byte("base aux")}, + machineTestFile{name: "config.json", data: []byte(`{"base":true}`)}, + ) + deltaLayer := deltaTestLayer(t, delta, derivedPath, bundle) + target := t.TempDir() + require.NoError(t, unpackMachineLayers2(ctx, target, baseLayer, deltaLayer)) + + got, err := os.ReadFile(filepath.Join(target, "disk.img")) + require.NoError(t, err) + require.Equal(t, derived, got) + aux, err := os.ReadFile(filepath.Join(target, "aux.img")) + require.NoError(t, err) + require.Equal(t, "derived aux", string(aux)) + config, err := os.ReadFile(filepath.Join(target, "config.json")) + require.NoError(t, err) + require.Equal(t, `{"derived":true}`, string(config)) +} + +func TestMacOSDeltaRejectsBaseOfDifferentSize(t *testing.T) { + dir := t.TempDir() + bundle := writeDeltaBundle(t, dir) + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) + delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} + baseLayer := machineFilesLayer(t, types.OCILayer, machineTestFile{name: "disk.img", data: deltaTestDisk(4)}) + err := unpackMachineLayers2(context.Background(), t.TempDir(), baseLayer, deltaTestLayer(t, delta, derivedPath, bundle)) + require.ErrorContains(t, err, "base disk does not match") +} + +func TestMacOSDeltaRejectsLayerWithoutBase(t *testing.T) { + dir := t.TempDir() + bundle := writeDeltaBundle(t, dir) + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) + delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} + err := unpackMachineLayers2(context.Background(), t.TempDir(), deltaTestLayer(t, delta, derivedPath, bundle)) + require.ErrorContains(t, err, "delta has no base disk") +} + +func TestMacOSDeltaRejectsIncompleteAndMisorderedLayers(t *testing.T) { + ctx := context.Background() + manifest := []byte(`{"version":1,"chunk_size":65536,"size":524288,"chunks":[]}`) + cases := map[string][]machineTestFile{ + "missing entries": {{name: "delta.json", data: manifest}}, + "wrong first entry": { + {name: "aux.img", data: []byte("x")}, + }, + "patch skipped": { + {name: "delta.json", data: manifest}, + {name: "aux.img", data: []byte("x")}, + }, + "extra entry": { + {name: "delta.json", data: manifest}, + {name: "disk.img.patch"}, + {name: "aux.img", data: []byte("x")}, + {name: "config.json", data: []byte("{}")}, + {name: "extra", data: []byte("x")}, + }, + } + for name, files := range cases { + t.Run(name, func(t *testing.T) { + target := t.TempDir() + writeDeltaTestFile(t, filepath.Join(target, "disk.img"), deltaTestDisk(8)) + layer := machineFilesLayer(t, MacOSDeltaMediaType, files...) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.Error(t, unpackMachineLayer(ctx, layer, diffID, target, unlimitedMachineBudget())) + }) + } +} + +func TestMacOSDeltaValidateRejectsBadManifests(t *testing.T) { + size := int64(8 * MacOSDeltaChunkSize) + cases := map[string]MacOSDelta{ + "version": {Version: 2, ChunkSize: MacOSDeltaChunkSize, Size: size}, + "chunk size": {Version: 1, ChunkSize: 4096, Size: size}, + "partial size": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size + 1}, + "unordered": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 1}}, + "duplicate": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 3}}, + "out of range": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{8}}, + } + for name, delta := range cases { + t.Run(name, func(t *testing.T) { + require.Error(t, delta.Validate()) + }) + } + valid := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{0, 7}} + require.NoError(t, valid.Validate()) +} + +// writeDeltaBundle writes the aux and config files a delta layer copies in full. +func writeDeltaBundle(t *testing.T, dir string) string { + t.Helper() + bundle := filepath.Join(dir, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("aux")) + writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte("{}")) + return bundle +} + +// unpackMachineLayers2 unpacks layers in order into one target directory with an +// unlimited budget, as the bundle unpack does. +func unpackMachineLayers2(ctx context.Context, target string, layers ...gcr.Layer) error { + budget := unlimitedMachineBudget() + for _, l := range layers { + diffID, err := l.DiffID() + if err != nil { + return err + } + if err := unpackMachineLayer(ctx, l, diffID, target, budget); err != nil { + return err + } + } + return nil +} diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index d0de8ee4d..0fa90c9e6 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "time" @@ -44,11 +43,8 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if !info.Mode().IsRegular() { return nil, fmt.Errorf("bundle file must be regular: %s", file) } - if e = exec.CommandContext(ctx, "lsof", "-t", file).Run(); e == nil { - return nil, fmt.Errorf("bundle storage is open: stop its VM first") - } - if exit, ok := e.(*exec.ExitError); !ok || exit.ExitCode() != 1 { - return nil, fmt.Errorf("cannot verify storage is closed: %v", e) + if e = RequireClosedFile(ctx, file); e != nil { + return nil, e } } root := p.ImageRepositoryDir(ref.Repository()) diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index ad08493ec..7b71ec481 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -70,7 +70,8 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi } // unpackMachineLayer extracts one layer, spending its decompressed bytes from the -// bundle's remaining budget. +// bundle's remaining budget. A delta layer patches the boot disk unpacked beneath it +// instead of writing files. func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targetDir string, remaining *int64) error { rc, err := l.Uncompressed() if err != nil { @@ -90,6 +91,12 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return err } archive := tar.NewReader(stream) + mediaType, err := l.MediaType() + if err != nil { + return err + } + delta := mediaType == MacOSDeltaMediaType + applier := &macOSDeltaApplier{targetDir: targetDir} for { header, err := archive.Next() if errors.Is(err, io.EOF) { @@ -102,6 +109,15 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe if !filepath.IsLocal(name) || strings.HasPrefix(filepath.Base(name), ".wh.") { return fmt.Errorf("machine layer entry %q is not a plain local path", header.Name) } + if delta { + if header.Typeflag != tar.TypeReg || isGNUSparse(header) { + return fmt.Errorf("delta layer entry %q must be a regular file", header.Name) + } + if err := applier.entry(name, header.Size, archive); err != nil { + return overLimit(err) + } + continue + } target := filepath.Join(targetDir, name) switch header.Typeflag { case tar.TypeDir: @@ -127,6 +143,11 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return fmt.Errorf("machine layer entry %q has unsupported type %q", header.Name, header.Typeflag) } } + if delta { + if err := applier.complete(); err != nil { + return overLimit(err) + } + } // Hash the trailing tar padding too, so the digest covers the whole stream. if _, err := io.Copy(io.Discard, stream); err != nil { return overLimit(err) diff --git a/lib/instances/create.go b/lib/instances/create.go index 9477b2202..e7ecdde66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,6 +99,9 @@ func (m *manager) createInstance( if err := validateExpiresAt(req.ExpiresAt, m.nowUTC()); err != nil { return nil, err } + if m.macOSOnly && strings.TrimSpace(req.Platform) == "" { + req.Platform = "darwin/arm64" + } if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index cd605597f..825196d8b 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -20,8 +20,22 @@ type createImageResolver interface { func resolveImageForCreate(ctx context.Context, imageManager createImageResolver, imageName, platform string, log *slog.Logger) (*images.Image, error) { if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { img, err := imageManager.GetImage(ctx, imageName) + if err != nil && !errors.Is(err, images.ErrNotFound) { + return nil, fmt.Errorf("get macOS image: %w", err) + } + if err != nil || img.MacOS == nil { + img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}) + if err != nil { + return nil, fmt.Errorf("pull macOS image: %w", err) + } + } + pinned, err := pinnedImageName(imageName, img.Digest) if err != nil { - return nil, fmt.Errorf("get imported macOS image: %w", err) + return nil, err + } + img, err = awaitReady(ctx, imageManager, img, pinned, log) + if err != nil { + return nil, err } if img.MacOS == nil { return nil, fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) diff --git a/lib/instances/create_image_test.go b/lib/instances/create_image_test.go index d964bafe2..fea162437 100644 --- a/lib/instances/create_image_test.go +++ b/lib/instances/create_image_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "strings" "testing" "github.com/kernel/hypeman/lib/images" @@ -30,6 +31,55 @@ func (f createImageResolverFake) WaitForReady(ctx context.Context, name string) return f.waitReady(ctx, name) } +func TestResolveImageForCreatePullsUncachedMacOSImage(t *testing.T) { + t.Parallel() + + const imageName = "registry.example/macos/app:latest" + digest := "sha256:" + strings.Repeat("a", 64) + pinnedName := "registry.example/macos/app@" + digest + createPlatform := "" + waitName := "" + resolver := createImageResolverFake{ + getImage: func(_ context.Context, name string) (*images.Image, error) { + if name == imageName { + return nil, images.ErrNotFound + } + if name != pinnedName { + t.Fatalf("unexpected image lookup %q", name) + } + return &images.Image{ + Name: imageName, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusReady, MacOS: &images.MacOSImage{}, + }, nil + }, + createImage: func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createPlatform = req.Platform + return &images.Image{ + Name: req.Name, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusPending, MacOS: &images.MacOSImage{}, + }, nil + }, + waitReady: func(_ context.Context, name string) error { + waitName = name + return nil + }, + } + + img, err := resolveImageForCreate(context.Background(), resolver, imageName, "darwin/arm64", slog.Default()) + if err != nil { + t.Fatalf("resolve image: %v", err) + } + if createPlatform != "darwin/arm64" { + t.Fatalf("expected Darwin pull, got %q", createPlatform) + } + if waitName != pinnedName { + t.Fatalf("expected readiness wait on %q, got %q", pinnedName, waitName) + } + if img.MacOS == nil || img.Platform != "darwin/arm64" { + t.Fatalf("expected ready macOS image, got %+v", img) + } +} + func TestResolveImageForCreateWithPlatformForcesPlatformResolve(t *testing.T) { t.Parallel() diff --git a/lib/instances/delete.go b/lib/instances/delete.go index d9ea69a40..5cc1ce936 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -114,7 +114,7 @@ func (m *manager) deleteInstanceWithOptions( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown = m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown = m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, false) if gracefulShutdown { gracefulSpanEnd(nil) } else { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index b7a10a72e..6fec8d6b1 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -551,7 +551,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe inst, err := m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false) return returnWithReadiness(inst, err, false) case StateStopped: - inst, err := m.stopInstance(ctx, forkID) + inst, err := m.stopInstance(ctx, forkID, false) return returnWithReadiness(inst, err, false) } } diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 9e8e4d901..909972aef 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -24,20 +24,17 @@ import ( // macOSAuxStorageFile is the instance's copy of the image's auxiliary storage. const macOSAuxStorageFile = "mac-aux.img" -// requireMacOSImageName refuses a create on a macOS-only server unless the image -// is an already imported macOS image, so a Linux image is never pulled there. +// requireMacOSImageName checks lookup errors before resolving the reference as +// darwin/arm64. The resolved image is validated by checkMacOSCreate. func (m *manager) requireMacOSImageName(ctx context.Context, name string) error { if !m.macOSOnly { return nil } - cached, err := m.imageManager.GetImage(ctx, name) - if err != nil && !errors.Is(err, images.ErrNotFound) { - return err - } - if err != nil || cached.MacOS == nil { - return fmt.Errorf("%w: macOS-only server requires an already imported macOS image", ErrInvalidRequest) + _, err := m.imageManager.GetImage(ctx, name) + if errors.Is(err, images.ErrNotFound) { + return nil } - return nil + return err } // checkMacOSCreate applies macOS machine defaults to a create from a macOS image. diff --git a/lib/instances/macos_agent_startup.go b/lib/instances/macos_agent_startup.go index 05451f57f..74025e0cb 100644 --- a/lib/instances/macos_agent_startup.go +++ b/lib/instances/macos_agent_startup.go @@ -83,7 +83,7 @@ func (m *manager) stopOverdueMacOSAgentStartup(ctx context.Context, id string, l reason += ": " + last.(string) } log.WarnContext(ctx, "stopping macOS guest whose agent did not start", "instance_id", id, "reason", reason) - if _, err := m.stopInstance(ctx, id); err != nil { + if _, err := m.stopInstance(ctx, id, false); err != nil { return err } meta, err = m.loadMetadata(id) diff --git a/lib/instances/macos_build_export.go b/lib/instances/macos_build_export.go new file mode 100644 index 000000000..13539cd15 --- /dev/null +++ b/lib/instances/macos_build_export.go @@ -0,0 +1,5 @@ +package instances + +// MacOSBuildExportReceipt describes one stop/export operation, not persisted state. +// A failed export may still confirm that instance cleanup is safe. +type MacOSBuildExportReceipt struct{ Graceful, VMMExited bool } diff --git a/lib/instances/macos_build_export_darwin.go b/lib/instances/macos_build_export_darwin.go new file mode 100644 index 000000000..c37a45424 --- /dev/null +++ b/lib/instances/macos_build_export_darwin.go @@ -0,0 +1,133 @@ +//go:build darwin + +package instances + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/images" +) + +// StopMacOSBuildInstance performs only the strict graceful stop used by build +// export. It copies no storage and never falls back to a forced hypervisor stop. +// Success requires the shutdown acknowledgement and confirmed VMM exit. +func (m *manager) StopMacOSBuildInstance(ctx context.Context, id string) (receipt MacOSBuildExportReceipt, err error) { + if err = ctx.Err(); err != nil { + return + } + if _, bounded := ctx.Deadline(); !bounded { + return receipt, fmt.Errorf("machine stop requires a deadline") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return receipt, err + } + if current.MacOS == nil || !current.GuestAgentEnabled() || (current.State != StateRunning && current.State != StateInitializing) { + return receipt, fmt.Errorf("strict stop requires an active macOS guest with a system agent") + } + if err = m.markRestartManualStopLocked(ctx, id); err != nil { + return + } + stopped, err := m.stopInstance(ctx, id, true) + if err != nil { + return receipt, err + } + receipt = MacOSBuildExportReceipt{Graceful: true, VMMExited: true} + m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) + return receipt, nil +} + +// ExportStoppedMacOSBuildInstance copies closed storage for an instance that is +// still Stopped when the copy starts. The instance lock excludes Start during the +// check and copy; a concurrently restarted instance fails export instead. +func (m *manager) ExportStoppedMacOSBuildInstance(ctx context.Context, id, destination string) error { + if err := ctx.Err(); err != nil { + return err + } + if _, bounded := ctx.Deadline(); !bounded { + return fmt.Errorf("machine export requires a deadline") + } + if !filepath.IsAbs(destination) { + return fmt.Errorf("export destination must be absolute") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return err + } + if current.State != StateStopped || current.MacOS == nil { + return fmt.Errorf("export requires a stopped macOS instance") + } + if rel, e := filepath.Rel(current.DataDir, destination); e != nil || filepath.IsLocal(rel) { + return fmt.Errorf("export must be outside instance storage") + } + return writeStoppedMacOSBuildBundle(ctx, current, m.paths.InstanceMacOSBootDisk(id), destination) +} + +func writeStoppedMacOSBuildBundle(ctx context.Context, inst *Instance, disk, destination string) error { + if inst.State != StateStopped || inst.MacOS == nil { + return fmt.Errorf("machine must be stopped") + } + aux := filepath.Join(inst.DataDir, "mac-aux.img") + var diskInfo os.FileInfo + for _, source := range []string{disk, aux} { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + if diskInfo != nil && os.SameFile(diskInfo, info) { + return fmt.Errorf("machine storage files must be distinct") + } + diskInfo = info + if err = images.RequireClosedFile(ctx, source); err != nil { + return err + } + } + for _, file := range []struct{ source, name string }{{disk, "disk.img"}, {aux, "aux.img"}} { + if err := ctx.Err(); err != nil { + return err + } + path := filepath.Join(destination, file.name) + if err := forkvm.CopyRegularFile(file.source, path); err != nil { + return err + } + if err := os.Chmod(path, 0600); err != nil { + return err + } + } + data, err := json.Marshal(inst.MacOS) + if err != nil { + return err + } + path := filepath.Join(destination, "config.json") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + if err = ctx.Err(); err != nil { + return err + } + _, err = images.ValidateMacOSBundle(destination) + return err +} diff --git a/lib/instances/macos_build_export_darwin_test.go b/lib/instances/macos_build_export_darwin_test.go new file mode 100644 index 000000000..0d357bed1 --- /dev/null +++ b/lib/instances/macos_build_export_darwin_test.go @@ -0,0 +1,53 @@ +//go:build darwin + +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildBundleRequiresClosedDistinctStorage(t *testing.T) { + root := t.TempDir() + disk := filepath.Join(root, paths.MacOSBootDiskFile) + aux := filepath.Join(root, "mac-aux.img") + require.NoError(t, os.WriteFile(disk, []byte("synthetic disk"), 0600)) + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + inst := &Instance{State: StateStopped, StoredMetadata: StoredMetadata{DataDir: root, MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}} + file, e := os.Open(disk) + require.NoError(t, e) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + require.NoError(t, file.Close()) + out := t.TempDir() + require.NoError(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, out)) + _, e = images.ValidateMacOSBundle(out) + require.NoError(t, e) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, e := os.Stat(filepath.Join(out, name)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + } + require.NoError(t, os.Remove(aux)) + require.NoError(t, os.Link(disk, aux)) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + inst.State = StateRunning + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) +} + +func TestMacOSBuildStopAndExportRequireBoundedInputs(t *testing.T) { + m := &manager{} + _, e := m.StopMacOSBuildInstance(context.Background(), "unused") + require.Error(t, e) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, e = m.StopMacOSBuildInstance(ctx, "unused") + require.ErrorIs(t, e, context.Canceled) + require.Error(t, m.ExportStoppedMacOSBuildInstance(context.Background(), "unused", t.TempDir())) + require.Error(t, m.ExportStoppedMacOSBuildInstance(ctx, "unused", t.TempDir())) +} diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 7e97f93cd..797468f83 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -186,7 +186,7 @@ func (m *manager) rekeyMacOSCloneOnStart(ctx context.Context, stored *StoredMeta // Teardown runs even if saving the pending flag fails. Otherwise the clone would keep // running, networked, with its source's SSH host keys. saveErr := m.saveMetadata(&metadata{StoredMetadata: *stored}) - _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id) + _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id, false) if saveErr != nil || stopErr != nil { return fmt.Errorf("macOS clone rekey failed and teardown needs attention: %w", errors.Join(err, saveErr, stopErr)) } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index b64d1c1c9..aa234f69a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -736,7 +736,7 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - inst, err := m.stopInstance(ctx, id) + inst, err := m.stopInstance(ctx, id, false) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventStop, inst) } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 0bf65d7af..88775bf14 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -255,7 +255,7 @@ func TestGracefulShutdownWaitsForSocketOwnerInsteadOfExitedStoredPID(t *testing. }} m := &manager{} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), "stop and delete must fall back to the hardened kill path while the socket owner is alive") assert.True(t, ProcessExists(owner.Process.Pid)) } @@ -297,7 +297,7 @@ func TestGracefulShutdownFallbackKillsCapturedPIDAfterListenerCloses(t *testing. VsockSocket: filepath.Join(t.TempDir(), "missing-vsock.sock"), }} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 0)) + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 0, false)) require.NoError(t, m.killHypervisor(context.Background(), inst)) assert.False(t, ProcessExists(owner.Process.Pid)) } @@ -752,7 +752,7 @@ func TestMacOSGracefulShutdownWithoutLiveVMMIsNotGraceful(t *testing.T) { sent = true return nil }} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), "a macOS stop with no VMM to signal must not produce a graceful receipt") assert.False(t, sent) } @@ -800,7 +800,7 @@ func TestMacOSGracefulShutdownRequiresGuestStopReceipt(t *testing.T) { return err }} - assert.Equal(t, guestStopped, m.tryGracefulGuestShutdown(context.Background(), inst, 5)) + assert.Equal(t, guestStopped, m.tryGuestShutdown(context.Background(), inst, 5, false)) assert.NoFileExists(t, receipt, "the receipt is consumed") }) } diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 227b6c76c..056bb076b 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -173,7 +173,7 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err return err } - stopped, err := m.stopInstance(ctx, id) + stopped, err := m.stopInstance(ctx, id, false) if err != nil { _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) return err diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 2d8ee7a39..4e4f23340 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -39,9 +39,10 @@ func hasVFIODevices(stored *StoredMetadata) bool { return storedVGPUDevicePath(stored) != "" || len(stored.Devices) > 0 } -// tryGracefulGuestShutdown asks guest init to shut down and waits for the -// hypervisor process to exit. Returns true if the process exited in time. -func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int) bool { +// tryGuestShutdown asks guest init to shut down and waits for the hypervisor +// process to exit. Returns true if the process exited in time. With strict, only +// an acknowledged shutdown within the context deadline counts. +func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int, strict bool) bool { log := logger.FromContext(ctx) if !inst.GuestAgentEnabled() { @@ -67,7 +68,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, if pid == 0 { // Nothing was signalled. Linux treats an already-gone VMM as stopped, but // macOS only counts a stop as graceful after a shutdown attempt. - return inst.MacOS == nil + return !strict && inst.MacOS == nil } inst.HypervisorProcessIdentity.Set(pid) if inst.MacOS != nil { @@ -104,6 +105,14 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // If we couldn't signal the guest, don't burn the full graceful timeout. waitTimeout = shutdownFailureFallbackWait } + if strict { + if ctx.Err() != nil { + return false + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < waitTimeout { + waitTimeout = max(0, time.Until(deadline)) + } + } if WaitForProcessExit(pid, waitTimeout) { // The shim exits after a forced stop too, so macOS also needs its receipt that @@ -113,7 +122,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, return false } log.DebugContext(ctx, "VM shut down gracefully", "instance_id", inst.Id) - return true + return shutdownSent || !strict } log.WarnContext(ctx, "graceful shutdown timed out, falling back to hypervisor shutdown", "instance_id", inst.Id) @@ -124,10 +133,9 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // Flow: send Shutdown RPC -> wait for VM to power off -> // fall back to hypervisor shutdown -> final SIGKILL if still alive. // Multi-hop orchestration: Running/Initializing → Shutdown → Stopped -func (m *manager) stopInstance( - ctx context.Context, - id string, -) (_ *Instance, retErr error) { +// stopInstance stops an instance. With strict, a failed graceful shutdown is an +// error instead of falling back to a forced hypervisor stop. +func (m *manager) stopInstance(ctx context.Context, id string, strict bool) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) log.InfoContext(ctx, "stopping instance", "instance_id", id) @@ -180,7 +188,7 @@ func (m *manager) stopInstance( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown := m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown := m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, strict) if gracefulShutdown { gracefulSpanEnd(nil) } else { @@ -189,6 +197,10 @@ func (m *manager) stopInstance( // 5. Fallback hypervisor shutdown if guest graceful shutdown didn't work if !gracefulShutdown { + // Export callers must never turn a forced stop into a graceful receipt. + if strict { + return nil, errGracefulShutdownFailed + } log.DebugContext(ctx, "shutting down hypervisor (fallback)", "instance_id", id) shutdownCtx, shutdownSpanEnd := m.startLifecycleStep(ctx, "shutdown_hypervisor", attribute.String("instance_id", id), diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 148967567..5aa8c610e 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -203,6 +203,26 @@ func extractRepoFromPath(path string) string { } // isWriteOperation returns true if the HTTP method implies a write operation +// authorizeBlobMount checks the source of a cross-repository blob mount. The blob +// store is shared by digest, so a mount is the only way a build could link another +// repository's blob into its own; the token must allow pulling from that source. +func authorizeBlobMount(claims *RegistryTokenClaims, r *http.Request) error { + query := r.URL.Query() + from := query.Get("from") + if query.Get("mount") == "" && from == "" { + return nil + } + if r.Method != http.MethodPost || query.Get("mount") == "" || from == "" { + return fmt.Errorf("blob mount requires POST with mount and from") + } + for _, perm := range claims.RepoAccess { + if perm.Repo == from && (perm.Scope == "pull" || perm.Scope == "push") { + return nil + } + } + return fmt.Errorf("%w: mount source %s", errRepoNotAllowed, from) +} + func isWriteOperation(method string) bool { return method == http.MethodPut || method == http.MethodPost || method == http.MethodPatch || method == http.MethodDelete } @@ -394,6 +414,10 @@ func JwtAuth(jwtSecret string) func(http.Handler) http.Handler { // Try to validate as a registry-scoped token registryClaims, err := validateRegistryToken(token, jwtSecret, r.URL.Path, r.Method) if err == nil { + if err := authorizeBlobMount(registryClaims, r); err != nil { + http.Error(w, err.Error(), http.StatusForbidden) + return + } // Valid registry token - set build ID as user for audit trail log.DebugContext(r.Context(), "registry token validated", "build_id", registryClaims.BuildID, diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 4c604c755..eb421b5b1 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -274,6 +274,12 @@ const ( SnapshotTargetStateStopped SnapshotTargetState = "Stopped" ) +// Defines values for CreateBuildMultipartBodyMachineNetworkMode. +const ( + Egress CreateBuildMultipartBodyMachineNetworkMode = "egress" + Isolated CreateBuildMultipartBodyMachineNetworkMode = "isolated" +) + // Defines values for GetInstanceLogsParamsSource. const ( App GetInstanceLogsParamsSource = "app" @@ -2002,6 +2008,21 @@ type CreateBuildMultipartBody struct { // Admin builds can populate the shared global cache that all tenant builds read from. IsAdminBuild *string `json:"is_admin_build,omitempty"` + // MachineBaseImage Experimental installed darwin/arm64 base pinned by sha256 digest. + // Requires server macos_only and macos_builds. The source must contain + // build.sh at its root, which runs as root in the guest. Machine source + // is limited to 64 MiB compressed; only machine_network_mode, + // timeout_seconds, memory_mb, cpus and tags are accepted alongside + // source and this selector. CPU/memory zero inherit the base. + // Linux/cache/secrets/admin options are rejected. Output retains the + // base identity and cannot run concurrently with it. + MachineBaseImage *string `json:"machine_base_image,omitempty"` + + // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables + // unrestricted VZ NAT for checksum-verified artifact downloads; it does not + // enforce domain allowlists. Linux builds do not use this selector. + MachineNetworkMode *CreateBuildMultipartBodyMachineNetworkMode `json:"machine_network_mode,omitempty"` + // MemoryMb Memory limit for builder VM in MB (default 2048) MemoryMb *int `json:"memory_mb,omitempty"` @@ -2021,6 +2042,9 @@ type CreateBuildMultipartBody struct { TimeoutSeconds *int `json:"timeout_seconds,omitempty"` } +// CreateBuildMultipartBodyMachineNetworkMode defines parameters for CreateBuild. +type CreateBuildMultipartBodyMachineNetworkMode string + // GetBuildEventsParams defines parameters for GetBuildEvents. type GetBuildEventsParams struct { // Follow Continue streaming new events after initial output @@ -19326,397 +19350,403 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", - "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", - "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", - "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", - "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", - "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", - "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", - "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", - "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", - "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", - "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", - "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", - "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", - "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", - "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", - "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", - "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", - "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", - "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", - "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", - "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", - "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", - "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", - "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", - "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", - "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", - "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", - "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", - "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", - "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", - "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", - "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", - "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", - "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", - "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", - "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", - "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", - "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", - "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", - "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", - "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", - "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", - "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", - "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", - "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", - "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", - "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", - "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", - "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", - "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", - "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", - "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", - "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", - "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", - "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", - "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", - "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", - "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", - "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", - "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", - "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", - "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", - "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", - "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", - "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", - "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", - "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", - "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", - "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", - "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", - "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", - "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", - "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", - "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", - "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", - "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", - "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", - "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", - "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", - "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", - "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", - "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", - "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", - "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", - "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", - "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", - "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", - "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", - "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", - "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", - "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", - "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", - "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", - "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", - "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", - "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", - "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", - "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", - "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", - "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", - "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", - "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", - "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", - "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", - "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", - "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", - "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", - "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", - "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", - "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", - "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", - "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", - "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", - "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", - "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", - "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", - "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", - "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", - "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", - "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", - "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", - "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", - "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", - "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", - "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", - "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", - "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", - "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", - "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", - "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", - "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", - "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", - "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", - "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", - "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", - "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", - "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", - "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", - "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", - "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", - "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", - "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", - "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", - "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", - "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", - "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", - "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", - "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", - "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", - "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", - "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", - "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", - "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", - "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", - "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", - "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", - "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", - "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", - "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", - "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", - "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", - "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", - "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", - "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", - "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", - "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", - "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", - "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", - "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", - "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", - "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", - "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", - "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", - "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", - "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", - "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", - "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", - "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", - "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", - "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", - "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", - "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", - "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", - "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", - "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", - "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", - "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", - "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", - "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", - "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", - "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", - "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", - "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", - "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", - "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", - "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", - "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", - "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", - "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", - "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", - "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", - "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", - "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", - "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", - "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", - "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", - "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", - "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", - "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", - "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", - "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", - "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", - "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", - "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", - "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", - "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", - "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", - "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", - "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", - "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", - "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", - "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", - "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", - "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", - "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", - "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", - "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", - "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", - "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", - "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", - "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", - "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", - "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", - "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", - "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", - "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", - "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", - "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", - "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", - "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", - "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", - "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", - "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", - "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", - "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", - "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", - "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", - "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", - "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", - "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", - "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", - "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", - "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", - "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", - "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", - "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", - "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", - "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", - "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", - "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", - "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", - "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", - "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", - "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", - "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", - "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", - "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", - "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", - "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", - "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", - "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", - "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", - "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", - "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", - "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", - "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", - "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", - "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", - "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", - "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", - "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", - "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", - "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", - "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", - "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", - "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", - "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", - "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", - "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", - "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", - "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", - "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", - "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", - "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", - "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", - "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", - "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", - "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", - "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", - "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", - "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", - "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", - "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", - "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", - "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", - "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", - "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", - "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", - "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", - "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", - "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", - "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", - "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", - "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", - "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", - "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", - "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", - "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", - "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", - "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", - "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", - "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", - "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", - "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", - "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", - "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", - "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", - "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", - "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", - "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", - "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", - "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", - "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", - "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", - "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", - "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", - "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", - "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", - "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", - "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", - "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", - "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", - "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", - "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", - "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", - "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", - "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", - "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", - "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", - "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", - "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", - "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", - "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", - "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", - "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", - "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", - "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", - "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", - "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", - "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", - "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", - "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", - "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", - "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", - "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", - "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", - "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", - "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", - "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", - "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", - "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", - "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", - "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", - "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", - "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", - "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", - "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", - "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", - "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", - "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", - "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", - "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", - "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", - "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", - "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", - "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", - "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", - "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", - "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", - "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", - "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV0s20qlvuPYjqMdK9a2bM+3J8yhwG6QxKgb6ABoyUzK", + "VeeveYCp84TzJKfWAtA3osmWbEn2xDVVE5mNOxYWFtblt/7oRTLNpGDC6N7RHz0dLVhK8c8nxtBo8U4m", + "ecpes99ypg38nCmZMWU4w0KpzIWZZNQs4F8x05HimeFS9I56p9QsyNWCKUYusRWiFzJPYjJlBOuxuNfv", + "sfc0zRLWO+rtpMLsxNTQXr9nlhn8pI3iYt770O8pRmMpkqXtZkbzxPSOZjTRrN/o9gSaJlQTqDLAOkV7", + "UykTRkXvA7b4W84Vi3tHv1Sn8WtRWE7/ziIDnT/JjTwzVMTT5alMeLRcnexLLvL32BuhuZEpNTwi2tYh", + "GVYiU6pZTKQgNDL8khEupjIXMXnz9JREUggWQWN6LORUM3XJYjJTMiVmwchCaoNljKLRBTF0mrDhWPT6", + "jf1gAr7Em1fprwtmFkwFBss1ca2QmVTELLgmXMDXiA2rG2ZUzlZXtt/jccImhqdM5mZ1oX6UVySRYo7T", + "8u2SNNeGLOglI78zJclvOU34bMnFvH2RpmwmFSM/LjOWUkGyhEZME24IF0b62dg1KmnsQRoiLj4XUrFJ", + "zLThgkL7k0wqo1dH/wr/oAmplMWhYXliFtR4KhfSkAvGsvpE6RW9qC/jL3t7/cej0ejXfo8bltpjRd/z", + "NE97R4cPHuw/6PdSLuy/d4vRc2HYnKneh+IXqhRdVqajZa4iNol4rNbNJEo4E4Y8PX72+oYT6O2Ohvi/", + "nUe9fm/38d5w9/AR/nv3sFed1srC10f+Yf3ROzPU5HqVB9nTNHGEMqkQyeqsf87TKVNEzkiUK8WESZYE", + "jxSLOxBdbdqj0FZEUsz4PFcsXu3bH7naci6oJlRYpjFo8IuysU7nLgImFssrMVEspVzAGq8M4rX/ROCE", + "EneIYEiRFEbJJAGmYAxLM6P9KeoDGxeEZlnCI2Q9tUN1kI50r98TeZLQ6coIy91mCZ9zLNBpabiubJKv", + "S4wkTBimihPeZWlqbLGt43K5g7tR8sVOXS5kEk9yYXiy2utb+NmtabUnrsmCJTGRs1mf8BmhBFqBny2N", + "15d9b7R3OBgdDEaHb3YfHo0eH40e/K3X782kSqnpHfViatgAdrnL3iDT1lxE4d0RTRJJ4UJSLLK7U1xY", + "tQ2cskimjEDTa0d+8PEjb71unpabCgWJK1jerIGtHjaujG7UnVBtCiaE+8XNckIDY3rDU6YNTTPgQzCG", + "ymK2cSHfYHMf/MqvXeDdj1pgwd6biVuh4HxC9MHeZyyCG1F6blIIGNCeI++CZd0JjStGtRRWTILL9Jde", + "LnSewdXN4kmWUAPt9vo9SwaTlGsNVYsfYq4tH+n3PJFPhDQTlQthCwpmrqS6qJZ0rUx41uv3FlRPLudZ", + "3uuvu7bqRI1dsIRmGttzO64mTCmpelY0Xk5mUvlN6v1aXcI1Ta2skC6u2MAK9fq92gIU7NzPxY+72NXg", + "4Po999dE2VcFNmcnszrwalurwy2Gtp6x21vECtF+m4mrrOscIOZ0LqQ2PNKd2DwKD7C9qYwDrPNZ0Rzh", + "MROGzzhTTq5mRMH9kDLiGyHQCOGC5LpxDgrRf8IumTB6cnkwMVG2uiiNh0118yqySXkjVm7lYvuLk7KB", + "SOtzDz6cLinHM/mMXfKIrcpubmsmseKXTAXYdyEAWFZoy5EtOOvAQoQUbLu2UuKSx5x2YQcxjmnCA9Rz", + "+vSY2M/k+BnZWrD39U72Hk4f9dqbFDQN0MKPeUrFAA4EDMu3j2Wrbb88CLXMZZrmk7mSebba8vGrk5O3", + "BD8SgRJutcVHeyFJNYv4hMaxYlqH5+8/Vsc2Go1GR3TvaDQajkKjvGQilqp1Se3n8JLujmK2pslOS+ra", + "X1nSn98dPzt+Qp5KlUmFb7aNB6e6PNV5Vcmmvish+v8+50m8SvVT+Jm1rBNTmmvDhCHf22JEMfuUI1cL", + "qRmJaLRgZGofLfg8x/a6kHzRsb+9QiN4xnUmkc+TdyelNIdPQ/aeRbmp9fstibk2XEROqKpMrsOIQP2U", + "MLh/Q5IFrgBxZbgUxHjR6cZCQKQY3dAdlOjU2er5zy15TVLd1rovQrggKU8SrlkkRayrfXBhDg/aJ1M5", + "xfbaXOnqOfxMUqY1nTOyhWopfG5YDk+4JjPKExZvd5Ow2ybzdzmt3Gu1M4dkMKDTaHdvP8jQUjpnk5jP", + "nV6xSYPwO8jH0I4hPG2dCEoY3eaBXSo2W+3vB7xPsBPFZkwxEX10d5mSl0xQ96T6D+y393/tlArXHadt", + "3cHFPC2Lf+j3fstZziaZ1NyOcIVNuC9ARrjUBGuEx4yf4u1OFKUNVevPB5b4BCexFDY3ro1T/YC8Recb", + "q7yBMk2GjuyoEHAqXKCVbz8HSSsgskhhmAiszks5JwkX9hnEhHF7gcLeMmPfJXK+3ftk61As/+rhh3Hf", + "gHnZH1pag2/94lWQyHl1NReMKjNltcVsuVddQ+XoWpf/tHZ8Ghco1WyynoOcciFYjDp3d7BtSZCtg28f", + "PEUX3Ewu4QYOnTkc1k/cEFeitalERhcznrDJguqFHTGNY24Vrqe1mQREyJoxg6KSwDeIog0+qs9+fLL3", + "4JC4DgJr6LS/UGB1JpXa0LwtSwxVU5okQdpoJ7fr39GrFBKmgFLh23b3FBToCdNyup7bTfd4z/XC/oW8", + "G0aFd1+v34uAvBL4e4Vw+733A2h1cEkVrjo0XxnWf/ueKr99X3Za+fW06L/y42s3lMpPP/hRVX57Wg7Q", + "rwpTq+fBPdNw0mGpUnJhBlzgFoAA7jRPU7t5xSvLaTKIFBUJj6ku7HydYPV0RaQiW69/eLq/v/94u7tw", + "xfXFRPPf2WQ+XSs3u0E7QRmqEahGuCBzPqfTpWF6SJ5SIaQBY2S0oGLOYkJnBmq5sdYe4A9Gm4wwrUIS", + "U20ikvn9gD5+9P49NY8P+ZV+/Hs6VfO/79Nem3Yx1y3rixo46nYTihJFRWATb3ZRp/Q9vPwtlYSMnyfW", + "XEWKQm4PdHMIQ1I+6Gf8Pay5Ibu1lQ4ubvj1V5iwhBSDXPDfctztLKHL1TegYTQdIEWEltfyDXuA9Obj", + "o+H82Dp+pleUGzg5hc3UTblPZBLDvTPjSpted2vYtQQDpj6ViFQ7ZWGJaZUimgvYeqkX42w9KwmfsWgZ", + "JWyVuXv2nancqVtjBncO/lkoELuycaYCnNj/eFp0Ufv5Wdlf7ffntvMP/d5TmtEpT7jnzHU+7WzyE6f4", + "27RL1cae2aqvXc0P/d6MUZMrFljLM/QPIFZPPEjYJUuIKw3Ee0SenB4TnasZmsyRWm1ZUNtLzfRYKDan", + "Kk6YRlK/WvBoUagrQfWgCRWlWwB5klzRpSaZYpoJczQW40JDrse9Phnbx5f721KJ/du6hvhCYq6Ydv8A", + "XcO414e2EjnX496Q/Ci1GcBz2clSR2Ts9DF63CNb9k+SUa3NQsl8vhiLlAo6ZykTpk/QSQPVwMCZkuU2", + "oSIm456SmhlDByzNE+T90NiTLEvYWJzxhEdSkJRGr85c3StuFuS1rVS5PHHGcFn3SeZUz5qmbCyoVYPC", + "Ti5JpuSUkYTmIlowTZiYSRWx7SE5ZWrgl9jvLaGKwWZYJS7JBRwRRsvN0L/8Spgwynk6dGcuMJPr0B8s", + "ffGI1tepeWxrfCisI9ep/LOrAqzKzTj4RlHLUpteoWd7Ixaqfn8d1SwB/bHAFcVd5VDqSpDals0SOkdS", + "qRwiS4x6LCKZpvjQc70QzS6ZokmxQ3DHSdQnwN3gbqSxiBKZx4PFMmPqkmup+nBDsAidEVSf/MbSvI+d", + "wl+DlEdKXqZEirFIgIx3aBofHmwPyROScA3E4efPLXE7EkNOgPoADqMd96jXyY97dmJcE7j4h+S5MIpb", + "miPaEtzU3qQN4uq6eRVmtXK34f5cp7EzW2PFwmF/dgTdX2GxFbopCbAg4woXDd1Zaxjwqgzu13W9McqN", + "r9itiAq3UzXqPCLcEJpljCpNuBiLkppEjFTqKmWK4XJobtzWpcyQrZQaZC+gtR0L5BDfaFLb/e0hQbnR", + "+oeVHAmqEMWSpR0QGws/5iueJKg4JDl6MCCHswcNNWMVDwo8TFQUvC8prV3O+OXsorUJ1I6I7fu/n5+8", + "JXqpDUvJlAuqltuE65LXWtJcNcyFRUa3e/hyR1GtPutYotcT1CVS1AXI5nndaEJwImhJF5so7EcZcrGk", + "KlqE3Oi0IU9P3xL4zA2LgIhr46UqPQyakqRuaQ56pSjE2vWuNYdMZ+OcJRwpHPGmyR4X90hYRCrM8avW", + "LqhZseOyhEXAq6w/jX2/MeKMy0SmyNIFW53NTusS+cb1pt412ZJ6Bya8XblZ4EyrXAzJK0FQjCA1KWIs", + "sCgXUZLHTJMKPyel+tbLFzU3qILsx+Jf/+//Z2WJmFxyWkgb5B1XBnxIfrev2JmiKQOmV7/SogWLLsai", + "KYYQaFVLQv2lUiwz19ULRfH5whAhr4ZjcWzHVAx4S8uZuaKK5Rk8M8lg4EY9cHLWtnXiwoa4abho/tLY", + "m8riXNOXsGbOKza0v0phm0j151JsqdPqnBp2RQMP4hdAeoNLrtFlDWnClSXHp0OCn9GUEC3sV+CiIMWS", + "LUsVXMzHAr84kXibOInWkpkzTQ7Jq5SbgviFLK4Xd9ONBTgYThkTaEVMLq0DckmqS9bcAPDm3AUD73A3", + "dDTmMPaJkZN5HtS7+rsu5opFhrw7GRg5eHdCjKKzGY+AjjKm3KjtBdMcdJCfg49B0rLSvqJ3oUiG8MhQ", + "PJ6DkAPL5UR/+5u9nDKmYFxvnpzCBYer/y0Z9wQ1tk7J6QdgyeExi8nPT97A1Rq9Ott2q+aeprbhHlw7", + "ISV4v6fzqWCmbfz2K/rghrcCXWk3MV+7RCtbtIm8P06igYvfNvDNGpmk5GAgnYAQW5DgkLysya+2QuG4", + "AiLoqkRftOAP8bekGO13KNCQlFGhCXWyfmVEhOux8NJHTeAASZu4FYWqKheWS9n7cFDhviiWWHkEJbLC", + "Sf8S/n9wqWV04Vw87Cuz4LSCpni6q0u3Illx00nI2agCqDxYAkvqe68JnvBG0IJmeiENvqr9P3YU00Yq", + "tg2Pc+cvBd8zmmtG/kJSlkq1HAtfvm/PWUKN9WEo6s6AMfXIVpRIwap6BOtAQMdCG5llLHaGiW/JzB1u", + "WjgRSlXoqm2hsXBd6B0rAWhSzAI3oLTAJMtykysz2UZVA86mmBbMOU/tsBfSZEk+H7h59shWwi+ZmzZM", + "kP/ObBtgRZFSDJw/JBQtdSf4iRiq5sw41bPG9pFmxr2x2MJzS/DfYFhJc8EjvM2x3DzLB1UVR49svTh9", + "u1P6LI1F5TNWAaXegMtBwlMO7G0LfiDHO6+Ighsaf+ZibkePhe10iklieTfFIXlHk9ydU6R2WN+x8NI/", + "jnRgVG4Wszw5Ipe/OyGsvh/FThZS0e5+32maEtQwtUoy706IppfMEyTWgie1NnCMaHzJlOHaaWMvyJYj", + "p4GzcyHdaSIYi4mQY1FtCw4ykikMsSnyE6uI0aSxSGNRim0NT3J/tXmTYaGK8IJPd43N+hdNi43h418t", + "13ginxWP+sZFkvF2qyroIvGg0Khcpsr7lZFXGRNQKpZRnjJhCBcz6YvWvchGw702z7SW7s+q10rRvV5I", + "ZcicG6IYiHFSOG427g1irgwcf53PZvw9viJz1AJZmQZb0n3gUONeLi6EvBLjXn2YDw/p3iye7m/cDT/s", + "fm0Ng/uATM9ppVvj9dbb0J6uNZiBqEmMhLdz9eXvKPwTmMvKkCS49NQAzkrCWVyh7b711p0zwYBzxdYH", + "1BAvobWYfAZBUfbWrUrXssS07Kj1nG3d0A1zmCdyiheen0jpdDokx/W16xOKH1Bgrq9yES1RcQwlW2w4", + "H8LtkkV8AJ6hA7o3GI0Goya1JwcD6/KeUWOYggH+P7/Qwe9PBn8bDR7/Wv45GQ5+/ct/BB/kHb1VvV3b", + "zXPLn6U+8YOturA2B7rJvfXGlrXq8NsPL6oWWnc6UgyPAU02jgE8Dp5WirfT+tPjVa83u1KxBD30kMud", + "hE8VVcsdMefi/RGIc9rU12192XUKlkBwjBWMfAFCNXEall92LqniVJhf7RjJuKocAPmN4K0B9/4zHBEZ", + "DHw7jn15z/uaAWBIfpaGUGeRhifdERZa7XeuaJpS1HZe0oTHeDicmU3zmFnlrMBv5NJKSYqZXAlyMBr5", + "L5PKoEBukNpPZdxzilMVLYhTCCmCqhAbIROxDLo0MqaNMKXKUqyeNDhmf9kp/tryfw5//cv2//6PT0rq", + "SGhraNyqM67Jzxp+3lCIbCXyiqmIakYSBrPVfRLzOTfaCm8xukahQu1bEllnD+u6KBVhwquosVydnNPl", + "gGZ84DQv1tz9kom5WfSODveDi7vl/hj8+p/+p5aFVXkSeq+9ljlqXvFzNfDaj6GT8cWvbp6g0SXl4thW", + "210VJz9uh/1E1u20fdS1bjXcMUVU0IaBrEbgg8dRGhIiLplScBQtX3t68gyeMBfMMTR4NZJxPhrtR1gA", + "/2TuFxDi4CmDv23XJR7bXEjY+aXHooVEN8gkkddRUvabTL3NS3CtjSy02iX3X/UpRCO6NYujs7srSKZK", + "XjCFJhlYYs40uWBLqy6Y1xSaTFySS6q8afANhiVgEf8JrUEMn8fwiMXgfIhlZcqxRfdms2FQjCYlt0wp", + "mLvwXakjmTFgqK4YTo2cM3F5TlKa2We9YlZpnlLDFIfnYqlYTFnMgUePBcODQTIKZz6KpAIVK+wt6mDL", + "VfhGk3P7TDzH5s+5ALI+twezoSr9o/fq7ZvvX739+dnk1enzn58cT356/j/ws63UO/rlj54F0ygcr75n", + "VDFF/uMPnO8H6zyLPn0Q6r6Qyr12vYke/VVoxocyY4LyYSTTXr/3n9V//vrhV+/uaSP3LntHwYF9CIqZ", + "TvcZ8MH3AVCaSFV6nlLEBUEW5dUOFXVDQ5VvJb9rHQl8onA5mWY6GJvS1FuUcujuaHTy/Y7zHnng/7E9", + "JM50i8MHHiSVE4/1AsinwOVAe1qSyMhFaMzaQvB9VyEGj7bWTHJhNjKnsugqjxoMyq/XYEU7Uy52NGzD", + "ILreuiPd3NhR+bm45EoKfJ6juDRNmK6flZ9fPXs+ef7zu94RXARxjnGEvX7v9NXrN72j3v5oNOqFCJS9", + "z7hiOuj4+GSqZZIbRrBQ6WM6JCe5RudOxzxmOepBUT7TJM1BsZQsCXsfJbkGPoXSgDFJKP740WBv983u", + "o6PR6Gg0+ltnh1Ug/Q3M+8XpWxuhjued0cQsJmia21TxRyz7FIt+6HvdJD7tA6tU7KnXVsLpdXXI1qIu", + "XdnJESTIcW//xff2QO2+GPcqhOhCgYtWbMON1XvxfWhdKtqoVUmv+GYj3ETsqX4VjMbq63Pt1ERVS/hA", + "UITYgZIxVTGZSqrib+u+NFgTFPnfaILaCpo4O5EVvbGOlSUl0/hW9j4+pbJV9xtLgJqf1KoSqSAMLaWX", + "HFjOIE259PXqbMkpU2o61GHFwBTQ41XchXr9HszM/cdPsNfvXf5ej6nerA50rjGd3oobHoE0ybhga16B", + "n4mwDzrlRNJ4sPuJZf2Kv1sDhsZ+qG938Tj1BL6C7jSlIr7isVlMICIehhy4Id0XUhQursn3MBOa/Osf", + "/3x3UipEdl9MM3dn7u49+Mg7s3FLQtPBWJtiInkWnsbbLDyJdyf/+sc//UzudxJWpLyReO72/7ltoQlQ", + "43C7rDd+i921kMPK+FzZYkvfgGEQunLlJVMJXVauEzem3u4IeXpjVIpbc6OrV1Ehr79coDUvrlWvF2Dz", + "M84S66NlXW/xfaG9hcjaDvG0AE/6RvsIJEamUho7AlgbRlewUcK30seppIbu5oikMJQLptx4ixvDXin2", + "xZIeHgzJ8/cZUxwEJpqQmKorLqzTi6/p3tyECgAESrhgA546C65dEjtJ5MnQMBjavMORtg5Hq1p7VFbH", + "xN2PtrJUDYXYbWqWFEPGPMkKJL11x+e1LV0++gMkGaDI76kuLLNd6LAgw929E/fnniXIKvEVh8ueNa4K", + "4rNdfQtiKDydDXaqLazdlBHoiVFtyAF5wb8nXEAof8LICf+eUHc7cUHe/c3uA/I63UmW0hc8s34eEzov", + "wlLXIf+dXfDMveexhvcysHdQnEPLeIaGY2GRfYBtIO2AR7633RoIItDWcQFuW7zPVmU00A6UN5otbs20", + "Khd9Ms0NPPmlYcQpXxzEAIwFC08ZyUsn0sYD3E1w1TECl+WCKcGSiX1g644rYysRV6l1cXCqM6odEpky", + "eVZfr2c/nZyRrWdLQVMekZ9sqycyzsEb0PKE7frq9cciUwjvAp3AWeCuXzkjMjcDORsYxZgfYoqNFXYY", + "B3Jx+eL0rYNJ0eCk9JrBwjIRO4dXL+wUHq/iG2NN4bVmq/03Fr0NCseb9zue7DNXvDza3TWS/Z4xSQjc", + "LrFsDYKIbBwFGK4K6vOhfX04gS9kicVgucGQvNXADUbg9iIVei2QqTSL6usSryVdfQGVEJplMXh/hp+Y", + "5Wu2cb4PFkG7dZTlddLd67diGl5ahwkQZ3RXznUJhSuMCwGXApxrryOjCoLNuCCf1R2z8LJVvZIdYLln", + "1NQBObqqwm3LCAa70TvUabRxSbprtCs61hXdtlcA/tExaGJ9+8fCD2StRr/UAX5EX2e2keYSubb7fmY3", + "WKXjYk0adoBPszxPdEVn2gm31co8Vp7RZOsc1KyOjkGxet4n5/9Z+wH4qZen4LlwRexqII8W8FO1fb3i", + "2bxBj3tj72aqb74fT3Qr6ha53CVGUaGtn/2CZmxIfsSLkRiWgrQINxWvuhYKefUtkfaN4quOBQxN130t", + "vTZf87kA1zN0mgeGE8dW5e80dt7/Rofgjr0ebkVit6NzoEXIy5y3PTn3qvfzOv9dVcyv6q2cpn5FYYG/", + "W0UFqnLMTpob6N6+XpxNWubGgpi5qTcEvLr6f5NjkBtLoYq8wf6fFeyiiSF9GdBYwOTkrLTXVAxHbfYZ", + "J/yFbUcXbIlb7u1EdMVSVDURhQ05zqXdiTKVqqg7tFcJ1xX7krUUOeMQOh/Xj2jQbLJpK2C9Oi9//eUf", + "jMMZ+MmWFOMe8x6f1HMhmJztD5WOmuHioybhyIZ1nfet6oOhfpEIIJbE++Y3m0e3ZHRiPHe/DF1r50WQ", + "4KfAHC+jP6pbSyo760VpbMaLYobF/bpsAOjUevOkrHvqosXsqdiV4p6ReYDLjiKvi5pJ3cPr4/RAzyuN", + "rXEI69rEapSsXd/KmB09WWBnH0Ri9wPtXzXE4ybwf9x4hlvkmHqX5zRJzsmWK7RNFPs7osK6vQIPu2L7", + "AeXWkUDhbvbupD8WlgucL4zJJvB/egKn+LzZmKvrT3gJyv5ohKLrwcF+PWbDDrjRbF1dHkTDad8a8Lu6", + "Jf+tFqW8R7/Hz8UNBIg3ZMvQORwdB3sE98T2xyrsudAsyhXbfNafoHjkoHdwTAnlYvDjmzenRLE51xhq", + "HAxqsK7xLRhtTllQuqv56RpZ19ru7R88OHz46PEwvlBDFqlhrgfwkBnsDmlKf5eCXmkQu3bSJc2yo8vd", + "jXzebkExvnaW71+27aQgU2DJ3iG5yyv5aVmlNBhecBF3beAnKLvZ6dU/4m/b7JIpNsizuaIx+8RGlxv7", + "NeFqtu/qhkwo6x2ac21AEVFCAG+t92OuL9alTAYxNfQ6bsx2uKsey+nSNmWf5KH2Wt3Dz5o+4TXcnmA+", + "hI91MvNjCW1LG6awVSSweGJkIHzXyw7Hz2DNfdlO4KkIgWzk5HLG5aYgfxhaPXQXq3v1BjQxyCLuNHXe", + "qsELrb9V1r07qfrWDMdigFLYEXlWdFA0WzRJ8YlBY2vl35KqFj+MhoApQJ+QdydD8qYY7TfaGwXsmEgR", + "OZqjQRGFooEVyaoDyDHQhptmdaeWtHjM2+hCJN23YZG1BRWghQIN9WVT3piPjevGjXJ3SRV6pqNCeB06", + "2Wt7H61DKKt5hDwYjHYHuw/e7I6u6xFyC5DTobae1HmL8/2vcp+nb4+f7Tm15/aNQck+OSh1mJU9K4MW", + "yFauWSUgV6pgqEIlIqAlFOHGEQa3FDBQwm+uK2tX4g2UvA207RAeGBbp3wAPu8kwN4KuVia3qtBZZqh+", + "KE9JRZC3O5pFPBh5DS6D3ytGLzCLwOq9jTgUE3ufhf0NESxkukRwLFRyKSnNTJMCQ6ZUfu8ePDx4tH94", + "8Gg0CuA5rxK8jPgELbKdBgAuOAldFtCGW+jGEJNpIqd1Qn+wf/jo4ejx7l7XcVjDfbd1KN7fvhbZcivy", + "F5/pzX+pDWpv7+Hh/v7+6PBw76DTqGxj3QblytZFkof7Dw92H+0ddFqF0LvuucfXbgrwoWQPT2x+JPjX", + "QGcs4jMeEcSlI1ABkQMWYMb3Vuj6mZzS2KfECN8dhvJEr/Xus525klbfmuaJ4VnC7DfckE4mCZz5M2wp", + "mGNMiCIFyfVacqjkG/2+/FyKIijOxGyaz+cO8q9YuhMXzV8KT5wl8REpnmxr+RzuZjmwX9vowM2hIzW8", + "lFcF+F6FCOzVZZEqFCMFndhNq83KhwtxkeVBkmhdyh9yZUE4sFFCpzI31oLvcsaUnVg8G4EW7VzE3dQd", + "P0h1sTG+A27iIjVOR4WBD/SfIRBAI8S/KvQVlnbrj+C+a/La1rCKwvLnLK/nBexjT06hKDw6QVzojV0z", + "XaXLsNyCOnPv1Gf7K3nnHXk0DmbWCezTvrDVnGFKKLNRYgFKsf5TZ1i8c7gYVNyoSOmw7oJd3cWiY8Dj", + "AMh2ACqU21nxdY7Tha6hLESkCzcAfE6YeVyBl2qctDOPuOGWfDgWLt6q+Mk5BUBFuw7WkUAqPuf1jut6", + "1jt0Xb4OZXriujF1Viu2+QzqwgwY4gEWcHpmPYGt92IV79btSa/fOytyZznGVF8a/2NgRcoAg1XoodO3", + "13VAzpSc8RAeEHodua/uoeZdc18ejM4Gu/9tvTuB/FBi48J6KoEdYdjIf4Tlu11EL07fnraNqUg+Raqj", + "W5nTGpSj0rOmjIopFsfFmnPtW64k6qWCGHrBQMOEJ65WzMJ3sXhInlpfG5Rr+Aw12sYXs/AqwIBs35fs", + "qAaVAz5hghWNWlNqnANVVXHeoBiIAtpbPREj7sXp2zpew+OQ+I2AK9McBjZJA/rAH+A7sQUKwIiT7+si", + "+N5BqOkw4z6tERC+9Gc0Au+AzhQS0CE2prEJkfHF6dvXLuFSG1Y1kFORlMnCVA/Jz0VKMlheXeJRDQPK", + "xQYJWtdzKDDRiQw5sPygGCP4rUyFCnSDRm+Lblw4GtAY0a2k8AkZwfffYQ3+llNFheGCxeTdD/pbMnJW", + "9nc/EBt/5HgW11W/y/rj8mFoS1vjCE8XSw26PbssNikKF1XFJnKBzi+T07KiUwEH3ifhdHye45AtSMKI", + "C3j2enD86t1OGrPLfm1M8NF66b44fbtdYcuXHsyiKFvnxpdtGia/VUV2yEmZkbJ5EBBH26k+m9QBezMi", + "UxZR79Td3LyoSOA8RXcNVJSBmdswQRR1emoqao0g6wLH+zxJho5n+iET1EzPkHVXsF/t7eaYmmIZxcdX", + "IXm9N9gpIzqPIsbihsdLc9QVejsiuUip0guaNCd3VAAsRAuqaGSYIt+8/6YXzsCErLTr3VChzuJy6kyW", + "lasoQI+Vc9d2xN/9UD+eVjqwYdVOp84FNwgTh9jgW8UgLZr5kJzQpVO9ZHjNOnopd4zOZhYstbhHFEsY", + "1Zg5XZN3P2z25TTS0KRtDm/go+NTWzAhOEUwzD7JamwAfq+ilFW7PQxeGeV8ApYp7LBp5hiSYxFme9Zh", + "CHEGLWSY1nwu4Dmonc8Q3Kd+JVf2rn53BjVJTVzGXn3hatPpBy6AEMWEbisbB7r6MlrN5Cov6mxKXmzO", + "Hmobae/3qQ9VbZjC3PPTn1uMaPXOS8RWdRzDKlQ1y6iihiVL+x4pqLOeDYKt+qFAtMA1YmWfQ/EPNgNQ", + "rtjELBTT4DNWU1Ds91ezSGuMSrhkLnGenVPFXGckSam6wFPmn98kF3YF6nGx+5vAsRbGZNeYFDhVWJ2c", + "YeqSJs2ILb3il/OMQZTWlJkrxoSfCtWEVr3imxHvuiV5GcTyMMVlfQ17+4F+z2ykBJkrGjFia/kc9AVb", + "g6PXdSl7+y3DggtH65b93V23v67qLE+67XFoWLubdthE19ngN09PfRKuIsm5X+a91VWGRBL2yLkaG7Z2", + "T69PB+e7EhZFe0VmnzLMD+dizapR7N4rEfPfCYzYq0SOV5iDrsY6u37wFNil6ttz/munN2HzuIfcbyDS", + "KJQsHj+4KL05gg1ibILK8eXFYysp2Tu5cj+7SCqQtLgAXOa633eUq6TX7w1mblZHOzvA7xPEWz3Y3320", + "s975e63Xv3NynMR8nVbIlXLOch4CwIJV4qTrJLFDs6yD3hxrbrofkD2tbIJNEA93W+WR5UNcwF7TcF58", + "TyPjU1qiIr3mKEGrxxZR0euCDDSY2ixV8Np9XD2fo6DnCrih1sl/Z4X2YWbWExloxCopG+toifz3oHQq", + "VShvoVTGhStPmX+0F/ehdwT2+N81k/roUXWWhw8e7D/YxIeQ2TSOuTt3gana0nVABlZZbnd+XQPwH12X", + "OdyX9WSF67KBpt48DZDU+gWVGRPXWs8HB/t711vPrhM59s6cDb4UwrMCACuUicrI4pQZGlND60wGNdDA", + "ZTAxFmUpBg7Ovl3PWlq8nqoAVTfN6vepfGZaEny+9nj2KRV8ho8kW7Las17QvQeHRzYNccxmBw8Oh8Ph", + "dWF7nhffum3FjnWtrSD4DPXi4/bhFtB5uszlj97pkzc/AiPLtbKX1o6ecnFU+Xfxz/ID/mH/OeUijOrT", + "JXM1n61krK5tb5Y72HQWH8FMhAvG8pJSB6++FhMSxhQglFgQI7PmW317YJgFjfN6zpNrgGKuCeUHcQU0", + "8q1rc+Mk1KLQhZpK8umqXqFDImpw/Vzn+eE1zljG9Wk1HUWO7lV/mBtlWddrE9Gu5CnMmChSzyaJ/SuS", + "AlHJQ3loa1ek/9YhjSFeIy5fYdFl9cei98qPT6sDqfzu0yBWfnIJaX+9piPbWoH0r6ty6GYu5MXRDYc5", + "rP4vboWuyb8diGYwguOe78KbOHvWe381/6/f/o8+ffj33d9evnv3P5cv/uvZz/x/3iWnrz4KoGk9fuq9", + "gqB+MtxT9HCsgZ92JaUTsMys0hM8/1pW2H2xGgcTLTBdMZmyI3DBfskNUzb5ZSNkGfI1MHwpYS0iBWYp", + "dGgW21D51PohQOU/vBj8odlG7GArlNuQAihJ59NYppSL7bEYC9cW8RPR+C6Av2IS0cxmsOQCFLPg2K0w", + "3agzDJed98kfNMs+bENKFGoIe29TAWQUMzbOCtuE8G4dblTWed0VZwXeso0jHoviti4ARK13wLDMNsJZ", + "0oz0a1mU9e8393J6NApBrWIUGmwkZtTCJ0hB2UBGRXgceTTaXn3PbXhjFDS0hvyc+4wNU4Yw4wAptoQ2", + "P2Mxj5Cv+OheW7ISWG0pzSnxMiXfL3FvXtuQ05jQ3CyYMM4xsk8iKS846+OW9gkYZ9FfC2taL5yFzAbT", + "5WAhswKGpbCFWz+W+iP7/wzcRAfvmOIz11MQTQNIJCB04pFxM8MypXZhZWJvbDIiodEo74rbrFbaJjKy", + "ARgmV8Ln2mGYcRfBhgrqYyGZ/FsSJRy1TnqBNrwFooUaaCYE+NkbFYFudBrFbNb8d90Rae/BIb5g/b/3", + "9zrHmdulW0dleRJ406ae9XXg2JZN4gCseDDxivANroNwA1pOTFBPYST+94z4hsoTVzASa+mzka/apW5J", + "dCXmdTsYnOiOgQMVmtDcdJ3d6imsxTp2aMH6FGG1pAOc0XOcB3nz8owYplKPerEVwe7gKbHYNQOude6y", + "CD55evJ8e9gLosTVTFq4VWtjIeuDDiCk2HPQ6jpV6mhoyvrk+BnGuLtrpXyLYVDSD1KRxN6K5WV0hHA+", + "dXUPtflHj585ATRZlo5KVmwZ97Z9i1nzejsir323hBZDKaKVS9ryTZaXCTbr3FZtxNRK643gdrSPueef", + "u48xPgqTs1pejEi3rfdXd52jW3H42NSQXftCqlRs139V9v5TY8J/ehl9/3oyurNCT7IF1SHqXlStmlho", + "Zd+rhuw6u1ftnn61fieaB8/WX32Kqso1ZBjcc7XqIRDkB4Pd3Te7B9dX310XzruO1FfB+CwQvbtDcd8G", + "pHUAoJqbSWtICIHPLgDE60XenUB0JSCvwceGdmR3/2EXpQT22jWYovIRRGocUsGlPPJeEQRgARAveJJY", + "AUbzuaAJeUy2zo5f/HT88uU2GZBXr06aW7GuRnB/boDsjbcArqN1jgqAr9WAPEgR8fvmzUv4EiUMg6as", + "HH5xc7zvjarFDvjffnAvTt9CjQXVE+9u3R5hTMsoffaea6NXcRc7RS18DN64rVoqxrpM0rbhnCo3gpb/", + "WEPVDqJ4bt8C2rgPOVlZznsA4r7P0N7PDwR8LWz3x2JvOz3DLUFvt15pIdjqBo7Jg7bb7eYg2rcynBoM", + "VIhtVSUcj7twY9Tqfo8HYs6fOD8+cnxapqkrjRG++cacHu8Ndw8fYabl3VEXxp7SaE3fJ0+edu98tGdv", + "mSM6PYriI3yw39Rm5QjbPkFocgWRuWP/SBz37IVZed1Wjq0t0y3mYhUc/GZY4E0xrgWWG8VZ77ikJ+na", + "REkdgoqbUIdp7kBjU54kXLNIiljXZeQF1URnFivZJgwqJPixwAH2SZG3HaUUQqNI5aXq0UnXTt7PM0f3", + "mLQ4k0Izl7XkJ7bUJOVoBC26R9dHTYrYtXgstpSPcywCGjFfcQw/YNRQn7inUB+9hjExElQYC73IDTCx", + "bQhAEZDeWDmtLJlytBhtE53bJy2OF1djCQxT85ipsYBiATTmP4rnydEhICP0e8VLDpK7jEYharpV4+fQ", + "wa5jCxgCQ4VLp4fe7ioXJBcxU+S11MwYSpglh6aL3DUNpx+Jce6rdxOvXPVSrgofzE2Q6J2EqY/GeMah", + "trzP0Tv0Bo/zBzcX0TtFCnr51cUIulqT63gwuJAMePFN4bazCjkWOzWkZi7Wo4i9eWsTB9en7lyPjQSj", + "ulqSdycnNbcHF6HRbeLIJVr2QWbX2oa9DTqSjaO5jnm5goi9GQX745Gpm5JKRUL85DjUVQujhy6wFFpT", + "bNUezuv0a/CqDMZAcWH3CYhmzQQbuaMt0EvpGFltVS/1zKGyOlHax1Q4RPoCtKU06uul3pnmeieL+I79", + "rncQUecRIuocBCEPYnY5yfPQ0wg+uX7I27cAe4O/ICK0DZCpdU/p4e6j0aPHg0fT3cPBQTzaHdDd/cPB", + "3gM6mu1HD/d39/bXBBV1iIG9eVhr8MUc8GMuvNYn3ns+5NTcFrvQkE2cP/YVF7G8ql1/QQfZau/O+XZT", + "96uu9Z2HEAwJSqg2Vn3RwslO8JJnEbSN5XycdZEnLqzoPHwz2v1Y7Q8OruWOeKNyYc2qWKA0IaSVAVc3", + "qz7Om7F8HJAPfNm0WtXOuy/a6OjB46MHH7toPnhj0xib5HSHm9vmEeaR0xvRIT5IuKJH8grKnpOJrFbf", + "Fl/2+r1cVP9GYaDhS1187hTE1XZg+2E2su5aaYE8OK69V9BTxSJnxkcgqfj3COZ8KYA1QAR6CrowUtHF", + "WSBBtMMdV94u0AyaxZyKzgYu22AMjIzk2sXzcQGMGO2P0IjDRzgiL7AsfqKpfda5QdjE3VXTG42X1l8G", + "zpfv2j6y1g/5zL2vsA48tgj8C6cNy+BUtuubsNLZEflZYp3itSdkU/dri+Mza7V4oyzZciD7Hu8GO3Oi", + "5hH5oRAvCwHVCaRbmrk/J45hlTBT2zWwD7fj4EhY2bkKcEW/Z1e01+/5hYI/A1AXb0uqXzl/VVIMOZIx", + "muBZLsP0c8MTB66PM+Ha8Ei74BHY3Daxx8VmsnhiH09tPqk29NWWIUUlL1W9OyFbiJ/6F+IU2/Cv7cJ/", + "tXbX7T0+eHz4cO/xYSeUtHKAm0Xjp4hMsDq4jXJylOUTpxtpmzok0cM1j6xWofB9AWtXSeaZksB6YOa+", + "wWrnj4ePq+BwscyBDxdDckiSFosaNyyIf1jwohY/yN94cslnM/Hb79HF3t8VT3ffH+q96W4L6LXtKKx2", + "O646F6zoqNl0YFN7hvG7kKCUboW4e800zoCcMUOQfgaERvi8KeKpHcnlurbiQcI62N/ff/TwwV4nunKj", + "qxycCSrhApeyG0FZkmBJsvX67IzsVAjOtumRYLIAzEHjnBGXoH1UF0iHu6P9EJW0XNwl1bi2L9PWJX/n", + "no9uUm7RMXi7eFqunPLgau/vjx4ePHj0oNsxdurhiXq/nsO4crY/nz6juvNbKE2+eXJKoHU1oxHksn9i", + "9cGj1oyKuWbk5ydvLLSTkCRjalDeiL6lepir8zW71vzMteaHSWRs8ocNU6x5qjw8hOit3W6ojyH3FYdn", + "Wjv6dS4YOL4B8grsa2ApVpl4v+3eCQlmllRfsyihPH0S+WCNxj1mkzxMlC1WbkKXK8a95VeuwA51O6nJ", + "CuWXDfXBOkQqkositdBws3H3k9ho2/m/vWg23w+hAB8Bq+fgyWy6xhssZabYJZe5/gQNSWOjb2eJlOpa", + "dduePq+ZBgueB8V6d/KNJo7WiDYsq7/GHDWuAXG74eSudZ5rJBIm8rbF6rQbXbZ+3YT7Lae2vw6ao8YN", + "WqETY+BcudjsRvqUJlGOybRosZ8wK7woEFwgy5KljRJIEikFiRZUoLlFudR9Yk4oARSGYdCnFr5MZkH/", + "DHlFEmlB3yHZj8szZQcB1UAYAs3SVjWboyWlRvrnB6llMi6TUJ0aH6ThTLVhdKYiqB7WkxpZwSO3VWo6", + "1UTONT4vDcY/DJtpMDKqbFgDFTZv2mVqX6UB1+3AEBvMPHSj4kc4mPap7IQXDFm3K0kjJXUFWOzdiR52", + "jZ4r4qE3u4bWB9uBdK1pNHCV4RWnO6dXDN2Pgcigj7khkYYx+nCN06VXc6ZU5Jh5qkLITqU/7OxYCXrt", + "SQGYdc3BajPBdDKgCixsYEX8fqFZ8mWC96JnbTdZLufBfKPaK1QVbqptgO08Nbii4dXqFzQYIuNVhLq1", + "oHglVGATFus6YJVl/hGusVVewSAkWxg9U2FLFSzJ7S7uNuHHL/Sz8u51mYjB4tEVo3E9JOMpNYtjMZOr", + "nOA6Nliv03Z+rxlTHiYxZoKz2L9KC2OsU5px7fLQxTlzK4fd1oD00FmC4svJadsgWKjO65sddlE02zGs", + "zzaD/bqCHXaS63Bo7huV41ohAQHxlEG6ndxJuZ6EDWOrDSs2zxOqEDqw45D1Mk24uOjSul6mU5nwiECF", + "poV9JiGEbAKf9Hc4l+1Os4MKk7bUZGd2cC7C0G5Io99yCt/BLLcb8c2o09mx9Xegfic/tKDP+Q88YQ7l", + "8q3g7yuEXk/HcLA3aou7b2m0FnG/CvN6Xc7tSDZ44nMdiFJcK+X4pGosdjkysDQmh8Oj1uFW8iDM3ph4", + "M9tQPQTl40BOnlp+3YA4IVOGEUR+aivNdWGLXaYSTC0D2fb+Lqd11WpXB+JAwsItUYJtKDYLRgrgjq5V", + "bdsSK2tS2d3roFkgW4WJYqVrgkRsSu1Yemq18ZPXK1kOF8wtGfdztBkPO2Tw8Z4kBRCC67U7IkIzH2XA", + "8xnxbrRZYmZnTKe1rCRZhTRlSiECPUg4UvjZIICLVHatPYIWxHUviWIp5WIsuCjUrQjLxohgl0xV4m2l", + "gkfWHII9/lp54sHvLM3M0iWDQDX8N5qAMc6PcSyqg4TGQY0+JE+E1SyqPDO1dLEsdq8+IBSfJF8xozB/", + "KjcLMlNML6pzD+XMBRnvSqq4NRnZkvgiMA2K1lpi5AUTVVZWNBN8GtqGJrbWqj+gTXiNX937k9RyUJNm", + "jun1/eWaqbCQWEypKNLJCaZyVCqPEzyQMACXW9T9ZVl8gafSAT2lbP6/fZPlT6dF4/XfGsUqCCkeM/yJ", + "VdsGVbBRlq+uTsMmt9HpBQPq1uHirFolyJZ3xvYJmvR2S6KkTvdkt5i+ZtiBH82OZlG994NHDx4edsxU", + "9VFmP4sD9qmNfJfpGuNey06ddLEgPXrw6PHj/YMHj/euZWHxESot+9MWpVLdH4ARMXBYk3/945/vTurU", + "svcAvblH1xpUnrUP6W3WYUDvTv71j3/6Ud14QCFGswr33+IB0Orvk1R30rsc1I2B3Yxka973T2pKAlqw", + "GbLFENQb3Hfsug3KwTQARbpJwTSjETfLACOnVz7BhSvSQO3u0HpjsCGR17bt8FWBc+l8WpQAHxZX4D+t", + "lblBC486J7zT+bTNov2q2SuWI6XVouot0cFZwVJEWMFdzOeK6ppreMys3aFfxKqtxu3YEuvxe5tBFTAU", + "hOSttRWFshY05ElXqbr9je2s2C1rap3miv+65hy2H8FraX0DN3JA6RtledeGSrkdLsCb1ZpMq6ko1+b6", + "rOWtLG/d6/fbIQ55NelJcYNdv79K6OV1KjZozdKjG4Nb8rLtfo0kWqipElUTUKDJhA0qaRCwNNG5tQjC", + "mXeA9YFY0OhCzmZ11NwH7SjrCCCEYWO+F2oMvEz6oPV3OosmRLdFCxr3Huhxj0gFIZnpuNcwWwUDMVP6", + "fuI6qKPEjNbBnpeB9I1Baj+DaSKjC5tlEZP3D8mIpIwKTXKBh79hVdsdrbcO9XtZZW/8OuLjbmUp3WaS", + "KVvQS46ZWZxNZV5z6WTvudHoeortHJFYWtyoWoppN0MoZsMkj8pJ46VDxdI1DA1COSm8b2xZFhV8M0xs", + "LX5nSvYd7AFw7FevTvrWgQGdGO3Aap6SfqJ2BL1+r+yikaih/D3siQzGChx3E/g/XV3HanQ7WlYV08xo", + "hwRekkODCCB8SZhmRoC020OuHqC2eiXlAt0GnfsHLKjv3RIIiVmEJ1KvnsU6od+AuBsRCG6lQyEI+yES", + "xkOxJpPPa2cQbg7AKhsq2eFtO1UPcWslnGgjs179VE/Y+4ixuAkdGi7S1eve1Qx63b+kDm2oSNzuSqPn", + "9OrshrcXKoZjbVvtanSAkGIAp67YUns4LXKdR72pE1p19avgGJMQUGuoQIfxC/Z+/Vr/zN4bRFqP84TF", + "q2tbHEnLqtxltGnFbxwj2XagpWIb03PeQtpK67l+o8SVtiq7j9yVrtSt5Ktc2Z0zZnzZM0dGrTtUTxlT", + "M2m5JogvUvexsaTUJ+6CJ7vpdoMEDxZhrYiD9+0Y7Qn6FjC+z/j7NcRjC9iHcR0gpTxInhqIRSrdSul7", + "cvAQk4jpxtgFny9Msqw74BwEUJk+KqmrYoYJryjssvPlbvqKq95ubjurrYeE47MKyNBKchQnkk7WIXA/", + "LYp57XxGl6jFaTUSPtw/GI3290Y3guD2w7rGcj0tq7gUqPV22oLzKvUIq1rMLAeqtlCEa6/m1baJ78rV", + "JNooRtMjDOHJaMRIwmYIt1eExm62LDa7Xj94J1C5eNyC/v1GuX3zNvh68p2iKzGv/MuqPnCudTSj6vcN", + "BtEWNhOtgPMFovf2B6PDN7v7Rw8Oj3Z3bwM2u1iktriRh7/vXj1M9ujsIHm0fPjb7uLhfC/dD77DLriI", + "u9LqT1C21cumvCTrqEg1lka23BwyppoJ0+taI0BzTrhgA13EWm0OeFzDC6z9feP5v56e385grexwVp9k", + "VYSghuggZd0PkpebzFrbRXM2x8/Wz+JGsUzNgYTprTkUKNVxMJjrYrf3kRgPueh4Db2tFOx8Ea2Nr9t0", + "FYUs7HjSg7vcsuIh8m5APFRmve4CX73kArrTuVTcLNL1t0VRrAAkR7/p37WJ68hRQ3I8F1Kx+s+Fm1z1", + "EQWVe/1e8vtB/cy437tjiDks44IA3VZXpYIObmQJu2TJ+lXAIuXDQ1lPdqoYLsR3u4Pdx+iHkPx+8N1o", + "8LjucdC3q1Vdvl1fuvbrqMsaVpMJ+iRUu4+v5XHt13MdBf3EQ6nwynvZoRw7Gi9z0/urw4fu1ja4/Lyy", + "xw1MoFYB9GMlPXe5TapCUwwa2RDKfUVRqxuvxyqRkSmbc6G76G33R4Xi9gEobskTBzVuE8LLop9q80Ar", + "VTrhacpiTg2OSpk1EQx7HXVxzbfE9bKc+FoBaW0YFtcebwZb2BRwtemaHH5EZO9HvX67vXjX4YCgXs0/", + "URGNDAv2CZ+BDrrevk97bR8dGBiJ/mpHHvKtJFnHA3QpB3o9SZ/MpSFlMH5HfVsu2vWCxfghR2e+Fn3D", + "EsTeJ4FWKaDI+Dr2dfyMZErGeVTGjyY46BI7ROVi2FXI3zi0W9VvYIg3KDk26zfaFBrd9JNt+93QTQLB", + "tm/17mjzVt+KUqTfy7N4Mw+zhbpxsGthwG8IQQyoaOrL3pAEK5P5tQNHf11dwdU3r9UlRyAS5Zk3sABN", + "rVJSwNyCJoaQX+8zljDDQo0QmcRllATXJRfdzFJ3Dx8t2kycaJFaHchPjGWEGoukhP2lYM0LDcwnMC3u", + "kq2Rxw3X1uA1sImP3GrVB/dwoyTWulVVFW5bMgbL5Rs6b4u8clFRf/vZ1ESzVSwVz/BrQtrr9mQCrqZ3", + "e2vVH9+GWu4+hbRXzvTQAID18MQFzrrvv/QFBrGuTrwHIfN8iCze0Dlm3dtgR2kllVCGSjRyUGKoS5OF", + "qTrhnwTzkWoP/eIFBKzuI3pd2kLCDaFz2sxB0Zbg0lqiN/sQtaeMeUPna9F9m+GwVfX7k8HfrLqdTIZH", + "O9/95f8e/PqfQbV7Q4GgmRrEbIYeVxdsObD5nEBZMaxj22IyiZ42dO7ODKMpKtMQN95xpep4H4wK7rn8", + "maYrU0BXtUoypt2NE/rLf7Q7elWW8S1eGBvP7kfnWrmNnLRG+nt5K2Vq7p3qfUTd9nAsMEjvgi01qaR4", + "my5rJ/YbXVSpuOKTc0vuQyYuz8mUY85MPRbwvKdRxMCePXRZgziORUlkw4rRpNqOSzXnz4uzzFrHCgif", + "XgFGfvX2zfev3v78bPLq9PnPT44nPz3/n95RT7Crge0hHgDtHTw4dOneqyu5G0w5cv3MGUNy4uIVnM/D", + "LMeXPUKfgY0UQtCSJWHvoyTX4DaBS29McvMcGatRyzfPOfGRoMrGJCH3DAf+DYDeMGZ7rzrvIq49MXKN", + "efKdlocLsiq6WMIZ9/BKMdJLEsEMJbAV4dUuN7a+6IeLbneJhZsNHHYM/YXDFtCivuQagTu8E0ylMNnC", + "EBqfzNdHEG9fD/72SdFg0AXzE+dsGj3+FHlV365NpHopkwG881qSTwTV6nYtgiEE2JQNzei1WV/m08Bj", + "xum453xOAwaXkGHlk+Q/9QPaGDq2sv+tieDCAR3Pmpk57LG0S9XIJNHQlmgzaI/3SEG8b8EyRrBmG8TL", + "K06GteZ3UmF2XJ7iEHBILBEffl3YdnnKPODkACttjkZe+8CszKwykva9OZF5aFvWLRDE4wOPVKyyEViB", + "xTdcMheg1AGOBumtjk7mKqN4jnZ3TbYKFZhfgiLsetUusD6jxQl9X/QAJeA10TTE4jzKjFq7L77H7AOv", + "3S4B+3RN4DAar9ww1n6ditatiaeq1c2oUtXqvG354MFzvGoN92s7Ww3iLPuokWaIHv9KuflBKnwXt4O/", + "3DpkP17+MVMIq9cE5O+EZs9TFk9kbtaffygG0qO98otMs2WmYq8DoEjEUS2uuY0XeHiScgyrKw3LwaJc", + "cbMEpZF7mUwxHtSnB8aFxI7w57JjTMn64QNqz2eByJkXTDDFI0x4a6EGBb6YALavzHtoU2CuQOCiCPTq", + "6bHTu3gUZXxHc4Ok5x1Qn5wegzGeKat76I2G+8MRHuaMCZrx3lFvf7g7HPXwVbXAKe5MIWG/CyR3L+ri", + "BX8cO0noe18IaiqaMoM1fglAIqADpiuOTxA6rzwiM8qVe0VmiYwLguFQGzM4+Av1yN7KfbvsnZXHGG+N", + "YUAse+U291cUlPHs4DT3RiOHFW/c9YtBTDZyYufvzo227LeTVOeWKJDQYOWZ52XLYuk/9HsHo91rjWnd", + "UPDshjp+K6iLZmb4On8wGt1+p8fCxie6aHPnF1Y9cUhI1bP2y6+wZzpPU6qWfsGqq5VJ3SYYM9BrubL2", + "HWc0iYBVYNolALRh9juoRKl14Va5wGzVviJQaP0U2Lb9JhdoTd/LePnJlrDWh9dRfKizMzguH1bo+dPR", + "TkHGqxvpPnnQcku1d0BA39Mi1fq9nZSD0ePb7xR0vwmPIFrVE7BzzOYafZ8ShGD3IExSkd9yaSgp4hq+", + "oCPtZNZpQW798ira+YPHH+zxTljIHnDKVEqFjRKxZTYc+pXjbG0z5XFee6t5wj9+1nM3lUcjshcVCnL1", + "I1q9tprC4Op1dNA7auvTTi++R8I/uIMT7iZbZNu9zyOHOU7BrvwlHSdna5yWQkhQlnvBzOdC86O7vLJc", + "XoY/4Sn6Ugj4BSskvHK3Vi6FnUzlwj6AgxLg6zJy09X7pi78vSm/VNyF0K4BTWOGEGMNzDReDolfU/vo", + "B6WvthD96tK6m9XP1ykM73M5YXt3ccJwxoWl6Os19fWaWnfKLbX4KeDBrJzyDjqIa2kg/nz6h2trH77q", + "HrrrHjppHgS7sqUBD3JInGtuJGNG9ALynYKx2QI/eSccQ9Vw/juhKlrwS4bofpj3Lk8Mz6hCF5uUgJXA", + "2tBbFRNr1RJFczvQ3MA7ZJYL3AT00GyCLjCTNiDO0hWTCwHKWqq910yJq7iSod2e/aCCvWiwvBrJ1UJq", + "VgAbClO5zTHOW9vXMTY7HIs3DvEWFhC9yj2v0SxB3N41+h8pCB0LV+Fbz0K8R5ymacm5qELwRG4hOu22", + "rEwVRzrRkQyBDr1hggoz0BmL+IxHbloXbOkcW4MNdkplBQP244S8MN5Ra287DFyHrkthlOJnxTfiKKlu", + "vxHSuCTfpZHLYylRBbkPghlK5omc0mRi1+eCBWyCL7CEW5TS4FJak4SMmc3Kny3NQgr7dz7Nhcnt31Ml", + "rzRTYHMaC4xIcWsN3gqFgEiuMDdemkllrJ8D9rljh7jzxwVbfhiOxZM45cLW0LYKTbQEMwXUM9KDh1ju", + "1UIP9jSF7eBPc21kWoWA9XRnhylzk+XGhdZoZvoh+NOxMJL84UEuP+z8Ufb4AY3FjMZAJ5UidkooW7eN", + "Wk8ozH6CRQPmdoYLMO7BRWrdPOaKCoNjKlE6yby6pVtFmghMQttc4YgKksnMpthAolpQILlaGwhaQZOE", + "GDxKvi4I7riTLfNJabTggk1KBhcwfr3PmOIpExDEiSbVBNxkY6quuNixydKR23nOt4QR7j04dIxvOBav", + "i3ADe/OkNJJ6giwJ9FP2n3bI9lXizg3CMsBhQ6BXy9ZgHQ2qtJSUxienQp5F7W81C/OQnNg5ujbHAmjT", + "ohHBThwekBP+fSUO+1vLKv3KeHjCVMasPxbOJueTPPVdoNkknfYJcCScD8hKpOozRihA6Wges7FwU8Ny", + "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1LcheYUnhSgGK4gX", + "wVjYm8k/4WAcERXAvFQuYLFLzBnUt3KzgXaqKxRgyNZ9yhX2dInOsLX0K1xLIPB4SJ6/B8mHG8IwP6eL", + "fdNjkQvFoHuYGnn3N0xGBozBpgoG9FamkKURqgyf0cgm1kQD4beEw7+YBjY9FkzMJOxCLFPKbWq5q4Rr", + "o4cEl9iP06FD5Zo1NqsGAuLHDj/hmIPpMApKaQX7RLJsXllckJPvKxfX6OBR+O5y9LDa+n+dvfqZoAQM", + "/M4WK10j7eoLoBjwWMM8PU5+eA6QZNYojLlQxz0ej3uF60S8jWOFxUFKGQzQfv8dDO07202fx98Nh9CU", + "dQ04Ir/8YVs5gnsrSy348Lj3oU8qH+bcLPJp8e3XMAG2YSae1S5dsmVFym3PSGCGFenaiqNwDKSTuIAr", + "kfK2rzqJTbmgarkuejmw9G4F5cx6DVcW448xegmPe0dj7yc87vXHPSYu8TfnTDzufQivQIMbtSjOC0eC", + "gogOR6PtzfD7bn0D/gEdjHCfWN/SqoEosgbDDjrs5z+XLe7fWtfzfaEdwSuqvJvs1eoNXff9hobe72Dz", + "f5YGxPOEpQy9twbV/IdwU2DezoEVYOA6vokVsvJsr6qjAnbIxtubiogl/u29Wdt79xZEoKKIgfR6tyfn", + "vs5NYSMv8pZ8UUZx3KzyfK+14d0zxY3u6rar2e7uh36/OCNawITmDGjs0sc7hLM2ISKX06cRWxgemWc4", + "psEZE4Y8x1+H7r9eAYQIs+eJnJ8fWf0dJCAhCRcuKKgSrYAPMLuWWMmCchX17D+LlJlbVsT51z/+iYPi", + "Yv6vf/zTZbr41z/+icd9x4JNYu7+8wWjykwZNedHBEJfBzSB95ibDGbXZpdMLcn+yBn+8BMMSbCoiKyQ", + "OaDTwkve5EpUgqds9krtGnT2QpgPFznTDtQMCvKZS61lHZwDylt/lu1S3umJ7gfAYXEGlQnArehpAJE1", + "ueAY7GXVUb2w3cTOuWY5afpqr3jsbuYvhr03lnoHdoDXZDC4xKFzhx/cpMnW2dnz7SHBZ6ClCkyfho+a", + "shn3vhl+5UmbeZLlKHWGgqtseVNEMzrlCfd2h5bcT/YIOpVKGWRQZF7wTRz5kQKPAX9r5w3dx6Jj8eps", + "B+0shkUmV6zvOIFyeMtlckjpgt2wB6cVQhfRgSs7FjNGoQ1y/MwygUpKgiI6umhYIKwROrpzU8tD2R8L", + "i6vttEpw8ECSTbAS9j+nhl3RZZ9UJF+bKyqhBl7qug+Fx8LGG7s1GCBwE6kMc4j8zA5p4N35XeCmYjNQ", + "9OBEbOo92/cWKDxcmEO/DC733dmQczssIp04DvOb4xNVWr0ktvTqzO/Gdp9oSaKEIzVEVIzFHL0BPZS5", + "FLVdLaJKF1TFg0jCJVCFtoMwhITF8zYe+7TSVu8WJZlaP4Hj9GOTXL804WKxOgE4xBauc735/pkr081+", + "71r8Mxnw7QJdx4JvzTjM8hu7ul+t+R2s+eF185b9kHn9mcejvT23f9vFPXn9e9pbXXP7pbJk96FqJFse", + "6AtNo1KR06fHhMaxYlpv/3srImGmlkpL+Q/uR2DF9+F/5sYild0Lr2+pE8iXwg5eu1ET6ufVzDZevd92", + "aqnIWm+6IitZeeXd/u3R6PQ610gp9Ja09vUm2ehxz3UE8ctVahmgaJSwQnwpzmmVijZpla0vf3HlrBWX", + "bClwlHIH8u70y67rXDTvhjtgis8aDPEeGWEdb4FQUcAyfEnU/LbYRTevdernz4s0R3cnBd21KjpE5l/S", + "czFuLBtwQYt21HqBvmDGYhzd5jvd9RCYOOi53am2A13aWRfTslWtA46dEGpi1r99j22Rbk9f296f6eWL", + "y3MdicUt+VcRpcNjt1yrdQ/cY5cQ//betzU4xzv2p3EEFlhk+OB004iNBjlmqV6KaPurS80np2hc7Moj", + "Vvl5s7jQZOP38p11V3LdE+E16jn4g+viXcsFmSUAFW2NADGfoROhqUAfWka0dwejfO38tomihjnfyS8x", + "+P80T6x1I5LikqkKXGv1St35Az3XNz+VPPNae7u+ff1ywEQk48J40i6Tui+f+MFk6b8W0H/3p+4LDGrn", + "XjxoExg/Yv9tRAkp8Hv/194PDsH3f+39QJOMC/a/9p8k1DBttm+NWEZ3ddPd9QPmCyY+eL/w+qKtsKYd", + "Q+fVYPQ6ZXoM6+tQZg172iu7i2i6T0Z9n17IawJ2d5Lw7ozu4SV117BFxz43id1SqTyEXQOX/N9b6Dur", + "EfSd6+tq3XMMhbHQDl8gVpJUHnqZOux67onb8SaBoTiblBFFqY76CF/+T6WSsJO+llKiWNeveokueonq", + "cq1VTdiCt6ucsH3ck/W9ILbQauOnr5h7d2DQcRRZwdyrWbhL1L2F1AY/fR4ALP9OkTi8OArV+6yjybTk", + "FGvvNX+mjp/1cYf7sKfHz0rghTsyoPpx3LlGwPV799LYk3TK57nMdRXTIaUmWjDt8E4SVr8ZvjRdRSk3", + "tGorPmMqHd3lnXbnyoivdH9LapLmhlrmbb0gNj1GfKmujxFXHjE3EAveYcK4zG19n9VzuyU6xaPBdyXj", + "Gmj+h36XcYUeSZAJR5PyHUPwaXM0Fv8bHka/GEbTX7/zAfH5aLR3iL8zcfnrdz4mXpx4UmHCKO6SMD/5", + "+Rm6nswxugDzNJdQN81xWHwRJD2fkOnf7uVWet90f7p5Kvz6dOv0dKss1/qnmy14y2+3elK3O3+8eXoL", + "Lbj99id9vv3Jbcq1p6XOZzMecSYwNR5qJfWKE7V9Yn41J99MP0qFc+KoeGDWJJHOz8iCa22Q0It865/U", + "xbHfmiFHEmoMSzMDEGsRg0zdNnRQL3IDwE8+YQ5O0Ode5OV8Qte7b2riGwlH7tYS+6wL3A2+dIuksXf9", + "1HUdf6Ghs9IGy8f+cVmKNu2vy/sl3tt9U3a4au/+Vfklk5h9vq0uXQYvhEACSGt/SnMbZ1zULLFjh+TN", + "m5c+ptgi87l0okb6HKI+z/xYVHOIIgyeT85qC9Sx/RwGAdT3WTljRuOEC4ZBGEyHwn/rmX/v9Vh8egk4", + "nNb4rm3vXY6l3dB7lIDvjRXciaxZOBsYZykujmYl43FxWry8iafmi+JXjgEFGE9I1tuhuZEDh1Kws5AW", + "vzYM4X2a0AgRvKGYBZd1wDAWTbraFKK9KJkkTFnQ4Cw3XtwC/G43OC6MLNLiO8nsHJqf5MLw5LzvoUgj", + "lmhCxdKh+Y1FrTMn8yF4AwKT4AgVy+yIGzm+YdAcVI1QCnEWql0SmlzRpR4LB/dgq6cSo6Iji6+dJEPy", + "o0SkHZvrvsJ4baLpb/RYnPM4YRMHlHNOuCZ6IZVhgsUklZdM1/tlVCWcKZzEUworp0lKlyXOqFsfmTEL", + "V1mD45HwbypijimLoediykdjQcneaERSRoV24BqazvDCcW0QHERtQN8SSg5Gj12txr5hagW//FtwmpRi", + "lzKiU8hODVQMQ+RmGzcwdSnEFdNGKti+GVfa7leh33S5YWsby7XPiB33SS5K+BDU9Vesa7BdJlcC5+nM", + "k4yr4hp0iElTFlFYTyHr/SBgtYyiXIUuSNjqSi77f0fBsTK9M1yqMDhHgiqDCGIAJCI/LvBMSzxK29+2", + "UFVJVH+OiyZ4SKQilFToutRosChH1riFoLPnZWJmMXHlz7e/9WcHjq9jBP74i/mXdD8hEcnZrHYAN19N", + "9gCvC4pbJeE/6zl96phejcXFnM6F1IZHnhl6/Pzqo/nrg7DTg3D9ygapeSbVRbsn9A9SXXR9gXmPzS/r", + "IVad4WdoiIDhoZPO/dsjUBtuHytANHf+SGvSV3FKUejiRhe+ygQyGDBV1crfudmg+qrbskibcJkqa+wu", + "cNHgETJxP9qk/hV/czQxRK7VP62zGhIbbHbpszZd2qdHiSR3PV4Jp6oK+GDfgtr6H/S9OIR05TdsC6X3", + "1e0KMlUAGt2I1Fp07mFJA1CtY/FW20Qq59b0dE4KHgwCrc3x4PKKAGwr/IbtW1RXmmXnBZT+9hF5gQe5", + "sm228y2bEgloTcuEWTTWyzQ9P1pNa//u5AQrYZmFTWB/fkR8Kvvi/tBQqgrDCrNIqDbkZwcuu1U8xnFH", + "z0GHWZnftgNoLVMdjEUIrBWwTm2DfEbOK7it5y2ggp7fvpRz/dmYispkUHYuRhL3dETaZCLutTl58CRs", + "+NkdjUJ5EzrCx9ph3DJ67MpgXsp5kYiqRso0y7qSrxsmUvFlmq6hYbJVchCiTSxz8xdtYqYUVnbU3Ubc", + "ZItG9h+GXgChOuRRf7C3x6JlqewMw0sFTLWSr8X+6zJNe/2eGw/UuzJZGsjc8vFwvM0GP/RDO1TB3P0T", + "i+f3diW6pGXFUfHJgqzE415QAOWK5wW39CbQv/UbqoL927junA6g/Rnx2hb405s4vaLxzsR396JpiE9B", + "+ri3U4xiVkMX/jlJ8pXx1vVmQhaoyjiyx3cxMoReJlEiNavZyb4c4EanL2yI3u36Nr/GAxhenPt0l10c", + "gc5c1TNf8zNQYWxyufFjJn66d+57szqCLxmEQq/MBthfA+1vk1POZ09In25LVqbahUK+0ub1lbWdCBOe", + "W6sswlWIXSrR3MiUGh5hOsBoIaWukH0BzW+T5DodfEGZqKGyygIXiHEOpHrutPnnTsQ8cppHQqufXB9D", + "rO7CN8I1/Keyxg8V5UrB8fv+BYWZaTSkTFOczUhGc81AzsxBI7aMEubzPzKwq0c0M7limEaakZQLnuZp", + "NecC7NglRXyo8930vE+muSEJaNlU8dH7LEUyTZmIrZw0FgtGLzlTsCgJNUxEy4FmQnN0jYK0D5izEzQ1", + "WUzRYIaBlIoBBWICi5QZGlNDUdA5hxM/sbFg5yRTDInIakcEe19SA9iOc/GtzaYDzZ77gZ4ThukiuF4U", + "mcsjGjMRBdMonH3ebOzTq/TPmGlO9J4crG7ES+/T46qquvbD+Tycsb5YyI8ObH6N0KvbH9X1IBpPRv+e", + "R9rO1c/xnux0xRKvO8Wfh4GuILrPxkh3/1Y4qUic2+4qpxLJ/M+qRywYSg0RAgJ07Tbe1L5WZGctlvla", + "PG/nD//n8Q20i58JJ+y3Puzb8gCWk/4cWK5b1Rvx3HtSqzpdUlUnd38s2A3q/sQnqSpc7ktRttaQ9wq+", + "XeVORlF8fUnxlW032bbzG7kp2/a62RXPiAoj52KArrZhDu7UuK2s2qkO/k2Dehqzq7DMe2eRpeXizmE0", + "PWvM6BJUJX8GX+s19qNIKmVRRKhhXxb0d0VrWI2yQN1cmWC074Ne352cbLdxCWXW8ghlvmAOUYlsgmpp", + "vKrAfXXJlOKxhzd9evKstMmqXAzJq5QbYiS5YCwrA4MwOBNQOws8lcawm8Ap/R4TRi0zyYXZOIqy6O0M", + "pvxBOliZz1CUdLkkvprDO5vDUbP/5bEz5DKEFhNY/zI11GzMcc3FTKrUymV0KnNoHXgQLBPsp3W2mPGE", + "6aU2LLXOnQAFAccN8xK53POunt3lPro2w8mxUYcZUynXmkuhx8KF3GRMQd9QHdqv+KkFDQKGFvz11DLJ", + "z8MHEgZj3f6oaVs1RL7CnNa9o94OzbKdmBra4mfnhvcRQ/oBnRqJXqZTCKAEr8gLTbYSfmGfJ+RSkwT+", + "2F7rFTnBep86s/7NTxas9LGYyWDeUkuzBTH/qYLjHFvzhsnPgq3d0zvzo70IX7DqwfW8cCZbWazeyGMV", + "o8kA8/J7LCaSG57w3y3bhUa4NjyyUWS02Md3JwWDH47FCTOKR9beG4H/OYJU4Et3J1My2gEcv/0o4wjo", + "t89wcMh82z+n2OPT07dYLmWpVMv+WFDhGn7z5NRammc0Ys2BCmbAkkyOd15t8Fk/w2X6N/aftBNcC0gR", + "3PCv5snrw8a0niHdckRltu7ZJrM/vYOvkya/6ji+TB0H0HA5m60Cq80DrIX1GZcyyVP4h/3jeBNUnaHR", + "4h0W/Wwkbzucjd34CX4Rh9LNKWY2x/O9GGDsgn2p/rOwcH4KKMTUPBODt8AT82ek7k9vSqiu42doenUr", + "6vOnfzZn665vPjcGD5pSXY8v5ZhbSvMzMXK9JuyK8nZN2PeJjC60Q9epqjCJYhQh8+HHEuLcmStRTMBg", + "X+JQqSz2GdP9sWgoQy2IkyaUGKZSLmiyg3O2jSBYu9eo0UvJMeY+wpiZgQZVdyaTBBHZEdEQZgMlZr6B", + "inVZuwyT1TJVw6iRANskU+YB7LdDT7e/Um5+kKqORv+58MU3lfVHlEfKUfe/AYC/vcePAuQ/oe/RbTvO", + "nXHbj2jrhSx/tGqpPsG9Gff2R3rc65Nxby8d92AHnlJU51JDHpCUi9wAsuUzq2vDqOrDEdEskiLWHkff", + "axP3R7pl5o4sWwJ2D7HeXYo9jqpwKV+7TkLsAcoRqI8BRGSreuDcmYz7eOhiInNjTQ/uXLlSMTOoHtm+", + "c7tx5Yx8fdt34eR/dce3xqNwl4FdVrbecvYs1wvWrnJ7aZNm5WYK61Ak2dYL8nc51X2IOrCaeUDMW+F7", + "UPvUdnAXuSOgq+vkjXBz/5o0okPSiHKtwvib1tkTrmRPHRaEk73PpDLwu8MtcDSELwkEA8FUmK+eHo9F", + "BKzIokUqlkrkTg7i3t7CT/56Rp4/fd0nzzDhM/kxn24PySuItMFWtbMXjYWVxCzziigAq9sZxKHr2Y4d", + "qec2Hdehg2tJ8XufztKDJyNg5fF75R3W+70FozFKJH/0XkrbWQBI+vVLOECwf65mse29tcIH6PDVcvBk", + "ZphabfbExWyJAgbFXdIeVdAJbtgxdqg9mF7Zp5UNLNrJ/l4vAH7y4Wsej9tPFH43Fjvs16V2hGi+LzVH", + "MB7EgjmGWGD1ui4yYbRFLDtetvaBgV0eP/skj4pbtVKv4121dAH/rqcLZ/rFGpqy2j4BERcZdDZaen2g", + "8sIiXNtqJKIZjbhZ9glNEndHuZug8I4ZFOLvVDF6AXr7IaCQuZ59cDHYavveUEtiri9sC84WOyTgHqfz", + "aTE4ggfNWo1xzSFE10gS0STKE2oYYbMZizAuGFPy6BZbbjGU3i2enbKTICxMxcM+/+LSFoZpAnevJIsm", + "xe3Yrd5RLEooT9vx5J2ghs6P6GowhUalAN+FxFIcjZTUmrimBizhcw5pNdBTAjJpgJqLpmwssoQKwRTJ", + "tfXQh6EPMsW0zhXzDcDd5SiqT0qsyExJ41wTEimVHgtP4e9OiDYsW0Nmr23LJzjnW5JtbeOup3tSUjfG", + "0K4KcUUIbIilFLvgQEegGroHt3g7oPuWEr+Ug/9G8Tn4vmZKUstkrV+SPdZ+Oe2hr0VPt6YwPStKdUth", + "WrRaiZCsRA+uxfqblPDpce96HoiBzi94Kxyk+3S9iOafoFLHvuuRs+FBuE8fOctQNtZ/x8SnZ5WAxa4K", + "rJLCvzR1UmXktaNaC/rdDPHVOcr3NqNuO2N53RuE15eM3EVrobxtD97PjxBGd4s4cdd5875s2qohb9Xe", + "pi3wA5sTJHwWFHg7mRHuGXHlBpkRPisMAKCde8RiCR7U+4rpr9mepfqa3OBWQ/lthgOEhmsL5bdczzmv", + "rn0ovXNluj2TXIt/JgneLtB15He/7F9f/R2eDJXF2mSCBoJnaWaWrpK3VZZOZ5r/zoYthuDCb/X2TME3", + "cOn8dOTh6bTVofMe0cvu0ZBzLz6jLhsk1+DmV2bd84HdXxjeYfXM1S6WHbh1BlRFC37J2pXu9RPslihT", + "bJDJDI0rsV0wtx7+LjNUDee/E9e8w391/8J0opjIgMUk5opFBrxFhJHIEWwf32iipDTuu1TLdi8Re0R+", + "UDJ94maz4T50Z8opw0o/w3Q5iKmhg0vPbdao0D7Cu9P7UwLDI1yQF9+TLfbeKJu0hMzg5QPR5n5J2fuI", + "sVgjTW5XB7w7Cg8RGp7Mp11GuSb9zCv8gyYkyrWRqd/742dkC9PZzZmAvQBRf4aSbKbkJY9ZXBsjOO/b", + "Vd1tWdDr6l1BqChyEfrHhR3cvcgwXS6k+e88q7OFwiVmygXFwW1M8FI/UxZQAPqjXHgHHLdHfhRfrzD3", + "8tvyjx2gREyt6hbRSGnhpre/XnNf8jVXDYbyd1rttvPuOeuV193iozqGLd1GEooidu5u1dbvPp+QHq6/", + "yGgepzq/LB6kbWrzz4sER3d3P9y1uvzdFxwCCsryy8ay2QbUZZhgXqJPd8wuWSKzFFPcY9lev5erpHfU", + "WxiTHe3soO/3QmpzdPD44X7vw68f/v8BAM9LG8e+AQIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 8de7c33d3..ff2334317 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/netip" "os" + "path/filepath" "strconv" "strings" "time" @@ -425,15 +426,20 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } - registryURL, err := builderRegistryURL(cfg.Registry.URL, networkManager) - if err != nil { - return nil, err + registryURL := cfg.Registry.URL + if !cfg.MacOSBuilds { + var err error + registryURL, err = builderRegistryURL(registryURL, networkManager) + if err != nil { + return nil, err + } } if registryURL != cfg.Registry.URL { log.Info("resolved registry URL for builder VMs", "original", cfg.Registry.URL, "resolved", registryURL) } buildConfig := builds.Config{ + MachineBuildOnly: cfg.MacOSOnly, MaxConcurrentBuilds: cfg.Build.MaxConcurrentSourceBuilds, BuilderImage: cfg.Build.BuilderImage, RegistryURL: registryURL, @@ -444,6 +450,17 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins DockerSocket: cfg.Build.DockerSocket, } + if cfg.MacOSBuilds { + workDir := filepath.Join(p.BuildsDir(), "machine-work") + if err := os.MkdirAll(workDir, 0700); err != nil { + return nil, err + } + buildConfig.MachineBuild = &builds.MachineBuildBackend{ + Driver: &builds.GuestMachineBuildDriver{Instances: instanceManager, WorkDir: workDir}, + Publisher: &builds.StreamingMachinePublisher{RegistryURL: registryURL, Tokens: builds.NewRegistryTokenGenerator(cfg.JwtSecret)}, + } + } + // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics) var secretProvider builds.SecretProvider if cfg.Build.SecretsDir != "" { diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 77249b920..3c3857673 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -111,6 +111,12 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodPost && req.URL.Query().Get("mount") != "" { + if matches := blobUploadPattern.FindStringSubmatch(req.URL.Path); matches != nil && r.mountBlob(w, req, matches[1]) { + return + } + } + // Intercept manifest PUT requests to store in blob store and trigger conversion if req.Method == http.MethodPut { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) @@ -161,6 +167,27 @@ func (r *Registry) Handler() http.Handler { }) } +// blobUploadPattern matches POST /v2/{name}/blobs/uploads/. +var blobUploadPattern = regexp.MustCompile(`^/v2/(.+)/blobs/uploads/?$`) + +// mountBlob answers a cross-repository mount when the blob is already stored, so +// the client does not upload it again. The middleware has already checked that the +// token may pull from the source repository. It returns false, leaving the request +// to the upload handler, when the blob is not stored. +func (r *Registry) mountBlob(w http.ResponseWriter, req *http.Request, repo string) bool { + hash, err := v1.NewHash(req.URL.Query().Get("mount")) + if err != nil { + return false + } + if _, err := r.blobStore.Stat(req.Context(), repo, hash); err != nil { + return false + } + w.Header().Set("Location", "/v2/"+repo+"/blobs/"+hash.String()) + w.Header().Set("Docker-Content-Digest", hash.String()) + w.WriteHeader(http.StatusCreated) + return true +} + // storeManifestBlob stores a manifest in the blob store by its digest. func (r *Registry) storeManifestBlob(digest string, data []byte) error { digestHex := strings.TrimPrefix(digest, "sha256:") diff --git a/lib/registrypush/macos_test.go b/lib/registrypush/macos_test.go new file mode 100644 index 000000000..a9acd4275 --- /dev/null +++ b/lib/registrypush/macos_test.go @@ -0,0 +1,48 @@ +package registrypush + +import ( + "context" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/ocicache/testutil" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestPushMacOSImagePreservesManifestAndPlatform(t *testing.T) { + host, _ := newTestRegistry(t) + img, err := random.Image(256, 1) + require.NoError(t, err) + config, err := img.ConfigFile() + require.NoError(t, err) + config.OS, config.Architecture = "darwin", "arm64" + config.Config.Labels = map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", + images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", + images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + } + img, err = mutate.ConfigFile(img, config) + require.NoError(t, err) + img = mutate.MediaType(img, types.OCIManifestSchema1) + img = mutate.ConfigMediaType(img, types.OCIConfigJSON) + p := paths.New(t.TempDir()) + digest, err := testutil.WriteImage(p, img) + require.NoError(t, err) + target := host + "/releases/macos:canary" + result, err := PushFromCache(context.Background(), p, digest, target, nil, Options{Insecure: true}) + require.NoError(t, err) + require.Equal(t, digest, result.Digest) + ref, err := name.ParseReference(host+"/releases/macos@"+digest, name.Insecure) + require.NoError(t, err) + published, err := remote.Image(ref) + require.NoError(t, err) + actual, err := published.ConfigFile() + require.NoError(t, err) + require.Equal(t, config, actual) +} diff --git a/openapi.yaml b/openapi.yaml index 73aa90cde..177cb24f3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4683,6 +4683,24 @@ paths: dockerfile: type: string description: Dockerfile content. Required if not included in the source tarball. + machine_base_image: + type: string + description: | + Experimental installed darwin/arm64 base pinned by sha256 digest. + Requires server macos_only and macos_builds. The source must contain + build.sh at its root, which runs as root in the guest. Machine source + is limited to 64 MiB compressed; only machine_network_mode, + timeout_seconds, memory_mb, cpus and tags are accepted alongside + source and this selector. CPU/memory zero inherit the base. + Linux/cache/secrets/admin options are rejected. Output retains the + base identity and cannot run concurrently with it. + machine_network_mode: + type: string + enum: [isolated, egress] + description: | + Native machine builds only. Defaults to isolated. Explicit egress enables + unrestricted VZ NAT for checksum-verified artifact downloads; it does not + enforce domain allowlists. Linux builds do not use this selector. base_image_digest: type: string description: Optional pinned base image digest From 87a6db8416e04e4d5a3f70e48f92fa4a7c04f102 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:03:07 +0000 Subject: [PATCH 02/33] Give each macOS build its own machine identity The builder instance is created with a fresh machine identifier and MAC, the same rebind a cold clone gets, and its SSH host keys are rekeyed on first start. The exported bundle carries that identity, so export validation now requires the hardware model and resources to match the base and the identity to differ from it. Builds on one base are no longer serialized, and an output can run beside its base. --- docs/macos-builds.md | 9 +- lib/builds/macos_build.go | 9 +- lib/builds/macos_build_failure_test.go | 4 +- lib/builds/macos_build_guest.go | 5 +- lib/builds/macos_build_manager.go | 13 +- lib/builds/macos_build_manager_test.go | 48 ++++-- lib/builds/macos_build_test.go | 9 +- lib/builds/manager.go | 6 +- lib/instances/create.go | 5 + lib/instances/macos_identity.go | 23 +++ lib/instances/macos_identity_test.go | 17 ++ lib/instances/types.go | 1 + lib/oapi/oapi.go | 213 +++++++++++++------------ openapi.yaml | 5 +- 14 files changed, 219 insertions(+), 148 deletions(-) diff --git a/docs/macos-builds.md b/docs/macos-builds.md index 81f2eb217..c5e488cee 100644 --- a/docs/macos-builds.md +++ b/docs/macos-builds.md @@ -2,8 +2,8 @@ A build job can provision a digest-pinned installed macOS base in an isolated VZ VM, run the source's `build.sh` in it, and publish the result as an OCI machine -image. This is not macOS installation, Linux rootfs conversion, memory forking or -identity rekeying. It depends on the shared system GuestService. +image. This is not macOS installation, Linux rootfs conversion or memory forking. +It depends on the shared system GuestService. The normal build manager owns the queue, persisted request, deadline, source staging, status, provenance and the image-readiness gate. `GuestMachineBuildDriver` @@ -50,8 +50,9 @@ Networking defaults to isolated. `machine_network_mode=egress` gives unrestricte VZ NAT for checksum-verified downloads; it is not a domain allowlist. Linux builder, cache, build-arg, secret and admin options are rejected. -Jobs on the same base digest are serialized. Identity is preserved, so concurrent -guests with the same identity are never allowed. +Each builder gets its own machine identifier and MAC, and its SSH host keys are +rekeyed on first start, the same way a cold clone is. The output keeps that identity. +Builds on one base run concurrently, and an output can run beside its base. ## Source diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go index 6eb8e9ef9..d58a55d27 100644 --- a/lib/builds/macos_build.go +++ b/lib/builds/macos_build.go @@ -282,8 +282,13 @@ func validateMachineExport(root string, base *images.MacOSImage) error { if err != nil { return err } - if base != nil && (!bytes.Equal(platform.HardwareModel, base.HardwareModel) || !bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || !strings.EqualFold(platform.MAC, base.MAC) || platform.CPUs != base.CPUs || platform.Memory != base.Memory) { - return fmt.Errorf("machine export changed preserved identity/resources") + if base != nil { + if !bytes.Equal(platform.HardwareModel, base.HardwareModel) || platform.CPUs != base.CPUs || platform.Memory != base.Memory { + return fmt.Errorf("machine export changed hardware model or resources") + } + if bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || strings.EqualFold(platform.MAC, base.MAC) { + return fmt.Errorf("machine export kept the base identity") + } } for _, entry := range entries { if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go index c0e4e6e0f..632bd7ee7 100644 --- a/lib/builds/macos_build_failure_test.go +++ b/lib/builds/macos_build_failure_test.go @@ -164,7 +164,7 @@ func TestMachineBuildMalformedExport(t *testing.T) { if err != nil { return err } - return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:01", "02:00:00:00:00:01:02:03", 1)), 0600) + return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:02", "02:00:00:00:00:02:03:04", 1)), 0600) case "identity", "unknown metadata": var raw map[string]any data, err := os.ReadFile(config) @@ -175,7 +175,7 @@ func TestMachineBuildMalformedExport(t *testing.T) { return err } if kind == "identity" { - raw["machine_identifier"] = []byte{9} + raw["machine_identifier"] = f.image.MacOS.MachineIdentifier // kept the base identity } else { raw["password"] = "must-not-publish" } diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go index ebf341318..3794b92ab 100644 --- a/lib/builds/macos_build_guest.go +++ b/lib/builds/macos_build_guest.go @@ -21,7 +21,8 @@ type machineExportManager interface { // GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system // GuestService. It never installs an agent or silently upgrades an unmanaged base. -// The backend validates the base and policy before Start. +// The backend validates the base and policy before Start. The builder gets its own +// machine identifier and MAC, so it and its output can run beside the base. type GuestMachineBuildDriver struct { Instances instances.Manager WorkDir string @@ -50,7 +51,7 @@ func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base * if err != nil { return nil, err } - inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) + inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, FreshMacOSIdentity: true, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) if inst == nil { _ = os.RemoveAll(dir) if err == nil { diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go index dcd5be191..eebd4b9f6 100644 --- a/lib/builds/macos_build_manager.go +++ b/lib/builds/macos_build_manager.go @@ -161,17 +161,10 @@ func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) return policy, nil } -// enqueueMachineBuild queues a machine build whose source is already staged. -func machineBuildSerialKey(req CreateBuildRequest) string { - ref, err := images.ParseNormalizedRef(req.MachineBaseImage) - if err != nil || !ref.IsDigest() { - return "machine:" + req.MachineBaseImage - } - return "machine:" + ref.Digest() -} - +// enqueueMachineBuild queues a machine build whose source is already staged. Each +// builder has its own identity, so builds on one base are not serialized. func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { - queuePos := m.queue.EnqueueSerial(meta.ID, req, machineBuildSerialKey(req), func() { + queuePos := m.queue.EnqueueSerial(meta.ID, req, "", func() { m.runBuild(context.Background(), meta.ID, req, policy) }) build := meta.toBuild() diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go index fdc2cab7d..c30799a76 100644 --- a/lib/builds/macos_build_manager_test.go +++ b/lib/builds/macos_build_manager_test.go @@ -104,10 +104,27 @@ func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { require.Error(t, err, "machine builds must not persist a builder config or registry token") require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) } -func TestMachineBuildSerializesByBaseDigest(t *testing.T) { - first := CreateBuildRequest{MachineBaseImage: "registry-a/base@" + machineTestDigest} - second := CreateBuildRequest{MachineBaseImage: "registry-b/another-name@" + machineTestDigest} - require.Equal(t, machineBuildSerialKey(first), machineBuildSerialKey(second)) +func TestMachineBuildsOnOneBaseRunConcurrently(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + var ids []string + for range 2 { + build, err := m.CreateBuild(context.Background(), req, []byte("source")) + require.NoError(t, err) + ids = append(ids, build.ID) + } + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 2 }, 5*time.Second, 10*time.Millisecond, "builds on one base must not serialize") + for _, id := range ids { + // Cancel only once the attempt has recorded its builder, so it is Building. + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, id) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), id)) + } + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) } func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { @@ -134,16 +151,19 @@ func TestQueuedMachineCancellationRemovesSource(t *testing.T) { fixture.waitForCancel = true m := newMachineTestManager(t, fixture, fixture, nil, false) req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} - first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) - require.NoError(t, err) - t.Cleanup(func() { - _ = m.CancelBuild(context.Background(), first.ID) - require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) - }) - require.Eventually(t, func() bool { - meta, err := readMetadata(m.paths, first.ID) - return err == nil && meta.BuilderInstance != nil - }, 5*time.Second, 10*time.Millisecond) + // Fill both concurrency slots so the next build stays queued. + for range m.config.MaxConcurrentBuilds { + first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) + require.NoError(t, err) + t.Cleanup(func() { + _ = m.CancelBuild(context.Background(), first.ID) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) + }) + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, first.ID) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + } queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) require.NoError(t, err) source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go index da848ce78..1b7f9c5ba 100644 --- a/lib/builds/macos_build_test.go +++ b/lib/builds/macos_build_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "strings" + "sync" "testing" "github.com/kernel/hypeman/lib/images" @@ -18,6 +19,7 @@ import ( const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" type machineFixture struct { + mu sync.Mutex image *images.Image calls []string fail string @@ -35,6 +37,8 @@ func newMachineFixture() *machineFixture { return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} } func (f *machineFixture) record(phase string) error { + f.mu.Lock() + defer f.mu.Unlock() f.calls = append(f.calls, phase) if f.fail == phase { return errors.New("sensitive-command-argument") @@ -84,7 +88,10 @@ func (f *machineFixture) Export(_ context.Context, root string) error { if err := f.record("export"); err != nil { return err } - data, _ := json.Marshal(*f.image.MacOS) + // The builder has its own identity; the export carries it, not the base's. + output := *f.image.MacOS + output.MachineIdentifier, output.MAC = []byte{3}, "02:00:00:00:00:02" + data, _ := json.Marshal(output) for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { return err diff --git a/lib/builds/manager.go b/lib/builds/manager.go index a4327e358..3584484bb 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -1546,11 +1546,7 @@ func (m *manager) RecoverPendingBuilds() { continue } - serialKey := meta.Request.BuilderID - if machineBuild(meta.Request) { - serialKey = machineBuildSerialKey(*meta.Request) - } - m.queue.EnqueueSerial(meta.ID, *meta.Request, serialKey, func() { + m.queue.EnqueueSerial(meta.ID, *meta.Request, meta.Request.BuilderID, func() { policy := DefaultBuildPolicy() if meta.Request.BuildPolicy != nil { policy = *meta.Request.BuildPolicy diff --git a/lib/instances/create.go b/lib/instances/create.go index e7ecdde66..8af4a9147 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -385,6 +385,11 @@ func (m *manager) createInstance( HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } + if req.FreshMacOSIdentity { + if err := m.freshMacOSIdentity(stored); err != nil { + return nil, err + } + } // 12. Ensure directories log.DebugContext(ctx, "creating instance directories", "instance_id", id) diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 797468f83..6c9ef6a7a 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -224,6 +224,29 @@ func (m *manager) clearMacOSStopReceipt(stored *StoredMetadata) error { return nil } +// freshMacOSIdentity gives a new macOS instance its own machine identifier and MAC +// instead of its image's, so it can run beside the image's other instances. The +// guest's SSH host keys are rekeyed on first start. +func (m *manager) freshMacOSIdentity(stored *StoredMetadata) error { + if stored.MacOS == nil { + return nil + } + if err := prepareMacOSClone(stored); err != nil { + return err + } + used, err := m.storedMACs(stored.Id) + if err != nil { + return fmt.Errorf("list MACs for macOS identity: %w", err) + } + mac, err := network.NewUniqueMAC(used) + if err != nil { + return fmt.Errorf("allocate macOS MAC: %w", err) + } + stored.MacOS.MAC = mac + stored.MAC = mac + return nil +} + // assignMacOSCloneMAC gives a networked clone a MAC no stored instance uses. It runs // under the fork admission lock, so concurrent forks cannot pick the same MAC. func (m *manager) assignMacOSCloneMAC(clone *StoredMetadata) error { diff --git a/lib/instances/macos_identity_test.go b/lib/instances/macos_identity_test.go index 2978e7da1..0d3f98506 100644 --- a/lib/instances/macos_identity_test.go +++ b/lib/instances/macos_identity_test.go @@ -4,10 +4,12 @@ package instances import ( "net" + "os" "testing" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) @@ -42,6 +44,21 @@ func TestPrepareMacOSCloneRebindsIdentityWithoutAliasingSource(t *testing.T) { require.False(t, source.IdentityRekeyPending) } +func TestFreshMacOSIdentityRebindsIdentifierAndMAC(t *testing.T) { + p := paths.New(t.TempDir()) + require.NoError(t, os.MkdirAll(p.GuestsDir(), 0700)) + m := &manager{paths: p} + stored := testMacOSSource() + stored.Id = "builder" + require.NoError(t, m.freshMacOSIdentity(stored)) + require.True(t, stored.IdentityRekeyPending) + require.NotEqual(t, []byte("source-identifier"), stored.MacOS.MachineIdentifier) + require.NotEqual(t, "a6:fd:95:db:a2:62", stored.MacOS.MAC) + require.Equal(t, stored.MacOS.MAC, stored.MAC) + _, err := net.ParseMAC(stored.MAC) + require.NoError(t, err) +} + func TestPrepareMacOSCloneRefusesWithoutGuestAgent(t *testing.T) { for _, disable := range []func(*StoredMetadata){ func(s *StoredMetadata) { s.MacOS.GuestAgent = false }, diff --git a/lib/instances/types.go b/lib/instances/types.go index bd8989885..7c50cb359 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -267,6 +267,7 @@ type CreateInstanceRequest struct { Name string // Required Image string // Required: OCI reference Platform string // Optional: target platform as os/arch[/variant]; drives image resolution and auto-pull. Empty means host platform. + FreshMacOSIdentity bool // Internal: give a macOS instance its own machine identifier and MAC instead of the image's. Size int64 // Base memory in bytes (default: 1GB) HotplugSize int64 // Hotplug memory in bytes (default: 0, set explicitly to enable) OverlaySize int64 // Overlay disk size in bytes (default: 10GB) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index eb421b5b1..b6b825e65 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -2014,8 +2014,9 @@ type CreateBuildMultipartBody struct { // is limited to 64 MiB compressed; only machine_network_mode, // timeout_seconds, memory_mb, cpus and tags are accepted alongside // source and this selector. CPU/memory zero inherit the base. - // Linux/cache/secrets/admin options are rejected. Output retains the - // base identity and cannot run concurrently with it. + // Linux/cache/secrets/admin options are rejected. The builder and its + // output get their own machine identifier and MAC, so the output can + // run beside the base. MachineBaseImage *string `json:"machine_base_image,omitempty"` // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables @@ -19643,110 +19644,110 @@ var swaggerSpec = []string{ "GDxKvi4I7riTLfNJabTggk1KBhcwfr3PmOIpExDEiSbVBNxkY6quuNixydKR23nOt4QR7j04dIxvOBav", "i3ADe/OkNJJ6giwJ9FP2n3bI9lXizg3CMsBhQ6BXy9ZgHQ2qtJSUxienQp5F7W81C/OQnNg5ujbHAmjT", "ohHBThwekBP+fSUO+1vLKv3KeHjCVMasPxbOJueTPPVdoNkknfYJcCScD8hKpOozRihA6Wges7FwU8Ny", - "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1LcheYUnhSgGK4gX", - "wVjYm8k/4WAcERXAvFQuYLFLzBnUt3KzgXaqKxRgyNZ9yhX2dInOsLX0K1xLIPB4SJ6/B8mHG8IwP6eL", - "fdNjkQvFoHuYGnn3N0xGBozBpgoG9FamkKURqgyf0cgm1kQD4beEw7+YBjY9FkzMJOxCLFPKbWq5q4Rr", - "o4cEl9iP06FD5Zo1NqsGAuLHDj/hmIPpMApKaQX7RLJsXllckJPvKxfX6OBR+O5y9LDa+n+dvfqZoAQM", - "/M4WK10j7eoLoBjwWMM8PU5+eA6QZNYojLlQxz0ej3uF60S8jWOFxUFKGQzQfv8dDO07202fx98Nh9CU", - "dQ04Ir/8YVs5gnsrSy348Lj3oU8qH+bcLPJp8e3XMAG2YSae1S5dsmVFym3PSGCGFenaiqNwDKSTuIAr", - "kfK2rzqJTbmgarkuejmw9G4F5cx6DVcW448xegmPe0dj7yc87vXHPSYu8TfnTDzufQivQIMbtSjOC0eC", - "gogOR6PtzfD7bn0D/gEdjHCfWN/SqoEosgbDDjrs5z+XLe7fWtfzfaEdwSuqvJvs1eoNXff9hobe72Dz", - "f5YGxPOEpQy9twbV/IdwU2DezoEVYOA6vokVsvJsr6qjAnbIxtubiogl/u29Wdt79xZEoKKIgfR6tyfn", - "vs5NYSMv8pZ8UUZx3KzyfK+14d0zxY3u6rar2e7uh36/OCNawITmDGjs0sc7hLM2ISKX06cRWxgemWc4", - "psEZE4Y8x1+H7r9eAYQIs+eJnJ8fWf0dJCAhCRcuKKgSrYAPMLuWWMmCchX17D+LlJlbVsT51z/+iYPi", - "Yv6vf/zTZbr41z/+icd9x4JNYu7+8wWjykwZNedHBEJfBzSB95ibDGbXZpdMLcn+yBn+8BMMSbCoiKyQ", - "OaDTwkve5EpUgqds9krtGnT2QpgPFznTDtQMCvKZS61lHZwDylt/lu1S3umJ7gfAYXEGlQnArehpAJE1", - "ueAY7GXVUb2w3cTOuWY5afpqr3jsbuYvhr03lnoHdoDXZDC4xKFzhx/cpMnW2dnz7SHBZ6ClCkyfho+a", - "shn3vhl+5UmbeZLlKHWGgqtseVNEMzrlCfd2h5bcT/YIOpVKGWRQZF7wTRz5kQKPAX9r5w3dx6Jj8eps", - "B+0shkUmV6zvOIFyeMtlckjpgt2wB6cVQhfRgSs7FjNGoQ1y/MwygUpKgiI6umhYIKwROrpzU8tD2R8L", - "i6vttEpw8ECSTbAS9j+nhl3RZZ9UJF+bKyqhBl7qug+Fx8LGG7s1GCBwE6kMc4j8zA5p4N35XeCmYjNQ", - "9OBEbOo92/cWKDxcmEO/DC733dmQczssIp04DvOb4xNVWr0ktvTqzO/Gdp9oSaKEIzVEVIzFHL0BPZS5", - "FLVdLaJKF1TFg0jCJVCFtoMwhITF8zYe+7TSVu8WJZlaP4Hj9GOTXL804WKxOgE4xBauc735/pkr081+", - "71r8Mxnw7QJdx4JvzTjM8hu7ul+t+R2s+eF185b9kHn9mcejvT23f9vFPXn9e9pbXXP7pbJk96FqJFse", - "6AtNo1KR06fHhMaxYlpv/3srImGmlkpL+Q/uR2DF9+F/5sYild0Lr2+pE8iXwg5eu1ET6ufVzDZevd92", - "aqnIWm+6IitZeeXd/u3R6PQ610gp9Ja09vUm2ehxz3UE8ctVahmgaJSwQnwpzmmVijZpla0vf3HlrBWX", - "bClwlHIH8u70y67rXDTvhjtgis8aDPEeGWEdb4FQUcAyfEnU/LbYRTevdernz4s0R3cnBd21KjpE5l/S", - "czFuLBtwQYt21HqBvmDGYhzd5jvd9RCYOOi53am2A13aWRfTslWtA46dEGpi1r99j22Rbk9f296f6eWL", - "y3MdicUt+VcRpcNjt1yrdQ/cY5cQ//betzU4xzv2p3EEFlhk+OB004iNBjlmqV6KaPurS80np2hc7Moj", - "Vvl5s7jQZOP38p11V3LdE+E16jn4g+viXcsFmSUAFW2NADGfoROhqUAfWka0dwejfO38tomihjnfyS8x", - "+P80T6x1I5LikqkKXGv1St35Az3XNz+VPPNae7u+ff1ywEQk48J40i6Tui+f+MFk6b8W0H/3p+4LDGrn", - "XjxoExg/Yv9tRAkp8Hv/194PDsH3f+39QJOMC/a/9p8k1DBttm+NWEZ3ddPd9QPmCyY+eL/w+qKtsKYd", - "Q+fVYPQ6ZXoM6+tQZg172iu7i2i6T0Z9n17IawJ2d5Lw7ozu4SV117BFxz43id1SqTyEXQOX/N9b6Dur", - "EfSd6+tq3XMMhbHQDl8gVpJUHnqZOux67onb8SaBoTiblBFFqY76CF/+T6WSsJO+llKiWNeveokueonq", - "cq1VTdiCt6ucsH3ck/W9ILbQauOnr5h7d2DQcRRZwdyrWbhL1L2F1AY/fR4ALP9OkTi8OArV+6yjybTk", - "FGvvNX+mjp/1cYf7sKfHz0rghTsyoPpx3LlGwPV799LYk3TK57nMdRXTIaUmWjDt8E4SVr8ZvjRdRSk3", - "tGorPmMqHd3lnXbnyoivdH9LapLmhlrmbb0gNj1GfKmujxFXHjE3EAveYcK4zG19n9VzuyU6xaPBdyXj", - "Gmj+h36XcYUeSZAJR5PyHUPwaXM0Fv8bHka/GEbTX7/zAfH5aLR3iL8zcfnrdz4mXpx4UmHCKO6SMD/5", - "+Rm6nswxugDzNJdQN81xWHwRJD2fkOnf7uVWet90f7p5Kvz6dOv0dKss1/qnmy14y2+3elK3O3+8eXoL", - "Lbj99id9vv3Jbcq1p6XOZzMecSYwNR5qJfWKE7V9Yn41J99MP0qFc+KoeGDWJJHOz8iCa22Q0It865/U", - "xbHfmiFHEmoMSzMDEGsRg0zdNnRQL3IDwE8+YQ5O0Ode5OV8Qte7b2riGwlH7tYS+6wL3A2+dIuksXf9", - "1HUdf6Ghs9IGy8f+cVmKNu2vy/sl3tt9U3a4au/+Vfklk5h9vq0uXQYvhEACSGt/SnMbZ1zULLFjh+TN", - "m5c+ptgi87l0okb6HKI+z/xYVHOIIgyeT85qC9Sx/RwGAdT3WTljRuOEC4ZBGEyHwn/rmX/v9Vh8egk4", - "nNb4rm3vXY6l3dB7lIDvjRXciaxZOBsYZykujmYl43FxWry8iafmi+JXjgEFGE9I1tuhuZEDh1Kws5AW", - "vzYM4X2a0AgRvKGYBZd1wDAWTbraFKK9KJkkTFnQ4Cw3XtwC/G43OC6MLNLiO8nsHJqf5MLw5LzvoUgj", - "lmhCxdKh+Y1FrTMn8yF4AwKT4AgVy+yIGzm+YdAcVI1QCnEWql0SmlzRpR4LB/dgq6cSo6Iji6+dJEPy", - "o0SkHZvrvsJ4baLpb/RYnPM4YRMHlHNOuCZ6IZVhgsUklZdM1/tlVCWcKZzEUworp0lKlyXOqFsfmTEL", - "V1mD45HwbypijimLoediykdjQcneaERSRoV24BqazvDCcW0QHERtQN8SSg5Gj12txr5hagW//FtwmpRi", - "lzKiU8hODVQMQ+RmGzcwdSnEFdNGKti+GVfa7leh33S5YWsby7XPiB33SS5K+BDU9Vesa7BdJlcC5+nM", - "k4yr4hp0iElTFlFYTyHr/SBgtYyiXIUuSNjqSi77f0fBsTK9M1yqMDhHgiqDCGIAJCI/LvBMSzxK29+2", - "UFVJVH+OiyZ4SKQilFToutRosChH1riFoLPnZWJmMXHlz7e/9WcHjq9jBP74i/mXdD8hEcnZrHYAN19N", - "9gCvC4pbJeE/6zl96phejcXFnM6F1IZHnhl6/Pzqo/nrg7DTg3D9ygapeSbVRbsn9A9SXXR9gXmPzS/r", - "IVad4WdoiIDhoZPO/dsjUBtuHytANHf+SGvSV3FKUejiRhe+ygQyGDBV1crfudmg+qrbskibcJkqa+wu", - "cNHgETJxP9qk/hV/czQxRK7VP62zGhIbbHbpszZd2qdHiSR3PV4Jp6oK+GDfgtr6H/S9OIR05TdsC6X3", - "1e0KMlUAGt2I1Fp07mFJA1CtY/FW20Qq59b0dE4KHgwCrc3x4PKKAGwr/IbtW1RXmmXnBZT+9hF5gQe5", - "sm228y2bEgloTcuEWTTWyzQ9P1pNa//u5AQrYZmFTWB/fkR8Kvvi/tBQqgrDCrNIqDbkZwcuu1U8xnFH", - "z0GHWZnftgNoLVMdjEUIrBWwTm2DfEbOK7it5y2ggp7fvpRz/dmYispkUHYuRhL3dETaZCLutTl58CRs", - "+NkdjUJ5EzrCx9ph3DJ67MpgXsp5kYiqRso0y7qSrxsmUvFlmq6hYbJVchCiTSxz8xdtYqYUVnbU3Ubc", - "ZItG9h+GXgChOuRRf7C3x6JlqewMw0sFTLWSr8X+6zJNe/2eGw/UuzJZGsjc8vFwvM0GP/RDO1TB3P0T", - "i+f3diW6pGXFUfHJgqzE415QAOWK5wW39CbQv/UbqoL927junA6g/Rnx2hb405s4vaLxzsR396JpiE9B", - "+ri3U4xiVkMX/jlJ8pXx1vVmQhaoyjiyx3cxMoReJlEiNavZyb4c4EanL2yI3u36Nr/GAxhenPt0l10c", - "gc5c1TNf8zNQYWxyufFjJn66d+57szqCLxmEQq/MBthfA+1vk1POZ09In25LVqbahUK+0ub1lbWdCBOe", - "W6sswlWIXSrR3MiUGh5hOsBoIaWukH0BzW+T5DodfEGZqKGyygIXiHEOpHrutPnnTsQ8cppHQqufXB9D", - "rO7CN8I1/Keyxg8V5UrB8fv+BYWZaTSkTFOczUhGc81AzsxBI7aMEubzPzKwq0c0M7limEaakZQLnuZp", - "NecC7NglRXyo8930vE+muSEJaNlU8dH7LEUyTZmIrZw0FgtGLzlTsCgJNUxEy4FmQnN0jYK0D5izEzQ1", - "WUzRYIaBlIoBBWICi5QZGlNDUdA5hxM/sbFg5yRTDInIakcEe19SA9iOc/GtzaYDzZ77gZ4ThukiuF4U", - "mcsjGjMRBdMonH3ebOzTq/TPmGlO9J4crG7ES+/T46qquvbD+Tycsb5YyI8ObH6N0KvbH9X1IBpPRv+e", - "R9rO1c/xnux0xRKvO8Wfh4GuILrPxkh3/1Y4qUic2+4qpxLJ/M+qRywYSg0RAgJ07Tbe1L5WZGctlvla", - "PG/nD//n8Q20i58JJ+y3Puzb8gCWk/4cWK5b1Rvx3HtSqzpdUlUnd38s2A3q/sQnqSpc7ktRttaQ9wq+", - "XeVORlF8fUnxlW032bbzG7kp2/a62RXPiAoj52KArrZhDu7UuK2s2qkO/k2Dehqzq7DMe2eRpeXizmE0", - "PWvM6BJUJX8GX+s19qNIKmVRRKhhXxb0d0VrWI2yQN1cmWC074Ne352cbLdxCWXW8ghlvmAOUYlsgmpp", - "vKrAfXXJlOKxhzd9evKstMmqXAzJq5QbYiS5YCwrA4MwOBNQOws8lcawm8Ap/R4TRi0zyYXZOIqy6O0M", - "pvxBOliZz1CUdLkkvprDO5vDUbP/5bEz5DKEFhNY/zI11GzMcc3FTKrUymV0KnNoHXgQLBPsp3W2mPGE", - "6aU2LLXOnQAFAccN8xK53POunt3lPro2w8mxUYcZUynXmkuhx8KF3GRMQd9QHdqv+KkFDQKGFvz11DLJ", - "z8MHEgZj3f6oaVs1RL7CnNa9o94OzbKdmBra4mfnhvcRQ/oBnRqJXqZTCKAEr8gLTbYSfmGfJ+RSkwT+", - "2F7rFTnBep86s/7NTxas9LGYyWDeUkuzBTH/qYLjHFvzhsnPgq3d0zvzo70IX7DqwfW8cCZbWazeyGMV", - "o8kA8/J7LCaSG57w3y3bhUa4NjyyUWS02Md3JwWDH47FCTOKR9beG4H/OYJU4Et3J1My2gEcv/0o4wjo", - "t89wcMh82z+n2OPT07dYLmWpVMv+WFDhGn7z5NRammc0Ys2BCmbAkkyOd15t8Fk/w2X6N/aftBNcC0gR", - "3PCv5snrw8a0niHdckRltu7ZJrM/vYOvkya/6ji+TB0H0HA5m60Cq80DrIX1GZcyyVP4h/3jeBNUnaHR", - "4h0W/Wwkbzucjd34CX4Rh9LNKWY2x/O9GGDsgn2p/rOwcH4KKMTUPBODt8AT82ek7k9vSqiu42doenUr", - "6vOnfzZn665vPjcGD5pSXY8v5ZhbSvMzMXK9JuyK8nZN2PeJjC60Q9epqjCJYhQh8+HHEuLcmStRTMBg", - "X+JQqSz2GdP9sWgoQy2IkyaUGKZSLmiyg3O2jSBYu9eo0UvJMeY+wpiZgQZVdyaTBBHZEdEQZgMlZr6B", - "inVZuwyT1TJVw6iRANskU+YB7LdDT7e/Um5+kKqORv+58MU3lfVHlEfKUfe/AYC/vcePAuQ/oe/RbTvO", - "nXHbj2jrhSx/tGqpPsG9Gff2R3rc65Nxby8d92AHnlJU51JDHpCUi9wAsuUzq2vDqOrDEdEskiLWHkff", - "axP3R7pl5o4sWwJ2D7HeXYo9jqpwKV+7TkLsAcoRqI8BRGSreuDcmYz7eOhiInNjTQ/uXLlSMTOoHtm+", - "c7tx5Yx8fdt34eR/dce3xqNwl4FdVrbecvYs1wvWrnJ7aZNm5WYK61Ak2dYL8nc51X2IOrCaeUDMW+F7", - "UPvUdnAXuSOgq+vkjXBz/5o0okPSiHKtwvib1tkTrmRPHRaEk73PpDLwu8MtcDSELwkEA8FUmK+eHo9F", - "BKzIokUqlkrkTg7i3t7CT/56Rp4/fd0nzzDhM/kxn24PySuItMFWtbMXjYWVxCzziigAq9sZxKHr2Y4d", - "qec2Hdehg2tJ8XufztKDJyNg5fF75R3W+70FozFKJH/0XkrbWQBI+vVLOECwf65mse29tcIH6PDVcvBk", - "ZphabfbExWyJAgbFXdIeVdAJbtgxdqg9mF7Zp5UNLNrJ/l4vAH7y4Wsej9tPFH43Fjvs16V2hGi+LzVH", - "MB7EgjmGWGD1ui4yYbRFLDtetvaBgV0eP/skj4pbtVKv4121dAH/rqcLZ/rFGpqy2j4BERcZdDZaen2g", - "8sIiXNtqJKIZjbhZ9glNEndHuZug8I4ZFOLvVDF6AXr7IaCQuZ59cDHYavveUEtiri9sC84WOyTgHqfz", - "aTE4ggfNWo1xzSFE10gS0STKE2oYYbMZizAuGFPy6BZbbjGU3i2enbKTICxMxcM+/+LSFoZpAnevJIsm", - "xe3Yrd5RLEooT9vx5J2ghs6P6GowhUalAN+FxFIcjZTUmrimBizhcw5pNdBTAjJpgJqLpmwssoQKwRTJ", - "tfXQh6EPMsW0zhXzDcDd5SiqT0qsyExJ41wTEimVHgtP4e9OiDYsW0Nmr23LJzjnW5JtbeOup3tSUjfG", - "0K4KcUUIbIilFLvgQEegGroHt3g7oPuWEr+Ug/9G8Tn4vmZKUstkrV+SPdZ+Oe2hr0VPt6YwPStKdUth", - "WrRaiZCsRA+uxfqblPDpce96HoiBzi94Kxyk+3S9iOafoFLHvuuRs+FBuE8fOctQNtZ/x8SnZ5WAxa4K", - "rJLCvzR1UmXktaNaC/rdDPHVOcr3NqNuO2N53RuE15eM3EVrobxtD97PjxBGd4s4cdd5875s2qohb9Xe", - "pi3wA5sTJHwWFHg7mRHuGXHlBpkRPisMAKCde8RiCR7U+4rpr9mepfqa3OBWQ/lthgOEhmsL5bdczzmv", - "rn0ovXNluj2TXIt/JgneLtB15He/7F9f/R2eDJXF2mSCBoJnaWaWrpK3VZZOZ5r/zoYthuDCb/X2TME3", - "cOn8dOTh6bTVofMe0cvu0ZBzLz6jLhsk1+DmV2bd84HdXxjeYfXM1S6WHbh1BlRFC37J2pXu9RPslihT", - "bJDJDI0rsV0wtx7+LjNUDee/E9e8w391/8J0opjIgMUk5opFBrxFhJHIEWwf32iipDTuu1TLdi8Re0R+", - "UDJ94maz4T50Z8opw0o/w3Q5iKmhg0vPbdao0D7Cu9P7UwLDI1yQF9+TLfbeKJu0hMzg5QPR5n5J2fuI", - "sVgjTW5XB7w7Cg8RGp7Mp11GuSb9zCv8gyYkyrWRqd/742dkC9PZzZmAvQBRf4aSbKbkJY9ZXBsjOO/b", - "Vd1tWdDr6l1BqChyEfrHhR3cvcgwXS6k+e88q7OFwiVmygXFwW1M8FI/UxZQAPqjXHgHHLdHfhRfrzD3", - "8tvyjx2gREyt6hbRSGnhpre/XnNf8jVXDYbyd1rttvPuOeuV193iozqGLd1GEooidu5u1dbvPp+QHq6/", - "yGgepzq/LB6kbWrzz4sER3d3P9y1uvzdFxwCCsryy8ay2QbUZZhgXqJPd8wuWSKzFFPcY9lev5erpHfU", - "WxiTHe3soO/3QmpzdPD44X7vw68f/v8BAM9LG8e+AQIA", + "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1Lf6C89pAsfCHh8C", + "BkSHcnEl/Hyr/AUqnDx52ifauoK6ehEVY6FycKOFGVWHt47EqgsZ4NvWy8qPwpEv+szWsrRwLeEcxEPy", + "/D0ISNwQhmk8XYicHotcKAbdwwqQd3/DnGXAP2xGYQB5ZQo5H6HK8BmNbP5NtCN+Szj8i2ng5mPBxEzC", + "ZsUypdxmoLtKuDZ6SHAn/DgdiFSuWWNPa1ghfuzwE445mDWjIKhWTFCk3ubNxgU5+b5yv40OHoWvOEc2", + "q63/19mrnwkKysAWbbHSg9KuvgDCAsc2TOfjxIzngFxmbceYMnXc4/G4V3hYxNs4VlgcVOAPBmjm/w6G", + "9p3tps/j74ZDaMp6EByRX/6wrRzB9ZalFqN43PvQJ5UPc24W+bT49muYANugFc9qdzPZspLntuc3MMOK", + "EG6lVjgS0glmwLxIKRRUfcmmXFC1XBfkHFh6t4JyZp2LK4vxxxidice9o7F3Jx73+uMeE5f4m/M5Hvc+", + "hFegwbRa9OuFv0FBRIej0fZmlH63vgE3gg62uk+slmlVVBTJhWEHHUT0n8tk92+tEvq+UKII6LyETbM3", + "sLeH3fdTG3q/g83/WRqQ4hOWMnTyGlTTJMJNgek9B1bOgev4JsbKyuu+qrUKmCsbT3QqIpb4J/pmpfDd", + "GxqBiiIGQu7dnpz7OjeFKb1Ib/JF2c5xs8rzvdbUd88UN7qr265m4rsf+v3ibG0BS5uzs7FLHxYRTu6E", + "wF1O7UZsYXiLnuGYBmdMGPIcfx26/3o9EQLRnidyfn5k1XyQp4QkXLjYoUpQA8gtbi2xksXuKurZfxaZ", + "NbesiPOvf/wTB8XF/F//+KdLiPGvf/wTj/uOxaTEFP/nC0aVmTJqzo8IRMgOaALvMTcZTMLNLplakv2R", + "sw/iJxiSYFERgCFzeGDCg9/kSlRirGySS+0adGZFmA8XOdMO+wwK8pnLwGX9oAM6Xn+W7VLe6YnuBzBk", + "cQaVCcCt6GkAATi54BgTZp/PvbB5xc65ZmBpunSvOPZu5i+GvTeWegd2gNdkMLjEoXOHH9ykydbZ2fPt", + "IcFnoKUKzLKGj5qyGfe+GX7lSZt5kuUodYaCq2x5U0QzOuUJ9+aJlhRR9gg6lUoZi1AkaPBNHPmRAo8B", + "t2znNN3HomPx6mwHzTGGRSZXrO84gXKwzGUOSeli4rAH5F/coCfpwJUdixmj0AY5fmaZQCVzQRFEXTQs", + "EP0I/eG5qaWr7I+Fhd92WiU4eCDJJlgJ+59Tw67osk8qkq9NKZVQAy913YfCY2HDkt0aDBDfiVSGOUR+", + "Zoc08F7/Lr5TsRkoenAiNkOf7XsLFB4uGqJfxqD77mxkuh0WkU4ch/nN8YkqrfoSW3p15ndjG/VwUcKR", + "GlAJN0enQY94LkVtV4vg0wVV8SCScAlUEfAgWiFh8byNxz6ttNW7RUmm1k/gOP3YJNcvTbhYrE4ADrFF", + "9Vxv5X/mynQz87sW/0x2frtA1zH0W2sPs/zGru5Xo38Ho3943bwDQMgK/8zD1t5edIDt4p6CAzztra65", + "/VJZsvtQNZItjweGFlSpyOnTY0LjWDGtt/+9FZEwU0ulpfwH9yOw4vtwU3Njkcruhde31AnkS2EHr92o", + "CfXzaiYlr95vO7WMZa03XZG8rLzybv/2aHR6nWukFHpLWvt6k2x0zOc6gjDnKrUMUDRKWCG+FOe0SkWb", + "tMrW5b+4ctaKS7YU+FO5A3l3+mXXdS6ad8MdMMVnDYZ4j4ywDstAqCjQG74kan5b7KKb1zr18+dFmqO7", + "k4LuWhUdIvMv6bkYN5YNuKAFRWq9QF8wY6GQbvOd7noITBz03O5U24Eu7ayLadmq1gHHTgg1Mevfvse2", + "SLenr23vz/TyxeW5jsTilvyriNLhsVuu1boH7rHLm39779sa6uMd+9M4AgssMnxwummEUINUtFQvRbT9", + "1aXmk1M0LnblEav8vFlcaLLxe/nOuiu57onwGvUc3MZ18a7lgswSQJS2RoCYz9CJ0FQQEi0j2ruDUb52", + "7t1EUcOc7+SXiBFwmifWuhFJcclUBdW1eqXu/IEO7pufSp55rb1d375+OWAiknFhPGmXSd2XT/xgsvRf", + "i/u/+1P3Bca+cy8etAmMH7H/NvCEFDC//2vvBwf0+7/2fqBJxgX7X/tPEmqYNtu3Riyju7rp7voB8wUT", + "H7xfeH3RVljTjqHzasx6nTI91PV1KLMGUe2V3UXQ3Sejvk8v5DVxvTtJeHdG9/CSumt0o2OfwsRuqVQe", + "6a4BX/7vLfSd1Qj6zvV1te45hsJYBIgvEFJJKo/QTB3EPffE7XiTwFCcTcqIolRHfYQv/6dSSdhJX0sp", + "UazrV71EF71EdbnWqiZswdtVTtg+7sn6XhBbaLXx01dovjsw6DiKrEDz1SzcJTjfQmqDnz4PnJZ/p0gc", + "XhyF6n3W0WRacoq195o/U8fP+rjDfdjT42clPsMdGVD9OO5cI+D6vXtp7Ek65fNc5roamp1SEy2YdrAo", + "CavfDF+arqKUG1q1FZ8xlY7u8k67c2XEV7q/JTVJc0Mt87ZeEJseI75U18eIK4/QHAgZ76BjXIK3vk/+", + "ud0SneJB47uScQ1b/0O/y7hCjyRImKNJ+Y4h+LQ5Gov/DQ+jXwyj6a/f+YD4fDTaO8Tfmbj89TsfEy9O", + "PKkwYRR3uZqf/PwMXU/mGF2A6ZxLRJzmOCwMCZKez9v0b/dyK71vuj/dPBV+fbp1erpVlmv9080WvOW3", + "Wz33250/3jy9hRbcfvuTPt/+5Dbl2tNS57MZjzgTmEEPtZJ6xYnaPjG/mpNvph+lwjlxVDwwa5JI52dk", + "wbU2SOhFWvZP6uLYb02kIwk1hqWZASS2iEFCbxs6qBe5AeAnn1cHJ+hTNPJyPqHr3Tc18Y2EI3dr+X/W", + "Be4GX7pFbtm7fuq6jr/Q0Flpg+Vj/7gsRZv21+X9Eu/tvik7XLV3/6r8kknMPt9Wly6DF0IgT6S1P6W5", + "jTMuapYQs0Py5s1LH1NsAfxc1lEjfapRn45+LKqpRhEGz+dwtQXqEIAOgwDq++SdMaNxwgXDIAymQ+G/", + "9QTB93osPr0EHM5+fNe29y7H0m7oPUrA98YK7kTWLJwNjLMUF0ezkhi5OC1e3sRT80XxK8eAAownJOvt", + "0NzIgUMp2FlIC3MbRvo+TWiEQN9QzGLQOmAYCzpdbQrRXpRMEqYstnCWGy9uAcy3GxwXRhbZ851kdg7N", + "T3JheHLetz6QUDbRhIqlQ/Mbi1pnTuZD8AYEJsERKpbZETdSgcOgOagaoRTiLFS7JDS5oks9Fg7uwVZP", + "JUZFRxaGO0mG5EeJSDs2JX6F8dp81N/osTjnccImDijnnHBN9EIqwwSLSSovma73y6hKOFM4iacUVk6T", + "lC5LnFG3PjJjFq6yBscj4d9UxBwzG0PPxZSPxoKSvdGIpIwK7cA1NJ3hhePaIDiI2oC+JZQcjB67Wo19", + "wwwMfvm34DQpxS5lRKeQxBqoGIbIzTZuYOoyjSumjVSwfTOutN2vQr/pUsjWNpZrnzg77pNclPAhqOuv", + "WNdgu0yuBM7TmScZV8U16BCTpiyisJ5C1vtBXGsZRbkKXZCw1ZWU9/+OgmNleme4VGFwjgRVBhHEAEhE", + "flzgmZZ4lLa/baGqkqj+HBdN8JBIRSip0HWp0WBRjqxxC0Fnz8v8zWLiyp9vf+vPDhxfxwj88RfzL+l+", + "QiKSs1ntAG6+muwBXhcUt0rCf9Zz+tQxvRqLizmdC6kNjzwz9DD71Ufz1wdhpwfh+pUNUvNMqot2T+gf", + "pLro+gLzHptf1kOsOsPP0BABw0Mnnfu3R6A23D5WgGju/JHWpK/ilKLQxY0ufJUJJDpgqqqVv3OzQfVV", + "t2WRNuEyVdbYXeCiwSNk4n60uf8r/uZoYohcq39aZzUkNtjs0mdturRPjxJJ7nq8Ek5VFfDBvgW19T/o", + "e3EI6cpv2BZK76vbFWSqADS6Eam16NzDkgagWsfirbbJLc6t6emcFDwYBFqb48GlHwHYVvgN27eorjTL", + "zgso/e0j8gIPcmXbbOdbNnMS0JqWCbNorJdpen60mv3+3ckJVsIyC5vn/vyI+Iz3xf2hoVQVhhVmkVBt", + "yM8OXHareIzjjp6DDrMyv20H0FqmOhiLEFgrYJ3aBvmMnFdwW89bQAU9v30p5/qzMRWVOaPsXIwk7umI", + "tMlE3Gtz8uBJ2PCzOxqF8iZ0hI+1w7hl9NiVwbyU8yJfVY2UaZZ1JV83TKTiyzRdQ8Nkq+QgRJtY5uYv", + "2sRMKazsqLuNuMkWjew/DL0AQnXIo/5gb49Fy1LZGYaXCphqJV+L/ddlmvb6PTceqHdlsjSQueXj4Xib", + "DX7oh3aogrn7JxbP7+1KdLnNiqPikwVZice9oADKFc8LbulNoH/rN1QF+7dx3TkdQPsz4rUt8Kc3cXpF", + "452J7+5F0xCfgvRxb6cYxayGLvxzkuQr463rzYQsUJVxZI/vYmQIvUyiRGpWs5N9OcCNTl/YEL3b9W1+", + "jQcwvDj3WTG7OAKduapnvuZnoMLY5HLjx0z8dO/c92Z1BF8yCIVemQ2wvwba3yannM+ekD7dlqxMtQuF", + "fKXN6ytrOxEmPLdWWYSrELuMo7mRKTU8wnSA0UJKXSH7Aprf5tJ1OviCMlFDZZUFLhDjHEj13Gnzz52I", + "eeQ0j4RWP7k+hljdhW+Ea/hPZY0fKsqVguP3/QsKM9NoSJmmOJuRjOaagZyZg0ZsGSXM539kYFePaGZy", + "xTDbNCMpFzzN02rOBdixS4r4UOe76XmfTHNDEtCyqeKj91mKZJoyEVs5aSwWjF5ypmBREmqYiJYDzYTm", + "6BoFaR8wZydoarKYosEMAykVAwrEBBYpMzSmhqKgcw4nfmJjwc5JphgSkdWOCPa+pIYY85t+a7PpQLPn", + "fqDnhGG6CK4XRYLziMZMRME0CmefNxv79Cr9M2aaE70nB6sb8dL79Liqqq79cD4PZ6wvFvKjA5tfI/Tq", + "9kd1PYjGk9G/55G2c/VzvCc7XbHE607x52GgK4juszHS3b8VTioS57a7yqlEMv+z6hELhlJDhIAAXbuN", + "N7WvFdlZi2W+Fs/b+cP/eXwD7eJnwgn7rQ/7tjyA5aQ/B5brVvVGPPee1KpOl1TVyd0fC3aDuj/xSaoK", + "l/tSlK015L2Cb1e5k1EUX19SfGXbTbbt/EZuyra9bnbFM6LCyLkYoKttmIM7NW4rq3aqg3/ToJ7G7Cos", + "895ZZGm5uHMYTc8aM7oEVcmfwdd6jf0okkpZFBFq2JcF/V3RGlajLFA3VyYY7fug13cnJ9ttXEKZtTxC", + "mS+YQ1Qim6BaGq8qcF9dMqV47OFNn548K22yKhdD8irlhhhJLhjLysAgDM4E1M4CT6Ux7CZwSr/HhFHL", + "THJhNo6iLHo7gyl/kA5W5jMUJV0uia/m8M7mcNTsf3nsDLkMocUE1r9MDTUbc1xzMZMqtXIZncocWgce", + "BMsE+2mdLWY8YXqpDUutcydAQcBxw7xELve8q2d3uY+uzXBybNRhxlTKteZS6LFwITcZU9A3VIf2K35q", + "QYOAoQV/PbVM8vPwgYTBWLc/atpWDZGvMKd176i3Q7NsJ6aGtvjZueF9xJB+QKdGopfpFAIowSvyQpOt", + "hF/Y5wm51CSBP7bXekVOsN6nzqx/85MFK30sZjKYt9TSbEHMf6rgOMfWvGHys2Br9/TO/GgvwhesenA9", + "L5zJVharN/JYxWgywLz8HouJ5IYn/HfLdqERrg2PbBQZLfbx3UnB4IdjccKM4pG190bgf44gFfjS3cmU", + "jHYAx28/yjgC+u0zHBwy3/bPKfb49PQtlktZKtWyPxZUuIbfPDm1luYZjVhzoIIZsCST451XG3zWz3CZ", + "/o39J+0E1wJSBDf8q3ny+rAxrWdItxxRma17tsnsT+/g66TJrzqOL1PHATRczmarwGrzAGthfcalTPIU", + "/mH/ON4EVWdotHiHRT8bydsOZ2M3foJfxKF0c4qZzfF8LwYYu2Bfqv8sLJyfAgoxNc/E4C3wxPwZqfvT", + "mxKq6/gZml7divr86Z/N2brrm8+NwYOmVNfjSznmltL8TIxcrwm7orxdE/Z9IqML7dB1qipMohhFyHz4", + "sYQ4d+ZKFBMw2Jc4VCqLfcZ0fywaylAL4qQJJYaplAua7OCcbSMI1u41avRScoy5jzBmZqBB1Z3JJEFE", + "dkQ0hNlAiZlvoGJd1i7DZLVM1TBqJMA2yZR5APvt0NPtr5SbH6Sqo9F/LnzxTWX9EeWRctT9bwDgb+/x", + "owD5T+h7dNuOc2fc9iPaeiHLH61aqk9wb8a9/ZEe9/pk3NtLxz3YgacU1bnUkAck5SI3gGz5zOraMKr6", + "cEQ0i6SItcfR99rE/ZFumbkjy5aA3UOsd5dij6MqXMrXrpMQe4ByBOpjABHZqh44dybjPh66mMjcWNOD", + "O1euVMwMqke279xuXDkjX9/2XTj5X93xrfEo3GVgl5Wtt5w9y/WCtavcXtqkWbmZwjoUSbb1gvxdTnUf", + "og6sZh4Q81b4HtQ+tR3cRe4I6Oo6eSPc3L8mjeiQNKJcqzD+pnX2hCvZU4cF4WTvM6kM/O5wCxwN4UsC", + "wUAwFearp8djEQErsmiRiqUSuZODuLe38JO/npHnT1/3yTNM+Ex+zKfbQ/IKIm2wVe3sRWNhJTHLvCIK", + "wOp2BnHoerZjR+q5Tcd16OBaUvzep7P04MkIWHn8XnmH9X5vwWiMEskfvZfSdhYAkn79Eg4Q7J+rWWx7", + "b63wATp8tRw8mRmmVps9cTFbooBBcZe0RxV0ght2jB1qD6ZX9mllA4t2sr/XC4CffPiax+P2E4XfjcUO", + "+3WpHSGa70vNEYwHsWCOIRZYva6LTBhtEcuOl619YGCXx88+yaPiVq3U63hXLV3Av+vpwpl+sYamrLZP", + "QMRFBp2Nll4fqLywCNe2GoloRiNuln1Ck8TdUe4mKLxjBoX4O1WMXoDefggoZK5nH1wMttq+N9SSmOsL", + "24KzxQ4JuMfpfFoMjuBBs1ZjXHMI0TWSRDSJ8oQaRthsxiKMC8aUPLrFllsMpXeLZ6fsJAgLU/Gwz7+4", + "tIVhmsDdK8miSXE7dqt3FIsSytN2PHknqKHzI7oaTKFRKcB3IbEURyMltSauqQFL+JxDWg30lIBMGqDm", + "oikbiyyhQjBFcm099GHog0wxrXPFfANwdzmK6pMSKzJT0jjXhERKpcfCU/i7E6INy9aQ2Wvb8gnO+ZZk", + "W9u46+melNSNMbSrQlwRAhtiKcUuONARqIbuwS3eDui+pcQv5eC/UXwOvq+ZktQyWeuXZI+1X0576GvR", + "060pTM+KUt1SmBatViIkK9GDa7H+JiV8ety7ngdioPML3goH6T5dL6L5J6jUse965Gx4EO7TR84ylI31", + "3zHx6VklYLGrAquk8C9NnVQZee2o1oJ+N0N8dY7yvc2o285YXvcG4fUlI3fRWihv24P38yOE0d0iTtx1", + "3rwvm7ZqyFu1t2kL/MDmBAmfBQXeTmaEe0ZcuUFmhM8KAwBo5x6xWIIH9b5i+mu2Z6m+Jje41VB+m+EA", + "oeHaQvkt13POq2sfSu9cmW7PJNfin0mCtwt0HfndL/vXV3+HJ0NlsTaZoIHgWZqZpavkbZWl05nmv7Nh", + "iyG48Fu9PVPwDVw6Px15eDptdei8R/SyezTk3IvPqMsGyTW4+ZVZ93xg9xeGd1g9c7WLZQdunQFV0YJf", + "snale/0EuyXKFBtkMkPjSmwXzK2Hv8sMVcP578Q17/Bf3b8wnSgmMmAxiblikQFvEWEkcgTbxzeaKCmN", + "+y7Vst1LxB6RH5RMn7jZbLgP3ZlyyrDSzzBdDmJq6ODSc5s1KrSP8O70/pTA8AgX5MX3ZIu9N8omLSEz", + "ePlAtLlfUvY+YizWSJPb1QHvjsJDhIYn82mXUa5JP/MK/6AJiXJtZOr3/vgZ2cJ0dnMmYC9A1J+hJJsp", + "ecljFtfGCM77dlV3Wxb0unpXECqKXIT+cWEHdy8yTJcLaf47z+psoXCJmXJBcXAbE7zUz5QFFID+KBfe", + "AcftkR/F1yvMvfy2/GMHKBFTq7pFNFJauOntr9fcl3zNVYOh/J1Wu+28e8565XW3+KiOYUu3kYSiiJ27", + "W7X1u88npIfrLzKax6nOL4sHaZva/PMiwdHd3Q93rS5/9wWHgIKy/LKxbLYBdRkmmJfo0x2zS5bILMUU", + "91i21+/lKukd9RbGZEc7O+j7vZDaHB08frjf+/Drh/9/ABXpBjflAQIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 177cb24f3..a51553ab5 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4692,8 +4692,9 @@ paths: is limited to 64 MiB compressed; only machine_network_mode, timeout_seconds, memory_mb, cpus and tags are accepted alongside source and this selector. CPU/memory zero inherit the base. - Linux/cache/secrets/admin options are rejected. Output retains the - base identity and cannot run concurrently with it. + Linux/cache/secrets/admin options are rejected. The builder and its + output get their own machine identifier and MAC, so the output can + run beside the base. machine_network_mode: type: string enum: [isolated, egress] From da6f1ee8ec4700686d6bc4a127a7f3ec5e4a0702 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:53:50 +0000 Subject: [PATCH 03/33] Skip uploading blobs the registry already stores The blob store is keyed by digest and shared across repositories, so a derivative's base layers and a retried publish only need a HEAD check. This replaces the cross-repository blob mount, its registry handler and the mount authorization in the auth middleware. --- lib/builds/macos_build_publisher.go | 12 ++++--- lib/builds/macos_delta_publish.go | 7 ++-- lib/builds/macos_delta_publish_test.go | 48 ++------------------------ lib/builds/macos_registry.go | 34 ++++++++---------- lib/middleware/oapi_auth.go | 23 ------------ lib/registry/registry.go | 27 --------------- 6 files changed, 28 insertions(+), 123 deletions(-) diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 4945020d3..59be90116 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -36,7 +36,6 @@ type StreamingMachinePublisher struct { type machineBlob struct { open func() (io.ReadCloser, error) descriptor v1.Descriptor - mountFrom string // set for a base layer mounted from this repository instead of uploaded } // Publish commits bundle as a standalone machine image in builds/{id}. @@ -112,10 +111,13 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri } digest := machineHash(manifestBytes).String() for _, blob := range append([]machineBlob{configBlob}, layers...) { - if blob.mountFrom != "" { - if err := mountMachineBlob(ctx, client, token, base, blob); err != nil { - return MachinePublication{}, err - } + // The blob store is keyed by digest, so a blob the registry already holds + // (a base layer, or a retried publish) is not uploaded again. + stored, e := machineBlobStored(ctx, client, token, base, blob.descriptor) + if e != nil { + return MachinePublication{}, e + } + if stored { continue } body, e := blob.open() diff --git a/lib/builds/macos_delta_publish.go b/lib/builds/macos_delta_publish.go index f0590eb47..f42e2282a 100644 --- a/lib/builds/macos_delta_publish.go +++ b/lib/builds/macos_delta_publish.go @@ -43,8 +43,9 @@ func (m MachineDeltaBase) validate() error { } // machineLayers returns a machine image's layer blobs and their diff IDs. A -// standalone image has one uploaded layer. A derivative mounts the base's layers -// and uploads one delta layer that patches the base disk. +// standalone image has one uploaded layer. A derivative lists the base's layers, +// which the registry already holds, and uploads one delta layer that patches the +// base disk. func machineLayers(ctx context.Context, client *http.Client, token, registry, bundle string, source *MachineDeltaBase) ([]machineBlob, []v1.Hash, error) { if source == nil { layer, err := machineTarballLayer(ctx, bundle) @@ -72,7 +73,7 @@ func machineLayers(ctx context.Context, client *http.Client, token, registry, bu } blobs := make([]machineBlob, 0, len(baseLayers)+1) for _, descriptor := range baseLayers { - blobs = append(blobs, machineBlob{descriptor: descriptor, mountFrom: source.Repo}) + blobs = append(blobs, machineBlob{descriptor: descriptor}) } diskPath := filepath.Join(bundle, "disk.img") delta, err := images.ComputeMacOSDelta(source.DiskPath, diskPath) diff --git a/lib/builds/macos_delta_publish_test.go b/lib/builds/macos_delta_publish_test.go index 034cff270..743f6167f 100644 --- a/lib/builds/macos_delta_publish_test.go +++ b/lib/builds/macos_delta_publish_test.go @@ -1,11 +1,9 @@ package builds_test import ( - "bytes" "context" "encoding/json" "fmt" - "io" "net/http" "net/http/httptest" "os" @@ -30,7 +28,6 @@ const deltaTestSecret = "synthetic registry secret" type deltaTestRegistry struct { server *httptest.Server tokens *builds.RegistryTokenGenerator - mounts atomic.Int32 uploads atomic.Int32 puts atomic.Int32 } @@ -42,9 +39,7 @@ func newDeltaTestRegistry(t *testing.T) *deltaTestRegistry { require.NoError(t, err) dr := &deltaTestRegistry{tokens: builds.NewRegistryTokenGenerator(deltaTestSecret)} handler := middleware.JwtAuth(deltaTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && r.URL.Query().Get("mount") != "" { - dr.mounts.Add(1) - } else if r.Method == http.MethodPost && r.URL.Query().Get("digest") != "" { + if r.Method == http.MethodPost && r.URL.Query().Get("digest") != "" { dr.uploads.Add(1) } if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { @@ -106,7 +101,7 @@ func (dr *deltaTestRegistry) pullManifest(t *testing.T, repo, digest string) v1. return manifest } -func TestStreamingMachinePublisherDerivedMountsBaseLayers(t *testing.T) { +func TestStreamingMachinePublisherDerivedReusesStoredBaseLayers(t *testing.T) { dr := newDeltaTestRegistry(t) publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -122,7 +117,6 @@ func TestStreamingMachinePublisherDerivedMountsBaseLayers(t *testing.T) { derived[3*64<<10+j] = 0xAA } derivedBundle := deltaTestBundle(t, derived, "derived aux") - dr.mounts.Store(0) dr.uploads.Store(0) result, err := publisher.PublishDerived(ctx, "derived-x", derivedBundle, builds.MachineDeltaBase{ Repo: "builds/base-x", @@ -130,8 +124,7 @@ func TestStreamingMachinePublisherDerivedMountsBaseLayers(t *testing.T) { DiskPath: filepath.Join(baseBundle, "disk.img"), }) require.NoError(t, err) - require.Equal(t, int32(1), dr.mounts.Load(), "the base layer is mounted, not uploaded") - require.Equal(t, int32(2), dr.uploads.Load(), "only the config and the delta layer are uploaded") + require.Equal(t, int32(2), dr.uploads.Load(), "only the config and the delta layer are uploaded; the base layer is already stored") derivedManifest := dr.pullManifest(t, "builds/derived-x", result.Digest) require.Len(t, derivedManifest.Layers, 2) @@ -165,38 +158,3 @@ func TestStreamingMachinePublisherDerivedRejectsNonBaseRepo(t *testing.T) { }) require.ErrorContains(t, err, "under builds/") } - -// mountRequest posts a blob mount with the given token and reports the status. -func (dr *deltaTestRegistry) mountRequest(t *testing.T, token, repo, digest, from string) int { - t.Helper() - target := dr.server.URL + "/v2/" + repo + "/blobs/uploads/?mount=" + digest + "&from=" + from - request, err := http.NewRequest(http.MethodPost, target, bytes.NewReader(nil)) - require.NoError(t, err) - request.Header.Set("Authorization", "Bearer "+token) - response, err := http.DefaultClient.Do(request) - require.NoError(t, err) - defer response.Body.Close() - _, _ = io.Copy(io.Discard, response.Body) - return response.StatusCode -} - -func TestBlobMountFallsThroughToUploadForMissingBlob(t *testing.T) { - dr := newDeltaTestRegistry(t) - token, err := dr.tokens.GenerateToken("mounter", []builds.RepoPermission{ - {Repo: "builds/target-x", Scope: "push"}, - {Repo: "builds/source-x", Scope: "pull"}, - }, time.Minute) - require.NoError(t, err) - status := dr.mountRequest(t, token, "builds/target-x", fmt.Sprintf("sha256:%064x", 3), "builds/source-x") - require.Equal(t, http.StatusAccepted, status, "a missing blob opens an upload session") -} - -func TestBlobMountRefusedFromUnauthorizedSource(t *testing.T) { - dr := newDeltaTestRegistry(t) - token, err := dr.tokens.GenerateToken("mounter", []builds.RepoPermission{ - {Repo: "builds/target-x", Scope: "push"}, - }, time.Minute) - require.NoError(t, err) - status := dr.mountRequest(t, token, "builds/target-x", fmt.Sprintf("sha256:%064x", 4), "builds/other-private") - require.Equal(t, http.StatusForbidden, status, "a token without pull on the source cannot mount from it") -} diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go index a9d2f35cd..078255fef 100644 --- a/lib/builds/macos_registry.go +++ b/lib/builds/macos_registry.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "net/http" - "net/url" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" @@ -79,27 +78,22 @@ func machineFetch(ctx context.Context, client *http.Client, token, target, diges return kept.Bytes(), nil } -// mountMachineBlob links a base layer into the build repository from its source. The -// registry answers 201 only when it already stores the blob; the size is then checked -// with a HEAD request. The bytes are not downloaded or hashed again. -func mountMachineBlob(ctx context.Context, client *http.Client, token, base string, blob machineBlob) error { - digest := blob.descriptor.Digest.String() - target := base + "/blobs/uploads/?mount=" + digest + "&from=" + url.QueryEscape(blob.mountFrom) - response, err := machineRegistryRequest(ctx, client, token, http.MethodPost, target, "", nil, 0) +// machineBlobStored reports whether the registry already holds a blob of the +// descriptor's digest and size. The store is shared by digest across repositories. +func machineBlobStored(ctx context.Context, client *http.Client, token, base string, descriptor v1.Descriptor) (bool, error) { + head, err := machineRegistryRequest(ctx, client, token, http.MethodHead, base+"/blobs/"+descriptor.Digest.String(), "", nil, 0) if err != nil { - return err - } - response.Body.Close() - if response.StatusCode != http.StatusCreated { - return fmt.Errorf("base layer %s was not mounted", digest) - } - head, err := machineRegistryRequest(ctx, client, token, http.MethodHead, base+"/blobs/"+digest, "", nil, 0) - if err != nil { - return err + return false, err } head.Body.Close() - if head.StatusCode != http.StatusOK || head.ContentLength != blob.descriptor.Size { - return fmt.Errorf("mounted base layer %s is not verified", digest) + switch head.StatusCode { + case http.StatusOK: + if head.ContentLength != descriptor.Size { + return false, fmt.Errorf("stored blob %s has size %d, want %d", descriptor.Digest, head.ContentLength, descriptor.Size) + } + return true, nil + case http.StatusNotFound: + return false, nil } - return nil + return false, fmt.Errorf("blob check returned status %d", head.StatusCode) } diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 5aa8c610e..9e8a64d25 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -203,25 +203,6 @@ func extractRepoFromPath(path string) string { } // isWriteOperation returns true if the HTTP method implies a write operation -// authorizeBlobMount checks the source of a cross-repository blob mount. The blob -// store is shared by digest, so a mount is the only way a build could link another -// repository's blob into its own; the token must allow pulling from that source. -func authorizeBlobMount(claims *RegistryTokenClaims, r *http.Request) error { - query := r.URL.Query() - from := query.Get("from") - if query.Get("mount") == "" && from == "" { - return nil - } - if r.Method != http.MethodPost || query.Get("mount") == "" || from == "" { - return fmt.Errorf("blob mount requires POST with mount and from") - } - for _, perm := range claims.RepoAccess { - if perm.Repo == from && (perm.Scope == "pull" || perm.Scope == "push") { - return nil - } - } - return fmt.Errorf("%w: mount source %s", errRepoNotAllowed, from) -} func isWriteOperation(method string) bool { return method == http.MethodPut || method == http.MethodPost || method == http.MethodPatch || method == http.MethodDelete @@ -414,10 +395,6 @@ func JwtAuth(jwtSecret string) func(http.Handler) http.Handler { // Try to validate as a registry-scoped token registryClaims, err := validateRegistryToken(token, jwtSecret, r.URL.Path, r.Method) if err == nil { - if err := authorizeBlobMount(registryClaims, r); err != nil { - http.Error(w, err.Error(), http.StatusForbidden) - return - } // Valid registry token - set build ID as user for audit trail log.DebugContext(r.Context(), "registry token validated", "build_id", registryClaims.BuildID, diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 3c3857673..77249b920 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -111,12 +111,6 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.Method == http.MethodPost && req.URL.Query().Get("mount") != "" { - if matches := blobUploadPattern.FindStringSubmatch(req.URL.Path); matches != nil && r.mountBlob(w, req, matches[1]) { - return - } - } - // Intercept manifest PUT requests to store in blob store and trigger conversion if req.Method == http.MethodPut { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) @@ -167,27 +161,6 @@ func (r *Registry) Handler() http.Handler { }) } -// blobUploadPattern matches POST /v2/{name}/blobs/uploads/. -var blobUploadPattern = regexp.MustCompile(`^/v2/(.+)/blobs/uploads/?$`) - -// mountBlob answers a cross-repository mount when the blob is already stored, so -// the client does not upload it again. The middleware has already checked that the -// token may pull from the source repository. It returns false, leaving the request -// to the upload handler, when the blob is not stored. -func (r *Registry) mountBlob(w http.ResponseWriter, req *http.Request, repo string) bool { - hash, err := v1.NewHash(req.URL.Query().Get("mount")) - if err != nil { - return false - } - if _, err := r.blobStore.Stat(req.Context(), repo, hash); err != nil { - return false - } - w.Header().Set("Location", "/v2/"+repo+"/blobs/"+hash.String()) - w.Header().Set("Docker-Content-Digest", hash.String()) - w.WriteHeader(http.StatusCreated) - return true -} - // storeManifestBlob stores a manifest in the blob store by its digest. func (r *Registry) storeManifestBlob(digest string, data []byte) error { digestHex := strings.TrimPrefix(digest, "sha256:") From 097d341a1a2d112653d993a439734a3e2add6a96 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:04:25 +0000 Subject: [PATCH 04/33] Attach an audio device to macOS guests Virtualization gives a guest audio only through a sound device, and the shim configured none, so macOS guests booted with no CoreAudio devices at all: system_profiler listed none, afplay failed with AudioQueueStart failed (-66680), and the live view's ScreenCaptureKit audio stream had an empty mix to tap. The output stream has no host sink - samples are discarded on the host, so nothing plays on the machine running the guests - while the guest gets a real output device. Verified on a macOS 27 guest: one Apple Virtual Sound Device, 48 kHz, afplay returns 0, and voxl delivers 283 Opus packets in 6 s while a sound plays against 0 when it stops. --- cmd/vz-shim/macos_vm.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmd/vz-shim/macos_vm.go b/cmd/vz-shim/macos_vm.go index 269e9af82..4241f57dd 100644 --- a/cmd/vz-shim/macos_vm.go +++ b/cmd/vz-shim/macos_vm.go @@ -45,6 +45,9 @@ func createMacVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.Virtual if err := configureMacPlatform(vmConfig, config); err != nil { return nil, nil, fmt.Errorf("configure Mac platform: %w", err) } + if err := configureMacAudio(vmConfig); err != nil { + return nil, nil, fmt.Errorf("configure audio: %w", err) + } vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() if err != nil { @@ -65,3 +68,26 @@ func createMacVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.Virtual } return vm, vmConfig, nil } + +// configureMacAudio gives the guest an audio device. Virtualization attaches audio +// only through a sound device, so without one the guest boots with no CoreAudio +// devices at all: `system_profiler SPAudioDataType` lists none and `afplay` fails +// with `AudioQueueStart failed (-66680)`. +// +// The output stream has no host sink. The samples are discarded on the host, which +// is what a headless browser VM wants -- nothing should play on the machine running +// the guests -- while the guest still has a real output device, so applications can +// render audio and ScreenCaptureKit has a mix to capture for the live view. +func configureMacAudio(vmConfig *vz.VirtualMachineConfiguration) error { + soundConfig, err := vz.NewVirtioSoundDeviceConfiguration() + if err != nil { + return fmt.Errorf("create sound device: %w", err) + } + output, err := vz.NewVirtioSoundDeviceOutputStreamConfiguration() + if err != nil { + return fmt.Errorf("create output stream: %w", err) + } + soundConfig.SetStreams(output) + vmConfig.SetAudioDevicesVirtualMachineConfiguration([]vz.AudioDeviceConfiguration{soundConfig}) + return nil +} From b67f024c2cf3edbd559e90754335e80a75bcddd8 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:05:28 +0000 Subject: [PATCH 05/33] Rewrite vz-shim in Swift with ASIF stack support The shim keeps its process model, control API, vsock protocol, stop receipt, identity lock and macOS audio device, and calls Virtualization.framework directly. A disk with parents is attached as a DiskImageKit stack: read-only ASIF layers with the writable ASIF overlay on top. vmnet networks get the subnet's host address so DHCP reservations are served. --- Makefile | 29 +- cmd/vz-shim/Config.swift | 100 ++++++ cmd/vz-shim/Log.swift | 45 +++ cmd/vz-shim/Shim.swift | 370 +++++++++++++++++++++ cmd/vz-shim/Socket.swift | 184 +++++++++++ cmd/vz-shim/VM.swift | 356 ++++++++++++++++++++ cmd/vz-shim/macos_arm64.go | 70 ---- cmd/vz-shim/macos_arm64_test.go | 24 -- cmd/vz-shim/macos_identity.go | 48 --- cmd/vz-shim/macos_identity_test.go | 25 -- cmd/vz-shim/macos_other.go | 16 - cmd/vz-shim/macos_vm.go | 93 ------ cmd/vz-shim/main.go | 219 ------------ cmd/vz-shim/main.swift | 137 ++++++++ cmd/vz-shim/rosetta_arm64.go | 45 --- cmd/vz-shim/rosetta_arm64_test.go | 81 ----- cmd/vz-shim/rosetta_other.go | 21 -- cmd/vz-shim/save_restore_arm64.go | 29 -- cmd/vz-shim/save_restore_unsupported.go | 22 -- cmd/vz-shim/server.go | 422 ------------------------ cmd/vz-shim/vm.go | 377 --------------------- lib/hypervisor/config.go | 5 +- lib/hypervisor/vz/shim_darwin_test.go | 93 ++++++ lib/hypervisor/vz/shimconfig/config.go | 3 + lib/hypervisor/vz/starter.go | 1 + 25 files changed, 1300 insertions(+), 1515 deletions(-) create mode 100644 cmd/vz-shim/Config.swift create mode 100644 cmd/vz-shim/Log.swift create mode 100644 cmd/vz-shim/Shim.swift create mode 100644 cmd/vz-shim/Socket.swift create mode 100644 cmd/vz-shim/VM.swift delete mode 100644 cmd/vz-shim/macos_arm64.go delete mode 100644 cmd/vz-shim/macos_arm64_test.go delete mode 100644 cmd/vz-shim/macos_identity.go delete mode 100644 cmd/vz-shim/macos_identity_test.go delete mode 100644 cmd/vz-shim/macos_other.go delete mode 100644 cmd/vz-shim/macos_vm.go delete mode 100644 cmd/vz-shim/main.go create mode 100644 cmd/vz-shim/main.swift delete mode 100644 cmd/vz-shim/rosetta_arm64.go delete mode 100644 cmd/vz-shim/rosetta_arm64_test.go delete mode 100644 cmd/vz-shim/rosetta_other.go delete mode 100644 cmd/vz-shim/save_restore_arm64.go delete mode 100644 cmd/vz-shim/save_restore_unsupported.go delete mode 100644 cmd/vz-shim/server.go delete mode 100644 cmd/vz-shim/vm.go create mode 100644 lib/hypervisor/vz/shim_darwin_test.go diff --git a/Makefile b/Makefile index 5e7a4cab0..1d4218df5 100644 --- a/Makefile +++ b/Makefile @@ -335,7 +335,7 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries # macOS tests (no sudo needed, adds e2fsprogs to PATH) # Uses 'go list' to discover compilable packages, then filters out packages # whose test files reference Linux-only symbols (devices, system/init). -DARWIN_EXCLUDE_PKGS := /lib/devices|/lib/system/init|/cmd/vz-shim +DARWIN_EXCLUDE_PKGS := /lib/devices|/lib/system/init test-darwin: build-embedded sign-vz-shim @VERBOSE_FLAG=""; \ if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ @@ -349,23 +349,6 @@ test-darwin: build-embedded sign-vz-shim PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) $$PKGS; \ fi - $(MAKE) test-vz-shim-signed VERBOSE="$(VERBOSE)" TEST="$(TEST)" TEST_TIMEOUT="$(TEST_TIMEOUT)" - -# cmd/vz-shim's tests call Virtualization.framework in-process, which requires the -# com.apple.security.virtualization entitlement. Plain `go test` binaries are -# unsigned, so compile the test binary, ad-hoc-sign it with the entitlement (as -# sign-vz-shim does for the shim), then run it. HYPEMAN_VZ_SIGNED=1 tells the -# tests not to skip for a missing entitlement. -.PHONY: test-vz-shim-signed -test-vz-shim-signed: - @set -e; \ - RUN_FLAG=""; \ - if [ -n "$(TEST)" ]; then RUN_FLAG="-test.run=$(TEST)"; fi; \ - BIN="$$(mktemp -d)/vz-shim.test"; \ - go test -tags containers_image_openpgp -c -o "$$BIN" ./cmd/vz-shim; \ - if [ ! -f "$$BIN" ]; then echo "no cmd/vz-shim test binary for this platform; skipping"; exit 0; fi; \ - codesign --sign - --entitlements $(ENTITLEMENTS_FILE) --force "$$BIN"; \ - HYPEMAN_VZ_SIGNED=1 "$$BIN" $$RUN_FLAG -test.v -test.timeout=$(TEST_TIMEOUT) # Manual-only guest memory policy integration tests (Linux hypervisors). test-guestmemory-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded @@ -424,8 +407,12 @@ release-prep: download-ch-binaries download-firecracker-binaries build-caddy-bin # Entitlements file for macOS codesigning ENTITLEMENTS_FILE ?= vz.entitlements -# Build vz-shim (subprocess that hosts vz VMs) -# Also copies to embed directory so it gets embedded in the hypeman binary +# Build vz-shim (subprocess that hosts vz VMs) from the Swift sources in +# cmd/vz-shim. Also copies to embed directory so it gets embedded in the hypeman +# binary. Stacked ASIF disks need DiskImageKit, so they need the macOS 27 SDK. +VZ_SHIM_SOURCES := $(wildcard cmd/vz-shim/*.swift) +VZ_SHIM_TARGET ?= $(shell uname -m)-apple-macos26.0 + # DARWIN_LDFLAGS silences the macOS (Xcode 15+) linker's "ignoring duplicate # libraries: '-lobjc'" warning: cgo deps that link Virtualization.framework # specify -lobjc more than once, which the new linker flags. Harmless; suppress. @@ -434,7 +421,7 @@ DARWIN_LDFLAGS := -ldflags=-extldflags=-Wl,-no_warn_duplicate_libraries .PHONY: build-vz-shim build-vz-shim: | $(BIN_DIR) @echo "Building vz-shim for macOS..." - go build $(DARWIN_LDFLAGS) -o $(BIN_DIR)/vz-shim ./cmd/vz-shim + swiftc -O -target $(VZ_SHIM_TARGET) -o $(BIN_DIR)/vz-shim $(VZ_SHIM_SOURCES) mkdir -p lib/hypervisor/vz/vz-shim cp $(BIN_DIR)/vz-shim lib/hypervisor/vz/vz-shim/vz-shim @echo "Build complete: $(BIN_DIR)/vz-shim" diff --git a/cmd/vz-shim/Config.swift b/cmd/vz-shim/Config.swift new file mode 100644 index 000000000..e2ad9fa7b --- /dev/null +++ b/cmd/vz-shim/Config.swift @@ -0,0 +1,100 @@ +import Foundation + +// These types mirror lib/hypervisor/vz/shimconfig. Every field is optional so a +// config written by any hypeman version decodes, and an absent field encodes as +// absent, which Go reads back as its zero value. + +struct ShimConfig: Codable { + var vcpus: Int? + var memoryBytes: Int64? + var disks: [DiskConfig]? + var networks: [NetworkConfig]? + var serialLogPath: String? + var kernelPath: String? + var initrdPath: String? + var kernelArgs: String? + var macHardwareModelData: String? + var macMachineIdentifierData: String? + var macAuxStoragePath: String? + var enableMemoryBalloon: Bool? + var requireMemoryBalloon: Bool? + var enableRosetta: Bool? + var controlSocket: String? + var vsockSocket: String? + var logPath: String? + var stopReceiptPath: String? + var machineIdentifierData: String? + var restoreMachineStatePath: String? + + enum CodingKeys: String, CodingKey { + case vcpus + case memoryBytes = "memory_bytes" + case disks, networks + case serialLogPath = "serial_log_path" + case kernelPath = "kernel_path" + case initrdPath = "initrd_path" + case kernelArgs = "kernel_args" + case macHardwareModelData = "mac_hardware_model_data" + case macMachineIdentifierData = "mac_machine_identifier_data" + case macAuxStoragePath = "mac_aux_storage_path" + case enableMemoryBalloon = "enable_memory_balloon" + case requireMemoryBalloon = "require_memory_balloon" + case enableRosetta = "enable_rosetta" + case controlSocket = "control_socket" + case vsockSocket = "vsock_socket" + case logPath = "log_path" + case stopReceiptPath = "stop_receipt_path" + case machineIdentifierData = "machine_identifier_data" + case restoreMachineStatePath = "restore_machine_state_path" + } + + var isMacGuest: Bool { + !(macHardwareModelData ?? "").isEmpty || !(macMachineIdentifierData ?? "").isEmpty || !(macAuxStoragePath ?? "").isEmpty + } +} + +struct DiskConfig: Codable { + var path: String + var readonly: Bool? + // parents, when set, are the read-only images below path, base first. path is + // then the ASIF overlay on top of them. + var parents: [String]? +} + +struct NetworkConfig: Codable { + var mac: String? + var ipv4Subnet: String? + var ipv4Address: String? + + enum CodingKeys: String, CodingKey { + case mac + case ipv4Subnet = "ipv4_subnet" + case ipv4Address = "ipv4_address" + } +} + +struct SnapshotManifest: Codable { + var hypervisor: String + var machineStateFile: String + var shimConfig: ShimConfig + + enum CodingKeys: String, CodingKey { + case hypervisor + case machineStateFile = "machine_state_file" + case shimConfig = "shim_config" + } +} + +let snapshotManifestFile = "config.json" +let snapshotMachineStateFile = "machine-state.vzm" +let rosettaMountTag = "rosetta" + +struct ShimError: Error, CustomStringConvertible { + let description: String + init(_ description: String) { self.description = description } +} + +// decodeBase64 accepts what Go's base64 decoder accepts, including line breaks. +func decodeBase64(_ s: String) -> Data? { + Data(base64Encoded: s, options: .ignoreUnknownCharacters) +} diff --git a/cmd/vz-shim/Log.swift b/cmd/vz-shim/Log.swift new file mode 100644 index 000000000..722ec84e6 --- /dev/null +++ b/cmd/vz-shim/Log.swift @@ -0,0 +1,45 @@ +import Foundation + +// Log writes JSON lines in the shape of Go's slog JSON handler, so the shim log +// that hypeman attaches to startup errors reads the same as before. +enum Log { + private static var handle = FileHandle.standardError + private static let lock = NSLock() + private static let formatter: ISO8601DateFormatter = { + let f = ISO8601DateFormatter() + f.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return f + }() + + static func setup(path: String?) throws { + guard let path, !path.isEmpty else { return } + try FileManager.default.createDirectory(atPath: (path as NSString).deletingLastPathComponent, withIntermediateDirectories: true) + let fd = open(path, O_CREAT | O_WRONLY | O_APPEND, 0o644) + guard fd >= 0 else { throw ShimError("open log file: \(String(cString: strerror(errno)))") } + handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + } + + static func debug(_ msg: String, _ fields: [String: Any] = [:]) { write("DEBUG", msg, fields) } + static func info(_ msg: String, _ fields: [String: Any] = [:]) { write("INFO", msg, fields) } + static func warn(_ msg: String, _ fields: [String: Any] = [:]) { write("WARN", msg, fields) } + static func error(_ msg: String, _ fields: [String: Any] = [:]) { write("ERROR", msg, fields) } + + private static func write(_ level: String, _ msg: String, _ fields: [String: Any]) { + var record = fields.mapValues { value -> Any in + if value is String || value is Int || value is Int64 || value is UInt32 || value is UInt64 || value is Bool { return value } + return String(describing: value) + } + record["time"] = formatter.string(from: Date()) + record["level"] = level + record["msg"] = msg + guard var line = try? JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]) else { return } + line.append(0x0a) + lock.lock() + handle.write(line) + lock.unlock() + } +} + +func printError(_ message: String) { + FileHandle.standardError.write((message + "\n").data(using: .utf8)!) +} diff --git a/cmd/vz-shim/Shim.swift b/cmd/vz-shim/Shim.swift new file mode 100644 index 000000000..a936e2bbf --- /dev/null +++ b/cmd/vz-shim/Shim.swift @@ -0,0 +1,370 @@ +import Foundation +import Virtualization + +// Shim owns the VM. The VM runs on the main queue, so every VM call below happens +// there; request handlers run on their own threads and wait for it. +final class Shim: NSObject, VZVirtualMachineDelegate { + let vm: VZVirtualMachine + let vmConfig: VZVirtualMachineConfiguration + let config: ShimConfig + // Serializes control operations, as the Go shim's mutex did. + private let ops = NSLock() + // forcedStop is set, on the main queue, before the shim stops the VM itself, so + // that stop is not mistaken for the guest powering off. + private var forcedStop = false + private var listeners: [UnixListener] = [] + private var stateObservation: NSKeyValueObservation? + + init(vm: VZVirtualMachine, vmConfig: VZVirtualMachineConfiguration, config: ShimConfig) { + self.vm = vm + self.vmConfig = vmConfig + self.config = config + super.init() + vm.delegate = self + stateObservation = vm.observe(\.state) { vm, _ in + Log.info("VM state changed", ["state": stateName(vm.state)]) + } + } + + // MARK: Lifecycle + + func listen() throws { + let control = try UnixListener(path: config.controlSocket ?? "") + let vsock = try UnixListener(path: config.vsockSocket ?? "") + listeners = [control, vsock] + control.serve { [self] fd in serveHTTP(fd, route) } + vsock.serve { [self] fd in proxyVsock(fd) } + Log.info("control API listening", ["socket": control.path]) + Log.info("vsock proxy listening", ["socket": vsock.path]) + } + + func guestDidStop(_ virtualMachine: VZVirtualMachine) { + if !forcedStop { + writeStopReceipt() + } + Log.info("VM stopped, shutting down shim") + finish() + } + + func virtualMachine(_ virtualMachine: VZVirtualMachine, didStopWithError error: Error) { + Log.error("VM stopped with error", ["error": error.localizedDescription]) + finish() + } + + // stopForSignal runs on the main queue. + func stopForSignal() { + if vm.state == .running && vm.canStop { + Log.info("stopping VM") + forcedStop = true + vm.stop { _ in self.finish() } + return + } + finish() + } + + private func finish() { + for listener in listeners { + listener.close() + } + Log.info("vz-shim shutdown complete") + exit(0) + } + + // writeStopReceipt records that the guest stopped the VM itself. The host counts + // a macOS stop as graceful only with this receipt. + private func writeStopReceipt() { + guard let path = config.stopReceiptPath, !path.isEmpty else { return } + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + let data = Data((formatter.string(from: Date()) + "\n").utf8) + let fd = open(path, O_CREAT | O_WRONLY | O_TRUNC, 0o600) + guard fd >= 0, writeAll(fd, data) else { + Log.error("failed to write stop receipt", ["error": errnoString(), "path": path]) + if fd >= 0 { close(fd) } + return + } + close(fd) + } + + // MARK: Main-queue helpers + + private func onMain(_ body: () throws -> T) rethrows -> T { + try DispatchQueue.main.sync(execute: body) + } + + // awaitMain starts an asynchronous VM operation on the main queue and waits for + // its completion handler. + private func awaitMain(_ start: @escaping (@escaping (Error?) -> Void) -> Void) -> Error? { + let done = DispatchSemaphore(value: 0) + var result: Error? + DispatchQueue.main.async { + start { error in + result = error + done.signal() + } + } + done.wait() + return result + } + + // MARK: Control API + + private static let routes: [String: Set] = [ + "/api/v1/vm.info": ["GET"], "/api/v1/vm.pause": ["PUT"], "/api/v1/vm.resume": ["PUT"], + "/api/v1/vm.shutdown": ["PUT"], "/api/v1/vm.snapshot": ["PUT"], "/api/v1/vm.snapshot-support": ["GET"], + "/api/v1/vm.balloon": ["GET", "PUT"], "/api/v1/vm.power-button": ["PUT"], + "/api/v1/vmm.ping": ["GET"], "/api/v1/vmm.shutdown": ["PUT"], + ] + + private func route(_ request: HTTPRequest) -> HTTPResponse { + guard let methods = Shim.routes[request.path] else { return .text(404, "404 page not found") } + guard methods.contains(request.method) else { return .text(405, "Method Not Allowed") } + switch (request.method, request.path) { + case ("GET", "/api/v1/vmm.ping"): + return HTTPResponse(status: 200, body: Data("OK".utf8)) + case ("PUT", "/api/v1/vmm.shutdown"): + return vmmShutdown() + default: + break + } + ops.lock() + defer { ops.unlock() } + switch (request.method, request.path) { + case ("GET", "/api/v1/vm.info"): return info() + case ("PUT", "/api/v1/vm.pause"): return pause() + case ("PUT", "/api/v1/vm.resume"): return resume() + case ("PUT", "/api/v1/vm.shutdown"): return requestStop("shutdown failed", "VM graceful shutdown requested (ACPI power button)") + case ("PUT", "/api/v1/vm.power-button"): return requestStop("power button failed", "power button sent") + case ("GET", "/api/v1/vm.snapshot-support"): return snapshotSupport() + case ("PUT", "/api/v1/vm.snapshot"): return snapshot(request.body) + case ("GET", "/api/v1/vm.balloon"): return getBalloon() + default: return setBalloon(request.body) + } + } + + private func info() -> HTTPResponse { + let (state, balloons) = onMain { (stateName(vm.state), vm.memoryBalloonDevices.count) } + var response: [String: Any] = ["state": state] + if balloons > 0 { + response["memory_balloon_devices"] = balloons + } + return .json(response) + } + + private func pause() -> HTTPResponse { + guard onMain({ vm.canPause }) else { return .text(400, "cannot pause VM") } + if let error = awaitMain({ done in self.vm.pause { done($0.error) } }) { + Log.error("failed to pause VM", ["error": error.localizedDescription]) + return .text(500, "pause failed: \(error.localizedDescription)") + } + Log.info("VM paused") + return .noContent() + } + + private func resume() -> HTTPResponse { + guard onMain({ vm.canResume }) else { return .text(400, "cannot resume VM") } + if let error = awaitMain({ done in self.vm.resume { done($0.error) } }) { + Log.error("failed to resume VM", ["error": error.localizedDescription]) + return .text(500, "resume failed: \(error.localizedDescription)") + } + Log.info("VM resumed") + return .noContent() + } + + // requestStop sends the guest a power-button event. The instance manager handles + // the timeout and forced stop if the guest does not shut down. + private func requestStop(_ failure: String, _ success: String) -> HTTPResponse { + do { + try onMain { try vm.requestStop() } + } catch { + Log.error("RequestStop failed", ["error": error.localizedDescription]) + return .text(500, "\(failure): \(error.localizedDescription)") + } + Log.info(success) + return .noContent() + } + + private func snapshotSupport() -> HTTPResponse { + var response: [String: Any] = ["supported": true] + if let reason = saveRestoreUnsupportedReason() { + response["supported"] = false + response["reason"] = reason + } + return .json(response) + } + + func saveRestoreUnsupportedReason() -> String? { + do { + try vmConfig.validateSaveRestoreSupport() + return nil + } catch { + return error.localizedDescription + } + } + + private func snapshot(_ body: Data) -> HTTPResponse { + let destination: String + do { + let object = try JSONSerialization.jsonObject(with: body) as? [String: Any] + destination = object?["destination_path"] as? String ?? "" + } catch { + return .text(400, "invalid snapshot request: \(error.localizedDescription)") + } + guard !destination.isEmpty else { return .text(400, "destination_path is required") } + guard onMain({ vm.state == .paused }) else { return .text(400, "vm must be paused before snapshot") } + if let reason = saveRestoreUnsupportedReason() { + return .text(400, "save/restore not supported: \(reason)") + } + let fm = FileManager.default + do { + try fm.createDirectory(atPath: destination, withIntermediateDirectories: true) + } catch { + return .text(500, "create snapshot dir failed: \(error.localizedDescription)") + } + var complete = false + defer { + if !complete { try? fm.removeItem(atPath: destination) } + } + let statePath = (destination as NSString).appendingPathComponent(snapshotMachineStateFile) + try? fm.removeItem(atPath: statePath) + if let error = awaitMain({ done in self.vm.saveMachineStateTo(url: URL(fileURLWithPath: statePath), completionHandler: done) }) { + return .text(500, "save machine state failed: \(error.localizedDescription)") + } + var shimConfig = config + // Runtime-only: the restore path is filled in by the caller on restore. + shimConfig.restoreMachineStatePath = nil + let manifestPath = (destination as NSString).appendingPathComponent(snapshotManifestFile) + do { + let manifest = SnapshotManifest(hypervisor: "vz", machineStateFile: snapshotMachineStateFile, shimConfig: shimConfig) + try JSONEncoder().encode(manifest).write(to: URL(fileURLWithPath: manifestPath + ".tmp")) + try? fm.removeItem(atPath: manifestPath) + try fm.moveItem(atPath: manifestPath + ".tmp", toPath: manifestPath) + } catch { + return .text(500, "write manifest failed: \(error.localizedDescription)") + } + complete = true + Log.info("VM snapshot saved", ["destination": destination, "machine_state": statePath]) + return .noContent() + } + + private func balloon() -> VZVirtioTraditionalMemoryBalloonDevice? { + vm.memoryBalloonDevices.compactMap { $0 as? VZVirtioTraditionalMemoryBalloonDevice }.first + } + + private func getBalloon() -> HTTPResponse { + guard let target = onMain({ balloon()?.targetVirtualMachineMemorySize }) else { + return .text(404, "no memory balloon device configured") + } + return .json(["target_guest_memory_bytes": Int64(target)]) + } + + private func setBalloon(_ body: Data) -> HTTPResponse { + guard onMain({ balloon() != nil }) else { return .text(404, "no memory balloon device configured") } + let object = try? JSONSerialization.jsonObject(with: body) as? [String: Any] + guard let target = (object?["target_guest_memory_bytes"] as? NSNumber)?.int64Value else { + return .text(400, "invalid balloon request: target_guest_memory_bytes is required") + } + guard target >= 0 else { return .text(400, "target_guest_memory_bytes must be non-negative") } + onMain { balloon()?.targetVirtualMachineMemorySize = UInt64(target) } + return .noContent() + } + + private func vmmShutdown() -> HTTPResponse { + Log.info("VMM shutdown requested") + var response = HTTPResponse.noContent() + response.after = { [self] in + // The process exits once the VM stops. + DispatchQueue.main.async { + guard self.vm.canStop else { return } + self.forcedStop = true + self.vm.stop { _ in self.finish() } + } + } + return response + } + + // MARK: Vsock proxy + + // proxyVsock speaks Cloud Hypervisor's vsock protocol: the client sends + // "CONNECT {port}\n", the shim answers "OK {port}\n", then proxies bytes. + private func proxyVsock(_ client: Int32) { + defer { close(client) } + guard let line = readLine(client, limit: 64) else { + Log.error("failed to read vsock handshake") + return + } + let fields = line.trimmingCharacters(in: .newlines).split(separator: " ") + guard fields.count == 2, fields[0] == "CONNECT", let port = UInt32(fields[1]) else { + Log.error("invalid vsock handshake", ["cmd": line]) + _ = writeAll(client, Data("ERR invalid command: \(line)".utf8)) + return + } + guard let device = onMain({ vm.socketDevices.first as? VZVirtioSocketDevice }) else { + Log.error("no vsock device configured") + _ = writeAll(client, Data("ERR no vsock device\n".utf8)) + return + } + var connection: VZVirtioSocketConnection? + let connectError = awaitMain { done in + device.connect(toPort: port) { result in + switch result { + case .success(let c): + connection = c + done(nil) + case .failure(let error): + done(error) + } + } + } + guard let connection else { + let message = connectError?.localizedDescription ?? "unknown error" + Log.error("failed to connect to guest vsock", ["port": port, "error": message]) + _ = writeAll(client, Data("ERR connect failed: \(message)\n".utf8)) + return + } + defer { connection.close() } + guard writeAll(client, Data("OK \(port)\n".utf8)) else { + Log.error("failed to send OK response") + return + } + Log.debug("vsock connection established", ["port": port]) + let guest = connection.fileDescriptor + // When either direction ends, shut both sockets down to wake the other copy, + // then wait for it before the descriptors are closed. + let finished = DispatchSemaphore(value: 0) + for (src, dst) in [(client, guest), (guest, client)] { + Thread.detachNewThread { + pump(from: src, to: dst) + shutdown(client, SHUT_RDWR) + shutdown(guest, SHUT_RDWR) + finished.signal() + } + } + finished.wait() + finished.wait() + } +} + +extension Result { + var error: Failure? { + if case .failure(let error) = self { return error } + return nil + } +} + +// stateName matches the Cloud Hypervisor state names the client expects. +func stateName(_ state: VZVirtualMachine.State) -> String { + switch state { + case .stopped: return "Shutdown" + case .running: return "Running" + case .paused: return "Paused" + case .error: return "Error" + case .starting: return "Starting" + case .pausing: return "Pausing" + case .resuming: return "Resuming" + case .stopping: return "Stopping" + case .saving: return "Saving" + case .restoring: return "Restoring" + @unknown default: return "Unknown" + } +} diff --git a/cmd/vz-shim/Socket.swift b/cmd/vz-shim/Socket.swift new file mode 100644 index 000000000..e7f950cf2 --- /dev/null +++ b/cmd/vz-shim/Socket.swift @@ -0,0 +1,184 @@ +import Foundation + +// UnixListener accepts stream connections on a Unix socket, each handled on its own +// thread. Closing it unlinks the socket path, as Go's listener does. +final class UnixListener { + let path: String + private let fd: Int32 + + init(path: String) throws { + unlink(path) + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + let capacity = MemoryLayout.size(ofValue: addr.sun_path) + guard path.utf8.count < capacity else { throw ShimError("socket path too long: \(path)") } + withUnsafeMutableBytes(of: &addr.sun_path) { buffer in + buffer.copyBytes(from: path.utf8) + buffer[path.utf8.count] = 0 + } + addr.sun_len = UInt8(MemoryLayout.size) + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { throw ShimError("socket: \(errnoString())") } + let bound = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(fd, $0, socklen_t(MemoryLayout.size)) } + } + guard bound == 0, listen(fd, 64) == 0 else { + let message = errnoString() + Darwin.close(fd) + throw ShimError("listen on \(path): \(message)") + } + self.path = path + self.fd = fd + } + + func serve(_ handle: @escaping (Int32) -> Void) { + Thread.detachNewThread { [fd] in + while true { + let conn = accept(fd, nil, nil) + if conn < 0 { + if errno == EINTR { continue } + return + } + var on: Int32 = 1 + setsockopt(conn, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout.size)) + Thread.detachNewThread { handle(conn) } + } + } + } + + func close() { + Darwin.close(fd) + unlink(path) + } +} + +func errnoString() -> String { + String(cString: strerror(errno)) +} + +func writeAll(_ fd: Int32, _ data: Data) -> Bool { + data.withUnsafeBytes { buffer -> Bool in + var offset = 0 + while offset < buffer.count { + let n = write(fd, buffer.baseAddress! + offset, buffer.count - offset) + if n < 0 { + if errno == EINTR { continue } + return false + } + offset += n + } + return true + } +} + +// readLine reads one newline-terminated line a byte at a time, so nothing after the +// newline is consumed. +func readLine(_ fd: Int32, limit: Int) -> String? { + var bytes: [UInt8] = [] + var byte: UInt8 = 0 + while bytes.count < limit { + let n = read(fd, &byte, 1) + if n < 0, errno == EINTR { continue } + if n <= 0 { return nil } + bytes.append(byte) + if byte == 0x0a { return String(decoding: bytes, as: UTF8.self) } + } + return nil +} + +// pump copies src to dst until either side fails or reaches EOF. +func pump(from src: Int32, to dst: Int32) { + var buffer = [UInt8](repeating: 0, count: 64 << 10) + while true { + let n = read(src, &buffer, buffer.count) + if n < 0, errno == EINTR { continue } + if n <= 0 { return } + if !writeAll(dst, Data(buffer[0.. Void)? + + static func noContent() -> HTTPResponse { HTTPResponse(status: 204) } + + static func text(_ status: Int, _ message: String) -> HTTPResponse { + HTTPResponse(status: status, body: (message + "\n").data(using: .utf8)!) + } + + static func json(_ object: Any) -> HTTPResponse { + let data = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])) ?? Data("{}".utf8) + return HTTPResponse(status: 200, contentType: "application/json", body: data + Data([0x0a])) + } +} + +private let reasons = [200: "OK", 204: "No Content", 400: "Bad Request", 404: "Not Found", 405: "Method Not Allowed", 413: "Payload Too Large", 500: "Internal Server Error"] + +func serveHTTP(_ fd: Int32, _ handle: (HTTPRequest) -> HTTPResponse) { + defer { close(fd) } + guard let request = readHTTPRequest(fd) else { + _ = writeHTTPResponse(fd, .text(400, "malformed request")) + return + } + let response = handle(request) + _ = writeHTTPResponse(fd, response) + response.after?() +} + +private func readHTTPRequest(_ fd: Int32) -> HTTPRequest? { + var data = Data() + var buffer = [UInt8](repeating: 0, count: 4096) + let separator = Data("\r\n\r\n".utf8) + var headerEnd: Range? + while headerEnd == nil { + guard data.count < 64 << 10 else { return nil } + let n = read(fd, &buffer, buffer.count) + if n < 0, errno == EINTR { continue } + if n <= 0 { return nil } + data.append(contentsOf: buffer[0..= 2 else { return nil } + var contentLength = 0 + for line in lines { + let parts = line.split(separator: ":", maxSplits: 1) + if parts.count == 2, parts[0].lowercased() == "content-length" { + contentLength = Int(parts[1].trimmingCharacters(in: .whitespaces)) ?? -1 + } + } + guard (0...(1 << 20)).contains(contentLength) else { return nil } + var body = Data(data[headerEnd!.upperBound...]) + while body.count < contentLength { + let n = read(fd, &buffer, min(buffer.count, contentLength - body.count)) + if n < 0, errno == EINTR { continue } + if n <= 0 { return nil } + body.append(contentsOf: buffer[0.. Bool { + var head = "HTTP/1.1 \(response.status) \(reasons[response.status] ?? "Status")\r\n" + if response.status != 204 { + head += "Content-Type: \(response.contentType)\r\nContent-Length: \(response.body.count)\r\n" + } + head += "Connection: close\r\n\r\n" + return writeAll(fd, Data(head.utf8) + response.body) +} diff --git a/cmd/vz-shim/VM.swift b/cmd/vz-shim/VM.swift new file mode 100644 index 000000000..d45725e11 --- /dev/null +++ b/cmd/vz-shim/VM.swift @@ -0,0 +1,356 @@ +import Foundation +import Virtualization +import vmnet +#if canImport(DiskImageKit) + import DiskImageKit +#endif + +// makeConfiguration builds a validated VM configuration. A Linux guest without a +// machine identifier gets a new one, recorded in config so snapshots keep it. +func makeConfiguration(_ config: inout ShimConfig) throws -> VZVirtualMachineConfiguration { + if config.isMacGuest { + return try makeMacConfiguration(config) + } + return try makeLinuxConfiguration(&config) +} + +// makeMacConfiguration boots an imported macOS disk with its platform identity. +// It has no serial console, entropy device, Rosetta share or memory balloon. +func makeMacConfiguration(_ c: ShimConfig) throws -> VZVirtualMachineConfiguration { + #if arch(arm64) + if c.enableRosetta == true || c.enableMemoryBalloon == true || c.requireMemoryBalloon == true { + throw ShimError("macOS guests do not support Linux Rosetta or memory ballooning") + } + guard let modelText = c.macHardwareModelData, !modelText.isEmpty, + let idText = c.macMachineIdentifierData, !idText.isEmpty, + let auxPath = c.macAuxStoragePath, !auxPath.isEmpty + else { + throw ShimError("create boot loader: macOS boot requires hardware model, machine identifier, and auxiliary storage") + } + guard let modelData = decodeBase64(modelText) else { + throw ShimError("configure Mac platform: decode Mac hardware model") + } + guard let model = VZMacHardwareModel(dataRepresentation: modelData) else { + throw ShimError("configure Mac platform: invalid Mac hardware model") + } + guard model.isSupported else { + throw ShimError("configure Mac platform: Mac hardware model is unsupported on this host") + } + guard let idData = decodeBase64(idText) else { + throw ShimError("configure Mac platform: decode Mac machine identifier") + } + guard let identifier = VZMacMachineIdentifier(dataRepresentation: idData) else { + throw ShimError("configure Mac platform: invalid Mac machine identifier") + } + let platform = VZMacPlatformConfiguration() + platform.hardwareModel = model + platform.machineIdentifier = identifier + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: URL(fileURLWithPath: auxPath)) + + let vc = VZVirtualMachineConfiguration() + vc.bootLoader = VZMacOSBootLoader() + vc.cpuCount = c.vcpus ?? 0 + vc.memorySize = UInt64(max(0, c.memoryBytes ?? 0)) + vc.platform = platform + Log.debug("macOS VM config", ["vcpus": vc.cpuCount, "memory_bytes": vc.memorySize]) + vc.networkDevices = try makeNetworkDevices(c.networks ?? []) + vc.storageDevices = try makeStorageDevices(c.disks ?? []) + + let graphics = VZMacGraphicsDeviceConfiguration() + // 3024x1964 pixels at 2x: 1512x982 points, the logical size of a 14-inch MacBook Pro. + graphics.displays = [VZMacGraphicsDisplayConfiguration(widthInPixels: 3024, heightInPixels: 1964, pixelsPerInch: 160)] + vc.graphicsDevices = [graphics] + vc.keyboards = [VZMacKeyboardConfiguration()] + vc.pointingDevices = [VZMacTrackpadConfiguration()] + vc.audioDevices = [makeMacAudio()] + vc.socketDevices = [VZVirtioSocketDeviceConfiguration()] + do { + try vc.validate() + } catch { + throw ShimError("invalid vm configuration: \(error.localizedDescription)") + } + return vc + #else + throw ShimError("macOS guests require Apple silicon") + #endif +} + +// makeMacAudio gives the guest an audio device. Virtualization attaches audio only +// through a sound device, so without one the guest boots with no CoreAudio devices: +// `system_profiler SPAudioDataType` lists none and `afplay` fails with +// `AudioQueueStart failed (-66680)`. +// +// The output stream has no host sink. The samples are discarded on the host, which +// is what a headless browser VM wants -- nothing should play on the machine running +// the guests -- while the guest still has a real output device, so applications can +// render audio and ScreenCaptureKit has a mix to capture for the live view. +func makeMacAudio() -> VZAudioDeviceConfiguration { + let sound = VZVirtioSoundDeviceConfiguration() + sound.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()] + return sound +} + +func makeLinuxConfiguration(_ c: inout ShimConfig) throws -> VZVirtualMachineConfiguration { + // vz uses hvc0 for the serial console. + var kernelArgs = c.kernelArgs ?? "" + if kernelArgs.isEmpty { + kernelArgs = "console=hvc0 root=/dev/vda" + } else { + kernelArgs = kernelArgs.replacingOccurrences(of: "console=ttyS0", with: "console=hvc0") + } + let bootLoader = VZLinuxBootLoader(kernelURL: URL(fileURLWithPath: c.kernelPath ?? "")) + bootLoader.commandLine = kernelArgs + if let initrd = c.initrdPath, !initrd.isEmpty { + bootLoader.initialRamdiskURL = URL(fileURLWithPath: initrd) + } + + let vc = VZVirtualMachineConfiguration() + vc.bootLoader = bootLoader + vc.cpuCount = cpuCount(c.vcpus ?? 0) + vc.memorySize = memorySize(UInt64(max(0, c.memoryBytes ?? 0))) + Log.debug("VM config", ["vcpus": vc.cpuCount, "memory_bytes": vc.memorySize, "kernel": c.kernelPath ?? "", "initrd": c.initrdPath ?? ""]) + + vc.serialPorts = [try makeSerialPort(c.serialLogPath)] + vc.networkDevices = try makeNetworkDevices(c.networks ?? []) + vc.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + vc.storageDevices = try makeStorageDevices(c.disks ?? []) + vc.directorySharingDevices = try makeRosettaShare(c) + vc.platform = try makeGenericPlatform(&c) + vc.socketDevices = [VZVirtioSocketDeviceConfiguration()] + if c.enableMemoryBalloon == true { + vc.memoryBalloonDevices = [VZVirtioTraditionalMemoryBalloonDeviceConfiguration()] + } + do { + try vc.validate() + } catch { + throw ShimError("invalid vm configuration: \(error.localizedDescription)") + } + return vc +} + +func makeGenericPlatform(_ c: inout ShimConfig) throws -> VZGenericPlatformConfiguration { + let platform = VZGenericPlatformConfiguration() + if let text = c.machineIdentifierData, !text.isEmpty { + guard let data = decodeBase64(text), let identifier = VZGenericMachineIdentifier(dataRepresentation: data) else { + throw ShimError("configure platform: recreate machine identifier") + } + platform.machineIdentifier = identifier + } else { + let identifier = VZGenericMachineIdentifier() + platform.machineIdentifier = identifier + c.machineIdentifierData = identifier.dataRepresentation.base64EncodedString() + } + return platform +} + +func makeSerialPort(_ logPath: String?) throws -> VZSerialPortConfiguration { + guard let reader = FileHandle(forReadingAtPath: "/dev/null") else { + throw ShimError("configure serial: open /dev/null for reading") + } + let writer: FileHandle + if let logPath, !logPath.isEmpty { + let fd = open(logPath, O_CREAT | O_WRONLY | O_APPEND, 0o644) + guard fd >= 0 else { throw ShimError("configure serial: open serial log file: \(String(cString: strerror(errno)))") } + writer = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + } else { + guard let null = FileHandle(forWritingAtPath: "/dev/null") else { + throw ShimError("configure serial: open /dev/null for writing") + } + writer = null + } + let port = VZVirtioConsoleDeviceSerialPortConfiguration() + port.attachment = VZFileHandleSerialPortAttachment(fileHandleForReading: reader, fileHandleForWriting: writer) + return port +} + +// makeRosettaShare attaches the Linux Rosetta virtio-fs share when requested. The +// host must have Rosetta installed; anything else is a startup error. +func makeRosettaShare(_ c: ShimConfig) throws -> [VZDirectorySharingDeviceConfiguration] { + guard c.enableRosetta == true else { return [] } + #if arch(arm64) + switch VZLinuxRosettaDirectoryShare.availability { + case .installed: + break + case .notInstalled: + throw ShimError("configure directory sharing: rosetta requested but not installed (run: softwareupdate --install-rosetta)") + default: + throw ShimError("configure directory sharing: rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") + } + let device = VZVirtioFileSystemDeviceConfiguration(tag: rosettaMountTag) + do { + device.share = try VZLinuxRosettaDirectoryShare() + } catch { + throw ShimError("configure directory sharing: create rosetta directory share: \(error.localizedDescription)") + } + Log.info("attached rosetta directory share", ["tag": rosettaMountTag]) + return [device] + #else + throw ShimError("configure directory sharing: rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") + #endif +} + +func makeStorageDevices(_ disks: [DiskConfig]) throws -> [VZStorageDeviceConfiguration] { + try disks.map { disk in + guard FileManager.default.fileExists(atPath: disk.path) else { + throw ShimError("configure storage: disk image not found: \(disk.path)") + } + if disk.path.hasSuffix(".qcow2") { + throw ShimError("configure storage: qcow2 not supported by vz, use raw format: \(disk.path)") + } + let readOnly = disk.readonly ?? false + let attachment: VZStorageDeviceAttachment + do { + if let parents = disk.parents, !parents.isEmpty { + attachment = try stackedAttachment(parents: parents, overlay: disk.path, readOnly: readOnly) + } else { + attachment = try VZDiskImageStorageDeviceAttachment(url: URL(fileURLWithPath: disk.path), readOnly: readOnly) + } + } catch let error as ShimError { + throw error + } catch { + throw ShimError("configure storage: create disk attachment for \(disk.path): \(error.localizedDescription)") + } + return VZVirtioBlockDeviceConfiguration(attachment: attachment) + } +} + +// stackedAttachment attaches an ASIF overlay above its read-only parents, base +// first. Each image must name the one below it as its parent; hypeman creates them. +func stackedAttachment(parents: [String], overlay: String, readOnly: Bool) throws -> VZStorageDeviceAttachment { + #if canImport(DiskImageKit) + if #available(macOS 27.0, *) { + for path in parents where !FileManager.default.fileExists(atPath: path) { + throw ShimError("configure storage: disk parent not found: \(path)") + } + var stacked = try DiskImage(opening: .open(url: URL(fileURLWithPath: parents[0]), mode: .readOnly)) + for path in parents.dropFirst() { + stacked = try stacked.appending(try openOverlay(path, mode: .readOnly)) + } + stacked = try stacked.appending(try openOverlay(overlay, mode: readOnly ? .readOnly : .readWrite)) + Log.info("attached stacked disk", ["base": parents[0], "overlay": overlay, "layers": (stacked as? any StackedImage)?.layers.count ?? 1]) + return try VZDiskImageStorageDeviceAttachment(diskImage: stacked, cachingMode: .automatic, synchronizationMode: .full) + } + #endif + throw ShimError("configure storage: stacked disks require DiskImageKit on macOS 27 or newer") +} + +#if canImport(DiskImageKit) + @available(macOS 27.0, *) + func openOverlay(_ path: String, mode: OpenConfiguration.Mode) throws -> DiskImage { + let image = try DiskImage(opening: .open(url: URL(fileURLWithPath: path), mode: mode)) + guard image.format == .asif else { + throw ShimError("configure storage: overlay must use ASIF format: \(path)") + } + return image + } +#endif + +func makeNetworkDevices(_ networks: [NetworkConfig]) throws -> [VZNetworkDeviceConfiguration] { + // No networks configured (NetworkEnabled=false): no NIC is attached. + try networks.map { n in + let device = VZVirtioNetworkDeviceConfiguration() + if let address = n.ipv4Address, !address.isEmpty { + device.attachment = try vmnetAttachment(n) + } else { + device.attachment = VZNATNetworkDeviceAttachment() + } + device.macAddress = macAddress(n.mac) + return device + } +} + +func macAddress(_ text: String?) -> VZMACAddress { + if let text, !text.isEmpty { + if let mac = VZMACAddress(string: text) { + Log.info("using specified MAC address", ["mac": text]) + return mac + } + Log.warn("failed to parse MAC address, generating random", ["mac": text]) + return VZMACAddress.randomLocallyAdministered() + } + let mac = VZMACAddress.randomLocallyAdministered() + Log.info("generated random MAC address", ["mac": mac.string]) + return mac +} + +// vmnetAttachment attaches the guest to its own shared-mode vmnet network with the +// given subnet and a DHCP reservation for the guest's MAC, so the guest's address is +// the one hypeman assigned. +func vmnetAttachment(_ n: NetworkConfig) throws -> VZNetworkDeviceAttachment { + guard #available(macOS 26.0, *) else { + throw ShimError("configure network: vmnet networks require macOS 26 or newer") + } + guard let macText = n.mac, let mac = VZMACAddress(string: macText) else { + throw ShimError("configure network: vmnet network needs the guest MAC") + } + guard let (network, bits) = parseIPv4Prefix(n.ipv4Subnet ?? ""), bits <= 30 else { + throw ShimError("configure network: parse guest subnet \(n.ipv4Subnet ?? "")") + } + guard let address = parseIPv4(n.ipv4Address ?? "") else { + throw ShimError("configure network: parse guest address \(n.ipv4Address ?? "")") + } + let mask: UInt32 = bits == 0 ? 0 : ~UInt32(0) << UInt32(32 - bits) + let subnetText = "\(formatIPv4(network & mask))/\(bits)" + + var status = vmnet_return_t.VMNET_SUCCESS + guard let config = vmnet_network_configuration_create(.VMNET_SHARED_MODE, &status), status == .VMNET_SUCCESS else { + throw ShimError("configure network: configure vmnet network \(subnetText): status \(status.rawValue)") + } + // vmnet needs the host address, the first in the subnet, to start DHCP-backed + // attachments; the network address alone leaves the guest without a lease. + var subnetAddr = in_addr(s_addr: ((network & mask) + 1).bigEndian) + var maskAddr = in_addr(s_addr: mask.bigEndian) + status = vmnet_network_configuration_set_ipv4_subnet(config, &subnetAddr, &maskAddr) + guard status == .VMNET_SUCCESS else { + throw ShimError("configure network: set vmnet subnet \(subnetText): status \(status.rawValue)") + } + var client = ether_addr_t() + withUnsafeMutableBytes(of: &client.octet) { $0.copyBytes(from: mac.ethernetAddress) } + var reserved = in_addr(s_addr: address.bigEndian) + status = vmnet_network_configuration_add_dhcp_reservation(config, &client, &reserved) + guard status == .VMNET_SUCCESS else { + throw ShimError("configure network: reserve \(formatIPv4(address)) on vmnet network \(subnetText): status \(status.rawValue)") + } + guard let network = vmnet_network_create(config, &status), status == .VMNET_SUCCESS else { + throw ShimError("configure network: create vmnet network \(subnetText): status \(status.rawValue)") + } + Log.info("attached guest vmnet network", ["subnet": subnetText, "address": formatIPv4(address), "mac": macText]) + return VZVmnetNetworkDeviceAttachment(network: network) +} + +extension VZMACAddress { + var ethernetAddress: [UInt8] { + string.split(separator: ":").compactMap { UInt8($0, radix: 16) } + } +} + +// parseIPv4 returns an address in host byte order. +func parseIPv4(_ text: String) -> UInt32? { + var addr = in_addr() + guard inet_pton(AF_INET, text, &addr) == 1 else { return nil } + return UInt32(bigEndian: addr.s_addr) +} + +func parseIPv4Prefix(_ text: String) -> (UInt32, Int)? { + let parts = text.split(separator: "/") + guard parts.count == 2, let address = parseIPv4(String(parts[0])), let bits = Int(parts[1]), (0...32).contains(bits) else { return nil } + return (address, bits) +} + +func formatIPv4(_ address: UInt32) -> String { + "\(address >> 24 & 0xff).\(address >> 16 & 0xff).\(address >> 8 & 0xff).\(address & 0xff)" +} + +func cpuCount(_ requested: Int) -> Int { + var count = requested + if count == 0 { + count = max(1, ProcessInfo.processInfo.processorCount - 1) + } + return min(max(count, VZVirtualMachineConfiguration.minimumAllowedCPUCount), VZVirtualMachineConfiguration.maximumAllowedCPUCount) +} + +func memorySize(_ requested: UInt64) -> UInt64 { + // 2 GiB safety default; the caller normally provides this. + let size = requested == 0 ? 2 << 30 : requested + return min(max(size, VZVirtualMachineConfiguration.minimumAllowedMemorySize), VZVirtualMachineConfiguration.maximumAllowedMemorySize) +} diff --git a/cmd/vz-shim/macos_arm64.go b/cmd/vz-shim/macos_arm64.go deleted file mode 100644 index e2887abad..000000000 --- a/cmd/vz-shim/macos_arm64.go +++ /dev/null @@ -1,70 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "encoding/base64" - "fmt" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -func newMacBootLoader(c *shimconfig.ShimConfig) (vz.BootLoader, error) { - if c.MacHardwareModelData == "" || c.MacMachineIdentifierData == "" || c.MacAuxStoragePath == "" { - return nil, fmt.Errorf("macOS boot requires hardware model, machine identifier, and auxiliary storage") - } - return vz.NewMacOSBootLoader() -} -func configureMacPlatform(vc *vz.VirtualMachineConfiguration, c *shimconfig.ShimConfig) error { - modelData, e := base64.StdEncoding.DecodeString(c.MacHardwareModelData) - if e != nil { - return fmt.Errorf("decode Mac hardware model: %w", e) - } - model, e := vz.NewMacHardwareModelWithData(modelData) - if e != nil { - return e - } - if !model.Supported() { - return fmt.Errorf("Mac hardware model is unsupported on this host") - } - idData, e := base64.StdEncoding.DecodeString(c.MacMachineIdentifierData) - if e != nil { - return fmt.Errorf("decode Mac machine identifier: %w", e) - } - id, e := vz.NewMacMachineIdentifierWithData(idData) - if e != nil { - return e - } - aux, e := vz.NewMacAuxiliaryStorage(c.MacAuxStoragePath) - if e != nil { - return e - } - platform, e := vz.NewMacPlatformConfiguration(vz.WithMacHardwareModel(model), vz.WithMacMachineIdentifier(id), vz.WithMacAuxiliaryStorage(aux)) - if e != nil { - return e - } - vc.SetPlatformVirtualMachineConfiguration(platform) - graphics, e := vz.NewMacGraphicsDeviceConfiguration() - if e != nil { - return e - } - // 3024x1964 pixels at 2x: 1512x982 points, the logical size of a 14-inch MacBook Pro. - display, e := vz.NewMacGraphicsDisplayConfiguration(3024, 1964, 160) - if e != nil { - return e - } - graphics.SetDisplays(display) - vc.SetGraphicsDevicesVirtualMachineConfiguration([]vz.GraphicsDeviceConfiguration{graphics}) - keyboard, e := vz.NewMacKeyboardConfiguration() - if e != nil { - return e - } - trackpad, e := vz.NewMacTrackpadConfiguration() - if e != nil { - return e - } - vc.SetKeyboardsVirtualMachineConfiguration([]vz.KeyboardConfiguration{keyboard}) - vc.SetPointingDevicesVirtualMachineConfiguration([]vz.PointingDeviceConfiguration{trackpad}) - return nil -} diff --git a/cmd/vz-shim/macos_arm64_test.go b/cmd/vz-shim/macos_arm64_test.go deleted file mode 100644 index 073c5387e..000000000 --- a/cmd/vz-shim/macos_arm64_test.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/stretchr/testify/require" - "testing" -) - -func TestMacBootRequiresCompleteIdentity(t *testing.T) { - for _, c := range []shimconfig.ShimConfig{ - {MacHardwareModelData: "AA=="}, - {MacMachineIdentifierData: "AA=="}, - {MacAuxStoragePath: "/missing"}, - } { - _, _, err := createVM(&c) - require.ErrorContains(t, err, "macOS boot requires") - } -} -func TestMacPlatformRejectsMalformedModel(t *testing.T) { - err := configureMacPlatform(nil, &shimconfig.ShimConfig{MacHardwareModelData: "not base64!"}) - require.ErrorContains(t, err, "decode Mac hardware model") -} diff --git a/cmd/vz-shim/macos_identity.go b/cmd/vz-shim/macos_identity.go deleted file mode 100644 index 43fd1dfdf..000000000 --- a/cmd/vz-shim/macos_identity.go +++ /dev/null @@ -1,48 +0,0 @@ -//go:build darwin - -package main - -import ( - "crypto/sha256" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "syscall" - - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// Preserve the identity lock until subprocess exit, including while paused. -// Never unlink a lock file: another waiter may already hold its inode open. -func lockMacOSIdentity(c *shimconfig.ShimConfig) (*os.File, error) { - if c.MacMachineIdentifierData == "" { - return nil, nil - } - data, err := base64.StdEncoding.DecodeString(c.MacMachineIdentifierData) - if err != nil { - return nil, fmt.Errorf("decode Mac identity for admission: %w", err) - } - hash := sha256.Sum256(data) - // Fixed directory, not TMPDIR: servers with different environments must contend on the same lock. - path := filepath.Join("/tmp", fmt.Sprintf("hypeman-mac-identity-%d-%x.lock", os.Getuid(), hash)) - // The shared /tmp name is predictable, so refuse symlinks and foreign-owned files. - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600) - if err != nil { - return nil, err - } - info, err := f.Stat() - if err != nil { - f.Close() - return nil, err - } - if st, ok := info.Sys().(*syscall.Stat_t); !ok || !info.Mode().IsRegular() || int(st.Uid) != os.Getuid() { - f.Close() - return nil, fmt.Errorf("Mac identity lock path is not an owned regular file") - } - if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - f.Close() - return nil, fmt.Errorf("Mac machine identity already in use: %w", err) - } - return f, nil -} diff --git a/cmd/vz-shim/macos_identity_test.go b/cmd/vz-shim/macos_identity_test.go deleted file mode 100644 index af56eb4c4..000000000 --- a/cmd/vz-shim/macos_identity_test.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build darwin - -package main - -import ( - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/stretchr/testify/require" - "testing" -) - -func TestMacOSIdentityLifetimeLock(t *testing.T) { - c := &shimconfig.ShimConfig{MacMachineIdentifierData: "aWRlbnRpdHk="} - first, err := lockMacOSIdentity(c) - require.NoError(t, err) - defer first.Close() - equivalent := *c - equivalent.MacMachineIdentifierData += "\n" - second, err := lockMacOSIdentity(&equivalent) - require.Error(t, err) - require.Nil(t, second) - require.NoError(t, first.Close()) - second, err = lockMacOSIdentity(c) - require.NoError(t, err) - second.Close() -} diff --git a/cmd/vz-shim/macos_other.go b/cmd/vz-shim/macos_other.go deleted file mode 100644 index ed256b850..000000000 --- a/cmd/vz-shim/macos_other.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build darwin && !arm64 - -package main - -import ( - "fmt" - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -func newMacBootLoader(*shimconfig.ShimConfig) (vz.BootLoader, error) { - return nil, fmt.Errorf("macOS guests require Apple silicon") -} -func configureMacPlatform(*vz.VirtualMachineConfiguration, *shimconfig.ShimConfig) error { - return fmt.Errorf("macOS guests require Apple silicon") -} diff --git a/cmd/vz-shim/macos_vm.go b/cmd/vz-shim/macos_vm.go deleted file mode 100644 index 4241f57dd..000000000 --- a/cmd/vz-shim/macos_vm.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build darwin - -package main - -import ( - "fmt" - "log/slog" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -func isMacGuest(config *shimconfig.ShimConfig) bool { - return config.MacHardwareModelData != "" || config.MacMachineIdentifierData != "" || config.MacAuxStoragePath != "" -} - -// createMacVM boots an imported macOS disk with its preserved platform identity. -// It has no serial console, entropy device, Rosetta share or memory balloon. -func createMacVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMachineConfiguration, error) { - if config.EnableRosetta || config.EnableMemoryBalloon || config.RequireMemoryBalloon { - return nil, nil, fmt.Errorf("macOS guests do not support Linux Rosetta or memory ballooning") - } - bootLoader, err := newMacBootLoader(config) - if err != nil { - return nil, nil, fmt.Errorf("create boot loader: %w", err) - } - - vcpus := uint(config.VCPUs) - memoryBytes := uint64(config.MemoryBytes) - slog.Debug("macOS VM config", "vcpus", vcpus, "memory_bytes", memoryBytes) - - vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, vcpus, memoryBytes) - if err != nil { - return nil, nil, fmt.Errorf("create vm configuration: %w", err) - } - if err := configureNetwork(vmConfig, config.Networks); err != nil { - return nil, nil, fmt.Errorf("configure network: %w", err) - } - if err := configureStorage(vmConfig, config.Disks); err != nil { - return nil, nil, fmt.Errorf("configure storage: %w", err) - } - if err := configureDirectorySharing(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure directory sharing: %w", err) - } - if err := configureMacPlatform(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure Mac platform: %w", err) - } - if err := configureMacAudio(vmConfig); err != nil { - return nil, nil, fmt.Errorf("configure audio: %w", err) - } - - vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() - if err != nil { - return nil, nil, fmt.Errorf("create vsock device: %w", err) - } - vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig}) - - validated, err := vmConfig.Validate() - if err != nil { - return nil, nil, fmt.Errorf("invalid vm configuration: %w", err) - } - if !validated { - return nil, nil, fmt.Errorf("invalid vm configuration") - } - vm, err := vz.NewVirtualMachine(vmConfig) - if err != nil { - return nil, nil, fmt.Errorf("create virtual machine: %w", err) - } - return vm, vmConfig, nil -} - -// configureMacAudio gives the guest an audio device. Virtualization attaches audio -// only through a sound device, so without one the guest boots with no CoreAudio -// devices at all: `system_profiler SPAudioDataType` lists none and `afplay` fails -// with `AudioQueueStart failed (-66680)`. -// -// The output stream has no host sink. The samples are discarded on the host, which -// is what a headless browser VM wants -- nothing should play on the machine running -// the guests -- while the guest still has a real output device, so applications can -// render audio and ScreenCaptureKit has a mix to capture for the live view. -func configureMacAudio(vmConfig *vz.VirtualMachineConfiguration) error { - soundConfig, err := vz.NewVirtioSoundDeviceConfiguration() - if err != nil { - return fmt.Errorf("create sound device: %w", err) - } - output, err := vz.NewVirtioSoundDeviceOutputStreamConfiguration() - if err != nil { - return fmt.Errorf("create output stream: %w", err) - } - soundConfig.SetStreams(output) - vmConfig.SetAudioDevicesVirtualMachineConfiguration([]vz.AudioDeviceConfiguration{soundConfig}) - return nil -} diff --git a/cmd/vz-shim/main.go b/cmd/vz-shim/main.go deleted file mode 100644 index 950602184..000000000 --- a/cmd/vz-shim/main.go +++ /dev/null @@ -1,219 +0,0 @@ -//go:build darwin - -// Package main implements hypeman-vz-shim, a subprocess that hosts vz VMs. -// This allows VMs to survive hypeman restarts by running in a separate process. -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "log/slog" - "net" - "net/http" - "os" - "os/signal" - "path/filepath" - "syscall" - "time" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -const controlIdleTimeout = 30 * time.Second - -func main() { - configJSON := flag.String("config", "", "VM configuration as JSON") - flag.Parse() - - if *configJSON == "" { - fmt.Fprintln(os.Stderr, "error: -config is required") - os.Exit(1) - } - - var config shimconfig.ShimConfig - if err := json.Unmarshal([]byte(*configJSON), &config); err != nil { - fmt.Fprintf(os.Stderr, "error: invalid config JSON: %v\n", err) - os.Exit(1) - } - - // Setup logging to file - if err := setupLogging(config.LogPath); err != nil { - fmt.Fprintf(os.Stderr, "error: setup logging: %v\n", err) - os.Exit(1) - } - - slog.Info("vz-shim starting", "control_socket", config.ControlSocket, "vsock_socket", config.VsockSocket) - - identityLock, err := lockMacOSIdentity(&config) - if err != nil { - fmt.Fprintf(os.Stderr, "macOS identity admission: %v\n", err) - os.Exit(1) - } - if identityLock != nil { - defer identityLock.Close() - } - - // Create the VM - vm, vmConfig, err := createVM(&config) - if err != nil { - slog.Error("failed to create VM", "error", err) - fmt.Fprintf(os.Stderr, "failed to create VM: %v\n", err) - os.Exit(1) - } - - if config.RestoreMachineStatePath != "" { - if err := validateSaveRestoreSupport(vmConfig); err != nil { - slog.Error("save/restore not supported for VM config", "error", err) - fmt.Fprintf(os.Stderr, "save/restore not supported for VM config: %v\n", err) - os.Exit(1) - } - if err := restoreMachineState(vm, config.RestoreMachineStatePath); err != nil { - slog.Error("failed to restore VM machine state", "error", err, "path", config.RestoreMachineStatePath) - fmt.Fprintf(os.Stderr, "failed to restore VM machine state: %v\n", err) - os.Exit(1) - } - slog.Info("VM restored from machine state", "path", config.RestoreMachineStatePath, "state", vm.State()) - } else { - if err := vm.Start(); err != nil { - slog.Error("failed to start VM", "error", err) - fmt.Fprintf(os.Stderr, "failed to start VM: %v\n", err) - os.Exit(1) - } - slog.Info("VM started", "vcpus", config.VCPUs, "memory_mb", config.MemoryBytes/1024/1024) - } - - // Create the shim server - server := NewShimServer(vm, vmConfig, config) - - // Start control socket listener (remove stale socket from previous run) - os.Remove(config.ControlSocket) - controlListener, err := net.Listen("unix", config.ControlSocket) - if err != nil { - slog.Error("failed to listen on control socket", "error", err, "path", config.ControlSocket) - os.Exit(1) - } - defer controlListener.Close() - - // Start vsock proxy listener (remove stale socket from previous run) - os.Remove(config.VsockSocket) - vsockListener, err := net.Listen("unix", config.VsockSocket) - if err != nil { - slog.Error("failed to listen on vsock socket", "error", err, "path", config.VsockSocket) - os.Exit(1) - } - defer vsockListener.Close() - - // Start HTTP server for control API - httpServer := &http.Server{ - Handler: server.Handler(), - IdleTimeout: controlIdleTimeout, - } - go func() { - slog.Info("control API listening", "socket", config.ControlSocket) - if err := httpServer.Serve(controlListener); err != nil && err != http.ErrServerClosed { - slog.Error("control API server error", "error", err) - } - }() - - // Start vsock proxy - go func() { - slog.Info("vsock proxy listening", "socket", config.VsockSocket) - server.ServeVsock(vsockListener) - }() - - // Wait for shutdown signal or VM stop - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) - - // Monitor VM state. Notifications can be dropped and their values can be stale, - // so each one, and the start, re-reads the current state. - stateChanged := make(chan vz.VirtualMachineState, 8) - stopNotify, err := vm.StateChangedNotify(stateChanged) - if err != nil { - slog.Error("failed to watch VM state", "error", err) - os.Exit(1) - } - defer stopNotify() - go func() { - for { - newState := vm.State() - if newState != vz.VirtualMachineStateStopped && newState != vz.VirtualMachineStateError { - select { - case <-ctx.Done(): - return - case <-stateChanged: - } - newState = vm.State() - slog.Info("VM state changed", "state", newState) - } - if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError { - if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() { - writeStopReceipt(config.StopReceiptPath) - } - slog.Info("VM stopped, shutting down shim") - cancel() - return - } - } - }() - - select { - case sig := <-sigChan: - slog.Info("received signal, shutting down", "signal", sig) - case <-ctx.Done(): - slog.Info("context cancelled, shutting down") - } - - // Graceful shutdown - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer shutdownCancel() - - httpServer.Shutdown(shutdownCtx) - - if vm.State() == vz.VirtualMachineStateRunning { - slog.Info("stopping VM") - if vm.CanStop() { - server.forcedStop.Store(true) - vm.Stop() - } - } - - slog.Info("vz-shim shutdown complete") -} - -// writeStopReceipt records that the guest stopped the VM itself. The host counts a -// macOS stop as graceful only with this receipt. -func writeStopReceipt(path string) { - if path == "" { - return - } - if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339Nano)+"\n"), 0600); err != nil { - slog.Error("failed to write stop receipt", "error", err, "path", path) - } -} - -func setupLogging(logPath string) error { - if logPath == "" { - // Log to stderr if no path specified - slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) - return nil - } - - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return fmt.Errorf("create log directory: %w", err) - } - - file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return fmt.Errorf("open log file: %w", err) - } - - slog.SetDefault(slog.New(slog.NewJSONHandler(file, &slog.HandlerOptions{Level: slog.LevelDebug}))) - return nil -} diff --git a/cmd/vz-shim/main.swift b/cmd/vz-shim/main.swift new file mode 100644 index 000000000..e5a3c0486 --- /dev/null +++ b/cmd/vz-shim/main.swift @@ -0,0 +1,137 @@ +// hypeman-vz-shim is a subprocess that hosts one vz VM, so VMs survive hypeman +// restarts. Hypeman starts it with -config and talks to it over +// the control and vsock sockets named in that config. +import CryptoKit +import Foundation +import Virtualization + +func configArgument() -> String? { + let args = CommandLine.arguments.dropFirst() + var iterator = args.makeIterator() + while let arg = iterator.next() { + if arg == "-config" || arg == "--config" { + return iterator.next() + } + for prefix in ["-config=", "--config="] where arg.hasPrefix(prefix) { + return String(arg.dropFirst(prefix.count)) + } + } + return nil +} + +// lockMacOSIdentity holds an exclusive lock on the Mac machine identity until the +// process exits, including while paused. The lock file is never unlinked: another +// waiter may already hold its inode open. +func lockMacOSIdentity(_ config: ShimConfig) throws -> Int32? { + guard let text = config.macMachineIdentifierData, !text.isEmpty else { return nil } + guard let data = decodeBase64(text) else { throw ShimError("decode Mac identity for admission") } + let hash = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + // Fixed directory, not TMPDIR: servers with different environments must contend + // on the same lock. The name is predictable, so refuse symlinks and files owned + // by anyone else. + let path = "/tmp/hypeman-mac-identity-\(getuid())-\(hash).lock" + let fd = open(path, O_CREAT | O_RDWR | O_NOFOLLOW, 0o600) + guard fd >= 0 else { throw ShimError("open \(path): \(errnoString())") } + var info = stat() + guard fstat(fd, &info) == 0, info.st_mode & S_IFMT == S_IFREG, info.st_uid == getuid() else { + close(fd) + throw ShimError("Mac identity lock path is not an owned regular file") + } + guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { + let message = errnoString() + close(fd) + throw ShimError("Mac machine identity already in use: \(message)") + } + return fd +} + +signal(SIGPIPE, SIG_IGN) + +guard let configJSON = configArgument(), !configJSON.isEmpty else { + printError("error: -config is required") + exit(1) +} +var config: ShimConfig +do { + config = try JSONDecoder().decode(ShimConfig.self, from: Data(configJSON.utf8)) +} catch { + printError("error: invalid config JSON: \(error)") + exit(1) +} +do { + try Log.setup(path: config.logPath) +} catch { + printError("error: setup logging: \(error)") + exit(1) +} +Log.info("vz-shim starting", ["control_socket": config.controlSocket ?? "", "vsock_socket": config.vsockSocket ?? ""]) + +let identityLock: Int32? +do { + identityLock = try lockMacOSIdentity(config) +} catch { + printError("macOS identity admission: \(error)") + exit(1) +} + +let vmConfig: VZVirtualMachineConfiguration +do { + vmConfig = try makeConfiguration(&config) +} catch { + Log.error("failed to create VM", ["error": "\(error)"]) + printError("failed to create VM: \(error)") + exit(1) +} +let shim = Shim(vm: VZVirtualMachine(configuration: vmConfig, queue: .main), vmConfig: vmConfig, config: config) + +func serveOrExit() { + do { + try shim.listen() + } catch { + Log.error("failed to listen", ["error": "\(error)"]) + exit(1) + } +} + +if let restorePath = config.restoreMachineStatePath, !restorePath.isEmpty { + if let reason = shim.saveRestoreUnsupportedReason() { + Log.error("save/restore not supported for VM config", ["error": reason]) + printError("save/restore not supported for VM config: \(reason)") + exit(1) + } + shim.vm.restoreMachineStateFrom(url: URL(fileURLWithPath: restorePath)) { error in + if let error { + Log.error("failed to restore VM machine state", ["error": error.localizedDescription, "path": restorePath]) + printError("failed to restore VM machine state: \(error.localizedDescription)") + exit(1) + } + Log.info("VM restored from machine state", ["path": restorePath, "state": stateName(shim.vm.state)]) + serveOrExit() + } +} else { + shim.vm.start { result in + if let error = result.error { + Log.error("failed to start VM", ["error": error.localizedDescription]) + printError("failed to start VM: \(error.localizedDescription)") + exit(1) + } + Log.info("VM started", ["vcpus": config.vcpus ?? 0, "memory_mb": (config.memoryBytes ?? 0) / 1024 / 1024]) + serveOrExit() + } +} + +var signalSources: [DispatchSourceSignal] = [] +for number in [SIGTERM, SIGINT] { + signal(number, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: number, queue: .main) + source.setEventHandler { + Log.info("received signal, shutting down", ["signal": number]) + shim.stopForSignal() + } + source.resume() + signalSources.append(source) +} + +withExtendedLifetime(identityLock) { + dispatchMain() +} diff --git a/cmd/vz-shim/rosetta_arm64.go b/cmd/vz-shim/rosetta_arm64.go deleted file mode 100644 index 5495f7731..000000000 --- a/cmd/vz-shim/rosetta_arm64.go +++ /dev/null @@ -1,45 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "fmt" - "log/slog" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// configureDirectorySharing attaches the Linux Rosetta virtio-fs share when -// EnableRosetta is set. The host must have Rosetta installed; NotInstalled and -// NotSupported are hard errors surfaced back through the shim-failure path. -func configureDirectorySharing(vmConfig *vz.VirtualMachineConfiguration, config *shimconfig.ShimConfig) error { - if !config.EnableRosetta { - return nil - } - - switch vz.LinuxRosettaDirectoryShareAvailability() { - case vz.LinuxRosettaAvailabilityInstalled: - case vz.LinuxRosettaAvailabilityNotInstalled: - return fmt.Errorf("rosetta requested but not installed (run: softwareupdate --install-rosetta)") - default: - return fmt.Errorf("rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") - } - - fsDevice, err := vz.NewVirtioFileSystemDeviceConfiguration(shimconfig.RosettaMountTag) - if err != nil { - return fmt.Errorf("create rosetta virtio-fs device (tag %q): %w", shimconfig.RosettaMountTag, err) - } - - share, err := vz.NewLinuxRosettaDirectoryShare() - if err != nil { - return fmt.Errorf("create rosetta directory share: %w", err) - } - fsDevice.SetDirectoryShare(share) - - vmConfig.SetDirectorySharingDevicesVirtualMachineConfiguration( - []vz.DirectorySharingDeviceConfiguration{fsDevice}, - ) - slog.Info("attached rosetta directory share", "tag", shimconfig.RosettaMountTag) - return nil -} diff --git a/cmd/vz-shim/rosetta_arm64_test.go b/cmd/vz-shim/rosetta_arm64_test.go deleted file mode 100644 index 25f3b5fad..000000000 --- a/cmd/vz-shim/rosetta_arm64_test.go +++ /dev/null @@ -1,81 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRosettaConfigureDirectorySharingDisabled(t *testing.T) { - // When Rosetta is not requested the call is a no-op and never touches the - // VM configuration, so a nil config is safe. - err := configureDirectorySharing(nil, &shimconfig.ShimConfig{EnableRosetta: false}) - assert.NoError(t, err) -} - -// TestRosettaConfigureDirectorySharingEnabled exercises the error paths -// unconditionally. The attach+validate path requires Rosetta to be installed on -// the runner; it is covered by TestRosettaConfigureDirectorySharingInstalled, -// which skips (visibly, under -v) when Rosetta is unavailable rather than -// silently asserting nothing. -func TestRosettaConfigureDirectorySharingEnabled(t *testing.T) { - cfg := &shimconfig.ShimConfig{EnableRosetta: true} - - switch vz.LinuxRosettaDirectoryShareAvailability() { - case vz.LinuxRosettaAvailabilityNotInstalled: - err := configureDirectorySharing(nil, cfg) - require.Error(t, err) - assert.Contains(t, err.Error(), "not installed") - case vz.LinuxRosettaAvailabilityNotSupported: - err := configureDirectorySharing(nil, cfg) - require.Error(t, err) - assert.Contains(t, err.Error(), "not supported") - } -} - -// TestRosettaConfigureDirectorySharingInstalled asserts that, when Rosetta is -// installed, configureDirectorySharing attaches the virtio-fs share and the -// resulting VM configuration validates. It skips otherwise so the log records -// whether the attach path actually ran on this runner. -func TestRosettaConfigureDirectorySharingInstalled(t *testing.T) { - if vz.LinuxRosettaDirectoryShareAvailability() != vz.LinuxRosettaAvailabilityInstalled { - t.Skip("rosetta not installed on this host; skipping attach+validate coverage") - } - - vmConfig := newTestVMConfiguration(t) - require.NoError(t, configureDirectorySharing(vmConfig, &shimconfig.ShimConfig{EnableRosetta: true})) - ok, err := vmConfig.Validate() - if err != nil && strings.Contains(err.Error(), "entitlement") { - if os.Getenv("HYPEMAN_VZ_SIGNED") != "" { - t.Fatalf("test binary should be signed with the virtualization entitlement (run via `make test-vz-shim-signed`): %v", err) - } - t.Skip("unsigned test binary lacks the virtualization entitlement; run via `make test-vz-shim-signed`") - } - require.NoError(t, err) - assert.True(t, ok) -} - -// newTestVMConfiguration builds a minimal valid VM configuration for tests that -// need to attach a device. The kernel file only needs to exist for the boot -// loader constructor. -func newTestVMConfiguration(t *testing.T) *vz.VirtualMachineConfiguration { - t.Helper() - - kernelPath := filepath.Join(t.TempDir(), "vmlinuz") - require.NoError(t, os.WriteFile(kernelPath, []byte("kernel"), 0o644)) - - bootLoader, err := vz.NewLinuxBootLoader(kernelPath, vz.WithCommandLine("console=hvc0")) - require.NoError(t, err) - - vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, 1, 256*1024*1024) - require.NoError(t, err) - return vmConfig -} diff --git a/cmd/vz-shim/rosetta_other.go b/cmd/vz-shim/rosetta_other.go deleted file mode 100644 index 5f45abb06..000000000 --- a/cmd/vz-shim/rosetta_other.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build darwin && !arm64 - -package main - -import ( - "fmt" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// configureDirectorySharing is the non-arm64 stub: Rosetta is Apple-silicon-only, -// so it errors when requested and is otherwise a no-op. cmd/vz-shim is arm64-only -// in practice (Code-Hex/vz v3.7.1 doesn't compile for darwin/amd64); this only -// satisfies the !arm64 type check, like save_restore_unsupported.go. -func configureDirectorySharing(_ *vz.VirtualMachineConfiguration, config *shimconfig.ShimConfig) error { - if config.EnableRosetta { - return fmt.Errorf("rosetta is only available on Apple silicon") - } - return nil -} diff --git a/cmd/vz-shim/save_restore_arm64.go b/cmd/vz-shim/save_restore_arm64.go deleted file mode 100644 index a5d522216..000000000 --- a/cmd/vz-shim/save_restore_arm64.go +++ /dev/null @@ -1,29 +0,0 @@ -//go:build darwin && arm64 - -package main - -import ( - "fmt" - - "github.com/Code-Hex/vz/v4" -) - -func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error { - ok, err := vmConfig.ValidateSaveRestoreSupport() - if err != nil { - return err - } - if !ok { - return fmt.Errorf("virtual machine configuration does not support save/restore") - } - return nil -} - -func saveMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - return vm.SaveMachineStateToPath(snapshotPath) -} - -func restoreMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - // The vz wrapper accepts a filesystem path and constructs a file URL internally. - return vm.RestoreMachineStateFromURL(snapshotPath) -} diff --git a/cmd/vz-shim/save_restore_unsupported.go b/cmd/vz-shim/save_restore_unsupported.go deleted file mode 100644 index 47ce10fd0..000000000 --- a/cmd/vz-shim/save_restore_unsupported.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build darwin && !arm64 - -package main - -import ( - "fmt" - "runtime" - - "github.com/Code-Hex/vz/v4" -) - -func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error { - return fmt.Errorf("save/restore is only supported on darwin/arm64 (current arch: %s)", runtime.GOARCH) -} - -func saveMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - return fmt.Errorf("save/restore is only supported on darwin/arm64 (current arch: %s)", runtime.GOARCH) -} - -func restoreMachineState(vm *vz.VirtualMachine, snapshotPath string) error { - return fmt.Errorf("save/restore is only supported on darwin/arm64 (current arch: %s)", runtime.GOARCH) -} diff --git a/cmd/vz-shim/server.go b/cmd/vz-shim/server.go deleted file mode 100644 index 100e69276..000000000 --- a/cmd/vz-shim/server.go +++ /dev/null @@ -1,422 +0,0 @@ -//go:build darwin - -package main - -import ( - "bufio" - "encoding/json" - "fmt" - "io" - "log/slog" - "net" - "net/http" - "os" - "path/filepath" - "sync" - "sync/atomic" - - "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// ShimServer handles control API and vsock proxy for a vz VM. -type ShimServer struct { - vm *vz.VirtualMachine - vmConfig *vz.VirtualMachineConfiguration - shimConfig shimconfig.ShimConfig - mu sync.RWMutex - // forcedStop is set before the shim stops the VM itself, so that stop is not - // mistaken for the guest powering off. - forcedStop atomic.Bool -} - -// NewShimServer creates a new shim server. -func NewShimServer(vm *vz.VirtualMachine, vmConfig *vz.VirtualMachineConfiguration, config shimconfig.ShimConfig) *ShimServer { - return &ShimServer{ - vm: vm, - vmConfig: vmConfig, - shimConfig: config, - } -} - -// VMInfoResponse matches the cloud-hypervisor VmInfo structure. -type VMInfoResponse struct { - State string `json:"state"` - MemoryBalloonDevices int `json:"memory_balloon_devices,omitempty"` -} - -type snapshotRequest struct { - DestinationPath string `json:"destination_path"` -} - -type balloonRequest struct { - TargetGuestMemoryBytes int64 `json:"target_guest_memory_bytes"` -} - -// Handler returns the HTTP handler for the control API. -func (s *ShimServer) Handler() http.Handler { - mux := http.NewServeMux() - - // Match cloud-hypervisor API patterns - mux.HandleFunc("GET /api/v1/vm.info", s.handleVMInfo) - mux.HandleFunc("PUT /api/v1/vm.pause", s.handlePause) - mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume) - mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown) - mux.HandleFunc("PUT /api/v1/vm.snapshot", s.handleSnapshot) - mux.HandleFunc("GET /api/v1/vm.snapshot-support", s.handleSnapshotSupport) - mux.HandleFunc("GET /api/v1/vm.balloon", s.handleGetBalloon) - mux.HandleFunc("PUT /api/v1/vm.balloon", s.handleSetBalloon) - mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton) - mux.HandleFunc("GET /api/v1/vmm.ping", s.handlePing) - mux.HandleFunc("PUT /api/v1/vmm.shutdown", s.handleVMMShutdown) - - return mux -} - -func (s *ShimServer) handleVMInfo(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() - - state := vzStateToString(s.vm.State()) - resp := VMInfoResponse{ - State: state, - MemoryBalloonDevices: len(s.vm.MemoryBalloonDevices()), - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) -} - -func (s *ShimServer) handlePause(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - if !s.vm.CanPause() { - http.Error(w, "cannot pause VM", http.StatusBadRequest) - return - } - - if err := s.vm.Pause(); err != nil { - slog.Error("failed to pause VM", "error", err) - http.Error(w, fmt.Sprintf("pause failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("VM paused") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleResume(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - if !s.vm.CanResume() { - http.Error(w, "cannot resume VM", http.StatusBadRequest) - return - } - - if err := s.vm.Resume(); err != nil { - slog.Error("failed to resume VM", "error", err) - http.Error(w, fmt.Sprintf("resume failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("VM resumed") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - // Send ACPI power button (graceful shutdown signal to guest). - // The caller (instance manager) handles timeout/force-kill if the guest - // doesn't shut down in time. Force-kill is in handleVMMShutdown / killHypervisor. - success, err := s.vm.RequestStop() - if err != nil || !success { - slog.Error("RequestStop failed", "error", err, "success", success) - http.Error(w, fmt.Sprintf("shutdown failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("VM graceful shutdown requested (ACPI power button)") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleSnapshotSupport(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() - response := struct { - Supported bool `json:"supported"` - Reason string `json:"reason,omitempty"` - }{Supported: true} - if err := validateSaveRestoreSupport(s.vmConfig); err != nil { - response.Supported = false - response.Reason = err.Error() - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) -} - -func (s *ShimServer) handleSnapshot(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - var req snapshotRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, fmt.Sprintf("invalid snapshot request: %v", err), http.StatusBadRequest) - return - } - if req.DestinationPath == "" { - http.Error(w, "destination_path is required", http.StatusBadRequest) - return - } - if s.vm.State() != vz.VirtualMachineStatePaused { - http.Error(w, "vm must be paused before snapshot", http.StatusBadRequest) - return - } - if err := validateSaveRestoreSupport(s.vmConfig); err != nil { - http.Error(w, fmt.Sprintf("save/restore not supported: %v", err), http.StatusBadRequest) - return - } - - if err := os.MkdirAll(req.DestinationPath, 0755); err != nil { - http.Error(w, fmt.Sprintf("create snapshot dir failed: %v", err), http.StatusInternalServerError) - return - } - snapshotComplete := false - defer func() { - if !snapshotComplete { - _ = os.RemoveAll(req.DestinationPath) - } - }() - - machineStatePath := filepath.Join(req.DestinationPath, shimconfig.SnapshotMachineStateFile) - if err := os.RemoveAll(machineStatePath); err != nil { - http.Error(w, fmt.Sprintf("prepare machine state path failed: %v", err), http.StatusInternalServerError) - return - } - if err := saveMachineState(s.vm, machineStatePath); err != nil { - http.Error(w, fmt.Sprintf("save machine state failed: %v", err), http.StatusInternalServerError) - return - } - - manifestPath := filepath.Join(req.DestinationPath, shimconfig.SnapshotManifestFile) - tmpManifestPath := manifestPath + ".tmp" - manifest := shimconfig.SnapshotManifest{ - Hypervisor: "vz", - MachineStateFile: shimconfig.SnapshotMachineStateFile, - ShimConfig: s.shimConfig, - } - // This field is runtime-only; restore path is populated by the caller on restore. - manifest.ShimConfig.RestoreMachineStatePath = "" - manifestBytes, err := json.Marshal(manifest) - if err != nil { - http.Error(w, fmt.Sprintf("marshal manifest failed: %v", err), http.StatusInternalServerError) - return - } - if err := os.WriteFile(tmpManifestPath, manifestBytes, 0644); err != nil { - http.Error(w, fmt.Sprintf("write manifest failed: %v", err), http.StatusInternalServerError) - return - } - if err := os.Rename(tmpManifestPath, manifestPath); err != nil { - http.Error(w, fmt.Sprintf("finalize manifest failed: %v", err), http.StatusInternalServerError) - return - } - - snapshotComplete = true - slog.Info("VM snapshot saved", "destination", req.DestinationPath, "machine_state", machineStatePath) - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handleGetBalloon(w http.ResponseWriter, r *http.Request) { - s.mu.RLock() - defer s.mu.RUnlock() - - device, err := s.getTraditionalBalloonDevice() - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(balloonRequest{ - TargetGuestMemoryBytes: int64(device.GetTargetVirtualMachineMemorySize()), - }) -} - -func (s *ShimServer) handleSetBalloon(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - device, err := s.getTraditionalBalloonDevice() - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - var req balloonRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, fmt.Sprintf("invalid balloon request: %v", err), http.StatusBadRequest) - return - } - if req.TargetGuestMemoryBytes < 0 { - http.Error(w, "target_guest_memory_bytes must be non-negative", http.StatusBadRequest) - return - } - - device.SetTargetVirtualMachineMemorySize(uint64(req.TargetGuestMemoryBytes)) - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) handlePowerButton(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - // RequestStop sends an ACPI power button event - success, err := s.vm.RequestStop() - if err != nil || !success { - slog.Error("failed to send power button", "error", err, "success", success) - http.Error(w, fmt.Sprintf("power button failed: %v", err), http.StatusInternalServerError) - return - } - - slog.Info("power button sent") - w.WriteHeader(http.StatusNoContent) -} - -func (s *ShimServer) getTraditionalBalloonDevice() (*vz.VirtioTraditionalMemoryBalloonDevice, error) { - for _, device := range s.vm.MemoryBalloonDevices() { - if traditional := vz.AsVirtioTraditionalMemoryBalloonDevice(device); traditional != nil { - return traditional, nil - } - } - return nil, fmt.Errorf("no memory balloon device configured") -} - -func (s *ShimServer) handlePing(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("OK")) -} - -func (s *ShimServer) handleVMMShutdown(w http.ResponseWriter, r *http.Request) { - slog.Info("VMM shutdown requested") - w.WriteHeader(http.StatusNoContent) - - // Stop the VM and exit - go func() { - s.mu.Lock() - defer s.mu.Unlock() - - if s.vm.CanStop() { - s.forcedStop.Store(true) - s.vm.Stop() - } - // Process will exit when VM stops (monitored in main) - }() -} - -func vzStateToString(state vz.VirtualMachineState) string { - switch state { - case vz.VirtualMachineStateStopped: - return "Shutdown" - case vz.VirtualMachineStateRunning: - return "Running" - case vz.VirtualMachineStatePaused: - return "Paused" - case vz.VirtualMachineStateError: - return "Error" - case vz.VirtualMachineStateStarting: - return "Starting" - case vz.VirtualMachineStatePausing: - return "Pausing" - case vz.VirtualMachineStateResuming: - return "Resuming" - case vz.VirtualMachineStateStopping: - return "Stopping" - case vz.VirtualMachineStateSaving: - return "Saving" - case vz.VirtualMachineStateRestoring: - return "Restoring" - default: - return "Unknown" - } -} - -// ServeVsock handles vsock proxy connections using the Cloud Hypervisor protocol. -// Protocol: Client sends "CONNECT {port}\n", server responds "OK {port}\n", then proxies. -func (s *ShimServer) ServeVsock(listener net.Listener) { - for { - conn, err := listener.Accept() - if err != nil { - slog.Debug("vsock listener closed", "error", err) - return - } - go s.handleVsockConnection(conn) - } -} - -func (s *ShimServer) handleVsockConnection(conn net.Conn) { - defer conn.Close() - - // Read the CONNECT command - reader := bufio.NewReader(conn) - cmd, err := reader.ReadString('\n') - if err != nil { - slog.Error("failed to read vsock handshake", "error", err) - return - } - - // Parse "CONNECT {port}\n" - var port uint32 - if _, err := fmt.Sscanf(cmd, "CONNECT %d\n", &port); err != nil { - slog.Error("invalid vsock handshake", "cmd", cmd, "error", err) - conn.Write([]byte(fmt.Sprintf("ERR invalid command: %s", cmd))) - return - } - - slog.Debug("vsock connect request", "port", port) - - // Get vsock device and connect to guest - s.mu.RLock() - socketDevices := s.vm.SocketDevices() - s.mu.RUnlock() - - if len(socketDevices) == 0 { - slog.Error("no vsock device configured") - conn.Write([]byte("ERR no vsock device\n")) - return - } - - guestConn, err := socketDevices[0].Connect(port) - if err != nil { - slog.Error("failed to connect to guest vsock", "port", port, "error", err) - conn.Write([]byte(fmt.Sprintf("ERR connect failed: %v\n", err))) - return - } - defer guestConn.Close() - - // Send OK response (matching CH protocol) - if _, err := conn.Write([]byte(fmt.Sprintf("OK %d\n", port))); err != nil { - slog.Error("failed to send OK response", "error", err) - return - } - - slog.Debug("vsock connection established", "port", port) - - // Proxy data bidirectionally - done := make(chan struct{}, 2) - - go func() { - io.Copy(guestConn, reader) - done <- struct{}{} - }() - - go func() { - io.Copy(conn, guestConn) - done <- struct{}{} - }() - - // Wait for one direction to close - <-done -} diff --git a/cmd/vz-shim/vm.go b/cmd/vz-shim/vm.go deleted file mode 100644 index 08b373e5f..000000000 --- a/cmd/vz-shim/vm.go +++ /dev/null @@ -1,377 +0,0 @@ -//go:build darwin - -package main - -import ( - "encoding/base64" - "fmt" - "log/slog" - "net" - "net/netip" - "os" - "runtime" - "strings" - - "github.com/Code-Hex/vz/v4" - "github.com/Code-Hex/vz/v4/vmnet" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" -) - -// createVM creates and configures a vz.VirtualMachine from ShimConfig. -func createVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMachineConfiguration, error) { - if isMacGuest(config) { - return createMacVM(config) - } - - // Prepare kernel command line (vz uses hvc0 for serial console) - kernelArgs := config.KernelArgs - if kernelArgs == "" { - kernelArgs = "console=hvc0 root=/dev/vda" - } else { - kernelArgs = strings.ReplaceAll(kernelArgs, "console=ttyS0", "console=hvc0") - } - - bootLoader, err := vz.NewLinuxBootLoader( - config.KernelPath, - vz.WithCommandLine(kernelArgs), - vz.WithInitrd(config.InitrdPath), - ) - if err != nil { - return nil, nil, fmt.Errorf("create boot loader: %w", err) - } - - vcpus := computeCPUCount(config.VCPUs) - memoryBytes := computeMemorySize(uint64(config.MemoryBytes)) - - slog.Debug("VM config", "vcpus", vcpus, "memory_bytes", memoryBytes, "kernel", config.KernelPath, "initrd", config.InitrdPath) - - vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, vcpus, memoryBytes) - if err != nil { - return nil, nil, fmt.Errorf("create vm configuration: %w", err) - } - - if err := configureSerialConsole(vmConfig, config.SerialLogPath); err != nil { - return nil, nil, fmt.Errorf("configure serial: %w", err) - } - - if err := configureNetwork(vmConfig, config.Networks); err != nil { - return nil, nil, fmt.Errorf("configure network: %w", err) - } - - entropyConfig, err := vz.NewVirtioEntropyDeviceConfiguration() - if err != nil { - return nil, nil, fmt.Errorf("create entropy device: %w", err) - } - vmConfig.SetEntropyDevicesVirtualMachineConfiguration([]*vz.VirtioEntropyDeviceConfiguration{entropyConfig}) - - if err := configureStorage(vmConfig, config.Disks); err != nil { - return nil, nil, fmt.Errorf("configure storage: %w", err) - } - - if err := configureDirectorySharing(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure directory sharing: %w", err) - } - - if err := configurePlatform(vmConfig, config); err != nil { - return nil, nil, fmt.Errorf("configure platform: %w", err) - } - - vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() - if err != nil { - return nil, nil, fmt.Errorf("create vsock device: %w", err) - } - vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig}) - - if config.EnableMemoryBalloon { - balloonConfig, err := vz.NewVirtioTraditionalMemoryBalloonDeviceConfiguration() - if err != nil { - if config.RequireMemoryBalloon { - return nil, nil, fmt.Errorf("create memory balloon device: %w", err) - } - slog.Warn("memory balloon unavailable, continuing without balloon", "error", err) - } else { - vmConfig.SetMemoryBalloonDevicesVirtualMachineConfiguration([]vz.MemoryBalloonDeviceConfiguration{ - balloonConfig, - }) - } - } - - if validated, err := vmConfig.Validate(); !validated || err != nil { - return nil, nil, fmt.Errorf("invalid vm configuration: %w", err) - } - - vm, err := vz.NewVirtualMachine(vmConfig) - if err != nil { - return nil, nil, fmt.Errorf("create virtual machine: %w", err) - } - - return vm, vmConfig, nil -} - -func configurePlatform(vmConfig *vz.VirtualMachineConfiguration, config *shimconfig.ShimConfig) error { - var machineID *vz.GenericMachineIdentifier - var err error - - if config.MachineIdentifierData != "" { - b, decodeErr := base64.StdEncoding.DecodeString(config.MachineIdentifierData) - if decodeErr != nil { - return fmt.Errorf("decode machine identifier data: %w", decodeErr) - } - machineID, err = vz.NewGenericMachineIdentifierWithData(b) - if err != nil { - return fmt.Errorf("recreate machine identifier: %w", err) - } - } else { - machineID, err = vz.NewGenericMachineIdentifier() - if err != nil { - return fmt.Errorf("create machine identifier: %w", err) - } - config.MachineIdentifierData = base64.StdEncoding.EncodeToString(machineID.DataRepresentation()) - } - - platformConfig, err := vz.NewGenericPlatformConfiguration( - vz.WithGenericMachineIdentifier(machineID), - ) - if err != nil { - return fmt.Errorf("create generic platform config: %w", err) - } - vmConfig.SetPlatformVirtualMachineConfiguration(platformConfig) - return nil -} - -func configureSerialConsole(vmConfig *vz.VirtualMachineConfiguration, logPath string) error { - var serialAttachment *vz.FileHandleSerialPortAttachment - - nullRead, err := os.OpenFile("/dev/null", os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("open /dev/null for reading: %w", err) - } - - if logPath != "" { - file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - nullRead.Close() - return fmt.Errorf("open serial log file: %w", err) - } - serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, file) - if err != nil { - nullRead.Close() - file.Close() - return fmt.Errorf("create serial attachment: %w", err) - } - } else { - nullWrite, err := os.OpenFile("/dev/null", os.O_WRONLY, 0) - if err != nil { - nullRead.Close() - return fmt.Errorf("open /dev/null for writing: %w", err) - } - serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, nullWrite) - if err != nil { - nullRead.Close() - nullWrite.Close() - return fmt.Errorf("create serial attachment: %w", err) - } - } - - consoleConfig, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(serialAttachment) - if err != nil { - return fmt.Errorf("create console config: %w", err) - } - vmConfig.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{ - consoleConfig, - }) - - return nil -} - -func configureNetwork(vmConfig *vz.VirtualMachineConfiguration, networks []shimconfig.NetworkConfig) error { - if len(networks) == 0 { - // No networks configured (NetworkEnabled=false) — do not attach any NIC. - return nil - } - - var devices []*vz.VirtioNetworkDeviceConfiguration - for _, netConfig := range networks { - var dev *vz.VirtioNetworkDeviceConfiguration - var err error - if netConfig.IPv4Address != "" { - dev, err = createVmnetNetworkDevice(netConfig) - } else { - dev, err = createNATNetworkDevice(netConfig.MAC) - } - if err != nil { - return err - } - devices = append(devices, dev) - } - vmConfig.SetNetworkDevicesVirtualMachineConfiguration(devices) - return nil -} - -func createNATNetworkDevice(macAddr string) (*vz.VirtioNetworkDeviceConfiguration, error) { - natAttachment, err := vz.NewNATNetworkDeviceAttachment() - if err != nil { - return nil, fmt.Errorf("create NAT attachment: %w", err) - } - - networkConfig, err := vz.NewVirtioNetworkDeviceConfiguration(natAttachment) - if err != nil { - return nil, fmt.Errorf("create network config: %w", err) - } - - mac, err := assignMACAddress(macAddr) - if err != nil { - return nil, err - } - networkConfig.SetMACAddress(mac) - - return networkConfig, nil -} - -// createVmnetNetworkDevice attaches the guest to its own shared-mode vmnet network -// with the given subnet and a DHCP reservation for the guest's MAC, so the guest's -// address is the one hypeman assigned. Saved state records this configuration, so a -// guest saved on the shared NAT (no address in its config) is restored on NAT. -func createVmnetNetworkDevice(netConfig shimconfig.NetworkConfig) (*vz.VirtioNetworkDeviceConfiguration, error) { - hw, err := net.ParseMAC(netConfig.MAC) - if err != nil { - return nil, fmt.Errorf("vmnet network needs the guest MAC: %w", err) - } - subnet, err := netip.ParsePrefix(netConfig.IPv4Subnet) - if err != nil { - return nil, fmt.Errorf("parse guest subnet: %w", err) - } - address, err := netip.ParseAddr(netConfig.IPv4Address) - if err != nil { - return nil, fmt.Errorf("parse guest address: %w", err) - } - cfg, err := vmnet.NewNetworkConfiguration(vmnet.ModeShared, vmnet.WithIPv4Subnet(subnet), vmnet.WithDHCPReservation(hw, address)) - if err != nil { - return nil, fmt.Errorf("configure vmnet network %s: %w", subnet, err) - } - network, err := vmnet.NewNetwork(cfg) - if err != nil { - return nil, fmt.Errorf("create vmnet network %s: %w", subnet, err) - } - attachment, err := vz.NewVMNetNetworkDeviceAttachment(network) - if err != nil { - return nil, fmt.Errorf("create vmnet attachment: %w", err) - } - dev, err := vz.NewVirtioNetworkDeviceConfiguration(attachment) - if err != nil { - return nil, fmt.Errorf("create network config: %w", err) - } - mac, err := assignMACAddress(netConfig.MAC) - if err != nil { - return nil, err - } - dev.SetMACAddress(mac) - slog.Info("attached guest vmnet network", "subnet", subnet.String(), "address", address.String(), "mac", netConfig.MAC) - return dev, nil -} - -func assignMACAddress(macAddr string) (*vz.MACAddress, error) { - if macAddr == "" { - mac, err := vz.NewRandomLocallyAdministeredMACAddress() - if err != nil { - return nil, fmt.Errorf("generate MAC address: %w", err) - } - slog.Info("generated random MAC address", "mac", mac.String()) - return mac, nil - } - - hwAddr, err := net.ParseMAC(macAddr) - if err != nil { - slog.Warn("failed to parse MAC address, generating random", "mac", macAddr, "error", err) - mac, err := vz.NewRandomLocallyAdministeredMACAddress() - if err != nil { - return nil, fmt.Errorf("generate MAC address: %w", err) - } - return mac, nil - } - - mac, err := vz.NewMACAddress(hwAddr) - if err != nil { - slog.Warn("failed to create MAC from parsed address, generating random", "mac", macAddr, "error", err) - mac, err := vz.NewRandomLocallyAdministeredMACAddress() - if err != nil { - return nil, fmt.Errorf("generate MAC address: %w", err) - } - return mac, nil - } - - slog.Info("using specified MAC address", "mac", macAddr) - return mac, nil -} - -func configureStorage(vmConfig *vz.VirtualMachineConfiguration, disks []shimconfig.DiskConfig) error { - var storageDevices []vz.StorageDeviceConfiguration - - for _, disk := range disks { - if _, err := os.Stat(disk.Path); os.IsNotExist(err) { - return fmt.Errorf("disk image not found: %s", disk.Path) - } - - if strings.HasSuffix(disk.Path, ".qcow2") { - return fmt.Errorf("qcow2 not supported by vz, use raw format: %s", disk.Path) - } - - attachment, err := vz.NewDiskImageStorageDeviceAttachment(disk.Path, disk.Readonly) - if err != nil { - return fmt.Errorf("create disk attachment for %s: %w", disk.Path, err) - } - - blockConfig, err := vz.NewVirtioBlockDeviceConfiguration(attachment) - if err != nil { - return fmt.Errorf("create block device config: %w", err) - } - - storageDevices = append(storageDevices, blockConfig) - } - - if len(storageDevices) > 0 { - vmConfig.SetStorageDevicesVirtualMachineConfiguration(storageDevices) - } - - return nil -} - -func computeCPUCount(requested int) uint { - virtualCPUCount := uint(requested) - if virtualCPUCount == 0 { - virtualCPUCount = uint(runtime.NumCPU() - 1) - if virtualCPUCount < 1 { - virtualCPUCount = 1 - } - } - - maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedCPUCount() - minAllowed := vz.VirtualMachineConfigurationMinimumAllowedCPUCount() - - if virtualCPUCount > maxAllowed { - virtualCPUCount = maxAllowed - } - if virtualCPUCount < minAllowed { - virtualCPUCount = minAllowed - } - - return virtualCPUCount -} - -func computeMemorySize(requested uint64) uint64 { - if requested == 0 { - requested = 2 * 1024 * 1024 * 1024 // 2GB safety default (caller normally provides this) - } - - maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedMemorySize() - minAllowed := vz.VirtualMachineConfigurationMinimumAllowedMemorySize() - - if requested > maxAllowed { - requested = maxAllowed - } - if requested < minAllowed { - requested = minAllowed - } - - return requested -} diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index da8c41daa..b0f457fc1 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -119,8 +119,9 @@ type DiskConfig struct { Path string Format DiskFormat Readonly bool - IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited) - IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps) + Parents []string // vz only: read-only images below Path, base first; Path is their ASIF overlay + IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited) + IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps) } // NetworkConfig represents a network interface attached to the VM diff --git a/lib/hypervisor/vz/shim_darwin_test.go b/lib/hypervisor/vz/shim_darwin_test.go new file mode 100644 index 000000000..a9bef337b --- /dev/null +++ b/lib/hypervisor/vz/shim_darwin_test.go @@ -0,0 +1,93 @@ +//go:build darwin + +package vz + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/stretchr/testify/require" +) + +// runShim runs the embedded shim with args and returns its stderr and exit code. +func runShim(t *testing.T, args ...string) (string, int) { + t.Helper() + bin, err := extractShim() + require.NoError(t, err) + var stderr bytes.Buffer + cmd := exec.Command(bin, args...) + cmd.Stderr = &stderr + err = cmd.Run() + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return stderr.String(), exitErr.ExitCode() + } + require.NoError(t, err) + return stderr.String(), 0 +} + +func runShimConfig(t *testing.T, config shimconfig.ShimConfig) (string, int) { + t.Helper() + config.LogPath = filepath.Join(t.TempDir(), "vz-shim.log") + data, err := json.Marshal(config) + require.NoError(t, err) + return runShim(t, "-config", string(data)) +} + +func TestShimRejectsMissingOrInvalidConfig(t *testing.T) { + stderr, code := runShim(t) + require.Equal(t, 1, code) + require.Contains(t, stderr, "-config is required") + stderr, code = runShim(t, "-config", "{") + require.Equal(t, 1, code) + require.Contains(t, stderr, "invalid config JSON") +} + +func TestShimMacBootRequiresCompleteIdentity(t *testing.T) { + for _, c := range []shimconfig.ShimConfig{ + {MacHardwareModelData: "AA=="}, + {MacMachineIdentifierData: "AA=="}, + {MacAuxStoragePath: "/missing"}, + } { + stderr, code := runShimConfig(t, c) + require.Equal(t, 1, code) + require.Contains(t, stderr, "macOS boot requires") + } +} + +func TestShimMacPlatformRejectsMalformedModel(t *testing.T) { + stderr, code := runShimConfig(t, shimconfig.ShimConfig{MacHardwareModelData: "not base64!", MacMachineIdentifierData: "AA==", MacAuxStoragePath: "/missing"}) + require.Equal(t, 1, code) + require.Contains(t, stderr, "decode Mac hardware model") +} + +// The shim holds a lock named for the identity's bytes, so encodings of the same +// identity contend for it. +func TestShimMacIdentityLockExcludesEquivalentIdentity(t *testing.T) { + identity := []byte("identity") + path := filepath.Join("/tmp", fmt.Sprintf("hypeman-mac-identity-%d-%x.lock", os.Getuid(), sha256.Sum256(identity))) + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + require.NoError(t, err) + defer f.Close() + require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)) + + encoded := base64.StdEncoding.EncodeToString(identity) + "\n" + stderr, code := runShimConfig(t, shimconfig.ShimConfig{MacMachineIdentifierData: encoded}) + require.Equal(t, 1, code) + require.Contains(t, stderr, "Mac machine identity already in use") + + require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_UN)) + stderr, code = runShimConfig(t, shimconfig.ShimConfig{MacMachineIdentifierData: encoded}) + require.Equal(t, 1, code) + require.Contains(t, stderr, "macOS boot requires", "with the lock free the shim gets past admission") +} diff --git a/lib/hypervisor/vz/shimconfig/config.go b/lib/hypervisor/vz/shimconfig/config.go index 16f413876..90b4b9e71 100644 --- a/lib/hypervisor/vz/shimconfig/config.go +++ b/lib/hypervisor/vz/shimconfig/config.go @@ -71,6 +71,9 @@ type ShimConfig struct { type DiskConfig struct { Path string `json:"path"` Readonly bool `json:"readonly"` + // Parents, when set, are the read-only images below Path, base first; Path is + // then the ASIF overlay on top of them. + Parents []string `json:"parents,omitempty"` } // NetworkConfig represents a network interface. diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index 321f832dd..70e09f137 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -232,6 +232,7 @@ func buildShimConfigFromVMConfig(config hypervisor.VMConfig, socketPath string) cfg.Disks = append(cfg.Disks, shimconfig.DiskConfig{ Path: disk.Path, Readonly: disk.Readonly, + Parents: disk.Parents, }) } for _, n := range config.Networks { From ea2feb04e52d4e353653579d56dcebd4aab92315 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:42:05 +0000 Subject: [PATCH 06/33] Store and publish macOS images as ASIF stacks Machine images are a chain of ASIF disks, one per OCI layer. Instances clone the chain and boot from an empty writable overlay above it. Builds publish the stopped builder's overlay as one new layer over the base's layers, so the 64 KiB delta format, its applier and the export copy are removed. --- docs/macos-builds.md | 37 +-- docs/macos-experimental.md | 45 +-- lib/builds/macos_build.go | 115 ++++---- lib/builds/macos_build_failure_test.go | 75 ++--- lib/builds/macos_build_guest.go | 10 +- lib/builds/macos_build_job_test.go | 2 +- lib/builds/macos_build_layer_test.go | 37 +-- lib/builds/macos_build_lifecycle_test.go | 4 +- lib/builds/macos_build_manager.go | 40 ++- lib/builds/macos_build_manager_test.go | 6 +- lib/builds/macos_build_publisher.go | 218 +++++++++++---- lib/builds/macos_build_publisher_test.go | 71 ++++- lib/builds/macos_build_recovery_test.go | 6 - lib/builds/macos_build_test.go | 78 ++++-- lib/builds/macos_delta_publish.go | 173 ------------ lib/builds/macos_delta_publish_test.go | 160 ----------- lib/builds/macos_registry.go | 3 + lib/diskutilization/diskutilization.go | 3 +- lib/diskutilization/diskutilization_test.go | 2 +- lib/images/finalize.go | 3 + lib/images/macos_asif.go | 127 +++++++++ lib/images/macos_asif_test.go | 57 ++++ lib/images/macos_bundle.go | 64 ++++- lib/images/macos_bundle_test.go | 12 +- lib/images/macos_delta.go | 261 ------------------ lib/images/macos_delta_test.go | 203 -------------- lib/images/macos_import_darwin.go | 38 ++- lib/images/macos_import_darwin_test.go | 17 +- lib/images/macos_machine.go | 60 ++-- lib/images/macos_machine_test.go | 200 +++++++++----- lib/images/macos_unpack.go | 29 +- lib/images/storage.go | 15 +- lib/instances/macos.go | 33 ++- lib/instances/macos_build_export.go | 46 ++- lib/instances/macos_build_export_darwin.go | 95 ++----- .../macos_build_export_darwin_test.go | 53 ++-- lib/instances/macos_darwin.go | 34 --- lib/instances/macos_darwin_test.go | 72 ----- lib/instances/macos_memory_snapshot_test.go | 8 +- lib/instances/macos_other.go | 9 - lib/instances/macos_storage.go | 62 +++++ lib/instances/macos_storage_test.go | 100 +++++++ lib/instances/snapshot_macos.go | 9 +- lib/instances/snapshot_macos_darwin_test.go | 4 +- lib/instances/snapshot_macos_test.go | 34 +-- lib/instances/storage.go | 3 +- lib/instances/types.go | 3 + lib/instances/vm_storage_path_test.go | 2 +- lib/paths/paths.go | 5 +- lib/registrypush/macos_test.go | 4 +- 50 files changed, 1266 insertions(+), 1481 deletions(-) delete mode 100644 lib/builds/macos_delta_publish.go delete mode 100644 lib/builds/macos_delta_publish_test.go create mode 100644 lib/images/macos_asif.go create mode 100644 lib/images/macos_asif_test.go delete mode 100644 lib/images/macos_delta.go delete mode 100644 lib/images/macos_delta_test.go delete mode 100644 lib/instances/macos_darwin.go delete mode 100644 lib/instances/macos_darwin_test.go delete mode 100644 lib/instances/macos_other.go create mode 100644 lib/instances/macos_storage.go create mode 100644 lib/instances/macos_storage_test.go diff --git a/docs/macos-builds.md b/docs/macos-builds.md index c5e488cee..ba8946e0b 100644 --- a/docs/macos-builds.md +++ b/docs/macos-builds.md @@ -69,22 +69,26 @@ and the host copy is removed. No registry token, host credential or secret is injected. Guest stdout and stderr are discarded, so build logs never contain guest output. Failures report the phase and, for the script, its exit code. -## Stop, export and cleanup +## Stop, publish and cleanup -Publication is two instance operations. `StopMacOSBuildInstance` does a strict +Publication follows two instance operations. `StopMacOSBuildInstance` does a strict graceful stop and rejects forced fallback, an instance already Stopped, non-macOS guests, and any stop without shutdown acknowledgement and confirmed VMM exit. -`ExportStoppedMacOSBuildInstance` then copies the closed storage of a still-Stopped -instance under the instance lock, so a concurrent Start makes export fail. The output -is a private 0700 directory holding only `disk.img`, `aux.img` and `config.json`. - -Failure cleanup uses a forced stop and exports nothing. Storage is deleted only after +`MacOSBuildStorage` then returns the closed storage of the still-Stopped instance, +checked under the instance lock: its read-only ASIF layers, its writable overlay +`boot.asif`, its auxiliary storage and its platform. Nothing is copied. The build +checks that the overlay opens as an ASIF stack on those layers, that the hardware +model and resources match the base and that the identity differs. A Start during +publication changes the overlay, so the registry's digest check fails the upload. + +Failure cleanup uses a forced stop and publishes nothing. Storage is deleted only after confirmed VMM exit; a VM with no confirmed exit stays quarantined for an operator. Cleanup has its own 2-minute budget. A re-enqueued attempt first reaps the builder left by the interrupted attempt, and fails rather than start a second one if that delete fails. Builders orphaned outside this path have no general reaper. -The builder is destroyed **before** publication. Source stays staged until a terminal +The builder is destroyed **after** the registry has verified the publication, since +its storage is the publication source. Source stays staged until a terminal status is persisted, so a job interrupted while pushing is rebuilt on recovery. Startup on a macOS-only server recovers machine jobs without preparing Linux builder state, reaps cancelled builders and removes terminal-job source. Cancellation @@ -97,21 +101,22 @@ The publisher uses a server-configured registry and a token scoped to `builds/{i Plaintext is allowed only on loopback; redirects are never followed and proxies are disabled. The layer is never written to disk: a first pass computes its digest, size and DiffID, the upload regenerates the same bytes, and the registry rejects any -mismatch. Blobs use monolithic POST uploads, every persisted blob is re-read and +mismatch. Blobs the registry already holds are skipped. Blobs use monolithic POST uploads, every persisted blob is re-read and hashed, the manifest is committed last, and the digest content is verified. Ambiguous errors never return success. -A derivative built from a base in the same registry publishes a delta: the base's -layers are mounted by digest and only a layer of changed 64 KiB disk chunks is -uploaded. See `lib/images/macos_delta.go`. Bases outside the registry publish -standalone. There is no garbage-collection rule yet; deleting a base that a derivative -references breaks a later re-pull of that derivative. +The image is the base's layers plus one layer holding the builder's overlay, +auxiliary storage and platform. The overlay holds only the build's writes, so for a +base in the same registry only that layer is uploaded; the base's layers are reused +by digest. A base outside the registry has its layers uploaded too. There is no +garbage-collection rule yet; deleting a base that a derivative references breaks a +later re-pull of that derivative. ## Status and gates Race tests cover admission, the shared queue, cancellation without publication, phase -failures and quarantine, export validation, and authenticated publication with -manifest-last ordering. Test disks are synthetic and prove nothing about bootability. +failures and quarantine, storage validation, and authenticated publication with +manifest-last ordering and base-layer reuse. Test disks are synthetic and prove nothing about bootability. A development base has been run end to end through the HTTP API on Apple silicon, including a cold boot of the result; this trimmed branch has not been re-run natively. diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 585453ee6..9a8cbb0db 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -6,11 +6,12 @@ Linux guests retain the existing defaults and boot path. ## Import -Use a stopped macOS bundle (`config.json`, `disk.img`, `aux.img`) on -Darwin/arm64. Configure its user, SSH, FileVault and desktop before import. -The importer verifies that storage is not open, clones disk and auxiliary -storage, hashes their contents and identity/configuration into a digest, and -publishes a local ready image. Import is an administrator CLI operation, not +Use a stopped macOS bundle (`config.json`, `aux.img`, and `disk.asif` or a raw +`disk.img`) on Darwin/arm64. Configure its user, SSH, FileVault and desktop before +import. The importer verifies that storage is not open, clones an ASIF disk or +converts a raw one to ASIF (all-zero 1 MiB chunks stay unallocated; a 100 GB disk +took about 10 minutes on the QA host), clones auxiliary storage, hashes their +contents and identity/configuration into a digest, and publishes a local ready image. Import is an administrator CLI operation, not an HTTP endpoint accepting arbitrary host paths. ```sh @@ -20,7 +21,8 @@ go run -tags containers_image_openpgp ./cmd/import-macos \ --name localhost/macos:spike ``` -The local importer needs APFS clonefile support. The data directory path must be +The local importer needs APFS clonefile support, and ASIF disks need macOS 26 or +later; the VZ shim needs macOS 27. The data directory path must be short enough that each guest's sockets fit the 103-byte macOS limit; a boot that would exceed it is refused. Images are `darwin/arm64` machine images, not OCI macOS containers. Registry pulls also support experimental complete-machine @@ -35,28 +37,31 @@ labels describe a complete installed machine: | Label | Value | |---|---| -| `io.hypeman.machine-image.version` | `1` | +| `io.hypeman.machine-image.version` | `2` | | `io.hypeman.machine-image.kind` | `macos-image` | -| `io.hypeman.machine-image.disk-format` | `raw` | -| `io.hypeman.machine-image.disk-path` | Relative installed boot disk path | +| `io.hypeman.machine-image.disk-format` | `asif` | +| `io.hypeman.machine-image.disk-path` | Relative ASIF disk path, in every layer | | `io.hypeman.machine-image.aux-path` | Relative auxiliary storage path | | `io.hypeman.machine-image.platform-path` | Relative macOS platform configuration JSON path | -All three files must be distinct, nonempty regular files inside the unpacked +Each layer carries one ASIF disk, and each layer's disk is an overlay on the +previous layer's, so the layers form the boot disk's chain, base first. The +auxiliary storage and platform come from the last layer that has them. A base +image is one layer; an image built on it adds one layer holding only the build's +writes. All files must be distinct, nonempty regular files inside the unpacked image. Platform configuration uses the local bundle's `MacOSImage` schema and is limited to 64 KiB. Absolute paths, traversal and symlink escapes are rejected. The installed disk is materialized directly rather than converted to a Linux root filesystem. Machine layers are unpacked sparsely (zero chunks become holes), accept only regular files (not GNU sparse tar entries) and directories, and are -checked against their diff IDs, so a mostly empty boot disk occupies only its -data. A bundle's layers may decompress to at most 1 TiB in total. The boot disk -is stored as `rootfs.raw`, as for a local import; machine images pulled before -that naming must be deleted and pulled again. The matching auxiliary storage and platform -identity are retained. Do not package live mutable guest storage. +checked against their diff IDs. A bundle's layers may decompress to at most 1 +TiB in total. The chain is stored as `disk.{n}.asif`, as for a local import, and +must open as an ASIF stack. Machine images from before ASIF (version `1`, `raw`) +are refused and must be rebuilt or reimported. The matching auxiliary storage and +platform identity are retained. Do not package live mutable guest storage. This schema is experimental pending reconciliation with other machine-image -platforms. Complete bundles only: no base/delta chain, identity rebinding, or -concurrent-template guarantee. Ordinary OCI transport does not imply sparse +platforms. There is no identity rebinding on pull or concurrent-template guarantee. Ordinary OCI transport does not imply sparse distribution efficiency, boot portability, or container execution semantics. ## API @@ -73,8 +78,10 @@ Use the usual JWT-authenticated API with a VZ-capable, signed build: ``` CPU and memory default to the imported template (minimum 2 CPUs and 4GB). -Each instance gets its own writable complete boot disk (`boot.raw`) and auxiliary -storage; instances created before that naming must be deleted and recreated. No +Each instance gets clones of the image's ASIF layers (`disk.{n}.asif`), its own +auxiliary storage, and an empty writable ASIF overlay (`boot.asif`) above the +layers; the guest's writes only grow the overlay. Instances created before ASIF +must be deleted and recreated. No Linux kernel, initrd, configuration disk, overlay filesystem or balloon is attached. A networked guest gets its own vmnet network (macOS 26 and later): hypeman assigns it a /29 from `network.macos_subnet_cidr` (default diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go index d58a55d27..06dd5e050 100644 --- a/lib/builds/macos_build.go +++ b/lib/builds/macos_build.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "strings" "time" @@ -17,16 +16,20 @@ import ( // MachineBuildRequest is the internal backend contract for one machine build. // The normal build manager owns ID, source staging and publish authorization. A concrete driver must require a provisioned system agent. type MachineBuildRequest struct { - ID string - BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. - SourceHash string // Optional expected lowercase SHA256 of the source archive. - Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. - onStarted func(string) error // Normal manager records the owned builder before provisioning. - onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. - deltaBase *MachineDeltaBase // Set when the base is in the publisher's registry; publishes a derivative. + ID string + BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. + SourceHash string // Optional expected lowercase SHA256 of the source archive. + Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. + onStarted func(string) error // Normal manager records the owned builder before provisioning. + onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. + registryBase *MachineBase // Set when the base is in the publisher's registry; its layers are reused. } type MachineStopReceipt = instances.MacOSBuildExportReceipt + +// MachineStorage is a stopped guest's closed ASIF disk chain, auxiliary storage +// and platform. Publication streams from these files. +type MachineStorage = instances.MacOSStorage type MachinePublication struct{ Reference, Digest string } type MachineBuildResult struct { Publication MachinePublication @@ -44,19 +47,21 @@ type MachineBuildSession interface { Provision(context.Context, string) error // Build-owned staged source, never a client host path. Sanitize(context.Context) error // Remove build credentials, source and secret injection artifacts. // Stop must await actual VMM exit. Only a strict graceful stop may produce a - // receipt that permits export. With force, a failed strict stop falls back to - // a forced stop, which is failure-only cleanup: nothing is exported after it. + // receipt that permits publication. With force, a failed strict stop falls back + // to a forced stop, which is failure-only cleanup: nothing is published after it. Stop(ctx context.Context, force bool) (MachineStopReceipt, error) - Export(context.Context, string) error // Stopped disk.img, aux.img, config.json only. - // Destroy removes instance storage only after proven VMM exit, not the export. + // Storage returns the stopped guest's closed storage. Only a strict stop permits it. + Storage(context.Context) (MachineStorage, error) + // Destroy removes instance storage only after proven VMM exit. Destroy(context.Context) error } type MachineBuildPublisher interface { // Publisher owns registry address, scoped token and builds/{id} destination. // It must stream blobs and commit/verify the manifest last. An ambiguous - // network error is not a verified successful publication. - Publish(context.Context, string, string) (MachinePublication, error) + // network error is not a verified successful publication. A non-nil base is + // the registry image the storage's layers came from. + Publish(ctx context.Context, id string, storage MachineStorage, base *MachineBase) (MachinePublication, error) } type MachineBuildBackend struct { @@ -113,9 +118,9 @@ func normalizeMachinePolicy(policy *BuildPolicy) error { // runPrepared executes only machine-specific phases. The shared build manager // owns resolution, source staging/hash verification, deadline, queue and status. -func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath, workDir string) (result *MachineBuildResult, err error) { - if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(workDir) || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { - return nil, fmt.Errorf("machine driver, publisher, private workspace and verified source required") +func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath string) (result *MachineBuildResult, err error) { + if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { + return nil, fmt.Errorf("machine driver, publisher and verified source required") } if _, bounded := ctx.Deadline(); !bounded { return nil, fmt.Errorf("prepared machine job requires a deadline") @@ -147,16 +152,6 @@ func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildR if e = policy.Validate(); e != nil { return nil, e } - workspace, e := os.MkdirTemp(workDir, "machine-build-") - if e != nil { - return nil, machineFailure("workspace", e) - } - defer func() { - if cleanupErr := os.RemoveAll(workspace); cleanupErr != nil { - err = errors.Join(err, machineFailure("cleanup_workspace", cleanupErr)) - result = nil - } - }() if e = verifyBuildSource(ctx, sourcePath, req.SourceHash); e != nil { return nil, machineFailure("verify_source", e) } @@ -210,27 +205,26 @@ func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildR if e != nil || !receipt.Graceful || !stopped { return nil, machineFailure("graceful_stop", e) } - bundle := filepath.Join(workspace, "bundle") - if e = os.Mkdir(bundle, 0700); e != nil { - return nil, machineFailure("export", e) - } - if e = session.Export(ctx, bundle); e != nil { - return nil, machineFailure("export", e) - } - if e = validateMachineExport(bundle, base.MacOS); e != nil { - return nil, machineFailure("validate_export", e) + storage, e := session.Storage(ctx) + if e != nil { + return nil, machineFailure("storage", e) } - if e = session.Destroy(ctx); e != nil { - return nil, machineFailure("cleanup", e) + if e = validateMachineStorage(storage, base.MacOS); e != nil { + return nil, machineFailure("validate_storage", e) } - destroyed = true if e = ctx.Err(); e != nil { return nil, e } - publication, e := r.publishBuild(ctx, req, bundle) + // The stopped builder's storage is the publication source; the builder is + // destroyed once the registry has verified the result. + publication, e := r.Publisher.Publish(ctx, req.ID, storage, req.registryBase) if e != nil { return nil, machineFailure("publish", e) } + if e = session.Destroy(ctx); e != nil { + return nil, machineFailure("cleanup", e) + } + destroyed = true output, e := images.ParseNormalizedRef(publication.Reference) if e != nil || !output.IsDigest() || !ValidMachineDigest(publication.Digest) || output.Digest() != publication.Digest { return nil, fmt.Errorf("machine publication receipt is unverified") @@ -257,42 +251,29 @@ func ValidMachineDigest(s string) bool { return strings.Trim(s[7:], "0123456789abcdef") == "" } -func validateMachineExport(root string, base *images.MacOSImage) error { - directory, err := os.Lstat(root) - if err != nil || !directory.IsDir() { - return fmt.Errorf("export root must be a real directory") - } - entries, err := os.ReadDir(root) - if err != nil { +// validateMachineStorage checks storage before publication: the platform is valid, +// the files are distinct regular files, and the overlay stacks on its layers as +// ASIF. Against base, the hardware model and resources must match while the +// identity differs. +func validateMachineStorage(storage MachineStorage, base *images.MacOSImage) error { + if err := storage.Platform.Validate(); err != nil { return err } - if len(entries) != 3 { - return fmt.Errorf("export must contain only machine payload files") - } - for _, entry := range entries { - if entry.Name() != "disk.img" && entry.Name() != "aux.img" && entry.Name() != "config.json" { - return fmt.Errorf("unexpected export file") - } - info, err := os.Lstat(filepath.Join(root, entry.Name())) - if err != nil || !info.Mode().IsRegular() { - return fmt.Errorf("export files must be regular") - } + if err := storage.CheckFiles(); err != nil { + return err } - platform, err := images.ValidateMacOSBundle(root) + disk, err := images.OpenMacOSDiskChain(append(append([]string(nil), storage.Layers...), storage.Overlay)) if err != nil { - return err + return fmt.Errorf("machine disk: %w", err) } + disk.Close() if base != nil { + platform := storage.Platform if !bytes.Equal(platform.HardwareModel, base.HardwareModel) || platform.CPUs != base.CPUs || platform.Memory != base.Memory { - return fmt.Errorf("machine export changed hardware model or resources") + return fmt.Errorf("machine storage changed hardware model or resources") } if bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || strings.EqualFold(platform.MAC, base.MAC) { - return fmt.Errorf("machine export kept the base identity") - } - } - for _, entry := range entries { - if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { - return err + return fmt.Errorf("machine storage kept the base identity") } } return nil diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go index 632bd7ee7..3c42f3e54 100644 --- a/lib/builds/macos_build_failure_test.go +++ b/lib/builds/macos_build_failure_test.go @@ -2,11 +2,9 @@ package builds import ( "context" - "encoding/json" "errors" "io" "os" - "path/filepath" "strings" "testing" "time" @@ -128,71 +126,46 @@ func TestMachineBuildPublicationReceipt(t *testing.T) { } } -func TestMachineBuildMalformedExport(t *testing.T) { - for _, kind := range []string{"empty disk", "empty aux", "symlink disk", "hardlink payloads", "truncated config", "oversized config", "resource change", "bad MAC", "identity", "extra file", "unknown metadata"} { +func TestMachineBuildMalformedStorage(t *testing.T) { + for _, kind := range []string{"missing overlay", "empty aux", "symlink overlay", "aliased aux", "unrelated overlay", "raw overlay", "resource change", "bad MAC", "identity"} { t.Run(kind, func(t *testing.T) { f := newMachineFixture() - f.mutateExport = func(root string) error { - disk, aux, config := filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"), filepath.Join(root, "config.json") + f.mutateStorage = func(s *MachineStorage) error { switch kind { - case "empty disk": - return os.Truncate(disk, 0) + case "missing overlay": + return os.Remove(s.Overlay) case "empty aux": - return os.Truncate(aux, 0) - case "symlink disk": - if err := os.Remove(disk); err != nil { + return os.Truncate(s.Aux, 0) + case "symlink overlay": + if err := os.Rename(s.Overlay, s.Overlay+".real"); err != nil { return err } - return os.Symlink(aux, disk) - case "hardlink payloads": - if err := os.Remove(aux); err != nil { + return os.Symlink(s.Overlay+".real", s.Overlay) + case "aliased aux": + if err := os.Remove(s.Aux); err != nil { return err } - return os.Link(disk, aux) - case "truncated config": - return os.WriteFile(config, []byte(`{"hardware_model":`), 0600) - case "oversized config": - return os.WriteFile(config, []byte(strings.Repeat(" ", 65<<10)), 0600) - case "resource change": - data, err := os.ReadFile(config) + return os.Link(s.Overlay, s.Aux) + case "unrelated overlay": + other, err := machineStorageFixture(t.TempDir()) if err != nil { return err } - return os.WriteFile(config, []byte(strings.Replace(string(data), `"cpus":4`, `"cpus":2`, 1)), 0600) + s.Overlay = other.Overlay + case "raw overlay": + return os.WriteFile(s.Overlay, []byte("not an asif image"), 0600) + case "resource change": + s.Platform.CPUs = 2 case "bad MAC": - data, err := os.ReadFile(config) - if err != nil { - return err - } - return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:02", "02:00:00:00:00:02:03:04", 1)), 0600) - case "identity", "unknown metadata": - var raw map[string]any - data, err := os.ReadFile(config) - if err == nil { - err = json.Unmarshal(data, &raw) - } - if err != nil { - return err - } - if kind == "identity" { - raw["machine_identifier"] = f.image.MacOS.MachineIdentifier // kept the base identity - } else { - raw["password"] = "must-not-publish" - } - data, _ = json.Marshal(raw) - return os.WriteFile(config, data, 0600) - case "extra file": - return os.WriteFile(filepath.Join(root, "secret.txt"), []byte("must-not-publish"), 0600) + s.Platform.MAC = "02:00:00:00:00:02:03:04" + case "identity": + s.Platform.MachineIdentifier = f.image.MacOS.MachineIdentifier // kept the base identity } return nil } _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) - if f.publisherCalled { - t.Fatal("malformed export published") - } - if strings.Contains(err.Error(), "must-not-publish") { - t.Fatal("export secret leaked in failure") - } + require.False(t, f.publisherCalled, "malformed storage published") + require.Contains(t, err.Error(), "validate_storage") }) } } diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go index 3794b92ab..01680e859 100644 --- a/lib/builds/macos_build_guest.go +++ b/lib/builds/macos_build_guest.go @@ -16,7 +16,7 @@ import ( type machineExportManager interface { StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) - ExportStoppedMacOSBuildInstance(context.Context, string, string) error + MacOSBuildStorage(context.Context, string) (instances.MacOSStorage, error) } // GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system @@ -166,13 +166,13 @@ func (s *guestMachineSession) Stop(ctx context.Context, force bool) (MachineStop return s.receipt, nil } -// Export copies only from closed storage of a Stopped instance, under the +// Storage returns the closed storage of the Stopped instance, checked under the // instance lock, after a strict graceful receipt and sanitation. -func (s *guestMachineSession) Export(ctx context.Context, destination string) error { +func (s *guestMachineSession) Storage(ctx context.Context) (MachineStorage, error) { if !s.receipt.Graceful || !s.receipt.VMMExited || !s.sanitized { - return fmt.Errorf("sanitized strict stop receipt required") + return MachineStorage{}, fmt.Errorf("sanitized strict stop receipt required") } - return s.exporter.ExportStoppedMacOSBuildInstance(ctx, s.ID(), destination) + return s.exporter.MacOSBuildStorage(ctx, s.ID()) } func (s *guestMachineSession) Destroy(ctx context.Context) error { diff --git a/lib/builds/macos_build_job_test.go b/lib/builds/macos_build_job_test.go index 12394b213..b1bcc616b 100644 --- a/lib/builds/macos_build_job_test.go +++ b/lib/builds/macos_build_job_test.go @@ -33,5 +33,5 @@ func (r *machineRunnerFixture) Run(ctx context.Context, req MachineBuildRequest, if req.SourceHash == "" { req.SourceHash = hash } - return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath, r.WorkDir) + return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath) } diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_build_layer_test.go index 6798c5582..5330dcaea 100644 --- a/lib/builds/macos_build_layer_test.go +++ b/lib/builds/macos_build_layer_test.go @@ -16,47 +16,36 @@ import ( func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { parent := t.TempDir() - bundle := filepath.Join(parent, "bundle") - require.NoError(t, os.Mkdir(bundle, 0700)) - disk := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk - copy(disk[1<<20:], "data") - for name, data := range map[string][]byte{"disk.img": disk, "aux.img": []byte("aux"), "config.json": []byte("{}")} { - require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) - } + disk := filepath.Join(parent, "disk.asif") + data := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk + copy(data[1<<20:], "data") + require.NoError(t, os.WriteFile(disk, data, 0600)) - layer, err := machineTarballLayer(context.Background(), bundle) - require.NoError(t, err) - diffID, err := layer.DiffID() + blob, diffID, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}, {name: "config.json", data: []byte("{}")}}) require.NoError(t, err) require.NotEmpty(t, diffID.Hex) - digest, err := layer.Digest() - require.NoError(t, err) - size, err := layer.Size() - require.NoError(t, err) for range 2 { - body, err := layer.Compressed() + body, err := blob.open() require.NoError(t, err) sum := sha256.New() n, err := io.Copy(sum, body) require.NoError(t, err) require.NoError(t, body.Close()) - require.Equal(t, size, n) - require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) + require.Equal(t, blob.descriptor.Size, n) + require.Equal(t, blob.descriptor.Digest.Hex, hex.EncodeToString(sum.Sum(nil))) } entries, err := os.ReadDir(parent) require.NoError(t, err) - require.Len(t, entries, 1, "nothing is spooled beside the bundle") + require.Len(t, entries, 1, "nothing is spooled beside the disk") } func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { - bundle := t.TempDir() - for name, data := range map[string][]byte{"disk.img": make([]byte, 8<<20), "aux.img": []byte("aux"), "config.json": []byte("{}")} { - require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) - } - layer, err := machineTarballLayer(context.Background(), bundle) + disk := filepath.Join(t.TempDir(), "disk.asif") + require.NoError(t, os.WriteFile(disk, make([]byte, 8<<20), 0600)) + blob, _, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}}) require.NoError(t, err) before := runtime.NumGoroutine() - body, err := layer.Compressed() + body, err := blob.open() require.NoError(t, err) _, err = io.CopyN(io.Discard, body, 16) require.NoError(t, err) diff --git a/lib/builds/macos_build_lifecycle_test.go b/lib/builds/macos_build_lifecycle_test.go index f7359e95c..44926de71 100644 --- a/lib/builds/macos_build_lifecycle_test.go +++ b/lib/builds/macos_build_lifecycle_test.go @@ -50,8 +50,8 @@ type abortExporter struct { func (e abortExporter) StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) { return e.receipt, e.err } -func (abortExporter) ExportStoppedMacOSBuildInstance(context.Context, string, string) error { - return errors.New("abort must not export") +func (abortExporter) MacOSBuildStorage(context.Context, string) (instances.MacOSStorage, error) { + return instances.MacOSStorage{}, errors.New("abort must not publish") } func TestMachineForcedStopCleanup(t *testing.T) { diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go index eebd4b9f6..d825c421a 100644 --- a/lib/builds/macos_build_manager.go +++ b/lib/builds/macos_build_manager.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "net/url" "os" "path/filepath" "strings" @@ -47,18 +48,15 @@ func (m *manager) executeMachineBuild(ctx context.Context, id string, req Create if err != nil { return nil, err } - // An interrupted attempt skips its own cleanup; drop its workspace first. - removeMachineWorkspaces(m.paths.BuildDir(id)) - defer removeMachineWorkspaces(m.paths.BuildDir(id)) job := MachineBuildRequest{ ID: id, BaseImage: req.MachineBaseImage, SourceHash: req.SourceHash, Policy: *policy, - deltaBase: m.machineDeltaBase(backend.Publisher, req.MachineBaseImage, base), - onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, + registryBase: machineRegistryBase(backend.Publisher, req.MachineBaseImage), + onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, onPublished: func(ctx context.Context, publication MachinePublication) error { return m.waitMachinePublication(ctx, id, publication.Digest) }, } - result, err := backend.runPrepared(ctx, job, base, sourcePath, filepath.Dir(filepath.Dir(sourcePath))) + result, err := backend.runPrepared(ctx, job, base, sourcePath) if err != nil { return nil, err } @@ -175,13 +173,31 @@ func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildReques return build } -// removeMachineWorkspaces deletes export workspaces under a build directory. Each -// holds a full exported disk. -func removeMachineWorkspaces(buildDir string) { - matches, _ := filepath.Glob(filepath.Join(buildDir, "machine-build-*")) - for _, dir := range matches { - _ = os.RemoveAll(dir) +// machineRegistryBase returns the base as a registry base when the publisher +// writes to the registry that stores it, so its layers are reused. Any other base +// publishes all of its layers. +func machineRegistryBase(publisher MachineBuildPublisher, baseImage string) *MachineBase { + streaming, ok := publisher.(*StreamingMachinePublisher) + if !ok { + return nil + } + address := streaming.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" { + return nil + } + ref, err := images.ParseNormalizedRef(baseImage) + if err != nil || !ref.IsDigest() { + return nil + } + repo, ok := strings.CutPrefix(ref.Repository(), registry.Host+"/") + if !ok { + return nil } + return &MachineBase{Repo: repo, Digest: ref.Digest()} } // removeMachineSource deletes a machine build's staged source after the job is terminal. diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go index c30799a76..39b29951a 100644 --- a/lib/builds/macos_build_manager_test.go +++ b/lib/builds/macos_build_manager_test.go @@ -58,15 +58,15 @@ type machineJobPublisher struct { sourcePath func(id string) string } -func (p *machineJobPublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { +func (p *machineJobPublisher) Publish(ctx context.Context, id string, storage MachineStorage, base *MachineBase) (MachinePublication, error) { p.publishedID = id if p.sourcePath != nil { if _, err := os.Stat(p.sourcePath(id)); err != nil { return MachinePublication{}, errors.New("source removed before publish") } } - // Reuse the fixture's export assertions with its fixed ID. - receipt, err := p.fixture.Publish(ctx, "build-test", bundle) + // Reuse the fixture's storage assertions with its fixed ID. + receipt, err := p.fixture.Publish(ctx, "build-test", storage, base) if err == nil { receipt.Reference = "localhost:4973/builds/" + id + "@" + receipt.Digest } diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 59be90116..30575268f 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -12,7 +12,6 @@ import ( "net/http" "net/url" "os" - "path/filepath" "strings" "time" @@ -23,8 +22,8 @@ import ( ) // StreamingMachinePublisher targets only the configured internal registry and a -// build-scoped repository. go-containerregistry hashes and compresses the layer -// from a reopenable tar stream, so it is never written to disk. Blobs use the +// build-scoped repository. go-containerregistry hashes and compresses each layer +// from a reopenable tar stream, so nothing is written to disk. Blobs use the // registry's disk-streaming monolithic upload and are independently verified before // the manifest is committed last. // No credentials, redirect destinations or arbitrary output repositories come from the source. @@ -33,26 +32,39 @@ type StreamingMachinePublisher struct { Tokens *RegistryTokenGenerator } -type machineBlob struct { - open func() (io.ReadCloser, error) - descriptor v1.Descriptor +// MachineBase is a pinned base image in the publisher's registry. A machine +// published over it reuses its layers, so only the new overlay is uploaded. +type MachineBase struct { + Repo string // builds/{id} repository holding the base manifest and layers + Digest string // canonical sha256 digest of the base manifest } -// Publish commits bundle as a standalone machine image in builds/{id}. -func (p *StreamingMachinePublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { - return p.publish(ctx, id, bundle, nil) +func (b MachineBase) validate() error { + if !strings.HasPrefix(b.Repo, "builds/") || strings.Contains(b.Repo, "..") || len(b.Repo) > 256 { + return fmt.Errorf("machine base repository must be under builds/") + } + if !ValidMachineDigest(b.Digest) { + return fmt.Errorf("machine base digest must be a canonical sha256 digest") + } + return nil } -// PublishDerived commits bundle as a delta over base. The base layers are mounted -// from base.Repo rather than uploaded; only the delta layer and config are uploaded. -func (p *StreamingMachinePublisher) PublishDerived(ctx context.Context, id, bundle string, base MachineDeltaBase) (MachinePublication, error) { - if err := base.validate(); err != nil { - return MachinePublication{}, err - } - return p.publish(ctx, id, bundle, &base) +// machineLabels describe a published machine image: one ASIF disk per layer, with +// the auxiliary storage and platform in the top layer. +var machineLabels = map[string]string{ + images.MacOSMachineVersionLabel: "2", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "asif", + images.MacOSMachineDiskLabel: "disk.asif", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", } -func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle string, source *MachineDeltaBase) (MachinePublication, error) { +type machineBlob struct { + open func() (io.ReadCloser, error) + descriptor v1.Descriptor +} + +// Publish commits storage as a machine image in builds/{id}. With a base, the +// storage's layers must be the base's and the base's registry layers are reused; +// without one, every layer is uploaded. +func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, storage MachineStorage, base *MachineBase) (MachinePublication, error) { deadline, ok := ctx.Deadline() if !ok || ctx.Err() != nil { return MachinePublication{}, fmt.Errorf("bounded publication required") @@ -60,6 +72,11 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil { return MachinePublication{}, fmt.Errorf("scoped machine publisher required") } + if base != nil { + if err := base.validate(); err != nil { + return MachinePublication{}, err + } + } address := p.RegistryURL if !strings.Contains(address, "://") { address = "https://" + address @@ -71,13 +88,13 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { return MachinePublication{}, fmt.Errorf("plaintext registry must be loopback") } - if err := validateMachineExport(bundle, nil); err != nil { + if err := validateMachineStorage(storage, nil); err != nil { return MachinePublication{}, err } repo := "builds/" + id access := []RepoPermission{{Repo: repo, Scope: "push"}} - if source != nil { - access = append(access, RepoPermission{Repo: source.Repo, Scope: "pull"}) + if base != nil { + access = append(access, RepoPermission{Repo: base.Repo, Scope: "pull"}) } token, err := p.Tokens.GenerateToken(id, access, time.Until(deadline)) if err != nil { @@ -87,14 +104,12 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri transport.Proxy = nil defer transport.CloseIdleConnections() client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} - base := strings.TrimRight(address, "/") + "/v2/" + repo - layers, diffIDs, err := machineLayers(ctx, client, token, address, bundle, source) + target := strings.TrimRight(address, "/") + "/v2/" + repo + layers, diffIDs, err := machineLayers(ctx, client, token, address, storage, base) if err != nil { return MachinePublication{}, err } - config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, Config: v1.Config{Labels: map[string]string{ - images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", - }}} + config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, Config: v1.Config{Labels: machineLabels}} configBytes, err := json.Marshal(config) if err != nil { return MachinePublication{}, err @@ -113,18 +128,21 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri for _, blob := range append([]machineBlob{configBlob}, layers...) { // The blob store is keyed by digest, so a blob the registry already holds // (a base layer, or a retried publish) is not uploaded again. - stored, e := machineBlobStored(ctx, client, token, base, blob.descriptor) + stored, e := machineBlobStored(ctx, client, token, target, blob.descriptor) if e != nil { return MachinePublication{}, e } if stored { continue } + if blob.open == nil { + return MachinePublication{}, fmt.Errorf("base layer %s is missing from the registry", blob.descriptor.Digest) + } body, e := blob.open() if e != nil { return MachinePublication{}, e } - response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, base+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) + response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, target+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) closeErr := body.Close() if e != nil { return MachinePublication{}, e @@ -138,12 +156,11 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri if status != http.StatusCreated { return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") } - if _, e = machineFetch(ctx, client, token, base+"/blobs/"+blob.descriptor.Digest.String(), blob.descriptor.Digest.String(), blob.descriptor.Size); e != nil { + if _, e = machineFetch(ctx, client, token, target+"/blobs/"+blob.descriptor.Digest.String(), blob.descriptor.Digest.String(), blob.descriptor.Size); e != nil { return MachinePublication{}, fmt.Errorf("machine blob verification failed: %w", e) } } - target := base + "/manifests/latest" - response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target, string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) + response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target+"/manifests/latest", string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) if err != nil { return MachinePublication{}, err } @@ -153,40 +170,143 @@ func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle stri return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") } // The digest covers manifestBytes exactly, so a matching fetch is the same bytes. - if _, err = machineFetch(ctx, client, token, base+"/manifests/"+digest, digest, int64(len(manifestBytes))); err != nil { + if _, err = machineFetch(ctx, client, token, target+"/manifests/"+digest, digest, int64(len(manifestBytes))); err != nil { return MachinePublication{}, fmt.Errorf("machine manifest verification failed: %w", err) } return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil } -func machineTarballLayer(ctx context.Context, bundle string) (v1.Layer, error) { - return tarball.LayerFromOpener(func() (io.ReadCloser, error) { +// machineLayers returns a machine image's layer blobs and diff IDs: the base's +// layers, from the registry or the storage's own layer files, then a top layer with +// the overlay, auxiliary storage and platform. +func machineLayers(ctx context.Context, client *http.Client, token, registry string, storage MachineStorage, base *MachineBase) ([]machineBlob, []v1.Hash, error) { + var blobs []machineBlob + var diffIDs []v1.Hash + if base != nil { + descriptors, ids, err := fetchMachineBase(ctx, client, token, registry, *base) + if err != nil { + return nil, nil, err + } + if len(descriptors) != len(storage.Layers) { + return nil, nil, fmt.Errorf("machine base has %d layers but the disk stacks on %d", len(descriptors), len(storage.Layers)) + } + for _, descriptor := range descriptors { + blobs = append(blobs, machineBlob{descriptor: descriptor}) + } + diffIDs = ids + } else { + for _, layer := range storage.Layers { + blob, diffID, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: layer}}) + if err != nil { + return nil, nil, err + } + blobs, diffIDs = append(blobs, blob), append(diffIDs, diffID) + } + } + platform, err := json.Marshal(storage.Platform) + if err != nil { + return nil, nil, err + } + top, diffID, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: storage.Overlay}, {name: "aux.img", path: storage.Aux}, {name: "config.json", data: platform}}) + if err != nil { + return nil, nil, err + } + return append(blobs, top), append(diffIDs, diffID), nil +} + +// fetchMachineBase reads the base manifest and config from the registry, checking +// each against its digest and that the base is an ASIF machine image. It returns +// the base's layer descriptors and diff IDs. +func fetchMachineBase(ctx context.Context, client *http.Client, token, registry string, base MachineBase) ([]v1.Descriptor, []v1.Hash, error) { + root := strings.TrimRight(registry, "/") + "/v2/" + base.Repo + manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+base.Digest, base.Digest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + var manifest v1.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + configDigest := manifest.Config.Digest.String() + configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + var config v1.ConfigFile + if err := json.Unmarshal(configBytes, &config); err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + if config.OS != "darwin" || config.Architecture != "arm64" || len(manifest.Layers) == 0 || len(config.RootFS.DiffIDs) != len(manifest.Layers) { + return nil, nil, fmt.Errorf("base is not a darwin/arm64 machine image with matching layers") + } + for label, want := range machineLabels { + if config.Config.Labels[label] != want { + return nil, nil, fmt.Errorf("base is not an ASIF machine image: %s=%q", label, config.Config.Labels[label]) + } + } + return manifest.Layers, config.RootFS.DiffIDs, nil +} + +// machineTarEntry is one file in a machine layer, from a path or in memory. +type machineTarEntry struct { + name string + path string + data []byte +} + +// machineLayer returns a gzipped layer of entries, with its descriptor and diff ID. +func machineLayer(ctx context.Context, entries []machineTarEntry) (machineBlob, v1.Hash, error) { + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { reader, writer := io.Pipe() - go func() { writer.CloseWithError(writeMachineTar(ctx, bundle, writer)) }() + go func() { writer.CloseWithError(writeMachineTar(ctx, entries, writer)) }() return reader, nil }, tarball.WithMediaType(types.OCILayer), tarball.WithCompressionLevel(gzip.DefaultCompression)) + if err != nil { + return machineBlob{}, v1.Hash{}, err + } + diffID, err := layer.DiffID() + if err != nil { + return machineBlob{}, v1.Hash{}, err + } + digest, err := layer.Digest() + if err != nil { + return machineBlob{}, v1.Hash{}, err + } + size, err := layer.Size() + if err != nil { + return machineBlob{}, v1.Hash{}, err + } + return machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: digest, Size: size}}, diffID, nil } -// writeMachineTar writes a deterministic tar of the exported bundle. -func writeMachineTar(ctx context.Context, bundle string, w io.Writer) error { +// writeMachineTar writes a deterministic tar of entries: fixed modes and no +// timestamps, so regenerating it yields the bytes that were hashed. +func writeMachineTar(ctx context.Context, entries []machineTarEntry, w io.Writer) error { archive := tar.NewWriter(w) - for _, name := range []string{"disk.img", "aux.img", "config.json"} { - source, e := os.Open(filepath.Join(bundle, name)) - if e != nil { - return e + for _, entry := range entries { + if entry.path == "" { + if err := archive.WriteHeader(&tar.Header{Name: entry.name, Mode: 0600, Size: int64(len(entry.data)), Typeflag: tar.TypeReg}); err != nil { + return err + } + if _, err := archive.Write(entry.data); err != nil { + return err + } + continue } - info, e := source.Stat() - if e != nil { - source.Close() - return e + source, err := os.Open(entry.path) + if err != nil { + return err + } + info, err := source.Stat() + if err == nil { + err = archive.WriteHeader(&tar.Header{Name: entry.name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) } - e = archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) - if e == nil { - _, e = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) + if err == nil { + _, err = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) } closeErr := source.Close() - if e != nil { - return e + if err != nil { + return err } if closeErr != nil { return closeErr diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go index c710a49e7..a3660de3f 100644 --- a/lib/builds/macos_build_publisher_test.go +++ b/lib/builds/macos_build_publisher_test.go @@ -2,7 +2,6 @@ package builds_test import ( "context" - "encoding/json" "net/http" "net/http/httptest" "os" @@ -12,6 +11,7 @@ import ( "testing" "time" + "github.com/Code-Hex/vz/v4/diskimagekit" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/middleware" @@ -30,16 +30,23 @@ func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, return &images.Image{Status: images.StatusReady, Digest: digest}, nil } -func machinePublisherBundle(t *testing.T) string { +func machinePublisherPlatform() images.MacOSImage { + return images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} +} + +// machinePublisherStorage writes a standalone ASIF disk and aux file, as a base +// bundle has. +func machinePublisherStorage(t *testing.T) builds.MachineStorage { t.Helper() root := t.TempDir() - platform := images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} - data, e := json.Marshal(platform) + storage := builds.MachineStorage{Overlay: filepath.Join(root, "disk.asif"), Aux: filepath.Join(root, "aux.img"), Platform: machinePublisherPlatform()} + img, e := diskimagekit.CreateASIF(storage.Overlay, (4<<20)/512) require.NoError(t, e) - for name, data := range map[string][]byte{"config.json": data, "disk.img": []byte("synthetic disk"), "aux.img": []byte("synthetic aux")} { - require.NoError(t, os.WriteFile(filepath.Join(root, name), data, 0600)) - } - return root + _, e = img.WriteAt([]byte("synthetic disk"), 0) + require.NoError(t, e) + require.NoError(t, img.Close()) + require.NoError(t, os.WriteFile(storage.Aux, []byte("synthetic aux"), 0600)) + return storage } func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { @@ -68,7 +75,7 @@ func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - result, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + result, e := publisher.Publish(ctx, "qa-build", machinePublisherStorage(t), nil) require.NoError(t, e) require.Contains(t, result.Reference, "/builds/qa-build@sha256:") select { @@ -90,7 +97,51 @@ func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing. publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic secret")} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + _, e := publisher.Publish(ctx, "qa-build", machinePublisherStorage(t), nil) require.Error(t, e) require.False(t, visited.Load()) } + +func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { + imports := &machineImportStub{imported: make(chan string, 2)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads atomic.Int32 + server := httptest.NewServer(middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + uploads.Add(1) + } + reg.Handler().ServeHTTP(w, r) + }))) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + baseStorage := machinePublisherStorage(t) + base, e := publisher.Publish(ctx, "qa-base", baseStorage, nil) + require.NoError(t, e) + require.Equal(t, int32(2), uploads.Load(), "config and one layer") + + // A build stacks a writable overlay on the base disk. + img, e := diskimagekit.Open(baseStorage.Overlay) + require.NoError(t, e) + derived := builds.MachineStorage{Layers: []string{baseStorage.Overlay}, Overlay: filepath.Join(t.TempDir(), "boot.asif"), Aux: filepath.Join(t.TempDir(), "aux.img"), Platform: machinePublisherPlatform()} + require.NoError(t, img.CreateLayer(derived.Overlay, diskimagekit.Overlay)) + _, e = img.WriteAt([]byte("build output"), 1<<20) + require.NoError(t, e) + require.NoError(t, img.Close()) + require.NoError(t, os.WriteFile(derived.Aux, []byte("build aux"), 0600)) + derived.Platform.MachineIdentifier = []byte{3} + + uploads.Store(0) + machineBase := &builds.MachineBase{Repo: "builds/qa-base", Digest: base.Digest} + result, e := publisher.Publish(ctx, "qa-derived", derived, machineBase) + require.NoError(t, e) + require.Equal(t, int32(2), uploads.Load(), "only the config and the overlay layer are uploaded") + require.NotEqual(t, base.Digest, result.Digest) + + derived.Layers = nil + _, e = publisher.Publish(ctx, "qa-derived", derived, machineBase) + require.ErrorContains(t, e, "machine base has 1 layers but the disk stacks on 0") +} diff --git a/lib/builds/macos_build_recovery_test.go b/lib/builds/macos_build_recovery_test.go index 411c12102..fa8096645 100644 --- a/lib/builds/macos_build_recovery_test.go +++ b/lib/builds/macos_build_recovery_test.go @@ -2,7 +2,6 @@ package builds import ( "context" - "os" "path/filepath" "testing" "time" @@ -20,10 +19,6 @@ func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { require.NoError(t, normalizeMachinePolicy(policy)) meta := &buildMetadata{ID: "build-test", Status: status, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: policy}} require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("synthetic compressed-source bytes"))) - // A workspace left by the interrupted attempt. - stale := filepath.Join(m.paths.BuildDir(meta.ID), "machine-build-interrupted") - require.NoError(t, os.MkdirAll(filepath.Join(stale, "bundle"), 0700)) - require.NoError(t, os.WriteFile(filepath.Join(stale, "bundle", "disk.img"), []byte("disk"), 0600)) require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "linux-job", Status: StatusQueued, Request: &CreateBuildRequest{BuilderID: "linux-builder"}})) t.Cleanup(func() { require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) @@ -39,7 +34,6 @@ func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { build, err := m.GetBuild(context.Background(), meta.ID) require.NoError(t, err) require.Equal(t, StatusReady, build.Status, "failure: %v", build.Error) - require.NoDirExists(t, stale, "the interrupted attempt's workspace is removed") require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz"), "source is removed once the image is ready") require.False(t, m.ReadyForBuilds(), "Linux builder image must not be prepared") var starts int diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go index 1b7f9c5ba..c1cfc0a5f 100644 --- a/lib/builds/macos_build_test.go +++ b/lib/builds/macos_build_test.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "os" "path/filepath" @@ -13,6 +12,7 @@ import ( "sync" "testing" + "github.com/Code-Hex/vz/v4/diskimagekit" "github.com/kernel/hypeman/lib/images" ) @@ -26,7 +26,8 @@ type machineFixture struct { stop MachineStopReceipt cancel context.CancelFunc waitForCancel bool - mutateExport func(string) error + mutateStorage func(*MachineStorage) error + storageDir string publicationOverride *MachinePublication publisherCalled bool cleanupContextLive bool @@ -84,29 +85,58 @@ func (f *machineFixture) Stop(ctx context.Context, force bool) (MachineStopRecei } return f.stop, f.record(phase) } -func (f *machineFixture) Export(_ context.Context, root string) error { - if err := f.record("export"); err != nil { - return err +func (f *machineFixture) Storage(context.Context) (MachineStorage, error) { + if err := f.record("storage"); err != nil { + return MachineStorage{}, err + } + dir, err := os.MkdirTemp("", "machine-storage-") + if err != nil { + return MachineStorage{}, err } - // The builder has its own identity; the export carries it, not the base's. - output := *f.image.MacOS - output.MachineIdentifier, output.MAC = []byte{3}, "02:00:00:00:00:02" - data, _ := json.Marshal(output) - for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { - if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { - return err + f.storageDir = dir + storage, err := machineStorageFixture(dir) + if err != nil { + return MachineStorage{}, err + } + storage.Platform = *f.image.MacOS + // The builder has its own identity; its storage carries it, not the base's. + storage.Platform.MachineIdentifier, storage.Platform.MAC = []byte{3}, "02:00:00:00:00:02" + if f.mutateStorage != nil { + if err := f.mutateStorage(&storage); err != nil { + return MachineStorage{}, err } } - if f.mutateExport != nil { - return f.mutateExport(root) + return storage, nil +} + +// machineStorageFixture writes a one-layer ASIF chain, its writable overlay and an +// aux file to dir, as a stopped builder has. The platform is left to the caller. +func machineStorageFixture(dir string) (MachineStorage, error) { + storage := MachineStorage{Layers: []string{filepath.Join(dir, "disk.0.asif")}, Overlay: filepath.Join(dir, "boot.asif"), Aux: filepath.Join(dir, "mac-aux.img")} + img, err := diskimagekit.CreateASIF(storage.Layers[0], (4<<20)/512) + if err != nil { + return storage, err } - return nil + defer img.Close() + if _, err := img.WriteAt([]byte("base"), 0); err != nil { + return storage, err + } + if err := img.CreateLayer(storage.Overlay, diskimagekit.Overlay); err != nil { + return storage, err + } + if _, err := img.WriteAt([]byte("build output"), 1<<20); err != nil { + return storage, err + } + return storage, os.WriteFile(storage.Aux, []byte("aux"), 0600) } func (f *machineFixture) Destroy(ctx context.Context) error { f.cleanupContextLive = ctx.Err() == nil + if f.storageDir != "" { + os.RemoveAll(f.storageDir) + } return f.record("destroy") } -func (f *machineFixture) Publish(ctx context.Context, id, root string) (MachinePublication, error) { +func (f *machineFixture) Publish(ctx context.Context, id string, storage MachineStorage, _ *MachineBase) (MachinePublication, error) { f.publisherCalled = true if err := ctx.Err(); err != nil { return MachinePublication{}, err @@ -114,10 +144,9 @@ func (f *machineFixture) Publish(ctx context.Context, id, root string) (MachineP if id != "build-test" { return MachinePublication{}, errors.New("wrong job namespace") } - for _, name := range []string{"disk.img", "aux.img", "config.json"} { - info, err := os.Stat(filepath.Join(root, name)) - if err != nil || info.Mode().Perm() != 0600 { - return MachinePublication{}, errors.New("unsafe export mode") + for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { + if _, err := os.Stat(path); err != nil { + return MachinePublication{}, errors.New("storage missing at publish") } } publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} @@ -141,7 +170,7 @@ func TestMachineBuildPublicationOrder(t *testing.T) { if err != nil { t.Fatal(err) } - expected := []string{"start", "provision", "sanitize", "stop", "export", "destroy", "publish"} + expected := []string{"start", "provision", "sanitize", "stop", "storage", "publish", "destroy"} if !reflect.DeepEqual(f.calls, expected) { t.Fatalf("wrong phase order: %v", f.calls) } @@ -165,8 +194,7 @@ func TestMachineBuildFailuresDoNotPublish(t *testing.T) { {name: "provision", phase: "provision"}, {name: "sanitize", phase: "sanitize"}, {name: "stop", phase: "stop"}, - {name: "export", phase: "export"}, - {name: "destroy", phase: "destroy"}, + {name: "storage", phase: "storage"}, {name: "unconfirmed stop", receipt: &MachineStopReceipt{}}, {name: "forced stop", receipt: &MachineStopReceipt{VMMExited: true}}, } { @@ -184,8 +212,8 @@ func TestMachineBuildFailuresDoNotPublish(t *testing.T) { t.Fatal("raw guest error leaked") } for _, phase := range f.calls { - if phase == "export" && tc.receipt != nil { - t.Fatal("unconfirmed stop exported") + if phase == "storage" && tc.receipt != nil { + t.Fatal("unconfirmed stop reached storage") } if phase == "destroy" && tc.receipt != nil && !tc.receipt.VMMExited { t.Fatal("destroyed unconfirmed live storage") diff --git a/lib/builds/macos_delta_publish.go b/lib/builds/macos_delta_publish.go deleted file mode 100644 index f42e2282a..000000000 --- a/lib/builds/macos_delta_publish.go +++ /dev/null @@ -1,173 +0,0 @@ -package builds - -import ( - "compress/gzip" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "path/filepath" - "strings" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/kernel/hypeman/lib/images" -) - -// maxMachineMetadataBytes bounds a base manifest or config fetched for a delta. -const maxMachineMetadataBytes = 4 << 20 - -// MachineDeltaBase is a pinned base that a derivative is published against. Its -// layers are mounted from Repo, so the base must already be in the registry the -// publisher writes to. -type MachineDeltaBase struct { - Repo string // builds/{id} repository holding the base manifest and layers - Digest string // canonical sha256 digest of the base manifest - DiskPath string // absolute path of the base boot disk on this host -} - -func (m MachineDeltaBase) validate() error { - if !strings.HasPrefix(m.Repo, "builds/") || strings.Contains(m.Repo, "..") || len(m.Repo) > 256 { - return fmt.Errorf("delta base repository must be under builds/") - } - if !ValidMachineDigest(m.Digest) { - return fmt.Errorf("delta base digest must be a canonical sha256 digest") - } - if !filepath.IsAbs(m.DiskPath) { - return fmt.Errorf("delta base disk path must be absolute") - } - return nil -} - -// machineLayers returns a machine image's layer blobs and their diff IDs. A -// standalone image has one uploaded layer. A derivative lists the base's layers, -// which the registry already holds, and uploads one delta layer that patches the -// base disk. -func machineLayers(ctx context.Context, client *http.Client, token, registry, bundle string, source *MachineDeltaBase) ([]machineBlob, []v1.Hash, error) { - if source == nil { - layer, err := machineTarballLayer(ctx, bundle) - if err != nil { - return nil, nil, err - } - diffID, err := layer.DiffID() - if err != nil { - return nil, nil, err - } - digest, err := layer.Digest() - if err != nil { - return nil, nil, err - } - size, err := layer.Size() - if err != nil { - return nil, nil, err - } - blob := machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: digest, Size: size}} - return []machineBlob{blob}, []v1.Hash{diffID}, nil - } - baseLayers, baseDiffIDs, err := fetchMachineSource(ctx, client, token, registry, source) - if err != nil { - return nil, nil, err - } - blobs := make([]machineBlob, 0, len(baseLayers)+1) - for _, descriptor := range baseLayers { - blobs = append(blobs, machineBlob{descriptor: descriptor}) - } - diskPath := filepath.Join(bundle, "disk.img") - delta, err := images.ComputeMacOSDelta(source.DiskPath, diskPath) - if err != nil { - return nil, nil, fmt.Errorf("compute disk delta: %w", err) - } - layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { - reader, writer := io.Pipe() - go func() { writer.CloseWithError(images.WriteMacOSDeltaTar(writer, delta, diskPath, bundle)) }() - return reader, nil - }, tarball.WithMediaType(images.MacOSDeltaMediaType), tarball.WithCompressionLevel(gzip.DefaultCompression)) - if err != nil { - return nil, nil, err - } - diffID, err := layer.DiffID() - if err != nil { - return nil, nil, err - } - digest, err := layer.Digest() - if err != nil { - return nil, nil, err - } - size, err := layer.Size() - if err != nil { - return nil, nil, err - } - blobs = append(blobs, machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: images.MacOSDeltaMediaType, Digest: digest, Size: size}}) - return blobs, append(baseDiffIDs, diffID), nil -} - -// fetchMachineSource reads the base manifest and config from the registry, checking -// each against its digest. It returns the base's layer descriptors and diff IDs. -func fetchMachineSource(ctx context.Context, client *http.Client, token, registry string, source *MachineDeltaBase) ([]v1.Descriptor, []v1.Hash, error) { - root := strings.TrimRight(registry, "/") + "/v2/" + source.Repo - manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+source.Digest, source.Digest, 0) - if err != nil { - return nil, nil, fmt.Errorf("base manifest: %w", err) - } - var manifest v1.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, nil, fmt.Errorf("base manifest: %w", err) - } - configDigest := manifest.Config.Digest.String() - configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest, 0) - if err != nil { - return nil, nil, fmt.Errorf("base config: %w", err) - } - var config v1.ConfigFile - if err := json.Unmarshal(configBytes, &config); err != nil { - return nil, nil, fmt.Errorf("base config: %w", err) - } - if config.OS != "darwin" || config.Architecture != "arm64" || len(manifest.Layers) == 0 || len(config.RootFS.DiffIDs) != len(manifest.Layers) { - return nil, nil, fmt.Errorf("base is not a darwin/arm64 machine image with matching layers") - } - return manifest.Layers, config.RootFS.DiffIDs, nil -} - -// machineDeltaBase returns the base as a delta source when the publisher writes to -// the registry that stores the base. Any other base publishes standalone. -func (m *manager) machineDeltaBase(publisher MachineBuildPublisher, baseImage string, base *images.Image) *MachineDeltaBase { - publisherWithRegistry, ok := publisher.(*StreamingMachinePublisher) - if !ok || base == nil { - return nil - } - address := publisherWithRegistry.RegistryURL - if !strings.Contains(address, "://") { - address = "https://" + address - } - registry, err := url.Parse(address) - if err != nil || registry.Host == "" { - return nil - } - ref, err := images.ParseNormalizedRef(baseImage) - if err != nil || !ref.IsDigest() { - return nil - } - prefix := registry.Host + "/" - if !strings.HasPrefix(ref.Repository(), prefix) { - return nil - } - disk, err := images.GetDiskPath(m.paths, baseImage, base.Digest) - if err != nil { - return nil - } - return &MachineDeltaBase{Repo: strings.TrimPrefix(ref.Repository(), prefix), Digest: base.Digest, DiskPath: disk} -} - -// publishBuild publishes bundle as a derivative when the base is a registry source, -// and as a standalone image otherwise. -func (r *MachineBuildBackend) publishBuild(ctx context.Context, req MachineBuildRequest, bundle string) (MachinePublication, error) { - if derived, ok := r.Publisher.(interface { - PublishDerived(context.Context, string, string, MachineDeltaBase) (MachinePublication, error) - }); ok && req.deltaBase != nil { - return derived.PublishDerived(ctx, req.ID, bundle, *req.deltaBase) - } - return r.Publisher.Publish(ctx, req.ID, bundle) -} diff --git a/lib/builds/macos_delta_publish_test.go b/lib/builds/macos_delta_publish_test.go deleted file mode 100644 index 743f6167f..000000000 --- a/lib/builds/macos_delta_publish_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package builds_test - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - "time" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/kernel/hypeman/lib/builds" - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/middleware" - "github.com/kernel/hypeman/lib/paths" - "github.com/kernel/hypeman/lib/registry" - "github.com/stretchr/testify/require" -) - -const deltaTestSecret = "synthetic registry secret" - -type deltaTestRegistry struct { - server *httptest.Server - tokens *builds.RegistryTokenGenerator - uploads atomic.Int32 - puts atomic.Int32 -} - -func newDeltaTestRegistry(t *testing.T) *deltaTestRegistry { - t.Helper() - imports := &machineImportStub{imported: make(chan string, 16)} - reg, err := registry.New(paths.New(t.TempDir()), imports) - require.NoError(t, err) - dr := &deltaTestRegistry{tokens: builds.NewRegistryTokenGenerator(deltaTestSecret)} - handler := middleware.JwtAuth(deltaTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && r.URL.Query().Get("digest") != "" { - dr.uploads.Add(1) - } - if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { - dr.puts.Add(1) - } - reg.Handler().ServeHTTP(w, r) - })) - dr.server = httptest.NewServer(handler) - t.Cleanup(dr.server.Close) - // Each manifest PUT starts a conversion goroutine that writes the OCI layout - // after the response. Its import call is the last step, so wait for every one - // before the TempDir cleanup removes the registry's data. - t.Cleanup(func() { - for range dr.puts.Load() { - select { - case <-imports.imported: - case <-time.After(10 * time.Second): - return - } - } - }) - return dr -} - -// deltaTestDisk is a disk of eight 64 KiB chunks, each a distinct byte. -func deltaTestDisk() []byte { - disk := make([]byte, 8*64<<10) - for i := range 8 { - for j := range 64 << 10 { - disk[i*64<<10+j] = byte(i + 1) - } - } - return disk -} - -func deltaTestBundle(t *testing.T, disk []byte, aux string) string { - t.Helper() - bundle := machinePublisherBundle(t) - require.NoError(t, os.WriteFile(filepath.Join(bundle, "disk.img"), disk, 0600)) - require.NoError(t, os.WriteFile(filepath.Join(bundle, "aux.img"), []byte(aux), 0600)) - return bundle -} - -// pullManifest fetches a manifest from the registry with a pull-scoped token. -func (dr *deltaTestRegistry) pullManifest(t *testing.T, repo, digest string) v1.Manifest { - t.Helper() - token, err := dr.tokens.GenerateToken("reader", []builds.RepoPermission{{Repo: repo, Scope: "pull"}}, time.Minute) - require.NoError(t, err) - request, err := http.NewRequest(http.MethodGet, dr.server.URL+"/v2/"+repo+"/manifests/"+digest, nil) - require.NoError(t, err) - request.Header.Set("Authorization", "Bearer "+token) - request.Header.Set("Accept", string(types.OCIManifestSchema1)) - response, err := http.DefaultClient.Do(request) - require.NoError(t, err) - defer response.Body.Close() - require.Equal(t, http.StatusOK, response.StatusCode) - var manifest v1.Manifest - require.NoError(t, json.NewDecoder(response.Body).Decode(&manifest)) - return manifest -} - -func TestStreamingMachinePublisherDerivedReusesStoredBaseLayers(t *testing.T) { - dr := newDeltaTestRegistry(t) - publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - baseBundle := deltaTestBundle(t, deltaTestDisk(), "base aux") - base, err := publisher.Publish(ctx, "base-x", baseBundle) - require.NoError(t, err) - baseManifest := dr.pullManifest(t, "builds/base-x", base.Digest) - - derived := deltaTestDisk() - for j := range 64 << 10 { - derived[3*64<<10+j] = 0xAA - } - derivedBundle := deltaTestBundle(t, derived, "derived aux") - dr.uploads.Store(0) - result, err := publisher.PublishDerived(ctx, "derived-x", derivedBundle, builds.MachineDeltaBase{ - Repo: "builds/base-x", - Digest: base.Digest, - DiskPath: filepath.Join(baseBundle, "disk.img"), - }) - require.NoError(t, err) - require.Equal(t, int32(2), dr.uploads.Load(), "only the config and the delta layer are uploaded; the base layer is already stored") - - derivedManifest := dr.pullManifest(t, "builds/derived-x", result.Digest) - require.Len(t, derivedManifest.Layers, 2) - require.Equal(t, baseManifest.Layers[0].Digest, derivedManifest.Layers[0].Digest) - require.Equal(t, images.MacOSDeltaMediaType, derivedManifest.Layers[1].MediaType) -} - -func TestStreamingMachinePublisherDerivedRejectsMissingBase(t *testing.T) { - dr := newDeltaTestRegistry(t) - publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - bundle := deltaTestBundle(t, deltaTestDisk(), "aux") - missing := fmt.Sprintf("sha256:%064x", 1) - _, err := publisher.PublishDerived(ctx, "derived-y", bundle, builds.MachineDeltaBase{ - Repo: "builds/never-published", - Digest: missing, - DiskPath: filepath.Join(bundle, "disk.img"), - }) - require.Error(t, err) -} - -func TestStreamingMachinePublisherDerivedRejectsNonBaseRepo(t *testing.T) { - dr := newDeltaTestRegistry(t) - publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} - bundle := deltaTestBundle(t, deltaTestDisk(), "aux") - _, err := publisher.PublishDerived(context.Background(), "derived-z", bundle, builds.MachineDeltaBase{ - Repo: "library/alpine", - Digest: fmt.Sprintf("sha256:%064x", 2), - DiskPath: filepath.Join(bundle, "disk.img"), - }) - require.ErrorContains(t, err, "under builds/") -} diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go index 078255fef..dad1345fe 100644 --- a/lib/builds/macos_registry.go +++ b/lib/builds/macos_registry.go @@ -13,6 +13,9 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) +// maxMachineMetadataBytes bounds a base manifest or config fetched from the registry. +const maxMachineMetadataBytes = 4 << 20 + // machineRegistryRequest sends one authenticated registry request. The caller // closes the response body. func machineRegistryRequest(ctx context.Context, client *http.Client, token, method, target, media string, body io.Reader, size int64) (*http.Response, error) { diff --git a/lib/diskutilization/diskutilization.go b/lib/diskutilization/diskutilization.go index 996e81642..e209c96db 100644 --- a/lib/diskutilization/diskutilization.go +++ b/lib/diskutilization/diskutilization.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "syscall" "github.com/kernel/hypeman/lib/paths" @@ -56,7 +57,7 @@ func Collect(p *paths.Paths) (Breakdown, error) { return false } name := entry.Name() - return name == "rootfs.erofs" || name == "rootfs.ext4" || name == "rootfs.raw" + return name == "rootfs.erofs" || name == "rootfs.ext4" || (strings.HasPrefix(name, "disk.") && strings.HasSuffix(name, ".asif")) }) if err != nil { return Breakdown{}, err diff --git a/lib/diskutilization/diskutilization_test.go b/lib/diskutilization/diskutilization_test.go index fab0a54bc..c84fabb9e 100644 --- a/lib/diskutilization/diskutilization_test.go +++ b/lib/diskutilization/diskutilization_test.go @@ -22,7 +22,7 @@ func TestCollect_UsesAllocatedBytesAndClassifiesSnapshots(t *testing.T) { require.NoError(t, createSparseTestFile(imagePath, 32*1024*1024, []sparseWrite{ {offset: 0, data: bytes.Repeat([]byte("i"), 4096)}, })) - machineImagePath := filepath.Join(p.ImagesDir(), "content", "def", "rootfs.raw") + machineImagePath := filepath.Join(p.ImagesDir(), "content", "def", "disk.0.asif") require.NoError(t, createSparseTestFile(machineImagePath, 32*1024*1024, []sparseWrite{ {offset: 0, data: bytes.Repeat([]byte("m"), 4096)}, })) diff --git a/lib/images/finalize.go b/lib/images/finalize.go index c16396702..2d77cffff 100644 --- a/lib/images/finalize.go +++ b/lib/images/finalize.go @@ -21,6 +21,9 @@ type stagedImageFiles struct { func (s stagedImageFiles) remove() { os.Remove(s.disk) if s.macos != nil { + for _, layer := range s.macos.layers { + os.Remove(layer) + } os.Remove(s.macos.aux) } } diff --git a/lib/images/macos_asif.go b/lib/images/macos_asif.go new file mode 100644 index 000000000..6aa28ac2f --- /dev/null +++ b/lib/images/macos_asif.go @@ -0,0 +1,127 @@ +package images + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/paths" +) + +// A macOS machine image's boot disk is an ASIF chain: disk.0.asif is the base and +// each later file is an overlay on the one before it. A pulled image has one file +// per OCI layer; an imported image has one. +func macOSDiskLayerFile(i int) string { + return fmt.Sprintf("disk.%d.asif", i) +} + +// MacOSDiskChain returns the boot disk chain of a ready macOS image, base first. +func MacOSDiskChain(p *paths.Paths, imageName, digest string) ([]string, error) { + ref, err := ParseNormalizedRef(imageName) + if err != nil { + return nil, fmt.Errorf("parse image name: %w", err) + } + layout := resolveImageLayout(p, ref.Repository(), strings.TrimPrefix(digest, "sha256:")) + meta, err := readMetadataAt(layout) + if err != nil { + return nil, err + } + if meta.MacOS == nil || meta.MacOSDiskLayers < 1 { + return nil, fmt.Errorf("%w: image is not a macOS disk chain", ErrInvalidPlatform) + } + chain := make([]string, meta.MacOSDiskLayers) + for i := range chain { + chain[i] = filepath.Join(layout.dir, macOSDiskLayerFile(i)) + } + return chain, nil +} + +// OpenMacOSDiskChain opens an ASIF chain read-only, base first. Each layer must name +// the one below it as its parent. +func OpenMacOSDiskChain(chain []string) (*diskimagekit.DiskImage, error) { + if len(chain) == 0 { + return nil, fmt.Errorf("empty macOS disk chain") + } + img, err := diskimagekit.Open(chain[0], diskimagekit.WithOpenMode(diskimagekit.ReadOnly)) + if err != nil { + return nil, fmt.Errorf("open %s: %w", filepath.Base(chain[0]), err) + } + for _, layer := range chain[1:] { + if err := img.Append(layer, diskimagekit.WithOpenMode(diskimagekit.ReadOnly)); err != nil { + img.Close() + return nil, fmt.Errorf("stack %s: %w", filepath.Base(layer), err) + } + } + for i, info := range img.Layers() { + if info.Format != diskimagekit.ASIF { + img.Close() + return nil, fmt.Errorf("macOS disk layer %d is not ASIF", i) + } + } + return img, nil +} + +// macOSChainSize validates a chain and returns its logical size in bytes. +func macOSChainSize(chain []string) (int64, error) { + img, err := OpenMacOSDiskChain(chain) + if err != nil { + return 0, err + } + defer img.Close() + return img.Size(), nil +} + +// ConvertRawToASIF writes a raw disk into a new ASIF image at dst with the same +// logical size. All-zero chunks are skipped, so they stay unallocated. +func ConvertRawToASIF(ctx context.Context, src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + const block = 512 + if info.Size() == 0 || info.Size()%block != 0 { + return fmt.Errorf("raw disk size %d is not a positive multiple of %d", info.Size(), block) + } + out, err := diskimagekit.CreateASIF(dst, info.Size()/block) + if err != nil { + return err + } + const chunk = 1 << 20 + buf, zero := make([]byte, chunk), make([]byte, chunk) + for offset := int64(0); offset < info.Size(); offset += chunk { + if err = ctx.Err(); err != nil { + break + } + n, readErr := io.ReadFull(io.NewSectionReader(in, offset, chunk), buf) + if readErr != nil && readErr != io.ErrUnexpectedEOF { + err = readErr + break + } + if bytes.Equal(buf[:n], zero[:n]) { + continue + } + if _, err = out.WriteAt(buf[:n], offset); err != nil { + break + } + } + if err == nil { + err = out.Sync() + } + if closeErr := out.Close(); err == nil { + err = closeErr + } + if err != nil { + os.Remove(dst) + } + return err +} diff --git a/lib/images/macos_asif_test.go b/lib/images/macos_asif_test.go new file mode 100644 index 000000000..2f21e4bd5 --- /dev/null +++ b/lib/images/macos_asif_test.go @@ -0,0 +1,57 @@ +package images + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConvertRawToASIF(t *testing.T) { + dir := t.TempDir() + raw := filepath.Join(dir, "disk.img") + data := make([]byte, 256<<20) + copy(data, "first chunk") + copy(data[100<<20+123:], "middle") + copy(data[len(data)-4:], "tail") + require.NoError(t, os.WriteFile(raw, data, 0600)) + asif := filepath.Join(dir, "disk.asif") + require.NoError(t, ConvertRawToASIF(context.Background(), raw, asif)) + + img, err := OpenMacOSDiskChain([]string{asif}) + require.NoError(t, err) + defer img.Close() + require.Equal(t, int64(len(data)), img.Size()) + got := make([]byte, len(data)) + _, err = img.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, data, got) + info, err := os.Stat(asif) + require.NoError(t, err) + require.Less(t, info.Size(), int64(len(data))/4, "zero chunks are not allocated") +} + +func TestConvertRawToASIFRejectsPartialBlocksAndCancellation(t *testing.T) { + dir := t.TempDir() + raw := filepath.Join(dir, "disk.img") + require.NoError(t, os.WriteFile(raw, make([]byte, 1000), 0600)) + require.Error(t, ConvertRawToASIF(context.Background(), raw, filepath.Join(dir, "a.asif"))) + + require.NoError(t, os.WriteFile(raw, make([]byte, 4<<20), 0600)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := filepath.Join(dir, "b.asif") + require.ErrorIs(t, ConvertRawToASIF(ctx, raw, out), context.Canceled) + require.NoFileExists(t, out, "a failed conversion leaves no image") +} + +func TestOpenMacOSDiskChainRejectsUnrelatedOverlay(t *testing.T) { + _, overlay := macOSFixtureChain(t, t.TempDir()) + otherBase, _ := macOSFixtureChain(t, t.TempDir()) + _, err := OpenMacOSDiskChain([]string{otherBase, overlay}) + require.Error(t, err) + _, err = OpenMacOSDiskChain(nil) + require.Error(t, err) +} diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 73b8a4e0a..4d0998e56 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -4,19 +4,22 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" "os/exec" "path/filepath" + "strconv" ) const maxMacOSMetadataBytes = 64 << 10 -// macOSMachinePayload is a complete cold-boot bundle, not a container rootfs. -// Admission of live storage and export sanitation belong to the caller. +// macOSMachinePayload is a complete cold-boot bundle, not a container rootfs: an +// ASIF boot disk chain, base first, with the auxiliary storage and platform of its +// top layer. Admission of live storage and export sanitation belong to the caller. type macOSMachinePayload struct { - Disk string + Disks []string Aux string Platform *MacOSImage } @@ -34,10 +37,11 @@ func RequireClosedFile(ctx context.Context, path string) error { return nil } -// ValidateMacOSBundle validates a fixed-layout export, rejecting unknown metadata -// fields so an accidental credential field cannot become part of an image. +// ValidateMacOSBundle validates a fixed-layout export of one ASIF disk, rejecting +// unknown metadata fields so an accidental credential field cannot become part of +// an image. func ValidateMacOSBundle(root string) (*MacOSImage, error) { - bundle, err := readMacOSMachineBundle(root, "disk.img", "aux.img", "config.json", true) + bundle, err := readMacOSMachineBundle(root, "disk.asif", "aux.img", "config.json", true) if err != nil { return nil, err } @@ -61,6 +65,52 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b if sameFile(disk, aux) || sameFile(disk, config) || sameFile(aux, config) { return nil, fmt.Errorf("machine bundle files must be distinct") } + platform, err := readMacOSPlatform(config, strict) + if err != nil { + return nil, err + } + return &macOSMachinePayload{Disks: []string{disk}, Aux: aux, Platform: platform}, nil +} + +// readMacOSMachineLayers reads a pulled image unpacked one layer per numbered +// directory. Every layer holds a disk; the auxiliary storage and platform come from +// the highest layer that has them. +func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSMachinePayload, error) { + var payload macOSMachinePayload + var config string + for i := 0; ; i++ { + dir := filepath.Join(root, strconv.Itoa(i)) + if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { + break + } + disk, err := machineBundleFile(dir, diskName) + if err != nil { + return nil, fmt.Errorf("machine layer %d: %w", i, err) + } + payload.Disks = append(payload.Disks, disk) + if aux, err := machineBundleFile(dir, auxName); err == nil { + payload.Aux = aux + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("machine layer %d: %w", i, err) + } + if c, err := machineBundleFile(dir, configName); err == nil { + config = c + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("machine layer %d: %w", i, err) + } + } + if len(payload.Disks) == 0 || payload.Aux == "" || config == "" { + return nil, fmt.Errorf("machine image needs a disk in every layer and auxiliary storage and platform in one") + } + platform, err := readMacOSPlatform(config, false) + if err != nil { + return nil, err + } + payload.Platform = platform + return &payload, nil +} + +func readMacOSPlatform(config string, strict bool) (*MacOSImage, error) { file, err := os.Open(config) if err != nil { return nil, err @@ -87,7 +137,7 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b if err := platform.Validate(); err != nil { return nil, err } - return &macOSMachinePayload{Disk: disk, Aux: aux, Platform: &platform}, nil + return &platform, nil } func machineBundleFile(root, relative string) (string, error) { diff --git a/lib/images/macos_bundle_test.go b/lib/images/macos_bundle_test.go index 0a5d883a0..409b7d1ae 100644 --- a/lib/images/macos_bundle_test.go +++ b/lib/images/macos_bundle_test.go @@ -17,20 +17,20 @@ func TestMacOSBundleValidation(t *testing.T) { for _, name := range []string{"valid", "empty-disk", "hardlink", "escape", "oversized", "truncated", "trailing", "unknown-field", "invalid-mac"} { t.Run(name, func(t *testing.T) { root := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(root, "disk.img"), []byte("disk"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "disk.asif"), []byte("disk"), 0600)) require.NoError(t, os.WriteFile(filepath.Join(root, "aux.img"), []byte("aux"), 0600)) config := append([]byte(nil), data...) switch name { case "empty-disk": - require.NoError(t, os.WriteFile(filepath.Join(root, "disk.img"), nil, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "disk.asif"), nil, 0600)) case "hardlink": require.NoError(t, os.Remove(filepath.Join(root, "aux.img"))) - require.NoError(t, os.Link(filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"))) + require.NoError(t, os.Link(filepath.Join(root, "disk.asif"), filepath.Join(root, "aux.img"))) case "escape": outside := filepath.Join(t.TempDir(), "disk") require.NoError(t, os.WriteFile(outside, []byte("outside"), 0600)) - require.NoError(t, os.Remove(filepath.Join(root, "disk.img"))) - require.NoError(t, os.Symlink(outside, filepath.Join(root, "disk.img"))) + require.NoError(t, os.Remove(filepath.Join(root, "disk.asif"))) + require.NoError(t, os.Symlink(outside, filepath.Join(root, "disk.asif"))) case "oversized": config = []byte(strings.Repeat(" ", maxMacOSMetadataBytes+1)) case "truncated": @@ -51,7 +51,7 @@ func TestMacOSBundleValidation(t *testing.T) { require.Error(t, err) } if name == "unknown-field" { - bundle, err := readMacOSMachineBundle(root, "disk.img", "aux.img", "config.json", false) + bundle, err := readMacOSMachineBundle(root, "disk.asif", "aux.img", "config.json", false) require.NoError(t, err, "import/OCI compatibility permits unknown non-export metadata") require.Equal(t, &platform, bundle.Platform) } diff --git a/lib/images/macos_delta.go b/lib/images/macos_delta.go deleted file mode 100644 index 17255ac55..000000000 --- a/lib/images/macos_delta.go +++ /dev/null @@ -1,261 +0,0 @@ -package images - -import ( - "archive/tar" - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/v1/types" -) - -// MacOSDeltaMediaType marks a machine layer that patches the boot disk unpacked -// from the layers beneath it. Its tar holds, in this order, delta.json, the changed -// chunks of disk.img, and aux.img and config.json in full. -const MacOSDeltaMediaType types.MediaType = "application/vnd.hypeman.machine.disk-delta.v1.tar+gzip" - -// MacOSDeltaChunkSize is the granularity at which a delta records changed bytes. -const MacOSDeltaChunkSize = 64 << 10 - -const ( - macOSDeltaVersion = 1 - macOSDeltaManifestEntry = "delta.json" - macOSDeltaPatchEntry = "disk.img.patch" - macOSDeltaDisk = "disk.img" - macOSDeltaAux = "aux.img" - macOSDeltaConfig = "config.json" - maxMacOSDeltaManifestBytes = 1 << 20 - macOSDeltaEntries = 4 -) - -// MacOSDelta lists the chunks of a boot disk that differ from the disk beneath it. -// Size is the disk's logical size, which the base disk must match. -type MacOSDelta struct { - Version int `json:"version"` - ChunkSize int64 `json:"chunk_size"` - Size int64 `json:"size"` - Chunks []int64 `json:"chunks"` -} - -// Validate checks a delta before any chunk is applied: a supported version and -// chunk size, a whole number of chunks, and changed chunks in ascending order. -func (d *MacOSDelta) Validate() error { - if d.Version != macOSDeltaVersion || d.ChunkSize != MacOSDeltaChunkSize { - return fmt.Errorf("unsupported machine disk delta version %d with chunk size %d", d.Version, d.ChunkSize) - } - if d.Size <= 0 || d.Size%d.ChunkSize != 0 { - return fmt.Errorf("machine disk delta size %d is not a positive multiple of %d", d.Size, d.ChunkSize) - } - count := d.Size / d.ChunkSize - previous := int64(-1) - for _, index := range d.Chunks { - if index <= previous || index >= count { - return fmt.Errorf("machine disk delta chunk %d is out of order or out of range", index) - } - previous = index - } - return nil -} - -// ComputeMacOSDelta compares a derived boot disk with its base chunk by chunk. Both -// files must be the same size and a whole number of chunks. -func ComputeMacOSDelta(basePath, derivedPath string) (MacOSDelta, error) { - base, err := os.Open(basePath) - if err != nil { - return MacOSDelta{}, err - } - defer base.Close() - derived, err := os.Open(derivedPath) - if err != nil { - return MacOSDelta{}, err - } - defer derived.Close() - baseInfo, err := base.Stat() - if err != nil { - return MacOSDelta{}, err - } - derivedInfo, err := derived.Stat() - if err != nil { - return MacOSDelta{}, err - } - if baseInfo.Size() != derivedInfo.Size() { - return MacOSDelta{}, fmt.Errorf("derived disk is %d bytes, base disk is %d", derivedInfo.Size(), baseInfo.Size()) - } - delta := MacOSDelta{Version: macOSDeltaVersion, ChunkSize: MacOSDeltaChunkSize, Size: derivedInfo.Size(), Chunks: []int64{}} - if err := delta.Validate(); err != nil { - return MacOSDelta{}, err - } - baseReader := bufio.NewReaderSize(base, 4<<20) - derivedReader := bufio.NewReaderSize(derived, 4<<20) - baseChunk := make([]byte, MacOSDeltaChunkSize) - derivedChunk := make([]byte, MacOSDeltaChunkSize) - for index := int64(0); index*MacOSDeltaChunkSize < delta.Size; index++ { - if _, err := io.ReadFull(baseReader, baseChunk); err != nil { - return MacOSDelta{}, fmt.Errorf("read base disk: %w", err) - } - if _, err := io.ReadFull(derivedReader, derivedChunk); err != nil { - return MacOSDelta{}, fmt.Errorf("read derived disk: %w", err) - } - if !bytes.Equal(baseChunk, derivedChunk) { - delta.Chunks = append(delta.Chunks, index) - } - } - return delta, nil -} - -// WriteMacOSDeltaTar writes a delta layer's tar. The changed chunks are read from -// derived, which must be the disk the delta was computed against. aux.img and -// config.json are copied in full from bundle. -func WriteMacOSDeltaTar(w io.Writer, delta MacOSDelta, derived, bundle string) error { - if err := delta.Validate(); err != nil { - return err - } - manifest, err := json.Marshal(delta) - if err != nil { - return err - } - disk, err := os.Open(derived) - if err != nil { - return err - } - defer disk.Close() - archive := tar.NewWriter(w) - if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaManifestEntry, Mode: 0600, Size: int64(len(manifest)), Typeflag: tar.TypeReg}); err != nil { - return err - } - if _, err := archive.Write(manifest); err != nil { - return err - } - patchSize := int64(len(delta.Chunks)) * delta.ChunkSize - if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaPatchEntry, Mode: 0600, Size: patchSize, Typeflag: tar.TypeReg}); err != nil { - return err - } - chunk := make([]byte, delta.ChunkSize) - for _, index := range delta.Chunks { - if _, err := disk.ReadAt(chunk, index*delta.ChunkSize); err != nil { - return fmt.Errorf("read changed chunk %d: %w", index, err) - } - if _, err := archive.Write(chunk); err != nil { - return err - } - } - for _, name := range []string{macOSDeltaAux, macOSDeltaConfig} { - if err := writeMacOSDeltaFile(archive, filepath.Join(bundle, name), name); err != nil { - return err - } - } - return archive.Close() -} - -func writeMacOSDeltaFile(archive *tar.Writer, path, name string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - info, err := file.Stat() - if err != nil { - return err - } - if err := archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}); err != nil { - return err - } - _, err = io.Copy(archive, file) - return err -} - -// macOSDeltaApplier applies the entries of a delta layer to the boot disk already -// unpacked into targetDir. The entries must arrive in the order WriteMacOSDeltaTar -// writes them, so a layer cannot name files outside the format. -type macOSDeltaApplier struct { - targetDir string - step int - manifest *MacOSDelta -} - -func (a *macOSDeltaApplier) entry(name string, size int64, archive io.Reader) error { - switch a.step { - case 0: - if name != macOSDeltaManifestEntry { - return fmt.Errorf("delta layer must start with %s, got %q", macOSDeltaManifestEntry, name) - } - if size > maxMacOSDeltaManifestBytes { - return fmt.Errorf("machine disk delta manifest is %d bytes", size) - } - decoder := json.NewDecoder(io.LimitReader(archive, size)) - decoder.DisallowUnknownFields() - var delta MacOSDelta - if err := decoder.Decode(&delta); err != nil { - return fmt.Errorf("decode machine disk delta: %w", err) - } - if err := delta.Validate(); err != nil { - return err - } - a.manifest = &delta - case 1: - if name != macOSDeltaPatchEntry { - return fmt.Errorf("delta layer entry %q must be %s", name, macOSDeltaPatchEntry) - } - if size != int64(len(a.manifest.Chunks))*a.manifest.ChunkSize { - return fmt.Errorf("machine disk patch is %d bytes, want %d", size, int64(len(a.manifest.Chunks))*a.manifest.ChunkSize) - } - if err := a.applyChunks(archive); err != nil { - return err - } - case 2, 3: - want := macOSDeltaAux - if a.step == 3 { - want = macOSDeltaConfig - } - if name != want { - return fmt.Errorf("delta layer entry %q must be %s", name, want) - } - if err := writeSparseFile(filepath.Join(a.targetDir, name), archive, size); err != nil { - return err - } - default: - return fmt.Errorf("delta layer has more than %d entries", macOSDeltaEntries) - } - a.step++ - return nil -} - -// applyChunks writes each changed chunk over the base disk in place. The base must -// be a regular file of the delta's size, so the offsets land where the diff computed. -func (a *macOSDeltaApplier) applyChunks(r io.Reader) error { - path := filepath.Join(a.targetDir, macOSDeltaDisk) - info, err := os.Lstat(path) - if err != nil { - return fmt.Errorf("delta has no base disk: %w", err) - } - if !info.Mode().IsRegular() || info.Size() != a.manifest.Size { - return fmt.Errorf("base disk does not match the delta's %d-byte size", a.manifest.Size) - } - file, err := os.OpenFile(path, os.O_WRONLY, 0) - if err != nil { - return err - } - buf := make([]byte, a.manifest.ChunkSize) - for _, index := range a.manifest.Chunks { - if _, err = io.ReadFull(r, buf); err != nil { - break - } - if _, err = file.WriteAt(buf, index*a.manifest.ChunkSize); err != nil { - break - } - } - return errors.Join(err, file.Close()) -} - -// complete reports an error if the layer ended before its last entry. -func (a *macOSDeltaApplier) complete() error { - if a.step != macOSDeltaEntries { - return fmt.Errorf("delta layer has %d of %d entries", a.step, macOSDeltaEntries) - } - return nil -} diff --git a/lib/images/macos_delta_test.go b/lib/images/macos_delta_test.go deleted file mode 100644 index 4e0227e85..000000000 --- a/lib/images/macos_delta_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package images - -import ( - "archive/tar" - "bytes" - "context" - "io" - "os" - "path/filepath" - "testing" - - gcr "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/stretchr/testify/require" -) - -type machineTestFile struct { - name string - data []byte -} - -// machineFilesLayer builds a layer of regular files with the given media type. -func machineFilesLayer(t *testing.T, mediaType types.MediaType, files ...machineTestFile) gcr.Layer { - t.Helper() - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - for _, f := range files { - require.NoError(t, tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0600, Size: int64(len(f.data)), Typeflag: tar.TypeReg})) - _, err := tw.Write(f.data) - require.NoError(t, err) - } - require.NoError(t, tw.Close()) - layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(mediaType)) - require.NoError(t, err) - return layer -} - -// deltaTestDisk returns a disk of n chunks, each filled with a distinct byte. -func deltaTestDisk(n int) []byte { - disk := make([]byte, n*MacOSDeltaChunkSize) - for i := range n { - for j := range MacOSDeltaChunkSize { - disk[i*MacOSDeltaChunkSize+j] = byte(i + 1) - } - } - return disk -} - -func writeDeltaTestFile(t *testing.T, path string, data []byte) { - t.Helper() - require.NoError(t, os.WriteFile(path, data, 0600)) -} - -// deltaTestLayer builds a delta layer from a manifest and a bundle directory. -func deltaTestLayer(t *testing.T, delta MacOSDelta, derived, bundle string) gcr.Layer { - t.Helper() - var buf bytes.Buffer - require.NoError(t, WriteMacOSDeltaTar(&buf, delta, derived, bundle)) - layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(MacOSDeltaMediaType)) - require.NoError(t, err) - return layer -} - -func TestMacOSDeltaRoundTripAppliesOnlyChangedChunks(t *testing.T) { - ctx := context.Background() - dir := t.TempDir() - base := deltaTestDisk(8) - derived := append([]byte(nil), base...) - for j := range MacOSDeltaChunkSize { - derived[2*MacOSDeltaChunkSize+j] = 0xAA - derived[5*MacOSDeltaChunkSize+j] = 0 - } - basePath := filepath.Join(dir, "base-disk.img") - derivedPath := filepath.Join(dir, "derived-disk.img") - writeDeltaTestFile(t, basePath, base) - writeDeltaTestFile(t, derivedPath, derived) - bundle := filepath.Join(dir, "bundle") - require.NoError(t, os.Mkdir(bundle, 0700)) - writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("derived aux")) - writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte(`{"derived":true}`)) - - delta, err := ComputeMacOSDelta(basePath, derivedPath) - require.NoError(t, err) - require.Equal(t, []int64{2, 5}, delta.Chunks) - - baseLayer := machineFilesLayer(t, types.OCILayer, - machineTestFile{name: "disk.img", data: base}, - machineTestFile{name: "aux.img", data: []byte("base aux")}, - machineTestFile{name: "config.json", data: []byte(`{"base":true}`)}, - ) - deltaLayer := deltaTestLayer(t, delta, derivedPath, bundle) - target := t.TempDir() - require.NoError(t, unpackMachineLayers2(ctx, target, baseLayer, deltaLayer)) - - got, err := os.ReadFile(filepath.Join(target, "disk.img")) - require.NoError(t, err) - require.Equal(t, derived, got) - aux, err := os.ReadFile(filepath.Join(target, "aux.img")) - require.NoError(t, err) - require.Equal(t, "derived aux", string(aux)) - config, err := os.ReadFile(filepath.Join(target, "config.json")) - require.NoError(t, err) - require.Equal(t, `{"derived":true}`, string(config)) -} - -func TestMacOSDeltaRejectsBaseOfDifferentSize(t *testing.T) { - dir := t.TempDir() - bundle := writeDeltaBundle(t, dir) - derivedPath := filepath.Join(dir, "derived-disk.img") - writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) - delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} - baseLayer := machineFilesLayer(t, types.OCILayer, machineTestFile{name: "disk.img", data: deltaTestDisk(4)}) - err := unpackMachineLayers2(context.Background(), t.TempDir(), baseLayer, deltaTestLayer(t, delta, derivedPath, bundle)) - require.ErrorContains(t, err, "base disk does not match") -} - -func TestMacOSDeltaRejectsLayerWithoutBase(t *testing.T) { - dir := t.TempDir() - bundle := writeDeltaBundle(t, dir) - derivedPath := filepath.Join(dir, "derived-disk.img") - writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) - delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} - err := unpackMachineLayers2(context.Background(), t.TempDir(), deltaTestLayer(t, delta, derivedPath, bundle)) - require.ErrorContains(t, err, "delta has no base disk") -} - -func TestMacOSDeltaRejectsIncompleteAndMisorderedLayers(t *testing.T) { - ctx := context.Background() - manifest := []byte(`{"version":1,"chunk_size":65536,"size":524288,"chunks":[]}`) - cases := map[string][]machineTestFile{ - "missing entries": {{name: "delta.json", data: manifest}}, - "wrong first entry": { - {name: "aux.img", data: []byte("x")}, - }, - "patch skipped": { - {name: "delta.json", data: manifest}, - {name: "aux.img", data: []byte("x")}, - }, - "extra entry": { - {name: "delta.json", data: manifest}, - {name: "disk.img.patch"}, - {name: "aux.img", data: []byte("x")}, - {name: "config.json", data: []byte("{}")}, - {name: "extra", data: []byte("x")}, - }, - } - for name, files := range cases { - t.Run(name, func(t *testing.T) { - target := t.TempDir() - writeDeltaTestFile(t, filepath.Join(target, "disk.img"), deltaTestDisk(8)) - layer := machineFilesLayer(t, MacOSDeltaMediaType, files...) - diffID, err := layer.DiffID() - require.NoError(t, err) - require.Error(t, unpackMachineLayer(ctx, layer, diffID, target, unlimitedMachineBudget())) - }) - } -} - -func TestMacOSDeltaValidateRejectsBadManifests(t *testing.T) { - size := int64(8 * MacOSDeltaChunkSize) - cases := map[string]MacOSDelta{ - "version": {Version: 2, ChunkSize: MacOSDeltaChunkSize, Size: size}, - "chunk size": {Version: 1, ChunkSize: 4096, Size: size}, - "partial size": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size + 1}, - "unordered": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 1}}, - "duplicate": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 3}}, - "out of range": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{8}}, - } - for name, delta := range cases { - t.Run(name, func(t *testing.T) { - require.Error(t, delta.Validate()) - }) - } - valid := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{0, 7}} - require.NoError(t, valid.Validate()) -} - -// writeDeltaBundle writes the aux and config files a delta layer copies in full. -func writeDeltaBundle(t *testing.T, dir string) string { - t.Helper() - bundle := filepath.Join(dir, "bundle") - require.NoError(t, os.Mkdir(bundle, 0700)) - writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("aux")) - writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte("{}")) - return bundle -} - -// unpackMachineLayers2 unpacks layers in order into one target directory with an -// unlimited budget, as the bundle unpack does. -func unpackMachineLayers2(ctx context.Context, target string, layers ...gcr.Layer) error { - budget := unlimitedMachineBudget() - for _, l := range layers { - diffID, err := l.DiffID() - if err != nil { - return err - } - if err := unpackMachineLayer(ctx, l, diffID, target, budget); err != nil { - return err - } - } - return nil -} diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index 0fa90c9e6..3608ba049 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -18,9 +18,9 @@ import ( ) // ImportMacOSImage is an offline, administrator-only import of a STOPPED macvm -// bundle. It intentionally is not exposed as an HTTP host-filesystem read API. -// Imports preserve identity: instances from this image may run sequentially, -// not concurrently, until rekeying is validated. +// bundle. It intentionally is not exposed as an HTTP host-filesystem read API. The +// bundle's boot disk is disk.asif, cloned as is, or a raw disk.img, converted to +// ASIF. The image is a one-layer chain. func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) (*Image, error) { ref, err := ParseNormalizedRef(name) if err != nil { @@ -29,12 +29,16 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if ref.IsDigest() { return nil, fmt.Errorf("%w: import requires a tagged name", ErrInvalidName) } - bundle, err := readMacOSMachineBundle(source, "disk.img", "aux.img", "config.json", false) + diskName := "disk.asif" + if _, err := os.Lstat(filepath.Join(source, diskName)); os.IsNotExist(err) { + diskName = "disk.img" + } + bundle, err := readMacOSMachineBundle(source, diskName, "aux.img", "config.json", false) if err != nil { return nil, err } mac := bundle.Platform - for _, file := range []string{"disk.img", "aux.img"} { + for _, file := range []string{diskName, "aux.img"} { file = filepath.Join(source, file) info, e := os.Lstat(file) if e != nil { @@ -58,21 +62,32 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) return nil, err } defer os.RemoveAll(stage) - for src, dst := range map[string]string{bundle.Disk: "rootfs.raw", bundle.Aux: "aux.img"} { + clones := map[string]string{bundle.Aux: "aux.img"} + if diskName == "disk.asif" { + clones[bundle.Disks[0]] = machineDiskFile + } else if err = ConvertRawToASIF(ctx, bundle.Disks[0], filepath.Join(stage, machineDiskFile)); err != nil { + return nil, fmt.Errorf("convert boot disk to ASIF: %w", err) + } + for src, dst := range clones { if err = unix.Clonefile(src, filepath.Join(stage, dst), unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY); err != nil { return nil, fmt.Errorf("clone bundle: %w", err) } - if err = os.Chmod(filepath.Join(stage, dst), 0600); err != nil { + } + for _, name := range []string{machineDiskFile, "aux.img"} { + if err = os.Chmod(filepath.Join(stage, name), 0600); err != nil { return nil, err } } + logicalSize, err := macOSChainSize([]string{filepath.Join(stage, machineDiskFile)}) + if err != nil { + return nil, fmt.Errorf("imported boot disk: %w", err) + } // Hash cloned content, not the caller's mutable paths. Per-file hashes and // logical sizes make the digest unambiguous, including sparse zero ranges. h := sha256.New() canonical, _ := json.Marshal(mac) h.Write(canonical) - var logicalSize int64 - for _, name := range []string{"rootfs.raw", "aux.img"} { + for _, name := range []string{machineDiskFile, "aux.img"} { f, e := os.Open(filepath.Join(stage, name)) if e != nil { return nil, e @@ -84,13 +99,10 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) return nil, e } fmt.Fprintf(h, "\n%s:%d:%x", name, n, fh.Sum(nil)) - if name == "rootfs.raw" { - logicalSize = n - } } digestHex := hex.EncodeToString(h.Sum(nil)) digest := "sha256:" + digestHex - meta := &imageMetadata{Name: ref.Repository() + "@" + digest, Digest: digest, Platform: "darwin/arm64", MacOS: mac, Status: StatusReady, SizeBytes: logicalSize, CreatedAt: time.Now().UTC()} + meta := &imageMetadata{Name: ref.Repository() + "@" + digest, Digest: digest, Platform: "darwin/arm64", MacOS: mac, MacOSDiskLayers: 1, Status: StatusReady, SizeBytes: logicalSize, CreatedAt: time.Now().UTC()} if err = writeMetadataFile(filepath.Join(stage, "metadata.json"), meta); err != nil { return nil, err } diff --git a/lib/images/macos_import_darwin_test.go b/lib/images/macos_import_darwin_test.go index f8cb4b660..ca04ca228 100644 --- a/lib/images/macos_import_darwin_test.go +++ b/lib/images/macos_import_darwin_test.go @@ -18,7 +18,9 @@ func TestMacOSOfflineImport(t *testing.T) { c := MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30} b, _ := json.Marshal(c) require.NoError(t, os.WriteFile(filepath.Join(source, "config.json"), b, 0600)) - require.NoError(t, os.WriteFile(filepath.Join(source, "disk.img"), []byte("boot disk"), 0600)) + raw := make([]byte, 4<<20) + copy(raw[1<<20:], "boot disk") + require.NoError(t, os.WriteFile(filepath.Join(source, "disk.img"), raw, 0600)) require.NoError(t, os.WriteFile(filepath.Join(source, "aux.img"), []byte("aux storage"), 0600)) img, err := ImportMacOSImage(context.Background(), p, "localhost/macos:test", source) require.NoError(t, err) @@ -28,9 +30,18 @@ func TestMacOSOfflineImport(t *testing.T) { loaded, err := m.GetImage(context.Background(), img.Name) require.NoError(t, err) require.NotNil(t, loaded.MacOS) - disk, err := GetDiskPath(p, img.Name, img.Digest) + chain, err := MacOSDiskChain(p, img.Name, img.Digest) require.NoError(t, err) - got, err := os.ReadFile(filepath.Join(filepath.Dir(disk), "aux.img")) + require.Len(t, chain, 1, "a raw disk imports as a one-layer ASIF chain") + disk, err := OpenMacOSDiskChain(chain) + require.NoError(t, err) + require.Equal(t, int64(len(raw)), disk.Size()) + buf := make([]byte, len("boot disk")) + _, err = disk.ReadAt(buf, 1<<20) + require.NoError(t, err) + require.Equal(t, "boot disk", string(buf)) + require.NoError(t, disk.Close()) + got, err := os.ReadFile(filepath.Join(filepath.Dir(chain[0]), "aux.img")) require.NoError(t, err) require.Equal(t, "aux storage", string(got)) _, err = m.TagImage(context.Background(), img.Name, "localhost/alias:tag") diff --git a/lib/images/macos_machine.go b/lib/images/macos_machine.go index 8ce357019..fc868458c 100644 --- a/lib/images/macos_machine.go +++ b/lib/images/macos_machine.go @@ -22,9 +22,10 @@ const ( MacOSMachinePlatformLabel = "io.hypeman.machine-image.platform-path" ) -// stagedMacOS is the part of a staged macOS machine image beyond its boot disk. +// stagedMacOS is the part of a staged macOS machine image beyond its base disk. type stagedMacOS struct { - aux string // staged auxiliary storage, beside its final path + layers []string // staged overlays above the base, in order, beside their final paths + aux string // staged auxiliary storage, beside its final path platform *MacOSImage } @@ -37,9 +38,9 @@ func (s stagedImageFiles) finalDiskPath(layout imageLayout) string { return layout.disk } -// installMacOSMachine installs a macOS machine image's auxiliary storage and records -// its platform on meta. It returns the paths installed, for rollback. Rootfs images -// install nothing here. +// installMacOSMachine installs a macOS machine image's overlays and auxiliary +// storage and records its platform and chain length on meta. It returns the paths +// installed, for rollback. Rootfs images install nothing here. func installMacOSMachine(layout imageLayout, staged stagedImageFiles, meta *imageMetadata, actualPlatform Platform) ([]string, error) { if actualPlatform.OS == "darwin" && staged.macos == nil { return nil, fmt.Errorf("macOS image requires a validated machine bundle") @@ -47,12 +48,21 @@ func installMacOSMachine(layout imageLayout, staged stagedImageFiles, meta *imag if staged.macos == nil { return nil, nil } + var installed []string + for i, layer := range staged.macos.layers { + path := filepath.Join(layout.dir, macOSDiskLayerFile(i+1)) + if err := os.Rename(layer, path); err != nil { + return installed, err + } + installed = append(installed, path) + } auxPath := filepath.Join(layout.dir, "aux.img") if err := os.Rename(staged.macos.aux, auxPath); err != nil { - return nil, err + return installed, err } meta.MacOS = staged.macos.platform - return []string{auxPath}, nil + meta.MacOSDiskLayers = len(staged.macos.layers) + 1 + return append(installed, auxPath), nil } // checkCachedMachinePlatform refuses to reuse a record whose platform does not match @@ -84,7 +94,7 @@ func checkCachedMachinePlatform(p *paths.Paths, ref *ResolvedRef, requested Plat return nil } -// stageMacOSOutputs stages a macOS machine image's raw boot disk and auxiliary +// stageMacOSOutputs stages a macOS machine image's boot disk chain and auxiliary // storage in layoutDir, outside the manager lock. For other images it stages nothing // and returns diskTemp for the rootfs export. func (m *manager) stageMacOSOutputs(ctx context.Context, digest, root string, meta *containerMetadata, layoutDir, diskTemp, buildID string) (stagedImageFiles, error) { @@ -94,29 +104,41 @@ func (m *manager) stageMacOSOutputs(ctx context.Context, digest, root string, me } start := time.Now() staged.disk = filepath.Join(layoutDir, machineDiskFile+".tmp-"+buildID) - err := stageMacOSMachine(&staged, root, meta, filepath.Join(layoutDir, "aux.img.tmp-"+buildID)) + err := stageMacOSMachine(&staged, root, meta, layoutDir, buildID) m.recordImageBuildPhase(ctx, digest, "filesystem_export", time.Since(start), phaseStatus(err), "not_applicable") return staged, err } -// readMacOSMachine reads the unpacked bundle that validated labels describe. +// readMacOSMachine reads the unpacked layers that validated labels describe. func readMacOSMachine(root string, meta *containerMetadata) (*macOSMachinePayload, error) { labels := meta.Labels - return readMacOSMachineBundle(root, labels[MacOSMachineDiskLabel], labels[MacOSMachineAuxLabel], labels[MacOSMachinePlatformLabel], false) + return readMacOSMachineLayers(root, labels[MacOSMachineDiskLabel], labels[MacOSMachineAuxLabel], labels[MacOSMachinePlatformLabel]) } -// stageMacOSMachine copies an unpacked bundle's files to the staging paths. The -// recorded size is the boot disk's logical size, as for a local import. -func stageMacOSMachine(staged *stagedImageFiles, root string, meta *containerMetadata, auxTemp string) error { +// stageMacOSMachine copies an unpacked image's disk chain and auxiliary storage to +// staging paths in layoutDir and checks that the chain stacks. The recorded size is +// the chain's logical size, as for a local import. +func stageMacOSMachine(staged *stagedImageFiles, root string, meta *containerMetadata, layoutDir, buildID string) error { payload, err := readMacOSMachine(root, meta) if err != nil { return err } - if staged.sizeBytes, err = stageMachineFile(payload.Disk, staged.disk); err != nil { - return fmt.Errorf("stage boot disk: %w", err) + staged.macos = &stagedMacOS{aux: filepath.Join(layoutDir, "aux.img.tmp-"+buildID), platform: payload.Platform} + chain := []string{staged.disk} + for i, disk := range payload.Disks { + if i > 0 { + layer := filepath.Join(layoutDir, macOSDiskLayerFile(i)+".tmp-"+buildID) + staged.macos.layers = append(staged.macos.layers, layer) + chain = append(chain, layer) + } + if _, err := stageMachineFile(disk, chain[i]); err != nil { + return fmt.Errorf("stage boot disk layer %d: %w", i, err) + } + } + if staged.sizeBytes, err = macOSChainSize(chain); err != nil { + return fmt.Errorf("machine boot disk: %w", err) } - staged.macos = &stagedMacOS{aux: auxTemp, platform: payload.Platform} - if _, err := stageMachineFile(payload.Aux, auxTemp); err != nil { + if _, err := stageMachineFile(payload.Aux, staged.macos.aux); err != nil { return fmt.Errorf("stage auxiliary storage: %w", err) } return nil @@ -188,7 +210,7 @@ func validateMacOSMachine(meta *containerMetadata) error { return fmt.Errorf("macOS machine requires darwin/arm64, not %s", normalized) } labels := meta.Labels - for _, want := range [][2]string{{MacOSMachineVersionLabel, "1"}, {MacOSMachineKindLabel, "macos-image"}, {MacOSMachineFormatLabel, "raw"}} { + for _, want := range [][2]string{{MacOSMachineVersionLabel, "2"}, {MacOSMachineKindLabel, "macos-image"}, {MacOSMachineFormatLabel, "asif"}} { if label := want[0]; labels[label] != want[1] { return fmt.Errorf("darwin artifact requires %s=%q, got %q; ordinary container images are not macOS machines", label, want[1], labels[label]) } diff --git a/lib/images/macos_machine_test.go b/lib/images/macos_machine_test.go index f2671a818..a32c1a0b5 100644 --- a/lib/images/macos_machine_test.go +++ b/lib/images/macos_machine_test.go @@ -4,22 +4,23 @@ import ( "archive/tar" "compress/gzip" "context" - "crypto/sha256" "encoding/json" - "fmt" "io" "maps" "net/http" "net/http/httptest" "os" "path/filepath" + "slices" "strings" "sync/atomic" "testing" "time" + "github.com/Code-Hex/vz/v4/diskimagekit" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" + gcr "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" @@ -30,26 +31,48 @@ import ( "github.com/stretchr/testify/require" ) +// macOSFixtureChain writes a small ASIF chain to dir: a base with data at two +// offsets and an overlay that replaces the second. It returns their paths. +func macOSFixtureChain(t *testing.T, dir string) (string, string) { + t.Helper() + base, overlay := filepath.Join(dir, "base.asif"), filepath.Join(dir, "overlay.asif") + img, err := diskimagekit.CreateASIF(base, (64<<20)/512) + require.NoError(t, err) + _, err = img.WriteAt([]byte("base data"), 0) + require.NoError(t, err) + _, err = img.WriteAt([]byte("replaced in overlay"), 32<<20) + require.NoError(t, err) + require.NoError(t, img.CreateLayer(overlay, diskimagekit.Overlay)) + _, err = img.WriteAt([]byte("overlay data"), 32<<20) + require.NoError(t, err) + require.NoError(t, img.Close()) + return base, overlay +} + +func macOSFixturePlatform(t *testing.T) []byte { + t.Helper() + b, err := json.Marshal(MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30}) + require.NoError(t, err) + return b +} + +// macOSFixture writes an unpacked one-layer image: root/0 holds the disk, the +// auxiliary storage and the platform. func macOSFixture(t *testing.T) string { t.Helper() root := t.TempDir() - c := MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30} - b, err := json.Marshal(c) - require.NoError(t, err) - // A mostly empty disk, like a real one, so transport can be checked for sparseness. - disk := make([]byte, 64<<20) - copy(disk, "not a real boot disk: synthetic OCI transport fixture") - copy(disk[len(disk)-4:], "tail") - for f, data := range map[string][]byte{"config.json": b, "disk.img": disk, "aux.img": []byte("synthetic aux")} { - require.NoError(t, os.WriteFile(filepath.Join(root, f), data, 0600)) + layer := filepath.Join(root, "0") + require.NoError(t, os.MkdirAll(layer, 0700)) + for f, data := range map[string][]byte{"config.json": macOSFixturePlatform(t), "disk.asif": []byte("disk"), "aux.img": []byte("synthetic aux")} { + require.NoError(t, os.WriteFile(filepath.Join(layer, f), data, 0600)) } return root } func macOSFixtureMetadata() *containerMetadata { return &containerMetadata{OS: "darwin", Architecture: "arm64", Labels: map[string]string{ - MacOSMachineVersionLabel: "1", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "raw", - MacOSMachineDiskLabel: "disk.img", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "config.json", + MacOSMachineVersionLabel: "2", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "asif", + MacOSMachineDiskLabel: "disk.asif", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "config.json", }} } @@ -69,10 +92,10 @@ func TestMacOSMachineValidation(t *testing.T) { require.NoError(t, err) require.NotNil(t, payload) for _, tc := range []struct{ name, key, value string }{ - {"unsupported version", MacOSMachineVersionLabel, "2"}, {"container not machine", MacOSMachineKindLabel, ""}, - {"wrong format", MacOSMachineFormatLabel, "qcow2"}, {"absolute path", MacOSMachineDiskLabel, "/etc/passwd"}, - {"traversal", MacOSMachineDiskLabel, "../disk.img"}, {"missing aux", MacOSMachineAuxLabel, "missing.img"}, - {"duplicate file", MacOSMachineAuxLabel, "disk.img"}, + {"unsupported version", MacOSMachineVersionLabel, "1"}, {"container not machine", MacOSMachineKindLabel, ""}, + {"raw format", MacOSMachineFormatLabel, "raw"}, {"absolute path", MacOSMachineDiskLabel, "/etc/passwd"}, + {"traversal", MacOSMachineDiskLabel, "../disk.asif"}, {"missing aux", MacOSMachineAuxLabel, "missing.img"}, + {"duplicate file", MacOSMachineAuxLabel, "disk.asif"}, } { t.Run(tc.name, func(t *testing.T) { meta := macOSFixtureMetadata() @@ -89,7 +112,7 @@ func TestMacOSMachineValidation(t *testing.T) { require.NoError(t, err, alias) require.NotNil(t, payload, alias) } - require.NoError(t, os.WriteFile(filepath.Join(root, "config.json"), []byte(`{}`), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "0", "config.json"), []byte(`{}`), 0600)) _, err = parseMacOSMachine(root, macOSFixtureMetadata()) require.Error(t, err) @@ -97,49 +120,54 @@ func TestMacOSMachineValidation(t *testing.T) { for _, mac := range []string{"02:00:00:00:00:00:00:01", strings.Repeat("00:", 19) + "01"} { b, err := json.Marshal(MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: mac, CPUs: 4, Memory: 8 << 30}) require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(root, "config.json"), b, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "0", "config.json"), b, 0600)) _, err = parseMacOSMachine(root, macOSFixtureMetadata()) require.Error(t, err, mac) } } -// TestMacOSMachineOCIRoundTrip pushes a machine bundle to a registry and pulls it -// through the image manager. It is not a VZ boot. -func TestMacOSMachineOCIRoundTrip(t *testing.T) { - allowMacOSMachines(t) - source := macOSFixture(t) - work := t.TempDir() - layerPath := filepath.Join(work, "bundle.tar.gz") - file, err := os.OpenFile(layerPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) +// machineFixtureLayer writes files into a gzipped tar layer and returns it. +func machineFixtureLayer(t *testing.T, dir, name string, files map[string]string) gcr.Layer { + t.Helper() + path := filepath.Join(dir, name+".tar.gz") + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) require.NoError(t, err) gz, err := gzip.NewWriterLevel(file, gzip.BestSpeed) require.NoError(t, err) tw := tar.NewWriter(gz) - hashes := map[string]string{} - for _, f := range []string{"disk.img", "aux.img", "config.json"} { - input, err := os.Open(filepath.Join(source, f)) + for _, entry := range slices.Sorted(maps.Keys(files)) { + input, err := os.Open(files[entry]) require.NoError(t, err) info, err := input.Stat() require.NoError(t, err) - // The registry controls modes; a permissive disk must not become the canonical image mode. - mode := int64(0600) - if f == "disk.img" { - mode = 0666 - } - require.NoError(t, tw.WriteHeader(&tar.Header{Name: f, Mode: mode, Size: info.Size(), Typeflag: tar.TypeReg})) - h := sha256.New() - _, err = io.Copy(io.MultiWriter(tw, h), input) + // The registry controls modes; a permissive file must not become the canonical image mode. + require.NoError(t, tw.WriteHeader(&tar.Header{Name: entry, Mode: 0666, Size: info.Size(), Typeflag: tar.TypeReg})) + _, err = io.Copy(tw, input) require.NoError(t, err) require.NoError(t, input.Close()) - hashes[f] = fmt.Sprintf("%x", h.Sum(nil)) } require.NoError(t, tw.Close()) require.NoError(t, gz.Close()) require.NoError(t, file.Close()) - layer, err := tarball.LayerFromFile(layerPath) + layer, err := tarball.LayerFromFile(path) require.NoError(t, err) - image, err := mutate.AppendLayers(empty.Image, layer) + return layer +} + +// TestMacOSMachineOCIRoundTrip pushes a two-layer machine image, a base disk and an +// overlay with the auxiliary storage and platform, and pulls it through the image +// manager. It is not a VZ boot. +func TestMacOSMachineOCIRoundTrip(t *testing.T) { + allowMacOSMachines(t) + work := t.TempDir() + base, overlay := macOSFixtureChain(t, work) + aux, config := filepath.Join(work, "aux.img"), filepath.Join(work, "config.json") + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + require.NoError(t, os.WriteFile(config, macOSFixturePlatform(t), 0600)) + image, err := mutate.AppendLayers(empty.Image, + machineFixtureLayer(t, work, "base", map[string]string{"disk.asif": base}), + machineFixtureLayer(t, work, "top", map[string]string{"disk.asif": overlay, "aux.img": aux, "config.json": config})) require.NoError(t, err) cfg, err := image.ConfigFile() require.NoError(t, err) @@ -166,26 +194,37 @@ func TestMacOSMachineOCIRoundTrip(t *testing.T) { require.Equal(t, StatusReady, pulled.Status) require.Equal(t, "darwin/arm64", pulled.Platform) require.NotNil(t, pulled.MacOS) - disk, err := GetDiskPath(p, pulled.Name, pulled.Digest) + require.Equal(t, int64(64<<20), *pulled.SizeBytes, "the recorded size is the chain's logical size") + + chain, err := MacOSDiskChain(p, pulled.Name, pulled.Digest) require.NoError(t, err) - require.Equal(t, machineDiskFile, filepath.Base(disk), "a raw machine disk is not named as a filesystem") - for f, path := range map[string]string{"disk.img": disk, "aux.img": filepath.Join(filepath.Dir(disk), "aux.img")} { - sum, err := fileSHA256(ctx, path) + require.Len(t, chain, 2) + for i, want := range []string{base, overlay} { + got, err := fileSHA256(ctx, chain[i]) require.NoError(t, err) - require.Equal(t, "sha256:"+hashes[f], sum, f) - info, err := os.Stat(path) + expected, err := fileSHA256(ctx, want) require.NoError(t, err) - require.Equal(t, os.FileMode(0600), info.Mode().Perm(), f) - if f == "disk.img" { - control := sparseControlBytes(t, filepath.Dir(path), info.Size()) - require.LessOrEqual(t, allocatedBytes(t, path), control+2*machineUnpackChunk, "the pulled disk stays sparse") - } + require.Equal(t, expected, got, "layer %d is installed byte for byte", i) + info, err := os.Stat(chain[i]) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) } - expected, err := os.ReadFile(filepath.Join(source, "config.json")) + stacked, err := OpenMacOSDiskChain(chain) require.NoError(t, err) - var config MacOSImage - require.NoError(t, json.Unmarshal(expected, &config)) - require.Equal(t, config, *pulled.MacOS) + for offset, want := range map[int64]string{0: "base data", 32 << 20: "overlay data"} { + buf := make([]byte, len(want)) + _, err := stacked.ReadAt(buf, offset) + require.NoError(t, err) + require.Equal(t, want, string(buf)) + } + require.NoError(t, stacked.Close()) + installedAux, err := os.ReadFile(filepath.Join(filepath.Dir(chain[0]), "aux.img")) + require.NoError(t, err) + require.Equal(t, "synthetic aux", string(installedAux)) + var platform MacOSImage + require.NoError(t, json.Unmarshal(macOSFixturePlatform(t), &platform)) + require.Equal(t, platform, *pulled.MacOS) + digest, err := image.Digest() require.NoError(t, err) require.Equal(t, digest.String(), pulled.Digest) @@ -196,20 +235,55 @@ func TestMacOSMachineOCIRoundTrip(t *testing.T) { require.NoError(t, err) require.Equal(t, pulled.Digest, tagged.Digest) require.Equal(t, pulled.MacOS, tagged.MacOS) - tagDisk, err := GetDiskPath(p, tagged.Name, tagged.Digest) + tagChain, err := MacOSDiskChain(p, tagged.Name, tagged.Digest) require.NoError(t, err) - require.Equal(t, disk, tagDisk) + require.Equal(t, chain, tagChain) _, err = manager.TagImage(ctx, ref.Name(), strings.TrimPrefix(server.URL, "http://")+"/other:stable") require.ErrorIs(t, err, ErrInvalidPlatform) _, err = manager.CreateImage(ctx, CreateImageRequest{Name: ref.Name(), Platform: "linux/arm64"}) require.ErrorIs(t, err, ErrInvalidPlatform) - t.Log("round-trip verified", pulled.Digest) +} + +// TestMacOSMachinePullRejectsBrokenChain pulls an image whose overlay layer does not +// belong to its base, and expects the pull to fail. +func TestMacOSMachinePullRejectsBrokenChain(t *testing.T) { + allowMacOSMachines(t) + work := t.TempDir() + _, overlay := macOSFixtureChain(t, work) + otherBase, _ := macOSFixtureChain(t, t.TempDir()) + aux, config := filepath.Join(work, "aux.img"), filepath.Join(work, "config.json") + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + require.NoError(t, os.WriteFile(config, macOSFixturePlatform(t), 0600)) + image, err := mutate.AppendLayers(empty.Image, + machineFixtureLayer(t, work, "base", map[string]string{"disk.asif": otherBase}), + machineFixtureLayer(t, work, "top", map[string]string{"disk.asif": overlay, "aux.img": aux, "config.json": config})) + require.NoError(t, err) + cfg, err := image.ConfigFile() + require.NoError(t, err) + cfg.OS, cfg.Architecture, cfg.Config.Labels = "darwin", "arm64", macOSFixtureMetadata().Labels + image, err = mutate.ConfigFile(image, cfg) + require.NoError(t, err) + server := httptest.NewServer(registry.New()) + defer server.Close() + ref, err := name.ParseReference(strings.TrimPrefix(server.URL, "http://") + "/macos:broken") + require.NoError(t, err) + require.NoError(t, remote.Write(ref, image)) + manager, err := NewManager(paths.New(filepath.Join(work, "hypeman-data")), 1, nil) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + pulled, err := manager.CreateImage(ctx, CreateImageRequest{Name: ref.Name(), Platform: "darwin/arm64"}) + require.NoError(t, err) + require.Error(t, manager.WaitForReady(ctx, pulled.Name)) + pulled, err = manager.GetImage(ctx, pulled.Name) + require.NoError(t, err) + require.Equal(t, StatusFailed, pulled.Status) } func TestValidateMacOSMachineRejectsBeforeUnpack(t *testing.T) { allowMacOSMachines(t) - valid := map[string]string{MacOSMachineVersionLabel: "1", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "raw", - MacOSMachineDiskLabel: "disk.img", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "platform.json"} + valid := map[string]string{MacOSMachineVersionLabel: "2", MacOSMachineKindLabel: "macos-image", MacOSMachineFormatLabel: "asif", + MacOSMachineDiskLabel: "disk.asif", MacOSMachineAuxLabel: "aux.img", MacOSMachinePlatformLabel: "platform.json"} require.False(t, isMacOSMachine(&containerMetadata{OS: "linux", Architecture: "amd64"})) require.True(t, isMacOSMachine(&containerMetadata{OS: "Darwin", Architecture: "aarch64"})) require.NoError(t, validateMacOSMachine(&containerMetadata{OS: "darwin", Architecture: "arm64", Labels: valid})) @@ -220,9 +294,9 @@ func TestValidateMacOSMachineRejectsBeforeUnpack(t *testing.T) { "missing disk path": func(l map[string]string) { delete(l, MacOSMachineDiskLabel) }, "missing aux path": func(l map[string]string) { delete(l, MacOSMachineAuxLabel) }, "missing platform path": func(l map[string]string) { delete(l, MacOSMachinePlatformLabel) }, - "absolute path": func(l map[string]string) { l[MacOSMachineDiskLabel] = "/disk.img" }, + "absolute path": func(l map[string]string) { l[MacOSMachineDiskLabel] = "/disk.asif" }, "escaping path": func(l map[string]string) { l[MacOSMachineAuxLabel] = "../aux.img" }, - "duplicate path": func(l map[string]string) { l[MacOSMachineAuxLabel] = "./disk.img" }, + "duplicate path": func(l map[string]string) { l[MacOSMachineAuxLabel] = "./disk.asif" }, } { labels := maps.Clone(valid) change(labels) diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index 7b71ec481..b3db67080 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -11,6 +11,7 @@ import ( "io" "os" "path/filepath" + "strconv" "strings" gcr "github.com/google/go-containerregistry/pkg/v1" @@ -28,7 +29,8 @@ var machineZeroChunk = make([]byte, machineUnpackChunk) // lower it. var maxMachineUnpackedBytes int64 = 1 << 40 -// unpackMachineLayers extracts a macOS machine image's layers into targetDir. Unlike +// unpackMachineLayers extracts each of a macOS machine image's layers into its own +// numbered directory under targetDir, since every layer carries a disk. Unlike // the rootfs unpack, it writes files sparsely: zero chunks become holes, so a // mostly empty boot disk occupies only its data. Machine bundles hold only regular // files and directories; any other entry is rejected. Each layer is checked @@ -62,7 +64,7 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi } remaining := maxMachineUnpackedBytes for i, l := range layers { - if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], targetDir, &remaining); err != nil { + if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], filepath.Join(targetDir, strconv.Itoa(i)), &remaining); err != nil { return fmt.Errorf("unpack machine layer %d: %w", i, err) } } @@ -70,8 +72,7 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi } // unpackMachineLayer extracts one layer, spending its decompressed bytes from the -// bundle's remaining budget. A delta layer patches the boot disk unpacked beneath it -// instead of writing files. +// bundle's remaining budget. func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targetDir string, remaining *int64) error { rc, err := l.Uncompressed() if err != nil { @@ -91,12 +92,6 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return err } archive := tar.NewReader(stream) - mediaType, err := l.MediaType() - if err != nil { - return err - } - delta := mediaType == MacOSDeltaMediaType - applier := &macOSDeltaApplier{targetDir: targetDir} for { header, err := archive.Next() if errors.Is(err, io.EOF) { @@ -109,15 +104,6 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe if !filepath.IsLocal(name) || strings.HasPrefix(filepath.Base(name), ".wh.") { return fmt.Errorf("machine layer entry %q is not a plain local path", header.Name) } - if delta { - if header.Typeflag != tar.TypeReg || isGNUSparse(header) { - return fmt.Errorf("delta layer entry %q must be a regular file", header.Name) - } - if err := applier.entry(name, header.Size, archive); err != nil { - return overLimit(err) - } - continue - } target := filepath.Join(targetDir, name) switch header.Typeflag { case tar.TypeDir: @@ -143,11 +129,6 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return fmt.Errorf("machine layer entry %q has unsupported type %q", header.Name, header.Typeflag) } } - if delta { - if err := applier.complete(); err != nil { - return overLimit(err) - } - } // Hash the trailing tar padding too, so the digest covers the whole stream. if _, err := io.Copy(io.Discard, stream); err != nil { return overLimit(err) diff --git a/lib/images/storage.go b/lib/images/storage.go index f9f9de89c..507652c3c 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -18,6 +18,7 @@ type imageMetadata struct { Digest string `json:"digest"` // Always present: sha256:... Platform string `json:"platform,omitempty"` MacOS *MacOSImage `json:"macos,omitempty"` + MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` // length of the macOS boot disk chain Status string `json:"status"` Error *string `json:"error,omitempty"` Request *CreateImageRequest `json:"request,omitempty"` @@ -154,15 +155,15 @@ func legacyLayout(p *paths.Paths, repository, digestHex string) imageLayout { return withMachineDisk(imageLayout{dir: p.ImageDigestDir(repository, digestHex), metadata: p.ImageMetadata(repository, digestHex), disk: p.ImageDigestPath(repository, digestHex)}) } -// machineDiskFile is a macOS machine image's raw boot disk, whether imported locally -// or pulled. Rootfs images use erofs or ext4 instead. -const machineDiskFile = "rootfs.raw" +// machineDiskFile is the base of a macOS machine image's ASIF boot disk chain, +// whether imported locally or pulled. Rootfs images use erofs or ext4 instead. +var machineDiskFile = macOSDiskLayerFile(0) -// withMachineDisk points layout at a machine image's raw disk when the directory has -// one. Rootfs layouts never do, so a stat is enough to pick the file. +// withMachineDisk points layout at a machine image's base disk when the directory +// has one. Rootfs layouts never do, so a stat is enough to pick the file. func withMachineDisk(layout imageLayout) imageLayout { - if raw := filepath.Join(layout.dir, machineDiskFile); pathExists(raw) { - layout.disk = raw + if base := filepath.Join(layout.dir, machineDiskFile); pathExists(base) { + layout.disk = base } return layout } diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 909972aef..b64cd6a41 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -58,7 +58,7 @@ func (m *manager) checkMacOSCreate(req *CreateInstanceRequest, img *images.Image config := hypervisor.VMConfig{ BootMode: hypervisor.BootModeMacOS, VCPUs: prepared.Vcpus, MemoryBytes: prepared.Size, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: img.MacOS.HardwareModel, MachineIdentifierData: img.MacOS.MachineIdentifier, AuxStoragePath: "mac-aux.img"}, - Disks: []hypervisor.DiskConfig{{Path: "disk.img"}}, + Disks: []hypervisor.DiskConfig{{Path: paths.MacOSBootDiskFile}}, } if err := validatePlannedVMConfig(starter, hvType, config); err != nil { return err @@ -159,18 +159,37 @@ func (m *manager) checkMacOSStart(stored *StoredMetadata, req StartInstanceReque return nil } -// createBootDisk creates the Linux overlay, or clones a macOS image's boot disk and -// aux storage into the instance. +// createBootDisk creates the Linux overlay, or the macOS boot disk: the image's +// ASIF chain and auxiliary storage cloned into the instance, with an empty +// writable overlay on top. func (m *manager) createBootDisk(stored *StoredMetadata, img *images.Image) error { if stored.MacOS == nil { return m.createOverlayDisk(stored.Id, stored.OverlaySize) } - disk, err := images.GetDiskPath(m.paths, img.Name, img.Digest) + chain, err := images.MacOSDiskChain(m.paths, img.Name, img.Digest) if err != nil { return err } - aux := filepath.Join(filepath.Dir(disk), "aux.img") - return cloneMacOSStorage(disk, aux, m.paths.InstanceMacOSBootDisk(stored.Id), filepath.Join(stored.DataDir, macOSAuxStorageFile)) + aux := filepath.Join(filepath.Dir(chain[0]), "aux.img") + if err := createMacOSStorage(chain, aux, stored.DataDir); err != nil { + return err + } + stored.MacOSDiskLayers = len(chain) + return nil +} + +// macOSDiskLayer is the ith read-only layer of a macOS guest's boot disk chain. +func macOSDiskLayer(dataDir string, i int) string { + return filepath.Join(dataDir, fmt.Sprintf("disk.%d.asif", i)) +} + +// macOSDiskParents returns the read-only layers below a macOS guest's boot overlay. +func macOSDiskParents(stored *StoredMetadata) []string { + parents := make([]string, stored.MacOSDiskLayers) + for i := range parents { + parents[i] = macOSDiskLayer(stored.DataDir, i) + } + return parents } // usesLinuxNetwork reports whether the instance's network is a hypeman TAP @@ -214,7 +233,7 @@ func (m *manager) admitMacOSBoot(ctx context.Context, stored *StoredMetadata) (f func (m *manager) macOSVMConfig(inst *Instance) hypervisor.VMConfig { c := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: inst.Vcpus, MemoryBytes: inst.Size, VsockCID: inst.VsockCID, VsockSocket: inst.VsockSocket, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: inst.MacOS.HardwareModel, MachineIdentifierData: inst.MacOS.MachineIdentifier, AuxStoragePath: filepath.Join(inst.DataDir, macOSAuxStorageFile)}, - Disks: []hypervisor.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(inst.Id)}}} + Disks: []hypervisor.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(inst.Id), Parents: macOSDiskParents(&inst.StoredMetadata)}}} if inst.NetworkEnabled { c.Networks = []hypervisor.NetworkConfig{{MAC: inst.MAC, IP: inst.IP, Netmask: macOSGuestNetmask}} } diff --git a/lib/instances/macos_build_export.go b/lib/instances/macos_build_export.go index 13539cd15..af30080f1 100644 --- a/lib/instances/macos_build_export.go +++ b/lib/instances/macos_build_export.go @@ -1,5 +1,47 @@ package instances -// MacOSBuildExportReceipt describes one stop/export operation, not persisted state. -// A failed export may still confirm that instance cleanup is safe. +import ( + "fmt" + "os" + + "github.com/kernel/hypeman/lib/images" +) + +// MacOSBuildExportReceipt describes one strict stop, not persisted state. type MacOSBuildExportReceipt struct{ Graceful, VMMExited bool } + +// MacOSStorage is a stopped macOS guest's closed boot disk and platform: the +// read-only ASIF layers below its writable overlay, base first, the overlay, and its +// auxiliary storage. A standalone base disk is an Overlay with no Layers. +type MacOSStorage struct { + Layers []string + Overlay string + Aux string + Platform images.MacOSImage +} + +// Files returns the storage's paths: the overlay, the auxiliary storage, then the layers. +func (s MacOSStorage) Files() []string { + return append([]string{s.Overlay, s.Aux}, s.Layers...) +} + +// CheckFiles requires every storage file to be a distinct nonempty regular file. +func (s MacOSStorage) CheckFiles() error { + var seen []os.FileInfo + for _, path := range s.Files() { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + for _, other := range seen { + if os.SameFile(other, info) { + return fmt.Errorf("machine storage files must be distinct") + } + } + seen = append(seen, info) + } + return nil +} diff --git a/lib/instances/macos_build_export_darwin.go b/lib/instances/macos_build_export_darwin.go index c37a45424..95381600f 100644 --- a/lib/instances/macos_build_export_darwin.go +++ b/lib/instances/macos_build_export_darwin.go @@ -4,12 +4,9 @@ package instances import ( "context" - "encoding/json" "fmt" - "os" "path/filepath" - "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/images" ) @@ -45,89 +42,49 @@ func (m *manager) StopMacOSBuildInstance(ctx context.Context, id string) (receip return receipt, nil } -// ExportStoppedMacOSBuildInstance copies closed storage for an instance that is -// still Stopped when the copy starts. The instance lock excludes Start during the -// check and copy; a concurrently restarted instance fails export instead. -func (m *manager) ExportStoppedMacOSBuildInstance(ctx context.Context, id, destination string) error { +// MacOSBuildStorage returns the closed storage of a Stopped macOS instance, checked +// under the instance lock. The caller publishes from these paths. A start in the +// meantime changes the overlay under the publisher, whose digest check then fails +// the upload instead of publishing it. +func (m *manager) MacOSBuildStorage(ctx context.Context, id string) (MacOSStorage, error) { if err := ctx.Err(); err != nil { - return err + return MacOSStorage{}, err } if _, bounded := ctx.Deadline(); !bounded { - return fmt.Errorf("machine export requires a deadline") - } - if !filepath.IsAbs(destination) { - return fmt.Errorf("export destination must be absolute") + return MacOSStorage{}, fmt.Errorf("machine storage requires a deadline") } lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() current, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { - return err + return MacOSStorage{}, err } if current.State != StateStopped || current.MacOS == nil { - return fmt.Errorf("export requires a stopped macOS instance") + return MacOSStorage{}, fmt.Errorf("machine storage requires a stopped macOS instance") + } + storage := MacOSStorage{ + Layers: macOSDiskParents(¤t.StoredMetadata), + Overlay: m.paths.InstanceMacOSBootDisk(id), + Aux: filepath.Join(current.DataDir, macOSAuxStorageFile), + Platform: *current.MacOS, } - if rel, e := filepath.Rel(current.DataDir, destination); e != nil || filepath.IsLocal(rel) { - return fmt.Errorf("export must be outside instance storage") + if err := checkClosedMacOSStorage(ctx, storage); err != nil { + return MacOSStorage{}, err } - return writeStoppedMacOSBuildBundle(ctx, current, m.paths.InstanceMacOSBootDisk(id), destination) + return storage, nil } -func writeStoppedMacOSBuildBundle(ctx context.Context, inst *Instance, disk, destination string) error { - if inst.State != StateStopped || inst.MacOS == nil { - return fmt.Errorf("machine must be stopped") - } - aux := filepath.Join(inst.DataDir, "mac-aux.img") - var diskInfo os.FileInfo - for _, source := range []string{disk, aux} { - info, err := os.Lstat(source) - if err != nil { - return err - } - if !info.Mode().IsRegular() || info.Size() == 0 { - return fmt.Errorf("machine storage must be nonempty regular files") - } - if diskInfo != nil && os.SameFile(diskInfo, info) { - return fmt.Errorf("machine storage files must be distinct") - } - diskInfo = info - if err = images.RequireClosedFile(ctx, source); err != nil { - return err - } +// checkClosedMacOSStorage requires every storage file to be a distinct nonempty +// regular file that no process holds open. +func checkClosedMacOSStorage(ctx context.Context, storage MacOSStorage) error { + if err := storage.CheckFiles(); err != nil { + return err } - for _, file := range []struct{ source, name string }{{disk, "disk.img"}, {aux, "aux.img"}} { - if err := ctx.Err(); err != nil { - return err - } - path := filepath.Join(destination, file.name) - if err := forkvm.CopyRegularFile(file.source, path); err != nil { + for _, path := range storage.Files() { + if err := images.RequireClosedFile(ctx, path); err != nil { return err } - if err := os.Chmod(path, 0600); err != nil { - return err - } - } - data, err := json.Marshal(inst.MacOS) - if err != nil { - return err - } - path := filepath.Join(destination, "config.json") - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - return err - } - _, writeErr := file.Write(data) - closeErr := file.Close() - if writeErr != nil { - return writeErr - } - if closeErr != nil { - return closeErr - } - if err = ctx.Err(); err != nil { - return err } - _, err = images.ValidateMacOSBundle(destination) - return err + return nil } diff --git a/lib/instances/macos_build_export_darwin_test.go b/lib/instances/macos_build_export_darwin_test.go index 0d357bed1..8f665c268 100644 --- a/lib/instances/macos_build_export_darwin_test.go +++ b/lib/instances/macos_build_export_darwin_test.go @@ -8,39 +8,36 @@ import ( "path/filepath" "testing" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) -func TestMacOSBuildBundleRequiresClosedDistinctStorage(t *testing.T) { +func TestMacOSBuildStorageRequiresClosedDistinctFiles(t *testing.T) { root := t.TempDir() - disk := filepath.Join(root, paths.MacOSBootDiskFile) - aux := filepath.Join(root, "mac-aux.img") - require.NoError(t, os.WriteFile(disk, []byte("synthetic disk"), 0600)) - require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) - inst := &Instance{State: StateStopped, StoredMetadata: StoredMetadata{DataDir: root, MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}} - file, e := os.Open(disk) - require.NoError(t, e) - require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) - require.NoError(t, file.Close()) - out := t.TempDir() - require.NoError(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, out)) - _, e = images.ValidateMacOSBundle(out) - require.NoError(t, e) - for _, name := range []string{"disk.img", "aux.img", "config.json"} { - info, e := os.Stat(filepath.Join(out, name)) - require.NoError(t, e) - require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + storage := MacOSStorage{ + Layers: []string{filepath.Join(root, "disk.0.asif")}, + Overlay: filepath.Join(root, paths.MacOSBootDiskFile), + Aux: filepath.Join(root, "mac-aux.img"), + } + for _, path := range []string{storage.Layers[0], storage.Overlay, storage.Aux} { + require.NoError(t, os.WriteFile(path, []byte("synthetic"), 0600)) } - require.NoError(t, os.Remove(aux)) - require.NoError(t, os.Link(disk, aux)) - require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) - inst.State = StateRunning - require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + require.NoError(t, checkClosedMacOSStorage(context.Background(), storage)) + + file, err := os.Open(storage.Layers[0]) + require.NoError(t, err) + require.Error(t, checkClosedMacOSStorage(context.Background(), storage), "an open layer is refused") + require.NoError(t, file.Close()) + + require.NoError(t, os.Remove(storage.Aux)) + require.NoError(t, os.Link(storage.Overlay, storage.Aux)) + require.Error(t, checkClosedMacOSStorage(context.Background(), storage), "aliased files are refused") + require.NoError(t, os.Remove(storage.Aux)) + require.NoError(t, os.WriteFile(storage.Aux, nil, 0600)) + require.Error(t, checkClosedMacOSStorage(context.Background(), storage), "empty files are refused") } -func TestMacOSBuildStopAndExportRequireBoundedInputs(t *testing.T) { +func TestMacOSBuildStopAndStorageRequireBoundedInputs(t *testing.T) { m := &manager{} _, e := m.StopMacOSBuildInstance(context.Background(), "unused") require.Error(t, e) @@ -48,6 +45,8 @@ func TestMacOSBuildStopAndExportRequireBoundedInputs(t *testing.T) { cancel() _, e = m.StopMacOSBuildInstance(ctx, "unused") require.ErrorIs(t, e, context.Canceled) - require.Error(t, m.ExportStoppedMacOSBuildInstance(context.Background(), "unused", t.TempDir())) - require.Error(t, m.ExportStoppedMacOSBuildInstance(ctx, "unused", t.TempDir())) + _, e = m.MacOSBuildStorage(context.Background(), "unused") + require.Error(t, e) + _, e = m.MacOSBuildStorage(ctx, "unused") + require.ErrorIs(t, e, context.Canceled) } diff --git a/lib/instances/macos_darwin.go b/lib/instances/macos_darwin.go deleted file mode 100644 index 884c794d7..000000000 --- a/lib/instances/macos_darwin.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build darwin - -package instances - -import ( - "os" - - "golang.org/x/sys/unix" -) - -func cloneMacOSStorage(rootDisk, rootAux, disk, aux string) (err error) { - var created []string - defer func() { - if err != nil { - // Only remove files this operation created, never pre-existing storage. - for _, path := range created { - os.Remove(path) - } - } - }() - for _, file := range []struct{ src, dst string }{ - {rootDisk, disk}, - {rootAux, aux}, - } { - if err = unix.Clonefile(file.src, file.dst, unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY); err != nil { - return err - } - created = append(created, file.dst) - if err = os.Chmod(file.dst, 0600); err != nil { - return err - } - } - return nil -} diff --git a/lib/instances/macos_darwin_test.go b/lib/instances/macos_darwin_test.go deleted file mode 100644 index 795a71199..000000000 --- a/lib/instances/macos_darwin_test.go +++ /dev/null @@ -1,72 +0,0 @@ -//go:build darwin - -package instances - -import ( - "errors" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/sys/unix" -) - -func TestMacOSStorageClone(t *testing.T) { - root := t.TempDir() - source := filepath.Join(root, "rootfs.raw") - auxSource := filepath.Join(root, "aux.img") - require.NoError(t, os.WriteFile(source, []byte("template disk"), 0644)) - require.NoError(t, os.WriteFile(auxSource, []byte("template aux"), 0644)) - probe := filepath.Join(root, "probe") - err := unix.Clonefile(source, probe, unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY) - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EXDEV) { - t.Skip("test filesystem does not support clonefile") - } - require.NoError(t, err) - require.NoError(t, os.Remove(probe)) - - t.Run("private writable copies", func(t *testing.T) { - dest := t.TempDir() - disk, aux := filepath.Join(dest, "disk"), filepath.Join(dest, "aux") - require.NoError(t, cloneMacOSStorage(source, auxSource, disk, aux)) - for path, expected := range map[string]string{disk: "template disk", aux: "template aux"} { - data, err := os.ReadFile(path) - require.NoError(t, err) - require.Equal(t, expected, string(data)) - info, err := os.Stat(path) - require.NoError(t, err) - require.Equal(t, os.FileMode(0600), info.Mode().Perm()) - require.NoError(t, os.WriteFile(path, []byte("instance changes"), 0600)) - } - for path, expected := range map[string]string{source: "template disk", auxSource: "template aux"} { - data, err := os.ReadFile(path) - require.NoError(t, err) - require.Equal(t, expected, string(data)) - } - }) - - t.Run("aux failure rolls back only new disk", func(t *testing.T) { - dest := t.TempDir() - disk, aux := filepath.Join(dest, "disk"), filepath.Join(dest, "aux") - require.NoError(t, os.WriteFile(aux, []byte("existing aux"), 0600)) - require.Error(t, cloneMacOSStorage(source, auxSource, disk, aux)) - _, err := os.Stat(disk) - require.ErrorIs(t, err, os.ErrNotExist) - data, err := os.ReadFile(aux) - require.NoError(t, err) - require.Equal(t, "existing aux", string(data)) - }) - - t.Run("existing disk is preserved", func(t *testing.T) { - dest := t.TempDir() - disk, aux := filepath.Join(dest, "disk"), filepath.Join(dest, "aux") - require.NoError(t, os.WriteFile(disk, []byte("existing disk"), 0600)) - require.Error(t, cloneMacOSStorage(source, auxSource, disk, aux)) - data, err := os.ReadFile(disk) - require.NoError(t, err) - require.Equal(t, "existing disk", string(data)) - _, err = os.Stat(aux) - require.ErrorIs(t, err, os.ErrNotExist) - }) -} diff --git a/lib/instances/macos_memory_snapshot_test.go b/lib/instances/macos_memory_snapshot_test.go index 28fb142b8..df3152b28 100644 --- a/lib/instances/macos_memory_snapshot_test.go +++ b/lib/instances/macos_memory_snapshot_test.go @@ -34,8 +34,10 @@ func macOSMemorySnapshotFixture(t *testing.T) (*manager, *metadata) { meta.MAC, meta.NetworkEnabled = meta.MacOS.MAC, true meta.SkipGuestAgent = true meta.StopGraceful = true + meta.MacOSDiskLayers = 1 require.NoError(t, m.saveMetadata(meta)) require.NoError(t, os.Rename(m.paths.InstanceOverlay(id), m.paths.InstanceMacOSBootDisk(id))) + require.NoError(t, os.WriteFile(macOSDiskLayer(meta.DataDir, 0), []byte("image layer"), 0400)) require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, macOSAuxStorageFile), []byte("aux-at-checkpoint"), 0600)) return m, meta } @@ -49,7 +51,7 @@ func writeMacOSMemorySnapshotPayload(t *testing.T, m *manager, guestDir string, VCPUs: stored.Vcpus, MemoryBytes: stored.Size, MacHardwareModelData: base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel), MacMachineIdentifierData: base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier), MacAuxStoragePath: filepath.Join(m.paths.InstanceDir(stored.Id), macOSAuxStorageFile), - Disks: []shimconfig.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(stored.Id)}}, + Disks: []shimconfig.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(stored.Id), Parents: macOSDiskParents(stored)}}, }} if stored.NetworkEnabled { manifest.ShimConfig.Networks = []shimconfig.NetworkConfig{{MAC: stored.MAC}} @@ -112,13 +114,13 @@ func TestMacOSMemorySnapshotCaptureRestoreAndIsolation(t *testing.T) { require.Equal(t, current.Phases.Cumulative[phasetracking.PhaseRunning], restored.Phases.Cumulative[phasetracking.PhaseRunning], "same-instance restore must not rewind host phase accounting") require.NotNil(t, restored.ExpiresAt) require.True(t, expiresAt.Equal(*restored.ExpiresAt)) - for name, want := range map[string]string{"boot.raw": "overlay", macOSAuxStorageFile: "aux-at-checkpoint", "snapshots/snapshot-latest/" + shimconfig.SnapshotMachineStateFile: "saved-memory"} { + for name, want := range map[string]string{paths.MacOSBootDiskFile: "overlay", macOSAuxStorageFile: "aux-at-checkpoint", "snapshots/snapshot-latest/" + shimconfig.SnapshotMachineStateFile: "saved-memory"} { data, err := os.ReadFile(filepath.Join(meta.DataDir, name)) require.NoError(t, err) require.Equal(t, want, string(data)) } require.NoError(t, os.WriteFile(m.paths.InstanceMacOSBootDisk(meta.Id), []byte("restored-write"), 0600)) - data, err := os.ReadFile(filepath.Join(m.paths.SnapshotGuestDir(snapshot.Id), "boot.raw")) + data, err := os.ReadFile(filepath.Join(m.paths.SnapshotGuestDir(snapshot.Id), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "overlay", string(data), "restoring must not alias the persistent snapshot") diff --git a/lib/instances/macos_other.go b/lib/instances/macos_other.go deleted file mode 100644 index 581e3ba18..000000000 --- a/lib/instances/macos_other.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !darwin - -package instances - -import "fmt" - -func cloneMacOSStorage(string, string, string, string) error { - return fmt.Errorf("macOS image clones require APFS on macOS") -} diff --git a/lib/instances/macos_storage.go b/lib/instances/macos_storage.go new file mode 100644 index 000000000..6d4ed400b --- /dev/null +++ b/lib/instances/macos_storage.go @@ -0,0 +1,62 @@ +package instances + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" +) + +// createMacOSStorage gives a macOS guest its own copy of an image's boot disk chain +// and auxiliary storage in dataDir, and an empty writable overlay above the chain. +// Copies are clones where the filesystem supports them. On failure only files this +// call created are removed. +func createMacOSStorage(chain []string, aux, dataDir string) (err error) { + var created []string + defer func() { + if err != nil { + for _, path := range created { + os.Remove(path) + } + } + }() + copies := make([]string, len(chain)) + for i, layer := range chain { + copies[i] = macOSDiskLayer(dataDir, i) + if err = copyMacOSStorageFile(layer, copies[i], 0400, &created); err != nil { + return err + } + } + if err = copyMacOSStorageFile(aux, filepath.Join(dataDir, macOSAuxStorageFile), 0600, &created); err != nil { + return err + } + img, err := images.OpenMacOSDiskChain(copies) + if err != nil { + return err + } + defer img.Close() + overlay := filepath.Join(dataDir, paths.MacOSBootDiskFile) + if _, err = os.Lstat(overlay); err == nil { + return fmt.Errorf("macOS boot overlay already exists: %s", overlay) + } + created = append(created, overlay) + if err = img.CreateLayer(overlay, diskimagekit.Overlay); err != nil { + return fmt.Errorf("create macOS boot overlay: %w", err) + } + return os.Chmod(overlay, 0600) +} + +func copyMacOSStorageFile(src, dst string, mode os.FileMode, created *[]string) error { + if _, err := os.Lstat(dst); err == nil { + return fmt.Errorf("macOS storage file already exists: %s", dst) + } + if err := forkvm.CopyRegularFile(src, dst); err != nil { + return err + } + *created = append(*created, dst) + return os.Chmod(dst, mode) +} diff --git a/lib/instances/macos_storage_test.go b/lib/instances/macos_storage_test.go new file mode 100644 index 000000000..d712f6480 --- /dev/null +++ b/lib/instances/macos_storage_test.go @@ -0,0 +1,100 @@ +package instances + +import ( + "crypto/sha256" + "os" + "path/filepath" + "testing" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// imageChainFixture writes a two-layer ASIF chain and an aux file, as an image has. +func imageChainFixture(t *testing.T) ([]string, string) { + t.Helper() + dir := t.TempDir() + chain := []string{filepath.Join(dir, "disk.0.asif"), filepath.Join(dir, "disk.1.asif")} + img, err := diskimagekit.CreateASIF(chain[0], (16<<20)/512) + require.NoError(t, err) + _, err = img.WriteAt([]byte("base"), 0) + require.NoError(t, err) + require.NoError(t, img.CreateLayer(chain[1], diskimagekit.Overlay)) + _, err = img.WriteAt([]byte("image"), 1<<20) + require.NoError(t, err) + require.NoError(t, img.Close()) + aux := filepath.Join(dir, "aux.img") + require.NoError(t, os.WriteFile(aux, []byte("aux"), 0600)) + return chain, aux +} + +func fileSum(t *testing.T, path string) [32]byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + return sha256.Sum256(data) +} + +func TestCreateMacOSStorage(t *testing.T) { + chain, aux := imageChainFixture(t) + before := []([32]byte){fileSum(t, chain[0]), fileSum(t, chain[1])} + dataDir := t.TempDir() + require.NoError(t, createMacOSStorage(chain, aux, dataDir)) + + stored := &StoredMetadata{DataDir: dataDir, MacOSDiskLayers: len(chain)} + parents := macOSDiskParents(stored) + for i, layer := range parents { + require.Equal(t, before[i], fileSum(t, layer), "layer %d is a copy of the image's", i) + info, err := os.Stat(layer) + require.NoError(t, err) + require.Equal(t, os.FileMode(0400), info.Mode().Perm(), "image layers are read-only in the instance") + } + overlay := filepath.Join(dataDir, paths.MacOSBootDiskFile) + info, err := os.Stat(overlay) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.Equal(t, []byte("aux"), must(os.ReadFile(filepath.Join(dataDir, macOSAuxStorageFile)))) + + // The guest's view: the image's data through the overlay, and writes land only in the overlay. + disk, err := images.OpenMacOSDiskChain(parents) + require.NoError(t, err) + require.NoError(t, disk.Append(overlay, diskimagekit.WithOpenMode(diskimagekit.ReadWrite))) + _, err = disk.WriteAt([]byte("guest"), 2<<20) + require.NoError(t, err) + for offset, want := range map[int64]string{0: "base", 1 << 20: "image", 2 << 20: "guest"} { + buf := make([]byte, len(want)) + _, err := disk.ReadAt(buf, offset) + require.NoError(t, err) + require.Equal(t, want, string(buf)) + } + require.NoError(t, disk.Close()) + for i, layer := range parents { + require.Equal(t, before[i], fileSum(t, layer), "writes do not reach layer %d", i) + } + require.Equal(t, before[0], fileSum(t, chain[0])) + require.Equal(t, before[1], fileSum(t, chain[1])) + + require.Error(t, createMacOSStorage(chain, aux, dataDir), "existing storage is never replaced") + require.FileExists(t, overlay) +} + +func TestCreateMacOSStorageRemovesOnlyWhatItCreated(t *testing.T) { + chain, _ := imageChainFixture(t) + dataDir := t.TempDir() + existing := filepath.Join(dataDir, "unrelated") + require.NoError(t, os.WriteFile(existing, []byte("keep"), 0600)) + require.Error(t, createMacOSStorage(chain, filepath.Join(t.TempDir(), "missing-aux.img"), dataDir)) + entries, err := os.ReadDir(dataDir) + require.NoError(t, err) + require.Len(t, entries, 1, "the copied layers are removed") + require.FileExists(t, existing) +} + +func must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go index e120fa38f..17eea7f72 100644 --- a/lib/instances/snapshot_macos.go +++ b/lib/instances/snapshot_macos.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "time" @@ -162,7 +163,7 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St if cfg.VCPUs != stored.Vcpus || cfg.MemoryBytes != stored.Size || cfg.MacHardwareModelData != base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel) || cfg.MacMachineIdentifierData != base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier) { return fmt.Errorf("%w: macOS snapshot configuration does not match its metadata", ErrInvalidRequest) } - if len(cfg.Disks) != 1 || cfg.Disks[0].Readonly || cfg.Disks[0].Path != m.paths.InstanceMacOSBootDisk(stored.Id) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { + if len(cfg.Disks) != 1 || cfg.Disks[0].Readonly || cfg.Disks[0].Path != m.paths.InstanceMacOSBootDisk(stored.Id) || !slices.Equal(cfg.Disks[0].Parents, macOSDiskParents(stored)) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { return fmt.Errorf("%w: macOS snapshot storage configuration does not match its instance", ErrInvalidRequest) } if cfg.KernelPath != "" || cfg.InitrdPath != "" || cfg.EnableMemoryBalloon || cfg.RequireMemoryBalloon || cfg.EnableRosetta { @@ -185,7 +186,11 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St } else if len(cfg.Networks) != 0 { return fmt.Errorf("%w: macOS snapshot unexpectedly enables networking", ErrInvalidRequest) } - for _, path := range []string{filepath.Join(guestDir, paths.MacOSBootDiskFile), filepath.Join(guestDir, macOSAuxStorageFile), filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile)} { + payload := []string{filepath.Join(guestDir, paths.MacOSBootDiskFile), filepath.Join(guestDir, macOSAuxStorageFile), filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile)} + for i := range stored.MacOSDiskLayers { + payload = append(payload, macOSDiskLayer(guestDir, i)) + } + for _, path := range payload { info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { return fmt.Errorf("%w: missing or invalid macOS snapshot payload %s", ErrInvalidRequest, filepath.Base(path)) diff --git a/lib/instances/snapshot_macos_darwin_test.go b/lib/instances/snapshot_macos_darwin_test.go index edf1db8a8..84c452589 100644 --- a/lib/instances/snapshot_macos_darwin_test.go +++ b/lib/instances/snapshot_macos_darwin_test.go @@ -36,7 +36,7 @@ func TestMacOSStoppedSnapshotForkRebindsIdentity(t *testing.T) { source.Phases.Record(phasetracking.PhaseStopped, now) require.NoError(t, vz.NewStarter().ValidateConfig(m.plannedStoredVMConfig(SnapshotKindStopped, source))) require.NoError(t, m.saveSnapshotRecord(&snapshotRecord{Snapshot: Snapshot{Id: snapshotID, Kind: SnapshotKindStopped, SourceInstanceID: source.Id, SourceName: source.Name, SourceHypervisor: source.HypervisorType, CreatedAt: now}, StoredMetadata: source})) - writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{"boot.raw": "snapshot-disk", macOSAuxStorageFile: "snapshot-aux"}) + writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{paths.MacOSBootDiskFile: "snapshot-disk", macOSAuxStorageFile: "snapshot-aux"}) first, err := m.ForkSnapshot(context.Background(), snapshotID, ForkSnapshotRequest{Name: "mac-snapshot-clone-1"}) require.NoError(t, err) @@ -67,7 +67,7 @@ func TestMacOSStoppedSnapshotForkRebindsIdentity(t *testing.T) { require.NoError(t, err) require.Equal(t, source.MacOS, persisted.StoredMetadata.MacOS) require.NoError(t, os.WriteFile(p.InstanceMacOSBootDisk(first.Id), []byte("clone-write"), 0600)) - data, err := os.ReadFile(filepath.Join(p.SnapshotGuestDir(snapshotID), "boot.raw")) + data, err := os.ReadFile(filepath.Join(p.SnapshotGuestDir(snapshotID), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "snapshot-disk", string(data)) } diff --git a/lib/instances/snapshot_macos_test.go b/lib/instances/snapshot_macos_test.go index 47cad876d..46a393427 100644 --- a/lib/instances/snapshot_macos_test.go +++ b/lib/instances/snapshot_macos_test.go @@ -76,13 +76,13 @@ func writeGuestFiles(t *testing.T, dir string, files map[string]string) { func TestMacOSRestoreSwapsPayloadWithMetadata(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} const id, snap = "mac-restore", "snap-1" - writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"boot.raw": "after", macOSAuxStorageFile: "aux-after"}) - writeGuestFiles(t, m.paths.SnapshotGuestDir(snap), map[string]string{"boot.raw": "before", macOSAuxStorageFile: "aux-before", "metadata.json": "stale-metadata"}) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{paths.MacOSBootDiskFile: "after", macOSAuxStorageFile: "aux-after"}) + writeGuestFiles(t, m.paths.SnapshotGuestDir(snap), map[string]string{paths.MacOSBootDiskFile: "before", macOSAuxStorageFile: "aux-before", "metadata.json": "stale-metadata"}) expiresAt := time.Now().Add(time.Hour) restored := &metadata{StoredMetadata: StoredMetadata{Id: id, Name: "current-name", ExpiresAt: &expiresAt, Vcpus: 4, Size: 8 << 30}} require.NoError(t, m.replaceMacOSInstanceWithSnapshotPayload(snap, restored)) - data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "boot.raw")) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "before", string(data)) require.NoDirExists(t, m.paths.InstanceRestoreStaging(id), "staging is removed after a successful swap") @@ -96,9 +96,9 @@ func TestMacOSRestoreSwapsPayloadWithMetadata(t *testing.T) { require.True(t, expiresAt.Equal(*got.ExpiresAt)) // A missing snapshot payload fails before the instance directory is retired. - writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"boot.raw": "current"}) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{paths.MacOSBootDiskFile: "current"}) require.Error(t, m.replaceMacOSInstanceWithSnapshotPayload("missing-snapshot", &metadata{StoredMetadata: StoredMetadata{Id: id}})) - data, err = os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "boot.raw")) + data, err = os.ReadFile(filepath.Join(m.paths.InstanceDir(id), paths.MacOSBootDiskFile)) require.NoError(t, err) require.Equal(t, "current", string(data), "a failed staging leaves the previous disk untouched") } @@ -110,7 +110,7 @@ func TestMacOSRestoreRecoveryRollsBackOrFinishes(t *testing.T) { return m, m.paths.InstanceRestoreStaging(id) } readBoot := func(t *testing.T, m *manager) string { - data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "boot.raw")) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), paths.MacOSBootDiskFile)) require.NoError(t, err) return string(data) } @@ -118,8 +118,8 @@ func TestMacOSRestoreRecoveryRollsBackOrFinishes(t *testing.T) { // Crash while retiring files, before the swap commits: the retired files come back. m, root := newManager(t) writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{macOSAuxStorageFile: "aux-previous"}) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "previous"}) - writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{"boot.raw": "restored"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "previous"}) + writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{paths.MacOSBootDiskFile: "restored"}) require.NoError(t, m.recoverMacOSRestoreStaging(id)) require.Equal(t, "previous", readBoot(t, m)) require.FileExists(t, filepath.Join(m.paths.InstanceDir(id), macOSAuxStorageFile)) @@ -128,28 +128,28 @@ func TestMacOSRestoreRecoveryRollsBackOrFinishes(t *testing.T) { // Crash while installing a committed swap: the rest of the payload is installed. m, root = newManager(t) writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{macOSAuxStorageFile: "aux-restored"}) - writeGuestFiles(t, filepath.Join(root, "installing"), map[string]string{"boot.raw": "restored"}) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "old", macOSAuxStorageFile: "aux-old"}) + writeGuestFiles(t, filepath.Join(root, "installing"), map[string]string{paths.MacOSBootDiskFile: "restored"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "old", macOSAuxStorageFile: "aux-old"}) require.NoError(t, m.recoverMacOSRestoreStaging(id)) require.Equal(t, "restored", readBoot(t, m)) require.NoDirExists(t, root) // Crash during the staging copy: nothing was retired, so the partial copy is removed. m, root = newManager(t) - writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"boot.raw": "current"}) - writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{"boot.raw": "partial"}) - writeGuestFiles(t, m.paths.InstanceDir("other"), map[string]string{"boot.raw": "untouched"}) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{paths.MacOSBootDiskFile: "current"}) + writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{paths.MacOSBootDiskFile: "partial"}) + writeGuestFiles(t, m.paths.InstanceDir("other"), map[string]string{paths.MacOSBootDiskFile: "untouched"}) require.NoError(t, m.recoverMacOSRestoreStaging("")) require.Equal(t, "current", readBoot(t, m)) require.NoDirExists(t, root) - require.FileExists(t, filepath.Join(m.paths.InstanceDir("other"), "boot.raw")) + require.FileExists(t, filepath.Join(m.paths.InstanceDir("other"), paths.MacOSBootDiskFile)) } func TestMacOSRestoreStagingIsDeletedWithInstance(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} const id = "mac-deleted" root := m.paths.InstanceRestoreStaging(id) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "previous"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "previous"}) require.Equal(t, m.paths.InstanceDir(id), filepath.Dir(root), "staging lives inside the instance directory") require.NoError(t, os.RemoveAll(m.paths.InstanceDir(id))) @@ -164,14 +164,14 @@ func TestMacOSRestoreRecoveryKeepsBackupWhenStagingIsUnreadable(t *testing.T) { const id = "mac-unknown" m := &manager{paths: paths.New(t.TempDir())} root := m.paths.InstanceRestoreStaging(id) - writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"boot.raw": "previous"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{paths.MacOSBootDiskFile: "previous"}) // An unreadable staging directory makes the swap's progress unknown. require.NoError(t, os.Chmod(root, 0)) t.Cleanup(func() { _ = os.Chmod(root, 0755) }) require.Error(t, m.recoverMacOSRestoreStaging(id)) require.NoError(t, os.Chmod(root, 0755)) - data, err := os.ReadFile(filepath.Join(root, "previous", "boot.raw")) + data, err := os.ReadFile(filepath.Join(root, "previous", paths.MacOSBootDiskFile)) require.NoError(t, err, "the parked backup must survive an inconclusive check") require.Equal(t, "previous", string(data)) } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 1ee166fc2..d26307166 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -26,7 +26,8 @@ const ( // {dataDir}/guests/{instance-id}/ // metadata.json # Instance metadata // overlay.raw # Configurable sparse overlay disk (default 10GB) -// boot.raw # macOS guests: complete boot disk, instead of overlay.raw +// boot.asif # macOS guests: writable ASIF overlay, instead of overlay.raw +// disk.{n}.asif # macOS guests: read-only ASIF layers below boot.asif, base first // config.ext4 # Read-only config disk (generated) // ch.sock # Hypervisor API socket (abbreviated name for SUN_LEN limit) // logs/ diff --git a/lib/instances/types.go b/lib/instances/types.go index 7c50cb359..68226bfbf 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -94,6 +94,9 @@ type StoredMetadata struct { // Read-only; echoed on the instance API. Platform string MacOS *images.MacOSImage `json:"MacOS,omitempty"` // Non-nil for experimental local macOS disk guests. + // MacOSDiskLayers is the number of read-only image layers below a macOS guest's + // boot overlay, cloned into its data directory. + MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` // IdentityRekeyPending marks a macOS clone whose SSH host keys still come from its // source. The next start rekeys them and clears the flag. IdentityRekeyPending bool `json:"identity_rekey_pending,omitempty"` diff --git a/lib/instances/vm_storage_path_test.go b/lib/instances/vm_storage_path_test.go index 62fd284e5..659b7f41b 100644 --- a/lib/instances/vm_storage_path_test.go +++ b/lib/instances/vm_storage_path_test.go @@ -10,6 +10,6 @@ import ( func TestVMStoragePathOnlyForMacOSGuests(t *testing.T) { require.Empty(t, (&StoredMetadata{DataDir: "/data/guests/i1"}).VMStoragePath()) mac := &StoredMetadata{DataDir: "/data/guests/i1", MacOS: &images.MacOSImage{}} - require.Equal(t, "/data/guests/i1/boot.raw", mac.VMStoragePath()) + require.Equal(t, "/data/guests/i1/boot.asif", mac.VMStoragePath()) require.Empty(t, (&StoredMetadata{MacOS: &images.MacOSImage{}}).VMStoragePath(), "no data dir, no path") } diff --git a/lib/paths/paths.go b/lib/paths/paths.go index f8fba4cb3..1940e4a99 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -251,8 +251,9 @@ func (p *Paths) InstanceOverlay(id string) string { return filepath.Join(p.InstanceDir(id), "overlay.raw") } -// MacOSBootDiskFile is a macOS guest's complete raw boot disk, cloned from its image. -const MacOSBootDiskFile = "boot.raw" +// MacOSBootDiskFile is a macOS guest's writable ASIF overlay, the top of its boot +// disk chain. The read-only layers below it are cloned from its image. +const MacOSBootDiskFile = "boot.asif" // InstanceMacOSBootDisk returns the path to a macOS guest's boot disk. func (p *Paths) InstanceMacOSBootDisk(id string) string { diff --git a/lib/registrypush/macos_test.go b/lib/registrypush/macos_test.go index a9acd4275..40b16d0e9 100644 --- a/lib/registrypush/macos_test.go +++ b/lib/registrypush/macos_test.go @@ -23,8 +23,8 @@ func TestPushMacOSImagePreservesManifestAndPlatform(t *testing.T) { require.NoError(t, err) config.OS, config.Architecture = "darwin", "arm64" config.Config.Labels = map[string]string{ - images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", - images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", + images.MacOSMachineVersionLabel: "2", images.MacOSMachineKindLabel: "macos-image", + images.MacOSMachineFormatLabel: "asif", images.MacOSMachineDiskLabel: "disk.asif", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", } img, err = mutate.ConfigFile(img, config) From 388341923fc441e5666052e0a0e524f64786b845 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:28:03 +0000 Subject: [PATCH 07/33] Clone macOS image layers a ready image already has A pull of a macOS image built on a base the host already has clones the base's installed disk instead of unpacking the layer again, so images on one base share its disk. Auxiliary storage and platform come only from the top layer, so a skipped lower layer cannot change them. --- docs/macos-experimental.md | 8 ++-- lib/images/macos_bundle.go | 28 +++++++------ lib/images/macos_machine.go | 34 ++++++++++++++-- lib/images/macos_machine_test.go | 68 ++++++++++++++++++++++++++++++++ lib/images/macos_unpack.go | 31 ++++++++++++++- lib/images/manager.go | 3 +- lib/images/oci.go | 2 + lib/images/storage.go | 3 +- 8 files changed, 154 insertions(+), 23 deletions(-) diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 9a8cbb0db..081d946ec 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -46,9 +46,11 @@ labels describe a complete installed machine: Each layer carries one ASIF disk, and each layer's disk is an overlay on the previous layer's, so the layers form the boot disk's chain, base first. The -auxiliary storage and platform come from the last layer that has them. A base -image is one layer; an image built on it adds one layer holding only the build's -writes. All files must be distinct, nonempty regular files inside the unpacked +auxiliary storage and platform come from the top layer; lower layers' copies are +ignored. A base image is one layer; an image built on it adds one layer holding +only the build's writes. A pull clones any lower layer whose diff ID a ready image +on the host already has instead of unpacking it, so images built on one base share +its disk. All files must be distinct, nonempty regular files inside the unpacked image. Platform configuration uses the local bundle's `MacOSImage` schema and is limited to 64 KiB. Absolute paths, traversal and symlink escapes are rejected. The installed disk is materialized directly rather than converted to a Linux diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 4d0998e56..b086ef959 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -77,7 +77,7 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b // the highest layer that has them. func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSMachinePayload, error) { var payload macOSMachinePayload - var config string + var top string for i := 0; ; i++ { dir := filepath.Join(root, strconv.Itoa(i)) if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { @@ -88,19 +88,21 @@ func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSM return nil, fmt.Errorf("machine layer %d: %w", i, err) } payload.Disks = append(payload.Disks, disk) - if aux, err := machineBundleFile(dir, auxName); err == nil { - payload.Aux = aux - } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("machine layer %d: %w", i, err) - } - if c, err := machineBundleFile(dir, configName); err == nil { - config = c - } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("machine layer %d: %w", i, err) - } + top = dir } - if len(payload.Disks) == 0 || payload.Aux == "" || config == "" { - return nil, fmt.Errorf("machine image needs a disk in every layer and auxiliary storage and platform in one") + if len(payload.Disks) == 0 { + return nil, fmt.Errorf("machine image needs a disk in every layer") + } + // Only the top layer's auxiliary storage and platform count, so a lower layer + // can be reused from another image without unpacking it. + aux, err := machineBundleFile(top, auxName) + if err != nil { + return nil, fmt.Errorf("machine image top layer: %w", err) + } + payload.Aux = aux + config, err := machineBundleFile(top, configName) + if err != nil { + return nil, fmt.Errorf("machine image top layer: %w", err) } platform, err := readMacOSPlatform(config, false) if err != nil { diff --git a/lib/images/macos_machine.go b/lib/images/macos_machine.go index fc868458c..2be88483a 100644 --- a/lib/images/macos_machine.go +++ b/lib/images/macos_machine.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "time" gcr "github.com/google/go-containerregistry/pkg/v1" @@ -27,6 +28,7 @@ type stagedMacOS struct { layers []string // staged overlays above the base, in order, beside their final paths aux string // staged auxiliary storage, beside its final path platform *MacOSImage + diffIDs []string // OCI diff ID of each chain layer, base first } // finalDiskPath is where finalization installs the staged disk: the layout's rootfs @@ -62,6 +64,7 @@ func installMacOSMachine(layout imageLayout, staged stagedImageFiles, meta *imag } meta.MacOS = staged.macos.platform meta.MacOSDiskLayers = len(staged.macos.layers) + 1 + meta.MacOSDiskDiffIDs = staged.macos.diffIDs return append(installed, auxPath), nil } @@ -94,17 +97,42 @@ func checkCachedMachinePlatform(p *paths.Paths, ref *ResolvedRef, requested Plat return nil } +// readyMacOSLayer returns the disk of a ready macOS image's chain layer that has +// diffID, or "" if no ready image has one. +func (m *manager) readyMacOSLayer(diffID string) string { + metas, err := listAllMetadata(m.paths) + if err != nil { + return "" + } + for _, meta := range metas { + if meta.Status != StatusReady || meta.MacOS == nil || len(meta.MacOSDiskDiffIDs) != meta.MacOSDiskLayers { + continue + } + i := slices.Index(meta.MacOSDiskDiffIDs, diffID) + if i < 0 { + continue + } + if chain, err := MacOSDiskChain(m.paths, meta.Name, meta.Digest); err == nil { + return chain[i] + } + } + return "" +} + // stageMacOSOutputs stages a macOS machine image's boot disk chain and auxiliary // storage in layoutDir, outside the manager lock. For other images it stages nothing // and returns diskTemp for the rootfs export. -func (m *manager) stageMacOSOutputs(ctx context.Context, digest, root string, meta *containerMetadata, layoutDir, diskTemp, buildID string) (stagedImageFiles, error) { +func (m *manager) stageMacOSOutputs(ctx context.Context, digest, root string, result *pullResult, layoutDir, diskTemp, buildID string) (stagedImageFiles, error) { staged := stagedImageFiles{disk: diskTemp} - if !isMacOSMachine(meta) { + if !isMacOSMachine(result.Metadata) { return staged, nil } start := time.Now() staged.disk = filepath.Join(layoutDir, machineDiskFile+".tmp-"+buildID) - err := stageMacOSMachine(&staged, root, meta, layoutDir, buildID) + err := stageMacOSMachine(&staged, root, result.Metadata, layoutDir, buildID) + if err == nil && result.Manifest != nil { + staged.macos.diffIDs = result.Manifest.Config.DiffIDs + } m.recordImageBuildPhase(ctx, digest, "filesystem_export", time.Since(start), phaseStatus(err), "not_applicable") return staged, err } diff --git a/lib/images/macos_machine_test.go b/lib/images/macos_machine_test.go index a32c1a0b5..2bae089c5 100644 --- a/lib/images/macos_machine_test.go +++ b/lib/images/macos_machine_test.go @@ -244,6 +244,74 @@ func TestMacOSMachineOCIRoundTrip(t *testing.T) { require.ErrorIs(t, err, ErrInvalidPlatform) } +// TestMacOSMachinePullReusesReadyLayers pulls two images that share a base layer +// and expects the second to clone the first's installed base disk, not unpack it. +func TestMacOSMachinePullReusesReadyLayers(t *testing.T) { + allowMacOSMachines(t) + work := t.TempDir() + base, overlay := macOSFixtureChain(t, work) + aux, config := filepath.Join(work, "aux.img"), filepath.Join(work, "config.json") + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + require.NoError(t, os.WriteFile(config, macOSFixturePlatform(t), 0600)) + baseLayer := machineFixtureLayer(t, work, "base", map[string]string{"disk.asif": base}) + server := httptest.NewServer(registry.New()) + defer server.Close() + p := paths.New(filepath.Join(work, "hypeman-data")) + mgr, err := NewManager(p, 1, nil) + require.NoError(t, err) + client := mgr.(*manager).ociClient + lookup := client.readyMachineLayer + var reused []string + client.readyMachineLayer = func(diffID string) string { + path := lookup(diffID) + if path != "" { + reused = append(reused, path) + } + return path + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + pull := func(repo string, top gcr.Layer) []string { + image, err := mutate.AppendLayers(empty.Image, baseLayer, top) + require.NoError(t, err) + cfg, err := image.ConfigFile() + require.NoError(t, err) + cfg.OS, cfg.Architecture = "darwin", "arm64" + cfg.Config.Labels = macOSFixtureMetadata().Labels + image, err = mutate.ConfigFile(image, cfg) + require.NoError(t, err) + ref, err := name.ParseReference(strings.TrimPrefix(server.URL, "http://") + "/" + repo + ":latest") + require.NoError(t, err) + require.NoError(t, remote.Write(ref, image)) + pulled, err := mgr.CreateImage(ctx, CreateImageRequest{Name: ref.Name(), Platform: "darwin/arm64"}) + require.NoError(t, err) + require.NoError(t, mgr.WaitForReady(ctx, pulled.Name)) + pulled, err = mgr.GetImage(ctx, pulled.Name) + require.NoError(t, err) + chain, err := MacOSDiskChain(p, pulled.Name, pulled.Digest) + require.NoError(t, err) + return chain + } + first := pull("first", machineFixtureLayer(t, work, "first", map[string]string{"disk.asif": overlay, "aux.img": aux, "config.json": config})) + require.Empty(t, reused, "no ready image has the base layer yet") + + img, err := diskimagekit.Open(base, diskimagekit.WithOpenMode(diskimagekit.ReadOnly)) + require.NoError(t, err) + second := filepath.Join(work, "second.asif") + require.NoError(t, img.CreateLayer(second, diskimagekit.Overlay)) + require.NoError(t, img.Close()) + chain := pull("second", machineFixtureLayer(t, work, "second", map[string]string{"disk.asif": second, "aux.img": aux, "config.json": config})) + require.Equal(t, []string{first[0]}, reused, "the shared base layer is cloned from the first image; the top layer is unpacked") + want, err := fileSHA256(ctx, base) + require.NoError(t, err) + got, err := fileSHA256(ctx, chain[0]) + require.NoError(t, err) + require.Equal(t, want, got) + stacked, err := OpenMacOSDiskChain(chain) + require.NoError(t, err) + require.NoError(t, stacked.Close()) +} + // TestMacOSMachinePullRejectsBrokenChain pulls an image whose overlay layer does not // belong to its base, and expects the pull to fail. func TestMacOSMachinePullRejectsBrokenChain(t *testing.T) { diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index b3db67080..6cd7bafbb 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -16,6 +16,7 @@ import ( gcr "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/kernel/hypeman/lib/forkvm" ) // machineUnpackChunk is the granularity of hole detection when writing machine files. @@ -34,7 +35,8 @@ var maxMachineUnpackedBytes int64 = 1 << 40 // the rootfs unpack, it writes files sparsely: zero chunks become holes, so a // mostly empty boot disk occupies only its data. Machine bundles hold only regular // files and directories; any other entry is rejected. Each layer is checked -// against its config diff ID. +// against its config diff ID. A layer below the top whose disk a ready image +// already has is cloned from that image instead of unpacked. func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDir string) error { path, err := layout.FromPath(c.cacheDir) if err != nil { @@ -64,13 +66,38 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi } remaining := maxMachineUnpackedBytes for i, l := range layers { - if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], filepath.Join(targetDir, strconv.Itoa(i)), &remaining); err != nil { + dir := filepath.Join(targetDir, strconv.Itoa(i)) + if i < len(layers)-1 && c.reuseMachineLayer(configFile.RootFS.DiffIDs[i], dir, configFile.Config.Labels[MacOSMachineDiskLabel]) { + continue + } + if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], dir, &remaining); err != nil { return fmt.Errorf("unpack machine layer %d: %w", i, err) } } return nil } +// reuseMachineLayer clones a ready image's disk with diffID into dir, where +// unpacking the layer would put it, and reports whether it did. A failed clone, as +// when that image is being deleted, leaves dir absent for an unpack. +func (c *ociClient) reuseMachineLayer(diffID gcr.Hash, dir, diskName string) bool { + if c.readyMachineLayer == nil { + return false + } + source := c.readyMachineLayer(diffID.String()) + if source == "" { + return false + } + target := filepath.Join(dir, diskName) + if err := os.MkdirAll(filepath.Dir(target), 0700); err == nil { + if err = forkvm.CopyRegularFile(source, target); err == nil { + return true + } + } + os.RemoveAll(dir) + return false +} + // unpackMachineLayer extracts one layer, spending its decompressed bytes from the // bundle's remaining budget. func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targetDir string, remaining *int64) error { diff --git a/lib/images/manager.go b/lib/images/manager.go index 6618b3176..5bfcd4453 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -110,6 +110,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string), } + ociClient.readyMachineLayer = m.readyMacOSLayer // Initialize metrics if meter is provided if meter != nil { @@ -522,7 +523,7 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials diskPath := layout.disk // Keep the temporary filesystem beside its final path so finalization stays // atomic even when system/builds and images are on different filesystems. - staged, err := m.stageMacOSOutputs(ctx, ref.Digest(), tempDir, result.Metadata, layout.dir, diskPath+".tmp-"+buildID, buildID) + staged, err := m.stageMacOSOutputs(ctx, ref.Digest(), tempDir, result, layout.dir, diskPath+".tmp-"+buildID, buildID) defer staged.remove() if err != nil { m.updateStatusByDigest(ref, StatusFailed, err, buildID) diff --git a/lib/images/oci.go b/lib/images/oci.go index 3a3ad743f..0ebb57636 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -25,6 +25,8 @@ import ( // ociClient handles OCI image operations without requiring Docker daemon type ociClient struct { cacheDir string + // readyMachineLayer returns a ready image's disk for a macOS layer diff ID, or "". + readyMachineLayer func(diffID string) string } type credentialedManifestInspector struct { diff --git a/lib/images/storage.go b/lib/images/storage.go index 507652c3c..b707d1b72 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -18,7 +18,8 @@ type imageMetadata struct { Digest string `json:"digest"` // Always present: sha256:... Platform string `json:"platform,omitempty"` MacOS *MacOSImage `json:"macos,omitempty"` - MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` // length of the macOS boot disk chain + MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` // length of the macOS boot disk chain + MacOSDiskDiffIDs []string `json:"macos_disk_diff_ids,omitempty"` // OCI diff ID of each chain layer; empty for local imports Status string `json:"status"` Error *string `json:"error,omitempty"` Request *CreateImageRequest `json:"request,omitempty"` From 45079a33958dfb7ea2318bbb32dd4315093f188f Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:02:14 +0000 Subject: [PATCH 08/33] Require the macOS 27 SDK for vz-shim The Swift shim links DiskImageKit and the vmnet network API, which older SDKs lack. Fail the build with the SDK version instead of a compiler error, drop the DiskImageKit import guard, build releases on the Xcode 27 runner, and print the macOS and SDK versions in test-darwin. --- .github/workflows/release.yml | 3 ++- .github/workflows/test.yml | 2 ++ Makefile | 4 ++- cmd/vz-shim/VM.swift | 48 +++++++++++++++-------------------- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b398a92c3..fa11e9551 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,8 @@ jobs: # cross-compiled from the Linux release job — it builds on a macOS arm64 runner. release-darwin: needs: release - runs-on: macos-14 + # vz-shim needs the macOS 27 SDK. + runs-on: xcode-27 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c99517e68..f1edda087 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -404,6 +404,8 @@ jobs: with: go-version-file: go.mod cache: false + - name: Show macOS and SDK versions + run: sw_vers && xcrun --sdk macosx --show-sdk-version - name: Ensure Docker is running run: | if docker info >/dev/null 2>&1; then diff --git a/Makefile b/Makefile index 1d4218df5..3f0732b07 100644 --- a/Makefile +++ b/Makefile @@ -409,7 +409,8 @@ ENTITLEMENTS_FILE ?= vz.entitlements # Build vz-shim (subprocess that hosts vz VMs) from the Swift sources in # cmd/vz-shim. Also copies to embed directory so it gets embedded in the hypeman -# binary. Stacked ASIF disks need DiskImageKit, so they need the macOS 27 SDK. +# binary. It links DiskImageKit and the vmnet network API, so it needs the macOS 27 +# SDK (Xcode 27); the binary runs on macOS 26, and stacked disks need macOS 27. VZ_SHIM_SOURCES := $(wildcard cmd/vz-shim/*.swift) VZ_SHIM_TARGET ?= $(shell uname -m)-apple-macos26.0 @@ -421,6 +422,7 @@ DARWIN_LDFLAGS := -ldflags=-extldflags=-Wl,-no_warn_duplicate_libraries .PHONY: build-vz-shim build-vz-shim: | $(BIN_DIR) @echo "Building vz-shim for macOS..." + @sdk=$$(xcrun --sdk macosx --show-sdk-version); case $$sdk in 2[7-9]*|[3-9][0-9]*) ;; *) echo "vz-shim needs the macOS 27 SDK (Xcode 27); found $$sdk" >&2; exit 1;; esac swiftc -O -target $(VZ_SHIM_TARGET) -o $(BIN_DIR)/vz-shim $(VZ_SHIM_SOURCES) mkdir -p lib/hypervisor/vz/vz-shim cp $(BIN_DIR)/vz-shim lib/hypervisor/vz/vz-shim/vz-shim diff --git a/cmd/vz-shim/VM.swift b/cmd/vz-shim/VM.swift index d45725e11..3c46f35dd 100644 --- a/cmd/vz-shim/VM.swift +++ b/cmd/vz-shim/VM.swift @@ -1,9 +1,7 @@ import Foundation import Virtualization +import DiskImageKit import vmnet -#if canImport(DiskImageKit) - import DiskImageKit -#endif // makeConfiguration builds a validated VM configuration. A Linux guest without a // machine identifier gets a new one, recorded in config so snapshots keep it. @@ -217,33 +215,29 @@ func makeStorageDevices(_ disks: [DiskConfig]) throws -> [VZStorageDeviceConfigu // stackedAttachment attaches an ASIF overlay above its read-only parents, base // first. Each image must name the one below it as its parent; hypeman creates them. func stackedAttachment(parents: [String], overlay: String, readOnly: Bool) throws -> VZStorageDeviceAttachment { - #if canImport(DiskImageKit) - if #available(macOS 27.0, *) { - for path in parents where !FileManager.default.fileExists(atPath: path) { - throw ShimError("configure storage: disk parent not found: \(path)") - } - var stacked = try DiskImage(opening: .open(url: URL(fileURLWithPath: parents[0]), mode: .readOnly)) - for path in parents.dropFirst() { - stacked = try stacked.appending(try openOverlay(path, mode: .readOnly)) - } - stacked = try stacked.appending(try openOverlay(overlay, mode: readOnly ? .readOnly : .readWrite)) - Log.info("attached stacked disk", ["base": parents[0], "overlay": overlay, "layers": (stacked as? any StackedImage)?.layers.count ?? 1]) - return try VZDiskImageStorageDeviceAttachment(diskImage: stacked, cachingMode: .automatic, synchronizationMode: .full) - } - #endif - throw ShimError("configure storage: stacked disks require DiskImageKit on macOS 27 or newer") + guard #available(macOS 27.0, *) else { + throw ShimError("configure storage: stacked disks require macOS 27 or newer") + } + for path in parents where !FileManager.default.fileExists(atPath: path) { + throw ShimError("configure storage: disk parent not found: \(path)") + } + var stacked = try DiskImage(opening: .open(url: URL(fileURLWithPath: parents[0]), mode: .readOnly)) + for path in parents.dropFirst() { + stacked = try stacked.appending(try openOverlay(path, mode: .readOnly)) + } + stacked = try stacked.appending(try openOverlay(overlay, mode: readOnly ? .readOnly : .readWrite)) + Log.info("attached stacked disk", ["base": parents[0], "overlay": overlay, "layers": (stacked as? any StackedImage)?.layers.count ?? 1]) + return try VZDiskImageStorageDeviceAttachment(diskImage: stacked, cachingMode: .automatic, synchronizationMode: .full) } -#if canImport(DiskImageKit) - @available(macOS 27.0, *) - func openOverlay(_ path: String, mode: OpenConfiguration.Mode) throws -> DiskImage { - let image = try DiskImage(opening: .open(url: URL(fileURLWithPath: path), mode: mode)) - guard image.format == .asif else { - throw ShimError("configure storage: overlay must use ASIF format: \(path)") - } - return image +@available(macOS 27.0, *) +func openOverlay(_ path: String, mode: OpenConfiguration.Mode) throws -> DiskImage { + let image = try DiskImage(opening: .open(url: URL(fileURLWithPath: path), mode: mode)) + guard image.format == .asif else { + throw ShimError("configure storage: overlay must use ASIF format: \(path)") } -#endif + return image +} func makeNetworkDevices(_ networks: [NetworkConfig]) throws -> [VZNetworkDeviceConfiguration] { // No networks configured (NetworkEnabled=false): no NIC is attached. From e8cf06d7225e59fe7f71dfc62c3899a80688de85 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:08:13 +0000 Subject: [PATCH 09/33] Remove redundant machine build checks and dead helpers The runner no longer revalidates what admission already checked, or the receipt the publisher builds itself. Machine and Linux builds share one enqueue helper, the publisher and base lookup share one registry parse, and fresh and cloned macOS identities share MAC assignment. Drop the unused export validator, its strict decoding, the staged-size return and the shim's unused power-button route. --- cmd/api/api/macos_builds.go | 3 -- cmd/vz-shim/Shim.swift | 11 +++---- lib/builds/macos_build.go | 19 ++--------- lib/builds/macos_build_failure_test.go | 42 +++++++----------------- lib/builds/macos_build_manager.go | 38 ++++----------------- lib/builds/macos_build_manager_test.go | 2 +- lib/builds/macos_build_publisher.go | 31 +++++++++++------ lib/builds/macos_build_publisher_test.go | 6 +--- lib/builds/macos_build_test.go | 34 +++++++------------ lib/builds/manager.go | 23 +++++++------ lib/images/macos_bundle.go | 22 +++---------- lib/images/macos_bundle_test.go | 11 ++----- lib/images/macos_import_darwin.go | 2 +- lib/images/macos_machine.go | 16 +++------ lib/images/macos_stage_test.go | 4 +-- lib/instances/macos_identity.go | 30 ++++++++--------- 16 files changed, 99 insertions(+), 195 deletions(-) diff --git a/cmd/api/api/macos_builds.go b/cmd/api/api/macos_builds.go index 9ec28c825..d27aab77b 100644 --- a/cmd/api/api/macos_builds.go +++ b/cmd/api/api/macos_builds.go @@ -82,9 +82,6 @@ func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildReque if err != nil || int64(len(data)) > limit { return req, nil, fmt.Errorf("invalid or oversized %s field", name) } - if err = part.Close(); err != nil { - return req, nil, fmt.Errorf("invalid multipart field") - } switch name { case "source": source = data diff --git a/cmd/vz-shim/Shim.swift b/cmd/vz-shim/Shim.swift index a936e2bbf..4613a1757 100644 --- a/cmd/vz-shim/Shim.swift +++ b/cmd/vz-shim/Shim.swift @@ -112,7 +112,7 @@ final class Shim: NSObject, VZVirtualMachineDelegate { private static let routes: [String: Set] = [ "/api/v1/vm.info": ["GET"], "/api/v1/vm.pause": ["PUT"], "/api/v1/vm.resume": ["PUT"], "/api/v1/vm.shutdown": ["PUT"], "/api/v1/vm.snapshot": ["PUT"], "/api/v1/vm.snapshot-support": ["GET"], - "/api/v1/vm.balloon": ["GET", "PUT"], "/api/v1/vm.power-button": ["PUT"], + "/api/v1/vm.balloon": ["GET", "PUT"], "/api/v1/vmm.ping": ["GET"], "/api/v1/vmm.shutdown": ["PUT"], ] @@ -133,8 +133,7 @@ final class Shim: NSObject, VZVirtualMachineDelegate { case ("GET", "/api/v1/vm.info"): return info() case ("PUT", "/api/v1/vm.pause"): return pause() case ("PUT", "/api/v1/vm.resume"): return resume() - case ("PUT", "/api/v1/vm.shutdown"): return requestStop("shutdown failed", "VM graceful shutdown requested (ACPI power button)") - case ("PUT", "/api/v1/vm.power-button"): return requestStop("power button failed", "power button sent") + case ("PUT", "/api/v1/vm.shutdown"): return requestStop() case ("GET", "/api/v1/vm.snapshot-support"): return snapshotSupport() case ("PUT", "/api/v1/vm.snapshot"): return snapshot(request.body) case ("GET", "/api/v1/vm.balloon"): return getBalloon() @@ -173,14 +172,14 @@ final class Shim: NSObject, VZVirtualMachineDelegate { // requestStop sends the guest a power-button event. The instance manager handles // the timeout and forced stop if the guest does not shut down. - private func requestStop(_ failure: String, _ success: String) -> HTTPResponse { + private func requestStop() -> HTTPResponse { do { try onMain { try vm.requestStop() } } catch { Log.error("RequestStop failed", ["error": error.localizedDescription]) - return .text(500, "\(failure): \(error.localizedDescription)") + return .text(500, "shutdown failed: \(error.localizedDescription)") } - Log.info(success) + Log.info("VM graceful shutdown requested (ACPI power button)") return .noContent() } diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go index 06dd5e050..e1786f412 100644 --- a/lib/builds/macos_build.go +++ b/lib/builds/macos_build.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "path/filepath" "strings" "time" @@ -119,23 +118,15 @@ func normalizeMachinePolicy(policy *BuildPolicy) error { // runPrepared executes only machine-specific phases. The shared build manager // owns resolution, source staging/hash verification, deadline, queue and status. func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath string) (result *MachineBuildResult, err error) { - if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { - return nil, fmt.Errorf("machine driver, publisher and verified source required") - } if _, bounded := ctx.Deadline(); !bounded { return nil, fmt.Errorf("prepared machine job requires a deadline") } - if len(req.ID) == 0 || len(req.ID) > 128 || strings.Trim(req.ID, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" { - return nil, fmt.Errorf("invalid machine build ID") - } + // Admission validated the request; the base may have changed since. ref, e := images.ParseNormalizedRef(req.BaseImage) - if e != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { - return nil, fmt.Errorf("machine base must be pinned by sha256 digest") - } - policy := req.Policy - if e = normalizeMachinePolicy(&policy); e != nil { + if e != nil { return nil, e } + policy := req.Policy if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || !base.MacOS.GuestAgent || base.Digest != ref.Digest() || base.MacOS.CPUs < 2 || base.MacOS.CPUs > MaxBuildCPUs || base.MacOS.Memory < 4<<30 || base.MacOS.Memory > uint64(MaxBuildMemoryMB)<<20 || base.MacOS.Memory%(1<<20) != 0 { return nil, fmt.Errorf("base is not the pinned ready macOS machine") } @@ -225,10 +216,6 @@ func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildR return nil, machineFailure("cleanup", e) } destroyed = true - output, e := images.ParseNormalizedRef(publication.Reference) - if e != nil || !output.IsDigest() || !ValidMachineDigest(publication.Digest) || output.Digest() != publication.Digest { - return nil, fmt.Errorf("machine publication receipt is unverified") - } if req.onPublished != nil { if e = req.onPublished(ctx, publication); e != nil { return nil, machineFailure("image_ready", e) diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go index 3c42f3e54..6581af11d 100644 --- a/lib/builds/macos_build_failure_test.go +++ b/lib/builds/macos_build_failure_test.go @@ -92,38 +92,18 @@ func TestMachineBuildInterruptedCleanup(t *testing.T) { }) } -func TestMachineBuildPublicationReceipt(t *testing.T) { - for _, kind := range []string{"ambiguous error", "tag receipt", "digest mismatch", "invalid digest"} { - t.Run(kind, func(t *testing.T) { - f := newMachineFixture() - switch kind { - case "ambiguous error": - f.fail = "publish" - case "tag receipt": - f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test:latest", Digest: machineTestDigest} - case "digest mismatch": - f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "sha256:" + strings.Repeat("b", 64)} - case "invalid digest": - f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "not-a-digest"} - } - _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) - if !f.publisherCalled { - t.Fatal("unverified receipt became ready") - } - if strings.Contains(err.Error(), "sensitive-command-argument") { - t.Fatal("publisher error leaked") - } - count := 0 - for _, phase := range f.calls { - if phase == "publish" { - count++ - } - } - if count != 1 { - t.Fatal("ambiguous publication was retried") - } - }) +func TestMachineBuildAmbiguousPublishIsNotRetried(t *testing.T) { + f := newMachineFixture() + f.fail = "publish" + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + require.NotContains(t, err.Error(), "sensitive-command-argument", "publisher error leaked") + count := 0 + for _, phase := range f.calls { + if phase == "publish" { + count++ + } } + require.Equal(t, 1, count, "ambiguous publication was retried") } func TestMachineBuildMalformedStorage(t *testing.T) { diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go index d825c421a..4985387c9 100644 --- a/lib/builds/macos_build_manager.go +++ b/lib/builds/macos_build_manager.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "net/url" "os" "path/filepath" "strings" @@ -53,7 +52,10 @@ func (m *manager) executeMachineBuild(ctx context.Context, id string, req Create registryBase: machineRegistryBase(backend.Publisher, req.MachineBaseImage), onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, onPublished: func(ctx context.Context, publication MachinePublication) error { - return m.waitMachinePublication(ctx, id, publication.Digest) + if err := m.imageManager.WaitForReady(ctx, "builds/"+id+"@"+publication.Digest); err != nil { + return fmt.Errorf("machine image import: %w", err) + } + return nil }, } result, err := backend.runPrepared(ctx, job, base, sourcePath) @@ -67,16 +69,6 @@ func (m *manager) executeMachineBuild(ctx context.Context, id string, req Create return &BuildResult{Success: true, ImageDigest: result.Publication.Digest, ImageRef: result.Publication.Reference, Provenance: result.Provenance}, nil } -func (m *manager) waitMachinePublication(ctx context.Context, id, digest string) error { - if !ValidMachineDigest(digest) { - return fmt.Errorf("canonical machine publication digest required") - } - if err := m.imageManager.WaitForReady(ctx, "builds/"+id+"@"+digest); err != nil { - return fmt.Errorf("machine image import: %w", err) - } - return nil -} - // reapPriorMachineBuilder removes the builder recorded by an interrupted attempt of // this same job before a re-enqueued attempt starts. Nothing from that builder is // published, so a forced stop is acceptable; a failed delete fails this attempt @@ -159,20 +151,6 @@ func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) return policy, nil } -// enqueueMachineBuild queues a machine build whose source is already staged. Each -// builder has its own identity, so builds on one base are not serialized. -func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { - queuePos := m.queue.EnqueueSerial(meta.ID, req, "", func() { - m.runBuild(context.Background(), meta.ID, req, policy) - }) - build := meta.toBuild() - if queuePos > 0 { - build.QueuePosition = &queuePos - } - m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) - return build -} - // machineRegistryBase returns the base as a registry base when the publisher // writes to the registry that stores it, so its layers are reused. Any other base // publishes all of its layers. @@ -181,12 +159,8 @@ func machineRegistryBase(publisher MachineBuildPublisher, baseImage string) *Mac if !ok { return nil } - address := streaming.RegistryURL - if !strings.Contains(address, "://") { - address = "https://" + address - } - registry, err := url.Parse(address) - if err != nil || registry.Host == "" { + registry, err := streaming.registry() + if err != nil { return nil } ref, err := images.ParseNormalizedRef(baseImage) diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go index 39b29951a..22fccd29b 100644 --- a/lib/builds/macos_build_manager_test.go +++ b/lib/builds/macos_build_manager_test.go @@ -28,7 +28,7 @@ func (m machineJobImages) GetImage(ctx context.Context, ref string) (*images.Ima return nil, m.resolveErr } if ref == "registry/base@"+machineTestDigest { - return m.fixture.ResolveBase(ctx, ref) + return m.fixture.image, nil } return &images.Image{Status: images.StatusReady, Digest: machineTestDigest}, nil } diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 30575268f..9219ae7ad 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -77,17 +77,11 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor return MachinePublication{}, err } } - address := p.RegistryURL - if !strings.Contains(address, "://") { - address = "https://" + address - } - registry, err := url.Parse(address) - if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { - return MachinePublication{}, fmt.Errorf("invalid internal registry address") - } - if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { - return MachinePublication{}, fmt.Errorf("plaintext registry must be loopback") + registry, err := p.registry() + if err != nil { + return MachinePublication{}, err } + address := registry.String() if err := validateMachineStorage(storage, nil); err != nil { return MachinePublication{}, err } @@ -176,6 +170,23 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil } +// registry parses the configured registry address, defaulting to HTTPS. Plaintext +// is allowed only on loopback. +func (p *StreamingMachinePublisher) registry() (*url.URL, error) { + address := p.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { + return nil, fmt.Errorf("invalid internal registry address") + } + if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { + return nil, fmt.Errorf("plaintext registry must be loopback") + } + return registry, nil +} + // machineLayers returns a machine image's layer blobs and diff IDs: the base's // layers, from the registry or the storage's own layer files, then a top layer with // the overlay, auxiliary storage and platform. diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go index a3660de3f..7b422f263 100644 --- a/lib/builds/macos_build_publisher_test.go +++ b/lib/builds/macos_build_publisher_test.go @@ -6,7 +6,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "strings" "sync/atomic" "testing" "time" @@ -54,7 +53,7 @@ func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { imports := &machineImportStub{imported: make(chan string, 1)} reg, e := registry.New(paths.New(t.TempDir()), imports) require.NoError(t, e) - var uploads, verified atomic.Int32 + var uploads atomic.Int32 handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.NotEqual(t, http.MethodPatch, r.Method, "uploads must use the disk-streaming monolithic endpoint") if r.Method == http.MethodPost { @@ -62,9 +61,6 @@ func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { require.Greater(t, r.ContentLength, int64(0)) uploads.Add(1) } - if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { - verified.Add(1) - } if r.Method == http.MethodPut { require.Equal(t, int32(2), uploads.Load(), "manifest must be last") } diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go index c1cfc0a5f..b82b5c6bd 100644 --- a/lib/builds/macos_build_test.go +++ b/lib/builds/macos_build_test.go @@ -19,19 +19,18 @@ import ( const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" type machineFixture struct { - mu sync.Mutex - image *images.Image - calls []string - fail string - stop MachineStopReceipt - cancel context.CancelFunc - waitForCancel bool - mutateStorage func(*MachineStorage) error - storageDir string - publicationOverride *MachinePublication - publisherCalled bool - cleanupContextLive bool - startGate chan struct{} + mu sync.Mutex + image *images.Image + calls []string + fail string + stop MachineStopReceipt + cancel context.CancelFunc + waitForCancel bool + mutateStorage func(*MachineStorage) error + storageDir string + publisherCalled bool + cleanupContextLive bool + startGate chan struct{} } func newMachineFixture() *machineFixture { @@ -46,12 +45,6 @@ func (f *machineFixture) record(phase string) error { } return nil } -func (f *machineFixture) ResolveBase(context.Context, string) (*images.Image, error) { - if err := f.record("resolve"); err != nil { - return nil, err - } - return f.image, nil -} func (f *machineFixture) Start(_ context.Context, _ string, _ *images.Image, p BuildPolicy) (MachineBuildSession, error) { if p.CPUs != 4 || p.MemoryMB != 8192 { return nil, errors.New("wrong resource defaults") @@ -150,9 +143,6 @@ func (f *machineFixture) Publish(ctx context.Context, id string, storage Machine } } publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} - if f.publicationOverride != nil { - publication = *f.publicationOverride - } return publication, f.record("publish") } func machineTestRunner(t *testing.T, f *machineFixture) *machineRunnerFixture { diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 3584484bb..ac0e2ac3e 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -503,7 +503,7 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc deleteBuild(m.paths, id) return nil, err } - return m.enqueueMachineBuild(meta, req, policy), nil + return m.enqueueBuild(meta, req, policy), nil } // Write initial metadata @@ -625,20 +625,23 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc return nil, fmt.Errorf("write build config: %w", err) } - // Enqueue the build. Builds sharing a builder are serialized by builder - // ID so a waiting same-builder build stays pending instead of holding a - // global concurrency slot while blocked on the builder's disk. - queuePos := m.queue.EnqueueSerial(id, req, req.BuilderID, func() { - m.runBuild(context.Background(), id, req, policy) - }) + return m.enqueueBuild(meta, req, policy), nil +} +// enqueueBuild queues a build whose inputs are stored. Builds sharing a builder +// are serialized by builder ID so a waiting same-builder build stays pending +// instead of holding a global concurrency slot while blocked on the builder's +// disk. Machine builds have no builder ID, so builds on one base run concurrently. +func (m *manager) enqueueBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { + queuePos := m.queue.EnqueueSerial(meta.ID, req, req.BuilderID, func() { + m.runBuild(context.Background(), meta.ID, req, policy) + }) build := meta.toBuild() if queuePos > 0 { build.QueuePosition = &queuePos } - - m.logger.Info("build created", "id", id, "queue_position", queuePos) - return build, nil + m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) + return build } // gcBoundsForDisk computes the BuildKit GC bounds for a builder disk: diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index b086ef959..55663beb0 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -37,18 +37,7 @@ func RequireClosedFile(ctx context.Context, path string) error { return nil } -// ValidateMacOSBundle validates a fixed-layout export of one ASIF disk, rejecting -// unknown metadata fields so an accidental credential field cannot become part of -// an image. -func ValidateMacOSBundle(root string) (*MacOSImage, error) { - bundle, err := readMacOSMachineBundle(root, "disk.asif", "aux.img", "config.json", true) - if err != nil { - return nil, err - } - return bundle.Platform, nil -} - -func readMacOSMachineBundle(root, diskName, auxName, configName string, strict bool) (*macOSMachinePayload, error) { +func readMacOSMachineBundle(root, diskName, auxName, configName string) (*macOSMachinePayload, error) { disk, err := machineBundleFile(root, diskName) if err != nil { return nil, err @@ -65,7 +54,7 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string, strict b if sameFile(disk, aux) || sameFile(disk, config) || sameFile(aux, config) { return nil, fmt.Errorf("machine bundle files must be distinct") } - platform, err := readMacOSPlatform(config, strict) + platform, err := readMacOSPlatform(config) if err != nil { return nil, err } @@ -104,7 +93,7 @@ func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSM if err != nil { return nil, fmt.Errorf("machine image top layer: %w", err) } - platform, err := readMacOSPlatform(config, false) + platform, err := readMacOSPlatform(config) if err != nil { return nil, err } @@ -112,7 +101,7 @@ func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSM return &payload, nil } -func readMacOSPlatform(config string, strict bool) (*MacOSImage, error) { +func readMacOSPlatform(config string) (*MacOSImage, error) { file, err := os.Open(config) if err != nil { return nil, err @@ -126,9 +115,6 @@ func readMacOSPlatform(config string, strict bool) (*MacOSImage, error) { return nil, fmt.Errorf("machine platform metadata is too large") } decoder := json.NewDecoder(bytes.NewReader(data)) - if strict { - decoder.DisallowUnknownFields() - } var platform MacOSImage if err := decoder.Decode(&platform); err != nil { return nil, err diff --git a/lib/images/macos_bundle_test.go b/lib/images/macos_bundle_test.go index 409b7d1ae..b35675643 100644 --- a/lib/images/macos_bundle_test.go +++ b/lib/images/macos_bundle_test.go @@ -43,18 +43,13 @@ func TestMacOSBundleValidation(t *testing.T) { config = []byte(strings.Replace(string(config), platform.MAC, "invalid", 1)) } require.NoError(t, os.WriteFile(filepath.Join(root, "config.json"), config, 0600)) - got, err := ValidateMacOSBundle(root) - if name == "valid" { + bundle, err := readMacOSMachineBundle(root, "disk.asif", "aux.img", "config.json") + if name == "valid" || name == "unknown-field" { require.NoError(t, err) - require.Equal(t, &platform, got) + require.Equal(t, &platform, bundle.Platform) } else { require.Error(t, err) } - if name == "unknown-field" { - bundle, err := readMacOSMachineBundle(root, "disk.asif", "aux.img", "config.json", false) - require.NoError(t, err, "import/OCI compatibility permits unknown non-export metadata") - require.Equal(t, &platform, bundle.Platform) - } }) } } diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index 3608ba049..2f3a4585b 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -33,7 +33,7 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if _, err := os.Lstat(filepath.Join(source, diskName)); os.IsNotExist(err) { diskName = "disk.img" } - bundle, err := readMacOSMachineBundle(source, diskName, "aux.img", "config.json", false) + bundle, err := readMacOSMachineBundle(source, diskName, "aux.img", "config.json") if err != nil { return nil, err } diff --git a/lib/images/macos_machine.go b/lib/images/macos_machine.go index 2be88483a..d7661870a 100644 --- a/lib/images/macos_machine.go +++ b/lib/images/macos_machine.go @@ -159,38 +159,30 @@ func stageMacOSMachine(staged *stagedImageFiles, root string, meta *containerMet staged.macos.layers = append(staged.macos.layers, layer) chain = append(chain, layer) } - if _, err := stageMachineFile(disk, chain[i]); err != nil { + if err := stageMachineFile(disk, chain[i]); err != nil { return fmt.Errorf("stage boot disk layer %d: %w", i, err) } } if staged.sizeBytes, err = macOSChainSize(chain); err != nil { return fmt.Errorf("machine boot disk: %w", err) } - if _, err := stageMachineFile(payload.Aux, staged.macos.aux); err != nil { + if err := stageMachineFile(payload.Aux, staged.macos.aux); err != nil { return fmt.Errorf("stage auxiliary storage: %w", err) } return nil } -func stageMachineFile(src, dst string) (int64, error) { +func stageMachineFile(src, dst string) error { // Copy helpers create the destination with the registry-supplied source mode, // and a multi-GB copy would expose contents before any later chmod. The copy // happens in installAtomically's fresh 0700 directory and is made 0600 before // it appears at the canonical staging path. - err := installAtomically(dst, func(path string) error { + return installAtomically(dst, func(path string) error { if err := forkvm.CopyRegularFile(src, path); err != nil { return err } return os.Chmod(path, 0600) }) - if err != nil { - return 0, err - } - info, err := os.Stat(dst) - if err != nil { - return 0, err - } - return info.Size(), nil } // macOSMachineHost is true where macOS machine images can boot. Other hosts diff --git a/lib/images/macos_stage_test.go b/lib/images/macos_stage_test.go index 5acd8e0ad..a369d7b89 100644 --- a/lib/images/macos_stage_test.go +++ b/lib/images/macos_stage_test.go @@ -14,9 +14,7 @@ func TestStageMachineFileNeverKeepsRegistryMode(t *testing.T) { require.NoError(t, os.WriteFile(src, []byte("synthetic"), 0644)) require.NoError(t, os.Chmod(src, 0644)) dst := filepath.Join(root, "staged.raw") - size, err := stageMachineFile(src, dst) - require.NoError(t, err) - require.EqualValues(t, len("synthetic"), size) + require.NoError(t, stageMachineFile(src, dst)) info, err := os.Stat(dst) require.NoError(t, err) require.Equal(t, os.FileMode(0600), info.Mode().Perm()) diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 6c9ef6a7a..4ad65c94f 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -234,17 +234,7 @@ func (m *manager) freshMacOSIdentity(stored *StoredMetadata) error { if err := prepareMacOSClone(stored); err != nil { return err } - used, err := m.storedMACs(stored.Id) - if err != nil { - return fmt.Errorf("list MACs for macOS identity: %w", err) - } - mac, err := network.NewUniqueMAC(used) - if err != nil { - return fmt.Errorf("allocate macOS MAC: %w", err) - } - stored.MacOS.MAC = mac - stored.MAC = mac - return nil + return m.assignMacOSMAC(stored) } // assignMacOSCloneMAC gives a networked clone a MAC no stored instance uses. It runs @@ -253,17 +243,23 @@ func (m *manager) assignMacOSCloneMAC(clone *StoredMetadata) error { if clone.MacOS == nil || !clone.NetworkEnabled { return nil } - used, err := m.storedMACs(clone.Id) + return m.assignMacOSMAC(clone) +} + +// assignMacOSMAC gives a macOS instance a MAC no stored instance uses. Its platform +// is copied first, since a clone's may be shared with its source. +func (m *manager) assignMacOSMAC(stored *StoredMetadata) error { + used, err := m.storedMACs(stored.Id) if err != nil { - return fmt.Errorf("list MACs for macOS clone: %w", err) + return fmt.Errorf("list MACs for macOS instance: %w", err) } mac, err := network.NewUniqueMAC(used) if err != nil { - return fmt.Errorf("allocate macOS clone MAC: %w", err) + return fmt.Errorf("allocate macOS MAC: %w", err) } - macos := *clone.MacOS + macos := *stored.MacOS macos.MAC = mac - clone.MacOS = &macos - clone.MAC = mac + stored.MacOS = &macos + stored.MAC = mac return nil } From 69a62a71bcc784aef29d31d29df52cb131b862a4 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:14:13 +0000 Subject: [PATCH 10/33] Stop re-reading uploaded machine blobs The registry hashes a monolithic upload and commits it only if it matches the digest, so reading every blob back, a second full pass over a base layer of tens of gigabytes, verified nothing new. machineFetch now only reads metadata. --- docs/macos-builds.md | 5 +++-- lib/builds/macos_build_publisher.go | 14 ++++++------- lib/builds/macos_registry.go | 32 ++++++----------------------- lib/builds/macos_registry_test.go | 20 ++++++------------ 4 files changed, 21 insertions(+), 50 deletions(-) diff --git a/docs/macos-builds.md b/docs/macos-builds.md index ba8946e0b..e6fb27fef 100644 --- a/docs/macos-builds.md +++ b/docs/macos-builds.md @@ -101,8 +101,9 @@ The publisher uses a server-configured registry and a token scoped to `builds/{i Plaintext is allowed only on loopback; redirects are never followed and proxies are disabled. The layer is never written to disk: a first pass computes its digest, size and DiffID, the upload regenerates the same bytes, and the registry rejects any -mismatch. Blobs the registry already holds are skipped. Blobs use monolithic POST uploads, every persisted blob is re-read and -hashed, the manifest is committed last, and the digest content is verified. Ambiguous +mismatch. Blobs the registry already holds are skipped. Blobs use monolithic POST uploads +that the registry commits only if they match their digest, the manifest is committed +last, and the manifest is read back by digest. Ambiguous errors never return success. The image is the base's layers plus one layer holding the builder's overlay, diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 9219ae7ad..3bbe8970f 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -24,8 +24,8 @@ import ( // StreamingMachinePublisher targets only the configured internal registry and a // build-scoped repository. go-containerregistry hashes and compresses each layer // from a reopenable tar stream, so nothing is written to disk. Blobs use the -// registry's disk-streaming monolithic upload and are independently verified before -// the manifest is committed last. +// registry's disk-streaming monolithic upload, which commits a blob only if it +// matches its digest, and the manifest is committed last. // No credentials, redirect destinations or arbitrary output repositories come from the source. type StreamingMachinePublisher struct { RegistryURL string @@ -147,12 +147,10 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor } status := response.StatusCode response.Body.Close() + // The registry hashes the body and commits it only if it matches the digest. if status != http.StatusCreated { return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") } - if _, e = machineFetch(ctx, client, token, target+"/blobs/"+blob.descriptor.Digest.String(), blob.descriptor.Digest.String(), blob.descriptor.Size); e != nil { - return MachinePublication{}, fmt.Errorf("machine blob verification failed: %w", e) - } } response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target+"/manifests/latest", string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) if err != nil { @@ -164,7 +162,7 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") } // The digest covers manifestBytes exactly, so a matching fetch is the same bytes. - if _, err = machineFetch(ctx, client, token, target+"/manifests/"+digest, digest, int64(len(manifestBytes))); err != nil { + if _, err = machineFetch(ctx, client, token, target+"/manifests/"+digest, digest); err != nil { return MachinePublication{}, fmt.Errorf("machine manifest verification failed: %w", err) } return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil @@ -230,7 +228,7 @@ func machineLayers(ctx context.Context, client *http.Client, token, registry str // the base's layer descriptors and diff IDs. func fetchMachineBase(ctx context.Context, client *http.Client, token, registry string, base MachineBase) ([]v1.Descriptor, []v1.Hash, error) { root := strings.TrimRight(registry, "/") + "/v2/" + base.Repo - manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+base.Digest, base.Digest, 0) + manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+base.Digest, base.Digest) if err != nil { return nil, nil, fmt.Errorf("base manifest: %w", err) } @@ -239,7 +237,7 @@ func fetchMachineBase(ctx context.Context, client *http.Client, token, registry return nil, nil, fmt.Errorf("base manifest: %w", err) } configDigest := manifest.Config.Digest.String() - configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest, 0) + configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest) if err != nil { return nil, nil, fmt.Errorf("base config: %w", err) } diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go index dad1345fe..5b29604b4 100644 --- a/lib/builds/macos_registry.go +++ b/lib/builds/macos_registry.go @@ -1,7 +1,6 @@ package builds import ( - "bytes" "context" "crypto/sha256" "encoding/hex" @@ -41,11 +40,9 @@ func machineHash(data []byte) v1.Hash { return v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum[:])} } -// machineFetch GETs a registry object and checks it against digest. A size of 0 -// marks metadata: the body is buffered up to maxMachineMetadataBytes and returned. -// Any other size must match exactly and is only hashed, never kept, so layers of -// tens of gigabytes are not held in memory. -func machineFetch(ctx context.Context, client *http.Client, token, target, digest string, size int64) ([]byte, error) { +// machineFetch GETs registry metadata, a manifest or config, and checks it against +// digest. The body is buffered up to maxMachineMetadataBytes and returned. +func machineFetch(ctx context.Context, client *http.Client, token, target, digest string) ([]byte, error) { response, err := machineRegistryRequest(ctx, client, token, http.MethodGet, target, "", nil, 0) if err != nil { return nil, err @@ -54,31 +51,14 @@ func machineFetch(ctx context.Context, client *http.Client, token, target, diges if response.StatusCode != http.StatusOK { return nil, fmt.Errorf("registry returned status %d", response.StatusCode) } - metadata := size == 0 - limit := size + 1 - if metadata { - limit = maxMachineMetadataBytes + 1 - } - sum := sha256.New() - var kept bytes.Buffer - out := io.Writer(sum) - if metadata { - out = io.MultiWriter(sum, &kept) - } - n, err := io.Copy(out, io.LimitReader(response.Body, limit)) + data, err := io.ReadAll(io.LimitReader(response.Body, maxMachineMetadataBytes+1)) if err != nil { return nil, err } - if n >= limit || (!metadata && n != size) { + if len(data) > maxMachineMetadataBytes || machineHash(data).String() != digest { return nil, fmt.Errorf("object does not match digest %s", digest) } - if (v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum.Sum(nil))}).String() != digest { - return nil, fmt.Errorf("object does not match digest %s", digest) - } - if !metadata { - return nil, nil - } - return kept.Bytes(), nil + return data, nil } // machineBlobStored reports whether the registry already holds a blob of the diff --git a/lib/builds/macos_registry_test.go b/lib/builds/macos_registry_test.go index 77ccbac79..4b069ccd4 100644 --- a/lib/builds/macos_registry_test.go +++ b/lib/builds/macos_registry_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestMachineFetchVerifiesSizeAndDigest(t *testing.T) { +func TestMachineFetchVerifiesDigest(t *testing.T) { body := []byte("synthetic object") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(body) @@ -27,28 +27,20 @@ func TestMachineFetchVerifiesSizeAndDigest(t *testing.T) { name string target string digest string - size int64 ok bool }{ - {"metadata matches digest", server.URL, digest, 0, true}, - {"sized object matches", server.URL, digest, int64(len(body)), true}, - {"wrong digest", server.URL, machineHash([]byte("other")).String(), 0, false}, - {"size shorter than body", server.URL, digest, int64(len(body)) - 1, false}, - {"size longer than body", server.URL, digest, int64(len(body)) + 1, false}, - {"metadata over cap", bigServer.URL, machineHash([]byte(oversize)).String(), 0, false}, + {"matches digest", server.URL, digest, true}, + {"wrong digest", server.URL, machineHash([]byte("other")).String(), false}, + {"over cap", bigServer.URL, machineHash([]byte(oversize)).String(), false}, } { t.Run(tc.name, func(t *testing.T) { - got, err := machineFetch(context.Background(), http.DefaultClient, "token", tc.target, tc.digest, tc.size) + got, err := machineFetch(context.Background(), http.DefaultClient, "token", tc.target, tc.digest) if !tc.ok { require.Error(t, err) return } require.NoError(t, err) - if tc.size == 0 { - require.Equal(t, body, got, "metadata is returned") - } else { - require.Nil(t, got, "sized objects are verified, never buffered") - } + require.Equal(t, body, got) }) } } From 4acfb19e5b3da92ab998eee04574d4cf4b9b58ec Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:24:09 +0000 Subject: [PATCH 11/33] Publish machine images with go-containerregistry The registry now writes chunked uploads to disk, hashing as they arrive, instead of holding each one in memory: pushing a 4 GiB blob through it previously grew the heap by gigabytes and now does not. With that, the machine publisher uses remote.Write instead of its own registry client: HEAD skips blobs the registry holds, so base layers are not uploaded, the manifest is committed last, and redirects are refused so the token never leaves the registry. --- lib/builds/macos_build_layer_test.go | 18 +- lib/builds/macos_build_publisher.go | 210 ++++++++--------------- lib/builds/macos_build_publisher_test.go | 10 +- lib/builds/macos_registry.go | 82 --------- lib/builds/macos_registry_test.go | 46 ----- lib/registry/blob_store.go | 10 ++ lib/registry/registry.go | 11 ++ lib/registry/uploads.go | 161 +++++++++++++++++ lib/registry/uploads_test.go | 140 +++++++++++++++ 9 files changed, 412 insertions(+), 276 deletions(-) delete mode 100644 lib/builds/macos_registry.go delete mode 100644 lib/builds/macos_registry_test.go create mode 100644 lib/registry/uploads.go create mode 100644 lib/registry/uploads_test.go diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_build_layer_test.go index 5330dcaea..54b59f8ef 100644 --- a/lib/builds/macos_build_layer_test.go +++ b/lib/builds/macos_build_layer_test.go @@ -21,18 +21,24 @@ func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { copy(data[1<<20:], "data") require.NoError(t, os.WriteFile(disk, data, 0600)) - blob, diffID, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}, {name: "config.json", data: []byte("{}")}}) + layer, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}, {name: "config.json", data: []byte("{}")}}) + require.NoError(t, err) + diffID, err := layer.DiffID() require.NoError(t, err) require.NotEmpty(t, diffID.Hex) + digest, err := layer.Digest() + require.NoError(t, err) + size, err := layer.Size() + require.NoError(t, err) for range 2 { - body, err := blob.open() + body, err := layer.Compressed() require.NoError(t, err) sum := sha256.New() n, err := io.Copy(sum, body) require.NoError(t, err) require.NoError(t, body.Close()) - require.Equal(t, blob.descriptor.Size, n) - require.Equal(t, blob.descriptor.Digest.Hex, hex.EncodeToString(sum.Sum(nil))) + require.Equal(t, size, n) + require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) } entries, err := os.ReadDir(parent) require.NoError(t, err) @@ -42,10 +48,10 @@ func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { disk := filepath.Join(t.TempDir(), "disk.asif") require.NoError(t, os.WriteFile(disk, make([]byte, 8<<20), 0600)) - blob, _, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}}) + layer, err := machineLayer(context.Background(), []machineTarEntry{{name: "disk.asif", path: disk}}) require.NoError(t, err) before := runtime.NumGoroutine() - body, err := blob.open() + body, err := layer.Compressed() require.NoError(t, err) _, err = io.CopyN(io.Discard, body, 16) require.NoError(t, err) diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 3bbe8970f..5df457b60 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -2,7 +2,6 @@ package builds import ( "archive/tar" - "bytes" "compress/gzip" "context" "encoding/json" @@ -15,7 +14,12 @@ import ( "strings" "time" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/kernel/hypeman/lib/images" @@ -23,9 +27,9 @@ import ( // StreamingMachinePublisher targets only the configured internal registry and a // build-scoped repository. go-containerregistry hashes and compresses each layer -// from a reopenable tar stream, so nothing is written to disk. Blobs use the -// registry's disk-streaming monolithic upload, which commits a blob only if it -// matches its digest, and the manifest is committed last. +// from a reopenable tar stream, so nothing is written to disk, skips blobs the +// registry already holds, and commits the manifest last. The registry stores each +// upload on disk and commits it only if it matches its digest. // No credentials, redirect destinations or arbitrary output repositories come from the source. type StreamingMachinePublisher struct { RegistryURL string @@ -56,11 +60,6 @@ var machineLabels = map[string]string{ images.MacOSMachineDiskLabel: "disk.asif", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", } -type machineBlob struct { - open func() (io.ReadCloser, error) - descriptor v1.Descriptor -} - // Publish commits storage as a machine image in builds/{id}. With a base, the // storage's layers must be the base's and the base's registry layers are reused; // without one, every layer is uploaded. @@ -81,7 +80,6 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor if err != nil { return MachinePublication{}, err } - address := registry.String() if err := validateMachineStorage(storage, nil); err != nil { return MachinePublication{}, err } @@ -97,75 +95,40 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil defer transport.CloseIdleConnections() - client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} - target := strings.TrimRight(address, "/") + "/v2/" + repo - layers, diffIDs, err := machineLayers(ctx, client, token, address, storage, base) - if err != nil { - return MachinePublication{}, err + options := []remote.Option{remote.WithContext(ctx), remote.WithAuth(&authn.Bearer{Token: token}), remote.WithTransport(refuseRedirects{transport})} + var nameOptions []name.Option + if registry.Scheme == "http" { + nameOptions = append(nameOptions, name.Insecure) } - config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, Config: v1.Config{Labels: machineLabels}} - configBytes, err := json.Marshal(config) + img, err := machineImage(ctx, registry.Host, storage, base, nameOptions, options) if err != nil { return MachinePublication{}, err } - configBlob := machineBlob{open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(configBytes)), nil }, descriptor: v1.Descriptor{MediaType: types.OCIConfigJSON, Digest: machineHash(configBytes), Size: int64(len(configBytes))}} - descriptors := make([]v1.Descriptor, 0, len(layers)) - for _, layer := range layers { - descriptors = append(descriptors, layer.descriptor) - } - manifest := v1.Manifest{SchemaVersion: 2, MediaType: types.OCIManifestSchema1, Config: configBlob.descriptor, Layers: descriptors} - manifestBytes, err := json.Marshal(manifest) + target, err := name.NewTag(registry.Host+"/"+repo+":latest", nameOptions...) if err != nil { return MachinePublication{}, err } - digest := machineHash(manifestBytes).String() - for _, blob := range append([]machineBlob{configBlob}, layers...) { - // The blob store is keyed by digest, so a blob the registry already holds - // (a base layer, or a retried publish) is not uploaded again. - stored, e := machineBlobStored(ctx, client, token, target, blob.descriptor) - if e != nil { - return MachinePublication{}, e - } - if stored { - continue - } - if blob.open == nil { - return MachinePublication{}, fmt.Errorf("base layer %s is missing from the registry", blob.descriptor.Digest) - } - body, e := blob.open() - if e != nil { - return MachinePublication{}, e - } - response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, target+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) - closeErr := body.Close() - if e != nil { - return MachinePublication{}, e - } - if closeErr != nil { - response.Body.Close() - return MachinePublication{}, closeErr - } - status := response.StatusCode - response.Body.Close() - // The registry hashes the body and commits it only if it matches the digest. - if status != http.StatusCreated { - return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") - } + if err = remote.Write(target, img, options...); err != nil { + return MachinePublication{}, fmt.Errorf("write machine image: %w", err) } - response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target+"/manifests/latest", string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) + digest, err := img.Digest() if err != nil { return MachinePublication{}, err } - status := response.StatusCode - response.Body.Close() - if status != http.StatusCreated { - return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") - } - // The digest covers manifestBytes exactly, so a matching fetch is the same bytes. - if _, err = machineFetch(ctx, client, token, target+"/manifests/"+digest, digest); err != nil { - return MachinePublication{}, fmt.Errorf("machine manifest verification failed: %w", err) + return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest.String(), Digest: digest.String()}, nil +} + +// refuseRedirects fails a redirected registry request rather than follow it, so +// the token never reaches another host. +type refuseRedirects struct{ http.RoundTripper } + +func (t refuseRedirects) RoundTrip(req *http.Request) (*http.Response, error) { + response, err := t.RoundTripper.RoundTrip(req) + if err == nil && response.StatusCode >= 300 && response.StatusCode < 400 { + response.Body.Close() + return nil, fmt.Errorf("registry redirect refused") } - return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil + return response, err } // registry parses the configured registry address, defaulting to HTTPS. Plaintext @@ -185,75 +148,65 @@ func (p *StreamingMachinePublisher) registry() (*url.URL, error) { return registry, nil } -// machineLayers returns a machine image's layer blobs and diff IDs: the base's -// layers, from the registry or the storage's own layer files, then a top layer with -// the overlay, auxiliary storage and platform. -func machineLayers(ctx context.Context, client *http.Client, token, registry string, storage MachineStorage, base *MachineBase) ([]machineBlob, []v1.Hash, error) { - var blobs []machineBlob - var diffIDs []v1.Hash +// machineImage returns the machine image for storage: the base's layers, from the +// registry or the storage's own layer files, then a top layer with the overlay, +// auxiliary storage and platform. +func machineImage(ctx context.Context, host string, storage MachineStorage, base *MachineBase, nameOptions []name.Option, options []remote.Option) (v1.Image, error) { + img := empty.Image if base != nil { - descriptors, ids, err := fetchMachineBase(ctx, client, token, registry, *base) + ref, err := name.NewDigest(host+"/"+base.Repo+"@"+base.Digest, nameOptions...) if err != nil { - return nil, nil, err + return nil, err } - if len(descriptors) != len(storage.Layers) { - return nil, nil, fmt.Errorf("machine base has %d layers but the disk stacks on %d", len(descriptors), len(storage.Layers)) + if img, err = remote.Image(ref, options...); err != nil { + return nil, fmt.Errorf("base image: %w", err) } - for _, descriptor := range descriptors { - blobs = append(blobs, machineBlob{descriptor: descriptor}) + config, err := img.ConfigFile() + if err != nil { + return nil, fmt.Errorf("base config: %w", err) + } + if config.OS != "darwin" || config.Architecture != "arm64" { + return nil, fmt.Errorf("base is not a darwin/arm64 machine image") + } + for label, want := range machineLabels { + if config.Config.Labels[label] != want { + return nil, fmt.Errorf("base is not an ASIF machine image: %s=%q", label, config.Config.Labels[label]) + } + } + if layers := len(config.RootFS.DiffIDs); layers != len(storage.Layers) { + return nil, fmt.Errorf("machine base has %d layers but the disk stacks on %d", layers, len(storage.Layers)) } - diffIDs = ids } else { - for _, layer := range storage.Layers { - blob, diffID, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: layer}}) + for _, path := range storage.Layers { + layer, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: path}}) if err != nil { - return nil, nil, err + return nil, err + } + if img, err = mutate.AppendLayers(img, layer); err != nil { + return nil, err } - blobs, diffIDs = append(blobs, blob), append(diffIDs, diffID) } } platform, err := json.Marshal(storage.Platform) if err != nil { - return nil, nil, err + return nil, err } - top, diffID, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: storage.Overlay}, {name: "aux.img", path: storage.Aux}, {name: "config.json", data: platform}}) + top, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: storage.Overlay}, {name: "aux.img", path: storage.Aux}, {name: "config.json", data: platform}}) if err != nil { - return nil, nil, err + return nil, err } - return append(blobs, top), append(diffIDs, diffID), nil -} - -// fetchMachineBase reads the base manifest and config from the registry, checking -// each against its digest and that the base is an ASIF machine image. It returns -// the base's layer descriptors and diff IDs. -func fetchMachineBase(ctx context.Context, client *http.Client, token, registry string, base MachineBase) ([]v1.Descriptor, []v1.Hash, error) { - root := strings.TrimRight(registry, "/") + "/v2/" + base.Repo - manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+base.Digest, base.Digest) - if err != nil { - return nil, nil, fmt.Errorf("base manifest: %w", err) + if img, err = mutate.AppendLayers(img, top); err != nil { + return nil, err } - var manifest v1.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, nil, fmt.Errorf("base manifest: %w", err) - } - configDigest := manifest.Config.Digest.String() - configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest) + config, err := img.ConfigFile() if err != nil { - return nil, nil, fmt.Errorf("base config: %w", err) - } - var config v1.ConfigFile - if err := json.Unmarshal(configBytes, &config); err != nil { - return nil, nil, fmt.Errorf("base config: %w", err) + return nil, err } - if config.OS != "darwin" || config.Architecture != "arm64" || len(manifest.Layers) == 0 || len(config.RootFS.DiffIDs) != len(manifest.Layers) { - return nil, nil, fmt.Errorf("base is not a darwin/arm64 machine image with matching layers") + config.OS, config.Architecture, config.Config.Labels = "darwin", "arm64", machineLabels + if img, err = mutate.ConfigFile(img, config); err != nil { + return nil, err } - for label, want := range machineLabels { - if config.Config.Labels[label] != want { - return nil, nil, fmt.Errorf("base is not an ASIF machine image: %s=%q", label, config.Config.Labels[label]) - } - } - return manifest.Layers, config.RootFS.DiffIDs, nil + return mutate.ConfigMediaType(mutate.MediaType(img, types.OCIManifestSchema1), types.OCIConfigJSON), nil } // machineTarEntry is one file in a machine layer, from a path or in memory. @@ -263,29 +216,14 @@ type machineTarEntry struct { data []byte } -// machineLayer returns a gzipped layer of entries, with its descriptor and diff ID. -func machineLayer(ctx context.Context, entries []machineTarEntry) (machineBlob, v1.Hash, error) { - layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { +// machineLayer returns a gzipped layer of entries, regenerated from the files +// each time it is read. +func machineLayer(ctx context.Context, entries []machineTarEntry) (v1.Layer, error) { + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { reader, writer := io.Pipe() go func() { writer.CloseWithError(writeMachineTar(ctx, entries, writer)) }() return reader, nil }, tarball.WithMediaType(types.OCILayer), tarball.WithCompressionLevel(gzip.DefaultCompression)) - if err != nil { - return machineBlob{}, v1.Hash{}, err - } - diffID, err := layer.DiffID() - if err != nil { - return machineBlob{}, v1.Hash{}, err - } - digest, err := layer.Digest() - if err != nil { - return machineBlob{}, v1.Hash{}, err - } - size, err := layer.Size() - if err != nil { - return machineBlob{}, v1.Hash{}, err - } - return machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: digest, Size: size}}, diffID, nil } // writeMachineTar writes a deterministic tar of entries: fixed modes and no diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go index 7b422f263..958ae0808 100644 --- a/lib/builds/macos_build_publisher_test.go +++ b/lib/builds/macos_build_publisher_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "sync/atomic" "testing" "time" @@ -55,13 +56,10 @@ func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { require.NoError(t, e) var uploads atomic.Int32 handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - require.NotEqual(t, http.MethodPatch, r.Method, "uploads must use the disk-streaming monolithic endpoint") - if r.Method == http.MethodPost { - require.NotEmpty(t, r.URL.Query().Get("digest")) - require.Greater(t, r.ContentLength, int64(0)) + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/blobs/uploads/") { uploads.Add(1) } - if r.Method == http.MethodPut { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { require.Equal(t, int32(2), uploads.Load(), "manifest must be last") } reg.Handler().ServeHTTP(w, r) @@ -104,7 +102,7 @@ func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { require.NoError(t, e) var uploads atomic.Int32 server := httptest.NewServer(middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/blobs/uploads/") { uploads.Add(1) } reg.Handler().ServeHTTP(w, r) diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go deleted file mode 100644 index 5b29604b4..000000000 --- a/lib/builds/macos_registry.go +++ /dev/null @@ -1,82 +0,0 @@ -package builds - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "net/http" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/types" -) - -// maxMachineMetadataBytes bounds a base manifest or config fetched from the registry. -const maxMachineMetadataBytes = 4 << 20 - -// machineRegistryRequest sends one authenticated registry request. The caller -// closes the response body. -func machineRegistryRequest(ctx context.Context, client *http.Client, token, method, target, media string, body io.Reader, size int64) (*http.Response, error) { - request, err := http.NewRequestWithContext(ctx, method, target, body) - if err != nil { - return nil, err - } - request.ContentLength = size - request.Header.Set("Authorization", "Bearer "+token) - if media != "" { - request.Header.Set("Content-Type", media) - } - request.Header.Set("Accept", string(types.OCIManifestSchema1)) - response, err := client.Do(request) - if err != nil { - return nil, fmt.Errorf("machine registry request failed") - } - return response, nil -} - -func machineHash(data []byte) v1.Hash { - sum := sha256.Sum256(data) - return v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum[:])} -} - -// machineFetch GETs registry metadata, a manifest or config, and checks it against -// digest. The body is buffered up to maxMachineMetadataBytes and returned. -func machineFetch(ctx context.Context, client *http.Client, token, target, digest string) ([]byte, error) { - response, err := machineRegistryRequest(ctx, client, token, http.MethodGet, target, "", nil, 0) - if err != nil { - return nil, err - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("registry returned status %d", response.StatusCode) - } - data, err := io.ReadAll(io.LimitReader(response.Body, maxMachineMetadataBytes+1)) - if err != nil { - return nil, err - } - if len(data) > maxMachineMetadataBytes || machineHash(data).String() != digest { - return nil, fmt.Errorf("object does not match digest %s", digest) - } - return data, nil -} - -// machineBlobStored reports whether the registry already holds a blob of the -// descriptor's digest and size. The store is shared by digest across repositories. -func machineBlobStored(ctx context.Context, client *http.Client, token, base string, descriptor v1.Descriptor) (bool, error) { - head, err := machineRegistryRequest(ctx, client, token, http.MethodHead, base+"/blobs/"+descriptor.Digest.String(), "", nil, 0) - if err != nil { - return false, err - } - head.Body.Close() - switch head.StatusCode { - case http.StatusOK: - if head.ContentLength != descriptor.Size { - return false, fmt.Errorf("stored blob %s has size %d, want %d", descriptor.Digest, head.ContentLength, descriptor.Size) - } - return true, nil - case http.StatusNotFound: - return false, nil - } - return false, fmt.Errorf("blob check returned status %d", head.StatusCode) -} diff --git a/lib/builds/macos_registry_test.go b/lib/builds/macos_registry_test.go deleted file mode 100644 index 4b069ccd4..000000000 --- a/lib/builds/macos_registry_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package builds - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMachineFetchVerifiesDigest(t *testing.T) { - body := []byte("synthetic object") - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write(body) - })) - defer server.Close() - digest := machineHash(body).String() - oversize := strings.Repeat("x", maxMachineMetadataBytes+1) - bigServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(oversize)) - })) - defer bigServer.Close() - - for _, tc := range []struct { - name string - target string - digest string - ok bool - }{ - {"matches digest", server.URL, digest, true}, - {"wrong digest", server.URL, machineHash([]byte("other")).String(), false}, - {"over cap", bigServer.URL, machineHash([]byte(oversize)).String(), false}, - } { - t.Run(tc.name, func(t *testing.T) { - got, err := machineFetch(context.Background(), http.DefaultClient, "token", tc.target, tc.digest) - if !tc.ok { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, body, got) - }) - } -} diff --git a/lib/registry/blob_store.go b/lib/registry/blob_store.go index e7d92fb02..5d3c2fb45 100644 --- a/lib/registry/blob_store.go +++ b/lib/registry/blob_store.go @@ -104,6 +104,16 @@ func (s *BlobStore) Put(_ context.Context, repo string, h v1.Hash, r io.ReadClos return nil } +// install moves a verified file into the store as the blob h, keeping an existing +// copy. The file must be on the store's filesystem. +func (s *BlobStore) install(path string, h v1.Hash) error { + target := s.blobPath(h.String()) + if _, err := os.Stat(target); err == nil { + return os.Remove(path) + } + return os.Rename(path, target) +} + func (s *BlobStore) Delete(_ context.Context, repo string, h v1.Hash) error { path := s.blobPath(h.String()) err := os.Remove(path) diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 77249b920..541d72dd0 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "os" + "path/filepath" "regexp" "strings" "sync" @@ -28,6 +29,7 @@ type Registry struct { paths *paths.Paths imageManager images.Manager blobStore *BlobStore + uploads *chunkedUploads handler http.Handler // pushedTags tracks pushed (repo, reference) tags to their manifest @@ -52,6 +54,11 @@ func New(p *paths.Paths, imgManager images.Manager) (*Registry, error) { return nil, err } + uploads, err := newChunkedUploads(filepath.Join(p.SystemOCICache(), "uploads"), blobStore) + if err != nil { + return nil, fmt.Errorf("create upload directory: %w", err) + } + // Create registry with custom blob handler regHandler := registry.New( registry.WithBlobHandler(blobStore), @@ -61,6 +68,7 @@ func New(p *paths.Paths, imgManager images.Manager) (*Registry, error) { paths: p, imageManager: imgManager, blobStore: blobStore, + uploads: uploads, handler: regHandler, pushedTags: make(map[string]string), } @@ -111,6 +119,9 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if r.uploads.serve(w, req) { + return + } // Intercept manifest PUT requests to store in blob store and trigger conversion if req.Method == http.MethodPut { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) diff --git a/lib/registry/uploads.go b/lib/registry/uploads.go new file mode 100644 index 000000000..07b2ccf6e --- /dev/null +++ b/lib/registry/uploads.go @@ -0,0 +1,161 @@ +package registry + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// uploadPattern matches /v2/{name}/blobs/uploads/ and /v2/{name}/blobs/uploads/{id}. +var uploadPattern = regexp.MustCompile(`^/v2/(.+)/blobs/uploads/([0-9a-f]*)$`) + +// chunkedUploads stores chunked blob uploads (POST, PATCH..., PUT ?digest=) in +// files beside the blob store, hashing as they arrive. The upstream registry keeps +// them in memory, which a multi-gigabyte layer cannot afford. Monolithic uploads +// (POST ?digest=) already stream and are left to it. +type chunkedUploads struct { + dir string + store *BlobStore + mu sync.Mutex + open map[string]*chunkedUpload +} + +type chunkedUpload struct { + mu sync.Mutex + file *os.File + hash hash.Hash + size int64 +} + +// newChunkedUploads removes uploads left by an earlier process: their sessions +// were in memory, so no client can finish them. +func newChunkedUploads(dir string, store *BlobStore) (*chunkedUploads, error) { + if err := os.RemoveAll(dir); err != nil { + return nil, err + } + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + return &chunkedUploads{dir: dir, store: store, open: make(map[string]*chunkedUpload)}, nil +} + +// serve handles req if it is part of a chunked upload and reports whether it did. +func (u *chunkedUploads) serve(w http.ResponseWriter, req *http.Request) bool { + match := uploadPattern.FindStringSubmatch(req.URL.Path) + if match == nil { + return false + } + repo, id := match[1], match[2] + switch { + case req.Method == http.MethodPost && id == "" && req.URL.Query().Get("digest") == "": + u.start(w, repo) + case id != "" && (req.Method == http.MethodPatch || req.Method == http.MethodPut || req.Method == http.MethodDelete): + u.mu.Lock() + upload := u.open[id] + u.mu.Unlock() + if upload == nil { + http.Error(w, "blob upload unknown", http.StatusNotFound) + return true + } + upload.mu.Lock() + defer upload.mu.Unlock() + switch req.Method { + case http.MethodPatch: + u.append(w, req, repo, id, upload) + case http.MethodPut: + u.commit(w, req, repo, id, upload) + default: + u.remove(id, upload) + w.WriteHeader(http.StatusNoContent) + } + default: + return false + } + return true +} + +func (u *chunkedUploads) start(w http.ResponseWriter, repo string) { + var raw [16]byte + rand.Read(raw[:]) + id := hex.EncodeToString(raw[:]) + file, err := os.OpenFile(filepath.Join(u.dir, id), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + http.Error(w, "start upload failed", http.StatusInternalServerError) + return + } + u.mu.Lock() + u.open[id] = &chunkedUpload{file: file, hash: sha256.New()} + u.mu.Unlock() + w.Header().Set("Location", "/v2/"+repo+"/blobs/uploads/"+id) + w.Header().Set("Range", "0-0") + w.WriteHeader(http.StatusAccepted) +} + +// append writes a chunk. A Content-Range must start where the upload ends. +func (u *chunkedUploads) append(w http.ResponseWriter, req *http.Request, repo, id string, upload *chunkedUpload) { + if r := req.Header.Get("Content-Range"); r != "" { + var start, end int64 + if _, err := fmt.Sscanf(r, "%d-%d", &start, &end); err != nil || start != upload.size { + http.Error(w, "content range does not continue the upload", http.StatusRequestedRangeNotSatisfiable) + return + } + } + n, err := io.Copy(io.MultiWriter(upload.file, upload.hash), req.Body) + upload.size += n + if err != nil { + u.remove(id, upload) + http.Error(w, "write upload failed", http.StatusInternalServerError) + return + } + w.Header().Set("Location", "/v2/"+repo+"/blobs/uploads/"+id) + w.Header().Set("Range", "0-"+strconv.FormatInt(upload.size-1, 10)) + w.WriteHeader(http.StatusAccepted) +} + +// commit writes the final chunk and installs the blob if it matches the digest. +// The upload ends either way. +func (u *chunkedUploads) commit(w http.ResponseWriter, req *http.Request, repo, id string, upload *chunkedUpload) { + defer u.remove(id, upload) + want, err := v1.NewHash(req.URL.Query().Get("digest")) + if err != nil { + http.Error(w, "invalid digest", http.StatusBadRequest) + return + } + if _, err = io.Copy(io.MultiWriter(upload.file, upload.hash), req.Body); err == nil { + err = upload.file.Close() + } + if err != nil { + http.Error(w, "write upload failed", http.StatusInternalServerError) + return + } + if got := "sha256:" + hex.EncodeToString(upload.hash.Sum(nil)); got != want.String() { + http.Error(w, "digest mismatch", http.StatusBadRequest) + return + } + if err = u.store.install(upload.file.Name(), want); err != nil { + http.Error(w, "store blob failed", http.StatusInternalServerError) + return + } + w.Header().Set("Docker-Content-Digest", want.String()) + w.Header().Set("Location", "/v2/"+repo+"/blobs/"+want.String()) + w.WriteHeader(http.StatusCreated) +} + +func (u *chunkedUploads) remove(id string, upload *chunkedUpload) { + u.mu.Lock() + delete(u.open, id) + u.mu.Unlock() + upload.file.Close() + os.Remove(upload.file.Name()) +} diff --git a/lib/registry/uploads_test.go b/lib/registry/uploads_test.go new file mode 100644 index 000000000..9dbb84e20 --- /dev/null +++ b/lib/registry/uploads_test.go @@ -0,0 +1,140 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "math/rand/v2" + "net/http" + "net/http/httptest" + "os" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// randomBlob is an incompressible blob generated on demand, so a large blob needs +// no memory to push. +type randomBlob struct { + size int64 + digest v1.Hash +} + +func newRandomBlob(t *testing.T, size int64) *randomBlob { + b := &randomBlob{size: size} + hash := sha256.New() + body, err := b.Compressed() + require.NoError(t, err) + _, err = io.Copy(hash, body) + require.NoError(t, err) + b.digest = v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(hash.Sum(nil))} + return b +} + +func (b *randomBlob) Compressed() (io.ReadCloser, error) { + return io.NopCloser(io.LimitReader(rand.NewChaCha8([32]byte{}), b.size)), nil +} +func (b *randomBlob) Uncompressed() (io.ReadCloser, error) { return b.Compressed() } +func (b *randomBlob) Digest() (v1.Hash, error) { return b.digest, nil } +func (b *randomBlob) DiffID() (v1.Hash, error) { return b.digest, nil } +func (b *randomBlob) Size() (int64, error) { return b.size, nil } +func (b *randomBlob) MediaType() (types.MediaType, error) { return types.OCILayer, nil } + +// TestChunkedUploadStreamsToDisk pushes a layer far larger than the heap bound +// through go-containerregistry's chunked upload, and checks it lands in the blob +// store without being held in memory. +func TestChunkedUploadStreamsToDisk(t *testing.T) { + const size = 768 << 20 + p := paths.New(t.TempDir()) + reg, err := New(p, nil) + require.NoError(t, err) + server := httptest.NewServer(reg.Handler()) + defer server.Close() + + layer := newRandomBlob(t, size) + ref, err := name.ParseReference(strings.TrimPrefix(server.URL, "http://")+"/cache/big:latest", name.Insecure) + require.NoError(t, err) + + runtime.GC() + var base runtime.MemStats + runtime.ReadMemStats(&base) + var peak atomic.Uint64 + done := make(chan struct{}) + go func() { + for { + var m runtime.MemStats + runtime.ReadMemStats(&m) + if m.HeapInuse > peak.Load() { + peak.Store(m.HeapInuse) + } + select { + case <-done: + return + case <-time.After(20 * time.Millisecond): + } + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + err = remote.WriteLayer(ref.Context(), layer, remote.WithContext(ctx)) + close(done) + require.NoError(t, err) + + stored, err := reg.blobStore.Stat(ctx, "cache/big", layer.digest) + require.NoError(t, err) + require.Equal(t, int64(size), stored) + grew := int64(peak.Load()) - int64(base.HeapInuse) + t.Logf("pushed %d MiB; heap grew %d MiB", size>>20, grew>>20) + require.Less(t, grew, int64(64<<20), "the upload is not buffered in memory") +} + +func TestChunkedUploadRejectsBadChunks(t *testing.T) { + reg, err := New(paths.New(t.TempDir()), nil) + require.NoError(t, err) + server := httptest.NewServer(reg.Handler()) + defer server.Close() + do := func(method, url, contentRange string, body string) *http.Response { + req, err := http.NewRequest(method, url, strings.NewReader(body)) + require.NoError(t, err) + if contentRange != "" { + req.Header.Set("Content-Range", contentRange) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + resp.Body.Close() + return resp + } + start := func() string { + resp := do(http.MethodPost, server.URL+"/v2/repo/blobs/uploads/", "", "") + require.Equal(t, http.StatusAccepted, resp.StatusCode) + return server.URL + resp.Header.Get("Location") + } + sum := sha256.Sum256([]byte("abcdef")) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + upload := start() + require.Equal(t, http.StatusAccepted, do(http.MethodPatch, upload, "0-2", "abc").StatusCode) + require.Equal(t, http.StatusRequestedRangeNotSatisfiable, do(http.MethodPatch, upload, "0-2", "def").StatusCode, "a chunk must continue the upload") + require.Equal(t, http.StatusAccepted, do(http.MethodPatch, upload, "3-5", "def").StatusCode) + require.Equal(t, http.StatusCreated, do(http.MethodPut, upload+"?digest="+digest, "", "").StatusCode) + require.Equal(t, http.StatusNotFound, do(http.MethodPatch, upload, "", "x").StatusCode, "a committed upload is closed") + + other := sha256.Sum256([]byte("xyz")) + upload = start() + require.Equal(t, http.StatusBadRequest, do(http.MethodPut, upload+"?digest=sha256:"+hex.EncodeToString(other[:]), "", "xyZ").StatusCode) + _, err = reg.blobStore.Stat(context.Background(), "repo", v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(other[:])}) + require.ErrorIs(t, err, ErrNotFound, "a mismatched upload is not stored") + entries, err := os.ReadDir(reg.uploads.dir) + require.NoError(t, err) + require.Empty(t, entries, "ended uploads leave no files") +} From 6bc80f3d95c3bb0e5d679f6858407384e82b4961 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:28:01 +0000 Subject: [PATCH 12/33] Serve stored manifests by digest after a registry restart The upstream registry keeps manifests in memory, so a build on a registry base failed once hypeman had restarted. Every pushed manifest is already kept in the blob store; serve digest requests from there. --- lib/registry/registry.go | 41 +++++++++++++++++++++++++++++++++++- lib/registry/uploads_test.go | 30 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 541d72dd0..12d72dc35 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -7,12 +7,14 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" "regexp" + "strconv" "strings" "sync" @@ -119,7 +121,7 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if r.uploads.serve(w, req) { + if r.uploads.serve(w, req) || r.serveStoredManifest(w, req) { return } // Intercept manifest PUT requests to store in blob store and trigger conversion @@ -172,6 +174,43 @@ func (r *Registry) Handler() http.Handler { }) } +// serveStoredManifest answers a manifest request by digest from the blob store, +// where every pushed manifest is kept. The upstream registry holds manifests in +// memory, so without this a manifest pushed before a restart is unknown by digest. +func (r *Registry) serveStoredManifest(w http.ResponseWriter, req *http.Request) bool { + matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) + if matches == nil || (req.Method != http.MethodGet && req.Method != http.MethodHead) { + return false + } + h, err := v1.NewHash(matches[2]) + if err != nil { + return false + } + path := r.blobStore.blobPath(h.String()) + if info, err := os.Stat(path); err != nil || info.Size() > maxStoredManifestBytes { + return false + } + data, err := os.ReadFile(path) + var manifest struct { + MediaType string `json:"mediaType"` + } + if err != nil || json.Unmarshal(data, &manifest) != nil || manifest.MediaType == "" || computeDigest(data) != h.String() { + return false + } + w.Header().Set("Content-Type", manifest.MediaType) + w.Header().Set("Docker-Content-Digest", h.String()) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.WriteHeader(http.StatusOK) + if req.Method == http.MethodGet { + w.Write(data) + } + return true +} + +// maxStoredManifestBytes bounds a stored blob served as a manifest, so a layer +// requested as one is not read into memory. +const maxStoredManifestBytes = 4 << 20 + // storeManifestBlob stores a manifest in the blob store by its digest. func (r *Registry) storeManifestBlob(digest string, data []byte) error { digestHex := strings.TrimPrefix(digest, "sha256:") diff --git a/lib/registry/uploads_test.go b/lib/registry/uploads_test.go index 9dbb84e20..afa1674fb 100644 --- a/lib/registry/uploads_test.go +++ b/lib/registry/uploads_test.go @@ -17,6 +17,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/kernel/hypeman/lib/paths" @@ -138,3 +139,32 @@ func TestChunkedUploadRejectsBadChunks(t *testing.T) { require.NoError(t, err) require.Empty(t, entries, "ended uploads leave no files") } + +func TestManifestByDigestSurvivesRestart(t *testing.T) { + p := paths.New(t.TempDir()) + serve := func() *httptest.Server { + reg, err := New(p, nil) + require.NoError(t, err) + return httptest.NewServer(reg.Handler()) + } + img, err := random.Image(64, 1) + require.NoError(t, err) + digest, err := img.Digest() + require.NoError(t, err) + + first := serve() + ref, err := name.ParseReference(strings.TrimPrefix(first.URL, "http://")+"/cache/app:latest", name.Insecure) + require.NoError(t, err) + require.NoError(t, remote.Write(ref, img)) + first.Close() + + second := serve() + defer second.Close() + byDigest, err := name.NewDigest(strings.TrimPrefix(second.URL, "http://")+"/cache/app@"+digest.String(), name.Insecure) + require.NoError(t, err) + pulled, err := remote.Image(byDigest) + require.NoError(t, err) + got, err := pulled.Digest() + require.NoError(t, err) + require.Equal(t, digest, got) +} From e9ddb63c588fd8b50f9f7a021d4dccce0743ebe8 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:43:48 +0000 Subject: [PATCH 13/33] Describe publication through remote.Write --- docs/macos-builds.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/macos-builds.md b/docs/macos-builds.md index e6fb27fef..9f89b16af 100644 --- a/docs/macos-builds.md +++ b/docs/macos-builds.md @@ -98,13 +98,14 @@ secrets are masked in strings and logs. ## Publication The publisher uses a server-configured registry and a token scoped to `builds/{id}`. -Plaintext is allowed only on loopback; redirects are never followed and proxies are -disabled. The layer is never written to disk: a first pass computes its digest, size -and DiffID, the upload regenerates the same bytes, and the registry rejects any -mismatch. Blobs the registry already holds are skipped. Blobs use monolithic POST uploads -that the registry commits only if they match their digest, the manifest is committed -last, and the manifest is read back by digest. Ambiguous -errors never return success. +Plaintext is allowed only on loopback; redirects are refused and proxies are +disabled. It writes the image with go-containerregistry's `remote.Write`. The layer is +never written to disk on the publishing side: a first pass computes its digest, size +and DiffID, and the upload regenerates the same bytes. Blobs the registry already +holds are skipped, and the manifest is committed last. The registry writes chunked +uploads to disk, hashing as they arrive, and commits a blob only if it matches its +digest. It serves manifests by digest from its blob store, so a base stays usable +after a restart. Ambiguous errors never return success. The image is the base's layers plus one layer holding the builder's overlay, auxiliary storage and platform. The overlay holds only the build's writes, so for a From 0b16b0014667af9519e7173947830e105bbf9b91 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:04:35 +0000 Subject: [PATCH 14/33] Remove the in-API macOS build path macOS images will be built with Packer and Tart outside hypeman. Drop the machine-build backend, guest driver, source staging, multipart endpoint, macos_builds config gate, build export, strict stop and fresh builder identity. The machine image publisher stays for importing built images. --- cmd/api/api/builds.go | 17 +- cmd/api/api/capabilities.go | 11 +- cmd/api/api/capabilities_machine_test.go | 7 +- cmd/api/api/macos_builds.go | 122 --- cmd/api/api/macos_builds_test.go | 226 ----- cmd/api/config/config.go | 24 +- cmd/api/config/macos_builds_test.go | 26 - cmd/api/main.go | 13 +- config.example.darwin.yaml | 7 +- docs/macos-builds.md | 133 --- docs/macos-support.md | 2 +- lib/builds/macos_build.go | 273 ------ lib/builds/macos_build_cancel.go | 70 -- lib/builds/macos_build_failure_test.go | 151 ---- lib/builds/macos_build_guest.go | 186 ---- lib/builds/macos_build_job_test.go | 37 - lib/builds/macos_build_lifecycle_test.go | 135 --- lib/builds/macos_build_manager.go | 182 ---- lib/builds/macos_build_manager_test.go | 248 ------ lib/builds/macos_build_policy_test.go | 44 - lib/builds/macos_build_publisher.go | 68 +- lib/builds/macos_build_recovery_test.go | 82 -- lib/builds/macos_build_source.go | 119 --- lib/builds/macos_build_source_test.go | 67 -- lib/builds/macos_build_test.go | 251 ------ lib/builds/manager.go | 120 +-- lib/builds/source.go | 75 -- lib/builds/source_test.go | 36 - lib/builds/storage.go | 8 +- lib/builds/types.go | 7 - lib/instances/create.go | 5 - lib/instances/delete.go | 2 +- lib/instances/fork.go | 2 +- lib/instances/macos_agent_startup.go | 2 +- lib/instances/macos_build_export.go | 47 - lib/instances/macos_build_export_darwin.go | 90 -- .../macos_build_export_darwin_test.go | 52 -- lib/instances/macos_identity.go | 15 +- lib/instances/macos_identity_test.go | 15 - lib/instances/manager.go | 2 +- lib/instances/process_identity_linux_test.go | 8 +- lib/instances/restart_policy.go | 2 +- lib/instances/stop.go | 32 +- lib/instances/types.go | 1 - lib/middleware/oapi_auth.go | 1 - lib/oapi/oapi.go | 813 +++++++++--------- lib/providers/providers.go | 23 +- openapi.yaml | 19 - 48 files changed, 518 insertions(+), 3360 deletions(-) delete mode 100644 cmd/api/api/macos_builds.go delete mode 100644 cmd/api/api/macos_builds_test.go delete mode 100644 cmd/api/config/macos_builds_test.go delete mode 100644 docs/macos-builds.md delete mode 100644 lib/builds/macos_build.go delete mode 100644 lib/builds/macos_build_cancel.go delete mode 100644 lib/builds/macos_build_failure_test.go delete mode 100644 lib/builds/macos_build_guest.go delete mode 100644 lib/builds/macos_build_job_test.go delete mode 100644 lib/builds/macos_build_lifecycle_test.go delete mode 100644 lib/builds/macos_build_manager.go delete mode 100644 lib/builds/macos_build_manager_test.go delete mode 100644 lib/builds/macos_build_policy_test.go delete mode 100644 lib/builds/macos_build_recovery_test.go delete mode 100644 lib/builds/macos_build_source.go delete mode 100644 lib/builds/macos_build_source_test.go delete mode 100644 lib/builds/macos_build_test.go delete mode 100644 lib/builds/source.go delete mode 100644 lib/builds/source_test.go delete mode 100644 lib/instances/macos_build_export.go delete mode 100644 lib/instances/macos_build_export_darwin.go delete mode 100644 lib/instances/macos_build_export_darwin_test.go diff --git a/cmd/api/api/builds.go b/cmd/api/api/builds.go index 2f4174b3b..ae8e74c27 100644 --- a/cmd/api/api/builds.go +++ b/cmd/api/api/builds.go @@ -44,15 +44,9 @@ func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequ // CreateBuild creates a new build job func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) { if s.Config != nil && s.Config.MacOSOnly { - if !s.Config.MacOSBuilds { - return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil - } - req, source, err := parseMachineBuildMultipart(request.Body) - if err != nil { - return oapi.CreateBuild400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil - } - return s.submitBuild(ctx, req, source) + return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil } + log := logger.FromContext(ctx) // Parse multipart form fields var sourceData []byte @@ -75,8 +69,6 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } switch part.FormName() { - case "machine_base_image", "machine_network_mode": - return oapi.CreateBuild400JSONResponse{Code: "unsupported", Message: "machine builds require macos_only and macos_builds"}, nil case "source": sourceData, err = io.ReadAll(part) if err != nil { @@ -264,11 +256,6 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } } - return s.submitBuild(ctx, domainReq, sourceData) -} - -func (s *ApiService) submitBuild(ctx context.Context, domainReq builds.CreateBuildRequest, sourceData []byte) (oapi.CreateBuildResponseObject, error) { - log := logger.FromContext(ctx) build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData) if err != nil { switch { diff --git a/cmd/api/api/capabilities.go b/cmd/api/api/capabilities.go index 63c82e31f..fa420f011 100644 --- a/cmd/api/api/capabilities.go +++ b/cmd/api/api/capabilities.go @@ -26,7 +26,6 @@ const ( featureDevices = "devices" featureRosettaEmulation = "rosetta-emulation" featureMacOSImages = "macos-machine-images" - featureMacOSBuilds = "macos-machine-builds-v1" ) // apiVersion is the API contract version from the embedded OpenAPI document. @@ -138,7 +137,7 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities Features: serverFeatures(runtime.GOOS, emulation), } - resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH, s.Config.MacOSBuilds)...) + resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH)...) return oapi.GetCapabilities200JSONResponse(resp), nil } @@ -225,15 +224,11 @@ func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi. return caps } -func machineImageFeatures(goos, goarch string, builds bool) []string { +func machineImageFeatures(goos, goarch string) []string { if goos != "darwin" || goarch != "arm64" { return nil } - features := []string{featureMacOSImages} - if builds { - features = append(features, featureMacOSBuilds) - } - return features + return []string{featureMacOSImages} } // serverFeatures builds the server-level feature ID list: API surfaces that diff --git a/cmd/api/api/capabilities_machine_test.go b/cmd/api/api/capabilities_machine_test.go index 705eabf86..220a1a812 100644 --- a/cmd/api/api/capabilities_machine_test.go +++ b/cmd/api/api/capabilities_machine_test.go @@ -27,8 +27,7 @@ func TestHostMachineImageCapabilities(t *testing.T) { } func TestMachineImageFeatureActivation(t *testing.T) { - require.Empty(t, machineImageFeatures("linux", "arm64", true)) - require.Empty(t, machineImageFeatures("darwin", "amd64", true)) - require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64", false)) - require.Equal(t, []string{"macos-machine-images", "macos-machine-builds-v1"}, machineImageFeatures("darwin", "arm64", true)) + require.Empty(t, machineImageFeatures("linux", "arm64")) + require.Empty(t, machineImageFeatures("darwin", "amd64")) + require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64")) } diff --git a/cmd/api/api/macos_builds.go b/cmd/api/api/macos_builds.go deleted file mode 100644 index d27aab77b..000000000 --- a/cmd/api/api/macos_builds.go +++ /dev/null @@ -1,122 +0,0 @@ -package api - -import ( - "fmt" - "io" - "mime/multipart" - "net/http" - "strconv" - - "github.com/kernel/hypeman/lib/builds" - "github.com/kernel/hypeman/lib/images" - mw "github.com/kernel/hypeman/lib/middleware" - "github.com/kernel/hypeman/lib/scopes" -) - -// maxConcurrentMachineBuildUploads bounds how many machine build bodies, each up -// to 65 MiB, are read into memory at once. -const maxConcurrentMachineBuildUploads = 4 - -// LimitMachineBuildBody must wrap the validator, which also consumes request bodies. -// A build upload is authenticated before it takes an upload slot, so unauthenticated -// requests cannot hold the slots. -func LimitMachineBuildBody(jwtSecret string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - uploads := make(chan struct{}, maxConcurrentMachineBuildUploads) - upload := mw.JwtAuth(jwtSecret)(scopes.RequireScope(scopes.BuildWrite)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - select { - case uploads <- struct{}{}: - defer func() { <-uploads }() - default: - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusTooManyRequests) - _, _ = w.Write([]byte(`{"code":"too_many_requests","message":"too many machine build uploads in progress"}`)) - return - } - r.Body = http.MaxBytesReader(w, r.Body, 65<<20) - next.ServeHTTP(w, r) - }))) - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && r.URL.Path == "/builds" { - upload.ServeHTTP(w, r) - return - } - next.ServeHTTP(w, r) - }) - } -} - -// Machine-only HTTP admission is bounded regardless of multipart field order. -// Linux retains its existing parser and source contract. -func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildRequest, []byte, error) { - req := builds.CreateBuildRequest{BuildPolicy: &builds.BuildPolicy{}} - policy := req.BuildPolicy - numbers := map[string]*int{"timeout_seconds": &policy.TimeoutSeconds, "memory_mb": &policy.MemoryMB, "cpus": &policy.CPUs} - var source []byte - seen := map[string]bool{} - if body == nil { - return req, nil, fmt.Errorf("multipart source is required") - } - for { - part, err := body.NextPart() - if err == io.EOF { - break - } - if err != nil { - return req, nil, fmt.Errorf("invalid multipart form") - } - name := part.FormName() - if seen[name] { - return req, nil, fmt.Errorf("duplicate multipart field") - } - seen[name] = true - limit := int64(64 << 10) - switch name { - case "source": - limit = 64 << 20 - case "machine_base_image", "machine_network_mode", "timeout_seconds", "memory_mb", "cpus", "tags": - default: - return req, nil, fmt.Errorf("unsupported machine build field: %s", name) - } - data, err := io.ReadAll(io.LimitReader(part, limit+1)) - if err != nil || int64(len(data)) > limit { - return req, nil, fmt.Errorf("invalid or oversized %s field", name) - } - switch name { - case "source": - source = data - case "machine_base_image": - req.MachineBaseImage = string(data) - case "machine_network_mode": - if string(data) != "isolated" && string(data) != "egress" { - return req, nil, fmt.Errorf("machine_network_mode must be isolated or egress") - } - req.BuildPolicy.NetworkMode = string(data) - case "tags": - req.Tags, err = parseTagsJSON(string(data)) - if err != nil { - return req, nil, fmt.Errorf("tags must be a JSON object of strings") - } - default: - value, err := strconv.Atoi(string(data)) - if err != nil || value < 0 { - return req, nil, fmt.Errorf("%s must be a nonnegative integer", name) - } - *numbers[name] = value - } - } - if len(source) == 0 { - return req, nil, fmt.Errorf("source is required") - } - ref, err := images.ParseNormalizedRef(req.MachineBaseImage) - if err != nil || !ref.IsDigest() || !builds.ValidMachineDigest(ref.Digest()) { - return req, nil, fmt.Errorf("machine_base_image must be pinned by sha256 digest") - } - if req.BuildPolicy.TimeoutSeconds > 86400 { - return req, nil, fmt.Errorf("timeout_seconds exceeds 86400") - } - if err = req.BuildPolicy.Validate(); err != nil { - return req, nil, err - } - return req, source, nil -} diff --git a/cmd/api/api/macos_builds_test.go b/cmd/api/api/macos_builds_test.go deleted file mode 100644 index 2fe07942f..000000000 --- a/cmd/api/api/macos_builds_test.go +++ /dev/null @@ -1,226 +0,0 @@ -package api - -import ( - "bytes" - "context" - "io" - "mime/multipart" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/golang-jwt/jwt/v5" - "github.com/kernel/hypeman/cmd/api/config" - "github.com/kernel/hypeman/lib/builds" - "github.com/kernel/hypeman/lib/oapi" - "github.com/stretchr/testify/require" -) - -func machineMultipart(t *testing.T, fields ...[2]string) *multipart.Reader { - t.Helper() - var data bytes.Buffer - writer := multipart.NewWriter(&data) - for _, field := range fields { - require.NoError(t, writer.WriteField(field[0], field[1])) - } - require.NoError(t, writer.Close()) - return multipart.NewReader(&data, writer.Boundary()) -} - -type machineZeroReader struct{} - -func (machineZeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil } - -func TestMachineBuildBodyBoundBeforeValidation(t *testing.T) { - for _, path := range []string{"/builds", "/images"} { - r := authorizedMachineBuildRequest(t, path, io.LimitReader(machineZeroReader{}, (65<<20)+1)) - LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - n, err := io.Copy(io.Discard, r.Body) - if path == "/builds" { - require.Error(t, err) - require.EqualValues(t, 65<<20, n) - } else { - require.NoError(t, err) - require.EqualValues(t, (65<<20)+1, n) - } - })).ServeHTTP(httptest.NewRecorder(), r) - } -} - -func TestMachineBuildUploadsAreBounded(t *testing.T) { - release := make(chan struct{}) - entered := make(chan struct{}, maxConcurrentMachineBuildUploads) - handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - entered <- struct{}{} - <-release - })) - done := make(chan struct{}) - for range maxConcurrentMachineBuildUploads { - go func() { - handler.ServeHTTP(httptest.NewRecorder(), authorizedMachineBuildRequest(t, "/builds", nil)) - done <- struct{}{} - }() - <-entered - } - - w := httptest.NewRecorder() - handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) - require.Equal(t, http.StatusTooManyRequests, w.Code) - - close(release) - for range maxConcurrentMachineBuildUploads { - <-done - } - w = httptest.NewRecorder() - go func() { <-entered }() - handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) - require.Equal(t, http.StatusOK, w.Code, "a slot frees once an upload finishes") -} - -func TestMachineBuildMultipartAdmission(t *testing.T) { - base := "example.test/base@sha256:" + strings.Repeat("a", 64) - valid := [][2]string{{"source", "synthetic"}, {"machine_base_image", base}} - for _, tc := range []struct { - name string - extra [][2]string - ok bool - }{ - {name: "inherit", ok: true}, - {name: "policy", extra: [][2]string{{"cpus", "4"}, {"memory_mb", "8192"}, {"timeout_seconds", "1200"}}, ok: true}, - {name: "isolated network", extra: [][2]string{{"machine_network_mode", "isolated"}}, ok: true}, - {name: "egress network", extra: [][2]string{{"machine_network_mode", "egress"}}, ok: true}, - {name: "invalid network", extra: [][2]string{{"machine_network_mode", "bridge"}}}, - {name: "empty network", extra: [][2]string{{"machine_network_mode", ""}}}, - {name: "duplicate", extra: [][2]string{{"source", "second"}}}, - {name: "linux", extra: [][2]string{{"dockerfile", "FROM scratch"}}}, - {name: "secret", extra: [][2]string{{"secrets", "[]"}}}, - {name: "invalid number", extra: [][2]string{{"cpus", "nan"}}}, - {name: "negative", extra: [][2]string{{"memory_mb", "-1"}}}, - {name: "timeout", extra: [][2]string{{"timeout_seconds", "86401"}}}, - {name: "metadata bound", extra: [][2]string{{"tags", strings.Repeat("x", (64<<10)+1)}}}, - } { - t.Run(tc.name, func(t *testing.T) { - req, source, err := parseMachineBuildMultipart(machineMultipart(t, append(append([][2]string{}, valid...), tc.extra...)...)) - if !tc.ok { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, base, req.MachineBaseImage) - require.Equal(t, []byte("synthetic"), source) - if tc.name == "egress network" { - require.Equal(t, "egress", req.BuildPolicy.NetworkMode) - } - }) - } - _, _, err := parseMachineBuildMultipart(machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", "example.test/base:latest"})) - require.Error(t, err) -} - -type machineHTTPManager struct { - builds.Manager - req builds.CreateBuildRequest - calls int -} - -func (m *machineHTTPManager) CreateBuild(_ context.Context, req builds.CreateBuildRequest, source []byte) (*builds.Build, error) { - m.calls++ - m.req = req - return &builds.Build{ID: "synthetic-build", Status: builds.StatusQueued}, nil -} - -func TestMachineBuildHTTPGateAndDispatch(t *testing.T) { - base := "example.test/base@sha256:" + strings.Repeat("a", 64) - for _, tc := range []struct { - name string - cfg *config.Config - accepted bool - }{ - {"disabled", &config.Config{MacOSOnly: true}, false}, - {"linux", &config.Config{}, false}, - {"enabled", &config.Config{MacOSOnly: true, MacOSBuilds: true}, true}, - } { - t.Run(tc.name, func(t *testing.T) { - manager := &machineHTTPManager{} - service := &ApiService{Config: tc.cfg, BuildManager: manager} - response, err := service.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{Body: machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", base})}) - require.NoError(t, err) - if tc.accepted { - require.IsType(t, oapi.CreateBuild202JSONResponse{}, response) - require.Equal(t, 1, manager.calls) - require.Equal(t, base, manager.req.MachineBaseImage) - } else { - // Disabled machine builds are unsupported (501) on macOS-only servers, and - // a Linux server rejects the machine selector as a bad request (400). - if tc.cfg.MacOSOnly { - require.IsType(t, oapi.CreateBuild501JSONResponse{}, response) - } else { - require.IsType(t, oapi.CreateBuild400JSONResponse{}, response) - } - require.Zero(t, manager.calls) - } - }) - } -} - -const machineBuildTestSecret = "machine-build-test-secret" - -func signedMachineBuildToken(t *testing.T, claims jwt.MapClaims) string { - t.Helper() - token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(machineBuildTestSecret)) - require.NoError(t, err) - return token -} - -func authorizedMachineBuildRequest(t *testing.T, path string, body io.Reader) *http.Request { - t.Helper() - token := signedMachineBuildToken(t, jwt.MapClaims{ - "sub": "user", - "iat": time.Now().Unix(), - "exp": time.Now().Add(time.Hour).Unix(), - }) - r := httptest.NewRequest(http.MethodPost, path, body) - r.Header.Set("Authorization", "Bearer "+token) - return r -} - -func TestMachineBuildUploadsRequireAuthBeforeTakingASlot(t *testing.T) { - readOnly := signedMachineBuildToken(t, jwt.MapClaims{ - "sub": "user", - "iat": time.Now().Unix(), - "exp": time.Now().Add(time.Hour).Unix(), - "permissions": []string{"build:read"}, - }) - for _, tc := range []struct { - name string - header string - status int - }{ - {"unauthenticated", "", http.StatusUnauthorized}, - {"read-only token", "Bearer " + readOnly, http.StatusForbidden}, - } { - t.Run(tc.name, func(t *testing.T) { - reached := false - handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - reached = true - })) - for range maxConcurrentMachineBuildUploads + 1 { - r := httptest.NewRequest(http.MethodPost, "/builds", nil) - if tc.header != "" { - r.Header.Set("Authorization", tc.header) - } - w := httptest.NewRecorder() - handler.ServeHTTP(w, r) - require.Equal(t, tc.status, w.Code) - } - require.False(t, reached, "rejected uploads must not reach processing") - - w := httptest.NewRecorder() - handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) - require.Equal(t, http.StatusOK, w.Code, "rejected requests hold no upload slot") - require.True(t, reached) - }) - } -} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 544a5d494..0bffb655b 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -285,8 +285,7 @@ type Config struct { JwtSecret string `koanf:"jwt_secret"` Env string `koanf:"env"` Version string `koanf:"version"` - MacOSBuilds bool `koanf:"macos_builds"` // Experimental: enable machine builds on the macOS-only server. - MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. + MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. Network NetworkConfig `koanf:"network"` Caddy CaddyConfig `koanf:"caddy"` @@ -607,9 +606,6 @@ func (c *Config) Validate() error { if err := c.validateAPIListenPorts(); err != nil { return err } - if err := c.validateMacOSBuilds(); err != nil { - return err - } if strings.TrimSpace(c.Metrics.ListenAddress) == "" { return fmt.Errorf("metrics.listen_address must not be empty") } @@ -856,21 +852,3 @@ func (c *Config) validateFirecrackerUFFDGraduation() error { func intPtr(v int) *int { return &v } - -// validateMacOSBuilds checks the experimental machine-build gate. The publisher -// validates the registry URL itself, since the base publication tool shares it. -func (c *Config) validateMacOSBuilds() error { - if !c.MacOSBuilds { - return nil - } - if !c.MacOSOnly { - return fmt.Errorf("macos_builds requires macos_only") - } - if c.Registry.URL == "" { - return fmt.Errorf("macos_builds requires an explicit registry URL") - } - if c.Registry.CACertFile != "" || c.Registry.Insecure { - return fmt.Errorf("macos_builds requires system-trusted registry TLS or loopback HTTP") - } - return nil -} diff --git a/cmd/api/config/macos_builds_test.go b/cmd/api/config/macos_builds_test.go deleted file mode 100644 index 6a4cd08d9..000000000 --- a/cmd/api/config/macos_builds_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMacOSBuildsFailClosedConfiguration(t *testing.T) { - require.False(t, (&Config{}).MacOSBuilds) - c := *defaultConfig() - c.MacOSBuilds = true - require.ErrorContains(t, c.validateMacOSBuilds(), "requires macos_only") - c.MacOSOnly = true - c.Hypervisor.Default = "vz" - c.Registry.URL = "" - require.ErrorContains(t, c.validateMacOSBuilds(), "explicit registry URL") - c.Registry.URL = "http://localhost:4973" - c.Registry.CACertFile = "custom.pem" - require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") - c.Registry.CACertFile = "" - c.Registry.Insecure = true - require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") - c.Registry.Insecure = false - require.NoError(t, c.validateMacOSBuilds()) -} diff --git a/cmd/api/main.go b/cmd/api/main.go index 1c8241323..347b395ff 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -535,11 +535,6 @@ func run() error { r.Use(api.RecordCreateFields) } - // Bound machine multipart input before OpenAPI validation reads the body. - if app.Config.MacOSBuilds { - r.Use(api.LimitMachineBuildBody(app.Config.JwtSecret)) - } - // OpenAPI request validation with authentication validatorOptions := &nethttpmiddleware.Options{ Options: openapi3filter.Options{ @@ -637,10 +632,10 @@ func run() error { logger.Error("failed to start builders manager", "error", err) return err } - } - if err := app.BuildManager.Start(gctx); err != nil { - logger.Error("failed to start build manager", "error", err) - return err + if err := app.BuildManager.Start(gctx); err != nil { + logger.Error("failed to start build manager", "error", err) + return err + } } grp.Go(func() error { diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index d838b9887..362b3d18d 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -27,13 +27,8 @@ data_dir: ~/Library/Application Support/hypeman # Server configuration port: "4973" -# Native macOS guests/builds are opt-in; these defaults retain Linux guests on VZ. +# Native macOS guests are opt-in; this default retains Linux guests on VZ. macos_only: false -macos_builds: false -# For a dedicated native Mac builder, set both true after the release gates in -# docs/macos-builds.md pass, and replace registry.url below with an explicit -# system-trusted HTTPS address reachable by the build and destination hosts. -# Machine publication runs on the host, not at the Linux guest NAT gateway. # ============================================================================= # Hypervisor Configuration (IMPORTANT FOR MACOS) diff --git a/docs/macos-builds.md b/docs/macos-builds.md deleted file mode 100644 index 9f89b16af..000000000 --- a/docs/macos-builds.md +++ /dev/null @@ -1,133 +0,0 @@ -# macOS image builds (experimental, opt-in) - -A build job can provision a digest-pinned installed macOS base in an isolated VZ -VM, run the source's `build.sh` in it, and publish the result as an OCI machine -image. This is not macOS installation, Linux rootfs conversion or memory forking. -It depends on the shared system GuestService. - -The normal build manager owns the queue, persisted request, deadline, source -staging, status, provenance and the image-readiness gate. `GuestMachineBuildDriver` -runs the guest and `StreamingMachinePublisher` writes the registry. - -## Enabling - -Off by default. Set `macos_only: true` and `macos_builds: true` on an Apple-silicon -VZ server, with `registry.url` set to an address the host can reach (not a guest -gateway), including its scheme: `http://localhost:4973` for loopback, or a -system-trusted `https://` endpoint. Credentials, paths, queries, fragments, custom -CAs and insecure HTTPS are rejected. Linux servers reject the machine selector. - -```sh -curl -H "Authorization: Bearer $TOKEN" \ - -F "machine_base_image=registry.example/base@sha256:$BASE_DIGEST" \ - -F "source=@source.tar.gz;type=application/gzip" \ - -F "timeout_seconds=1200" "$API/builds" -``` - -The base must already be Ready. A 202 queues an ordinary job; status, cancellation -and provenance use the shared `/builds` APIs. Only `source`, `machine_base_image`, -`machine_network_mode`, `timeout_seconds`, `memory_mb`, `cpus` and `tags` are -accepted. Duplicate or unknown fields, malformed numbers and mutable bases return -400. Limits: 64 KiB per metadata field, 64 MiB source, 65 MiB total body. Linux -source parsing is unchanged. - -## Using the result - -`GET /builds/{id}` returns a digest-pinned `image_ref` for instance create. Other -Mac hosts need `registry.url` reachable from them; loopback only works on the build -host. A host that hasn't cached the image pulls on create, which can exceed the -readiness wait for large images, so retry or pre-pull. Distribute by digest with -`POST /pushes`. Consumers should pin the digest, not a tag. - -## Base and policy - -The base must be pinned by canonical SHA256, be `darwin/arm64`, and declare a -provisioned system agent on port 2222. The driver also checks guest UID 0 and never -installs an agent or falls back to host SSH. CPU and memory match the base exactly; -zero request values inherit them. Timeout defaults to 600 s, capped at 24 h. - -Networking defaults to isolated. `machine_network_mode=egress` gives unrestricted -VZ NAT for checksum-verified downloads; it is not a domain allowlist. Linux builder, -cache, build-arg, secret and admin options are rejected. - -Each builder gets its own machine identifier and MAC, and its SSH host keys are -rekeyed on first start, the same way a cold clone is. The output keeps that identity. -Builds on one base run concurrently, and an output can run beside its base. - -## Source - -The source tar.gz is staged privately and SHA256-verified, including on recovery, -and `provenance.source_hash` pins exactly what ran. Input is capped at 64 MiB -compressed, 128 MiB uncompressed, 1024 entries and 64 MiB per file. Only regular -files and directories are accepted; links, devices, duplicates and path escapes fail -before transfer. The archive must contain a non-empty `build.sh` at its root. It is -copied to `/var/root/hypeman-build-{id}` in the guest and run as root with -`/bin/sh build.sh`. - -Before stopping, the guest's build directory is removed and its absence verified, -and the host copy is removed. No registry token, host credential or secret is -injected. Guest stdout and stderr are discarded, so build logs never contain guest -output. Failures report the phase and, for the script, its exit code. - -## Stop, publish and cleanup - -Publication follows two instance operations. `StopMacOSBuildInstance` does a strict -graceful stop and rejects forced fallback, an instance already Stopped, non-macOS -guests, and any stop without shutdown acknowledgement and confirmed VMM exit. -`MacOSBuildStorage` then returns the closed storage of the still-Stopped instance, -checked under the instance lock: its read-only ASIF layers, its writable overlay -`boot.asif`, its auxiliary storage and its platform. Nothing is copied. The build -checks that the overlay opens as an ASIF stack on those layers, that the hardware -model and resources match the base and that the identity differs. A Start during -publication changes the overlay, so the registry's digest check fails the upload. - -Failure cleanup uses a forced stop and publishes nothing. Storage is deleted only after -confirmed VMM exit; a VM with no confirmed exit stays quarantined for an operator. -Cleanup has its own 2-minute budget. A re-enqueued attempt first reaps the builder -left by the interrupted attempt, and fails rather than start a second one if that -delete fails. Builders orphaned outside this path have no general reaper. - -The builder is destroyed **after** the registry has verified the publication, since -its storage is the publication source. Source stays staged until a terminal -status is persisted, so a job interrupted while pushing is rebuilt on recovery. -Startup on a macOS-only server recovers machine jobs without preparing Linux builder -state, reaps cancelled builders and removes terminal-job source. Cancellation -interrupts the running job and keeps Cancelled terminal. Guest errors that may carry -secrets are masked in strings and logs. - -## Publication - -The publisher uses a server-configured registry and a token scoped to `builds/{id}`. -Plaintext is allowed only on loopback; redirects are refused and proxies are -disabled. It writes the image with go-containerregistry's `remote.Write`. The layer is -never written to disk on the publishing side: a first pass computes its digest, size -and DiffID, and the upload regenerates the same bytes. Blobs the registry already -holds are skipped, and the manifest is committed last. The registry writes chunked -uploads to disk, hashing as they arrive, and commits a blob only if it matches its -digest. It serves manifests by digest from its blob store, so a base stays usable -after a restart. Ambiguous errors never return success. - -The image is the base's layers plus one layer holding the builder's overlay, -auxiliary storage and platform. The overlay holds only the build's writes, so for a -base in the same registry only that layer is uploaded; the base's layers are reused -by digest. A base outside the registry has its layers uploaded too. There is no -garbage-collection rule yet; deleting a base that a derivative references breaks a -later re-pull of that derivative. - -## Status and gates - -Race tests cover admission, the shared queue, cancellation without publication, phase -failures and quarantine, storage validation, and authenticated publication with -manifest-last ordering and base-layer reuse. Test disks are synthetic and prove nothing about bootability. -A development base has been run end to end through the HTTP API on Apple silicon, -including a cold boot of the result; this trimmed branch has not been re-run natively. - -Before production use: a reviewed fresh base with approved artifact pins (base -publication tooling is a separate PR), cross-host publish and pull with sequential -cold boot, disk-space monitoring (this change has no disk admission, so run machine -builds serially on a dedicated host), SDK and CLI release, and Apple licence review. -Keep `macos_builds` off until then. - -**Rollback:** stop new submissions, let or cancel running jobs through the normal -APIs, repoint workloads to the prior validated digest, then set `macos_builds: false`. -Disabling does not delete persisted jobs or images. diff --git a/docs/macos-support.md b/docs/macos-support.md index 13afe299e..8646a1146 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -14,7 +14,7 @@ owns the behavior. | Stats | Partial | CPU and memory from the VM process. Network counters are 0 under NAT. | PR 1 | | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 | -| Builds | Partial | Opt-in `macos_builds`. Runs the source's build.sh as root in the guest. Linux Dockerfile builds unsupported. | PR 6 | +| Builds | Unsupported | Linux Dockerfile builds unsupported. | PR 6 | | Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | | Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | | Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go deleted file mode 100644 index e1786f412..000000000 --- a/lib/builds/macos_build.go +++ /dev/null @@ -1,273 +0,0 @@ -package builds - -import ( - "bytes" - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/instances" -) - -// MachineBuildRequest is the internal backend contract for one machine build. -// The normal build manager owns ID, source staging and publish authorization. A concrete driver must require a provisioned system agent. -type MachineBuildRequest struct { - ID string - BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. - SourceHash string // Optional expected lowercase SHA256 of the source archive. - Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. - onStarted func(string) error // Normal manager records the owned builder before provisioning. - onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. - registryBase *MachineBase // Set when the base is in the publisher's registry; its layers are reused. -} - -type MachineStopReceipt = instances.MacOSBuildExportReceipt - -// MachineStorage is a stopped guest's closed ASIF disk chain, auxiliary storage -// and platform. Publication streams from these files. -type MachineStorage = instances.MacOSStorage -type MachinePublication struct{ Reference, Digest string } -type MachineBuildResult struct { - Publication MachinePublication - Provenance BuildProvenance -} - -type MachineBuildDriver interface { - // Start owns an isolated instance and identity lease, named name. A partially - // created session must be returned even with an error so cleanup can stop it. - Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) -} - -type MachineBuildSession interface { - ID() string - Provision(context.Context, string) error // Build-owned staged source, never a client host path. - Sanitize(context.Context) error // Remove build credentials, source and secret injection artifacts. - // Stop must await actual VMM exit. Only a strict graceful stop may produce a - // receipt that permits publication. With force, a failed strict stop falls back - // to a forced stop, which is failure-only cleanup: nothing is published after it. - Stop(ctx context.Context, force bool) (MachineStopReceipt, error) - // Storage returns the stopped guest's closed storage. Only a strict stop permits it. - Storage(context.Context) (MachineStorage, error) - // Destroy removes instance storage only after proven VMM exit. - Destroy(context.Context) error -} - -type MachineBuildPublisher interface { - // Publisher owns registry address, scoped token and builds/{id} destination. - // It must stream blobs and commit/verify the manifest last. An ambiguous - // network error is not a verified successful publication. A non-nil base is - // the registry image the storage's layers came from. - Publish(ctx context.Context, id string, storage MachineStorage, base *MachineBase) (MachinePublication, error) -} - -type MachineBuildBackend struct { - Driver MachineBuildDriver - Publisher MachineBuildPublisher -} - -// machinePhaseError keeps potentially secret-bearing guest/command errors out of -// ordinary log/error text, while retaining errors.Is/As for internal handling. -type machinePhaseError struct { - phase string - cause error -} - -func (e *machinePhaseError) Error() string { - if exit := (*machineExitError)(nil); errors.As(e.cause, &exit) { - return "machine build " + e.phase + " failed: " + exit.Error() - } - // Host admission errors, such as a template identity already in use, carry - // no guest output. - if errors.Is(e.cause, instances.ErrInsufficientResources) || errors.Is(e.cause, instances.ErrInvalidState) { - return "machine build " + e.phase + " failed: " + e.cause.Error() - } - return "machine build " + e.phase + " failed" -} -func (e *machinePhaseError) Unwrap() error { return e.cause } -func machineFailure(phase string, err error) error { return &machinePhaseError{phase, err} } - -// machineExitError is the entrypoint's nonzero exit code. It holds no guest -// output, so it is safe to show. -type machineExitError struct{ code int } - -func (e *machineExitError) Error() string { - return fmt.Sprintf("%s exited with code %d", machineEntrypoint, e.code) -} - -// normalizeMachinePolicy applies machine build policy defaults and rejects what -// machine builds do not support. Admission, the runner and the guest driver share it. -func normalizeMachinePolicy(policy *BuildPolicy) error { - if policy.TimeoutSeconds == 0 { - policy.TimeoutSeconds = 600 - } - if policy.TimeoutSeconds < 1 || policy.TimeoutSeconds > 86400 { - return fmt.Errorf("invalid machine build timeout") - } - if policy.NetworkMode == "" { - policy.NetworkMode = "isolated" - } - if len(policy.AllowedDomains) != 0 || (policy.NetworkMode != "isolated" && policy.NetworkMode != "egress") { - return fmt.Errorf("unsupported machine network policy") - } - return nil -} - -// runPrepared executes only machine-specific phases. The shared build manager -// owns resolution, source staging/hash verification, deadline, queue and status. -func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath string) (result *MachineBuildResult, err error) { - if _, bounded := ctx.Deadline(); !bounded { - return nil, fmt.Errorf("prepared machine job requires a deadline") - } - // Admission validated the request; the base may have changed since. - ref, e := images.ParseNormalizedRef(req.BaseImage) - if e != nil { - return nil, e - } - policy := req.Policy - if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || !base.MacOS.GuestAgent || base.Digest != ref.Digest() || base.MacOS.CPUs < 2 || base.MacOS.CPUs > MaxBuildCPUs || base.MacOS.Memory < 4<<30 || base.MacOS.Memory > uint64(MaxBuildMemoryMB)<<20 || base.MacOS.Memory%(1<<20) != 0 { - return nil, fmt.Errorf("base is not the pinned ready macOS machine") - } - if base.MacOS.Validate() != nil { - return nil, fmt.Errorf("base has invalid machine identity") - } - memory := int(base.MacOS.Memory >> 20) - cpus := int(base.MacOS.CPUs) - if (policy.MemoryMB != 0 && policy.MemoryMB != memory) || (policy.CPUs != 0 && policy.CPUs != cpus) { - return nil, fmt.Errorf("machine resources must match installed base") - } - policy.MemoryMB = memory - policy.CPUs = cpus - if e = policy.Validate(); e != nil { - return nil, e - } - if e = verifyBuildSource(ctx, sourcePath, req.SourceHash); e != nil { - return nil, machineFailure("verify_source", e) - } - if e = ctx.Err(); e != nil { - return nil, e - } - session, e := r.Driver.Start(ctx, machineBuilderName(req.ID), base, policy) - if session == nil { - if e == nil { - e = fmt.Errorf("driver returned no session") - } - return nil, machineFailure("start", e) - } - stopped, destroyed := false, false - defer func() { - cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), machineCleanupTimeout) - defer cancel() - if !stopped { - receipt, stopErr := session.Stop(cleanup, true) - stopped = receipt.VMMExited - if !stopped { - err = errors.Join(err, machineFailure("quarantine_unconfirmed_vmm_exit", stopErr)) - result = nil - return - } - } - if !destroyed { - if destroyErr := session.Destroy(cleanup); destroyErr != nil { - err = errors.Join(err, machineFailure("cleanup", destroyErr)) - result = nil - } - } - }() - instanceID := session.ID() - if req.onStarted != nil { - if recordErr := req.onStarted(instanceID); recordErr != nil { - return nil, machineFailure("record_builder", recordErr) - } - } - if e != nil { - return nil, machineFailure("start", e) - } - if e = session.Provision(ctx, sourcePath); e != nil { - return nil, machineFailure("provision", e) - } - if e = session.Sanitize(ctx); e != nil { - return nil, machineFailure("sanitize", e) - } - receipt, e := session.Stop(ctx, false) - stopped = receipt.VMMExited - if e != nil || !receipt.Graceful || !stopped { - return nil, machineFailure("graceful_stop", e) - } - storage, e := session.Storage(ctx) - if e != nil { - return nil, machineFailure("storage", e) - } - if e = validateMachineStorage(storage, base.MacOS); e != nil { - return nil, machineFailure("validate_storage", e) - } - if e = ctx.Err(); e != nil { - return nil, e - } - // The stopped builder's storage is the publication source; the builder is - // destroyed once the registry has verified the result. - publication, e := r.Publisher.Publish(ctx, req.ID, storage, req.registryBase) - if e != nil { - return nil, machineFailure("publish", e) - } - if e = session.Destroy(ctx); e != nil { - return nil, machineFailure("cleanup", e) - } - destroyed = true - if req.onPublished != nil { - if e = req.onPublished(ctx, publication); e != nil { - return nil, machineFailure("image_ready", e) - } - } - provenance := BuildProvenance{BaseImageDigest: base.Digest, SourceHash: req.SourceHash, Timestamp: time.Now().UTC()} - return &MachineBuildResult{Publication: publication, Provenance: provenance}, nil -} - -// machineCleanupTimeout covers a strict graceful stop attempt (30s power-off -// budget) plus forced fallback and storage deletion. Export is outside cleanup. -const machineCleanupTimeout = 2 * time.Minute - -// ValidMachineDigest reports whether s is a canonical lowercase sha256 digest. -// It is the single validation rule for machine base and output digests. -func ValidMachineDigest(s string) bool { - if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { - return false - } - return strings.Trim(s[7:], "0123456789abcdef") == "" -} - -// validateMachineStorage checks storage before publication: the platform is valid, -// the files are distinct regular files, and the overlay stacks on its layers as -// ASIF. Against base, the hardware model and resources must match while the -// identity differs. -func validateMachineStorage(storage MachineStorage, base *images.MacOSImage) error { - if err := storage.Platform.Validate(); err != nil { - return err - } - if err := storage.CheckFiles(); err != nil { - return err - } - disk, err := images.OpenMacOSDiskChain(append(append([]string(nil), storage.Layers...), storage.Overlay)) - if err != nil { - return fmt.Errorf("machine disk: %w", err) - } - disk.Close() - if base != nil { - platform := storage.Platform - if !bytes.Equal(platform.HardwareModel, base.HardwareModel) || platform.CPUs != base.CPUs || platform.Memory != base.Memory { - return fmt.Errorf("machine storage changed hardware model or resources") - } - if bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || strings.EqualFold(platform.MAC, base.MAC) { - return fmt.Errorf("machine storage kept the base identity") - } - } - return nil -} - -// machineBuilderName names a build's builder instance after the build, so a retry -// can find and reap a builder that an interrupted attempt created but never recorded. -func machineBuilderName(buildID string) string { - return "macos-build-" + strings.ToLower(buildID) -} diff --git a/lib/builds/macos_build_cancel.go b/lib/builds/macos_build_cancel.go deleted file mode 100644 index 09b984000..000000000 --- a/lib/builds/macos_build_cancel.go +++ /dev/null @@ -1,70 +0,0 @@ -package builds - -import ( - "context" - "fmt" - "sync" - "time" -) - -type machineBuildControl struct { - mu sync.Mutex - cancel context.CancelFunc -} - -func (m *manager) recordMachineBuilder(id, instanceID string) error { - if active, ok := m.machineBuilds.Load(id); ok { - control := active.(*machineBuildControl) - control.mu.Lock() - defer control.mu.Unlock() - } - meta, err := readMetadata(m.paths, id) - if err != nil { - return err - } - if meta.Status == StatusCancelled { - return context.Canceled - } - meta.BuilderInstance = &instanceID - return writeMetadata(m.paths, meta) -} - -func (m *manager) cancelQueuedMachineBuild(meta *buildMetadata) error { - meta.Status = StatusCancelled - now := time.Now() - meta.CompletedAt = &now - if err := writeMetadata(m.paths, meta); err != nil { - return err - } - m.removeMachineSource(meta.ID) - m.notifyStatusChange(meta.ID, StatusCancelled) - return nil -} - -func (m *manager) cancelMachineBuild(id string) error { - active, ok := m.machineBuilds.Load(id) - if !ok { - return ErrBuildInProgress - } - control := active.(*machineBuildControl) - control.mu.Lock() - defer control.mu.Unlock() - meta, err := readMetadata(m.paths, id) - if err != nil { - return err - } - if meta.Status != StatusBuilding && meta.Status != StatusPushing { - return fmt.Errorf("machine build no longer running") - } - // Persist cancellation before interrupting phases. The backend performs bounded - // graceful cleanup or quarantine; Cancel must not force-delete mutable storage. - meta.Status = StatusCancelled - now := time.Now() - meta.CompletedAt = &now - if err := writeMetadata(m.paths, meta); err != nil { - return err - } - control.cancel() - m.notifyStatusChange(id, StatusCancelled) - return nil -} diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go deleted file mode 100644 index 6581af11d..000000000 --- a/lib/builds/macos_build_failure_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package builds - -import ( - "context" - "errors" - "io" - "os" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func runMachineBuildFailure(t *testing.T, f *machineFixture, ctx context.Context, req MachineBuildRequest, source io.Reader) (*machineRunnerFixture, error) { - t.Helper() - runner := machineTestRunner(t, f) - result, err := runner.Run(ctx, req, source) - if err == nil || result != nil { - t.Fatal("failed build returned a result") - } - return runner, err -} - -type failingMachineSource struct{} - -func (failingMachineSource) Read(p []byte) (int, error) { - return copy(p, "partial"), errors.New("secret-source-error") -} - -type cancelMachineSource struct{ cancel context.CancelFunc } - -func (s cancelMachineSource) Read(p []byte) (int, error) { - s.cancel() - return copy(p, "source"), io.EOF -} - -func TestMachineBuildSourceFailureDoesNotStart(t *testing.T) { - for _, tc := range []struct { - name string - source func(context.CancelFunc) io.Reader - wantCancel bool - }{ - {name: "read failure", source: func(context.CancelFunc) io.Reader { return failingMachineSource{} }}, - {name: "canceled", source: func(cancel context.CancelFunc) io.Reader { return cancelMachineSource{cancel} }, wantCancel: true}, - } { - t.Run(tc.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - f := newMachineFixture() - r, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), tc.source(cancel)) - if tc.wantCancel { - require.ErrorIs(t, err, context.Canceled) - } else { - require.NotContains(t, err.Error(), "secret-source-error", "source failure was not masked") - } - require.Empty(t, f.calls, "source failure started guest") - entries, err := os.ReadDir(r.WorkDir) - require.NoError(t, err) - require.Empty(t, entries, "partial source retained") - }) - } -} - -// Cancellation and timeout both interrupt a running build. Cleanup must still -// run to completion on a live context, and nothing may publish. -func TestMachineBuildInterruptedCleanup(t *testing.T) { - assertCleanup := func(t *testing.T, f *machineFixture, err, want error) { - t.Helper() - require.ErrorIs(t, err, want) - require.False(t, f.publisherCalled, "interrupted build published") - require.True(t, f.cleanupContextLive, "cleanup ran on a cancelled context") - require.Equal(t, []string{"start", "abort", "destroy"}, f.calls) - } - t.Run("cancellation", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - f := newMachineFixture() - f.cancel = cancel - _, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), strings.NewReader("source")) - assertCleanup(t, f, err, context.Canceled) - }) - t.Run("timeout", func(t *testing.T) { - f := newMachineFixture() - f.waitForCancel = true - req := machineTestRequest() - req.Policy.TimeoutSeconds = 1 - parent, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - _, err := runMachineBuildFailure(t, f, parent, req, strings.NewReader("source")) - assertCleanup(t, f, err, context.DeadlineExceeded) - }) -} - -func TestMachineBuildAmbiguousPublishIsNotRetried(t *testing.T) { - f := newMachineFixture() - f.fail = "publish" - _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) - require.NotContains(t, err.Error(), "sensitive-command-argument", "publisher error leaked") - count := 0 - for _, phase := range f.calls { - if phase == "publish" { - count++ - } - } - require.Equal(t, 1, count, "ambiguous publication was retried") -} - -func TestMachineBuildMalformedStorage(t *testing.T) { - for _, kind := range []string{"missing overlay", "empty aux", "symlink overlay", "aliased aux", "unrelated overlay", "raw overlay", "resource change", "bad MAC", "identity"} { - t.Run(kind, func(t *testing.T) { - f := newMachineFixture() - f.mutateStorage = func(s *MachineStorage) error { - switch kind { - case "missing overlay": - return os.Remove(s.Overlay) - case "empty aux": - return os.Truncate(s.Aux, 0) - case "symlink overlay": - if err := os.Rename(s.Overlay, s.Overlay+".real"); err != nil { - return err - } - return os.Symlink(s.Overlay+".real", s.Overlay) - case "aliased aux": - if err := os.Remove(s.Aux); err != nil { - return err - } - return os.Link(s.Overlay, s.Aux) - case "unrelated overlay": - other, err := machineStorageFixture(t.TempDir()) - if err != nil { - return err - } - s.Overlay = other.Overlay - case "raw overlay": - return os.WriteFile(s.Overlay, []byte("not an asif image"), 0600) - case "resource change": - s.Platform.CPUs = 2 - case "bad MAC": - s.Platform.MAC = "02:00:00:00:00:02:03:04" - case "identity": - s.Platform.MachineIdentifier = f.image.MacOS.MachineIdentifier // kept the base identity - } - return nil - } - _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) - require.False(t, f.publisherCalled, "malformed storage published") - require.Contains(t, err.Error(), "validate_storage") - }) - } -} diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go deleted file mode 100644 index 01680e859..000000000 --- a/lib/builds/macos_build_guest.go +++ /dev/null @@ -1,186 +0,0 @@ -package builds - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - "time" - - "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/instances" -) - -type machineExportManager interface { - StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) - MacOSBuildStorage(context.Context, string) (instances.MacOSStorage, error) -} - -// GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system -// GuestService. It never installs an agent or silently upgrades an unmanaged base. -// The backend validates the base and policy before Start. The builder gets its own -// machine identifier and MAC, so it and its output can run beside the base. -type GuestMachineBuildDriver struct { - Instances instances.Manager - WorkDir string -} - -func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) { - exporter, ok := d.Instances.(machineExportManager) - if !ok || !filepath.IsAbs(d.WorkDir) { - return nil, fmt.Errorf("strict machine export and private workspace required") - } - if _, ok = ctx.Deadline(); !ok { - return nil, fmt.Errorf("machine start requires a deadline") - } - ref, err := images.ParseNormalizedRef(base.Name) - if err != nil { - return nil, err - } - // Workspaces are named per builder, so one left by an interrupted attempt - // of this job is removed here. - prefix := "guest-machine-" + name + "-" - stale, _ := filepath.Glob(filepath.Join(d.WorkDir, prefix+"*")) - for _, dir := range stale { - _ = os.RemoveAll(dir) - } - dir, err := os.MkdirTemp(d.WorkDir, prefix) - if err != nil { - return nil, err - } - inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, FreshMacOSIdentity: true, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) - if inst == nil { - _ = os.RemoveAll(dir) - if err == nil { - err = fmt.Errorf("instance creation returned no instance") - } - return nil, err - } - session := &guestMachineSession{manager: d.Instances, exporter: exporter, instance: inst, workspace: dir, guestDir: "/var/root/hypeman-build-" + inst.Id} - return session, err -} - -type guestMachineSession struct { - manager instances.Manager - exporter machineExportManager - instance *instances.Instance - workspace, guestDir string - receipt MachineStopReceipt - sanitized bool -} - -func (s *guestMachineSession) ID() string { return s.instance.Id } -func (s *guestMachineSession) dialer() (hypervisor.VsockDialer, error) { - return hypervisor.NewVsockDialer(s.instance.HypervisorType, s.instance.VsockSocket, s.instance.VsockCID) -} -func (s *guestMachineSession) exec(ctx context.Context, command []string, cwd string, env map[string]string, wait time.Duration) error { - dialer, err := s.dialer() - if err != nil { - return err - } - // Drain output without retaining or publishing it. Redaction cannot reliably - // make arbitrary guest output safe: even successful commands may print secrets. - exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{Command: command, Cwd: cwd, Env: env, WaitForAgent: wait, Stdout: io.Discard, Stderr: io.Discard}) - if err != nil { - return err - } - if exit == nil { - return fmt.Errorf("machine guest command did not succeed") - } - if exit.Code != 0 { - return &machineExitError{code: exit.Code} - } - return nil -} - -func (s *guestMachineSession) Provision(ctx context.Context, source string) error { - root := filepath.Join(s.workspace, "source") - if err := extractMachineSource(ctx, source, root); err != nil { - return err - } - err := s.exec(ctx, []string{"/bin/sh", "-c", `test "$(/usr/bin/id -u)" -eq 0`}, "", nil, 30*time.Second) - if err != nil { - return err - } - if err = s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 30*time.Second); err != nil { - return err - } - if err = s.exec(ctx, []string{"/bin/mkdir", "-m", "700", s.guestDir}, "", nil, 0); err != nil { - return err - } - dialer, err := s.dialer() - if err != nil { - return err - } - if err = guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{SrcPath: root, DstPath: s.guestDir}); err != nil { - return err - } - return s.exec(ctx, []string{"/bin/sh", machineEntrypoint}, s.guestDir, nil, 0) -} - -func (s *guestMachineSession) Sanitize(ctx context.Context) error { - if err := s.exec(ctx, []string{"/bin/rm", "-rf", "--", s.guestDir}, "", nil, 0); err != nil { - return err - } - if err := s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 0); err != nil { - return err - } - if err := os.RemoveAll(filepath.Join(s.workspace, "source")); err != nil { - return err - } - s.sanitized = true - return nil -} - -// Stop is the publication path: only an acknowledged graceful power-off with -// confirmed VMM exit may produce a receipt that permits export. With force, a -// failed strict stop falls back to the ordinary forced hypervisor stop. Nothing -// from a failed build is exported, and Destroy still requires confirmed VMM exit, -// so a non-exited instance remains quarantined. -func (s *guestMachineSession) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { - if s.receipt.VMMExited { - return s.receipt, nil - } - receipt, err := s.exporter.StopMacOSBuildInstance(ctx, s.ID()) - if !force { - s.receipt = receipt - return s.receipt, err - } - if err == nil && receipt.VMMExited { - s.receipt = receipt - return s.receipt, nil - } - if inst, err := s.manager.GetInstance(ctx, s.ID()); err != nil || inst.State != instances.StateStopped { - inst, err = s.manager.StopInstance(ctx, s.ID()) - if err != nil { - return s.receipt, err - } - if inst == nil || inst.State != instances.StateStopped { - return s.receipt, fmt.Errorf("forced stop did not confirm VMM exit") - } - } - s.receipt = MachineStopReceipt{VMMExited: true} - return s.receipt, nil -} - -// Storage returns the closed storage of the Stopped instance, checked under the -// instance lock, after a strict graceful receipt and sanitation. -func (s *guestMachineSession) Storage(ctx context.Context) (MachineStorage, error) { - if !s.receipt.Graceful || !s.receipt.VMMExited || !s.sanitized { - return MachineStorage{}, fmt.Errorf("sanitized strict stop receipt required") - } - return s.exporter.MacOSBuildStorage(ctx, s.ID()) -} - -func (s *guestMachineSession) Destroy(ctx context.Context) error { - if !s.receipt.VMMExited { - return fmt.Errorf("unconfirmed VMM exit: instance quarantined") - } - if err := s.manager.DeleteInstance(ctx, s.ID()); err != nil { - return err - } - return os.RemoveAll(s.workspace) -} diff --git a/lib/builds/macos_build_job_test.go b/lib/builds/macos_build_job_test.go deleted file mode 100644 index b1bcc616b..000000000 --- a/lib/builds/macos_build_job_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package builds - -import ( - "context" - "io" - "os" - "path/filepath" - "time" -) - -type machineRunnerFixture struct { - MachineBuildBackend - WorkDir string -} - -func (r *machineRunnerFixture) Run(ctx context.Context, req MachineBuildRequest, source io.Reader) (*MachineBuildResult, error) { - timeout := req.Policy.TimeoutSeconds - if timeout == 0 { - timeout = 600 - } - ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) - defer cancel() - job, err := os.MkdirTemp(r.WorkDir, "prepared-job-") - if err != nil { - return nil, err - } - defer os.RemoveAll(job) - sourcePath := filepath.Join(job, "source.tar.gz") - hash, err := stageBuildSource(ctx, source, sourcePath, maxBuildSourceBytes) - if err != nil { - return nil, machineFailure("stage_source", err) - } - if req.SourceHash == "" { - req.SourceHash = hash - } - return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath) -} diff --git a/lib/builds/macos_build_lifecycle_test.go b/lib/builds/macos_build_lifecycle_test.go deleted file mode 100644 index 44926de71..000000000 --- a/lib/builds/macos_build_lifecycle_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package builds - -import ( - "context" - "errors" - "log/slog" - "testing" - - "github.com/kernel/hypeman/lib/instances" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/require" -) - -type machineLifecycleInstances struct { - instances.Manager - lookedUp string - state, stoppedState instances.State - missing, stopped bool - deleted bool - stopErr, deleteErr error -} - -func (m *machineLifecycleInstances) GetInstance(_ context.Context, id string) (*instances.Instance, error) { - m.lookedUp = id - if m.missing { - return nil, instances.ErrNotFound - } - return &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "prior"}, State: m.state}, nil -} -func (m *machineLifecycleInstances) StopInstance(context.Context, string) (*instances.Instance, error) { - m.stopped = true - if m.stopErr != nil { - return nil, m.stopErr - } - if m.stoppedState != "" { - m.state = m.stoppedState - } - return &instances.Instance{State: m.state}, nil -} -func (m *machineLifecycleInstances) DeleteInstance(context.Context, string) error { - m.deleted = true - return m.deleteErr -} - -type abortExporter struct { - receipt instances.MacOSBuildExportReceipt - err error -} - -func (e abortExporter) StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) { - return e.receipt, e.err -} -func (abortExporter) MacOSBuildStorage(context.Context, string) (instances.MacOSStorage, error) { - return instances.MacOSStorage{}, errors.New("abort must not publish") -} - -func TestMachineForcedStopCleanup(t *testing.T) { - for _, tc := range []struct { - name string - state, stoppedState instances.State - graceful, forceError, wantExit bool - }{ - {"graceful exit", instances.StateRunning, "", true, false, true}, - {"already stopped", instances.StateStopped, "", false, false, true}, - {"force running VM", instances.StateRunning, instances.StateStopped, false, false, true}, - {"unconfirmed stop", instances.StateRunning, "", false, false, false}, - {"failed force stop", instances.StateRunning, "", false, true, false}, - } { - t.Run(tc.name, func(t *testing.T) { - m := &machineLifecycleInstances{state: tc.state, stoppedState: tc.stoppedState} - stop := MachineStopReceipt{Graceful: tc.graceful, VMMExited: tc.graceful} - var stopErr error - if !tc.graceful { - stopErr = errors.New("agent never became ready") - } - if tc.forceError { - m.stopErr = errors.New("hypervisor still alive") - } - s := &guestMachineSession{manager: m, exporter: abortExporter{receipt: stop, err: stopErr}, instance: &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "builder"}}} - receipt, err := s.Stop(context.Background(), true) - if tc.wantExit { - require.NoError(t, err) - require.Equal(t, MachineStopReceipt{Graceful: tc.graceful, VMMExited: true}, receipt) - } else { - wantErr := "forced stop did not confirm VMM exit" - if tc.forceError { - wantErr = "hypervisor still alive" - } - require.EqualError(t, err, wantErr) - require.False(t, receipt.VMMExited) - require.Error(t, s.Destroy(context.Background()), "unconfirmed exit must remain quarantined") - } - require.Equal(t, !tc.graceful && tc.state != instances.StateStopped, m.stopped) - }) - } -} - -func TestReapPriorMachineBuilder(t *testing.T) { - for _, tc := range []struct { - name string - state instances.State - missing bool - unrecorded bool - deleteErr error - wantStop bool - wantDelete bool - wantError bool - }{ - {"stop and delete", instances.StateRunning, false, false, nil, true, true, false}, - {"delete failure blocks retry", instances.StateStopped, false, false, errors.New("storage still open"), false, true, true}, - {"missing builder", "", true, false, nil, false, false, false}, - {"find unrecorded builder", instances.StateRunning, false, true, nil, true, true, false}, - } { - t.Run(tc.name, func(t *testing.T) { - p := paths.New(t.TempDir()) - meta := &buildMetadata{ID: "job", Status: StatusQueued} - if !tc.unrecorded { - prior := "prior" - meta.BuilderInstance = &prior - } - require.NoError(t, writeMetadata(p, meta)) - fake := &machineLifecycleInstances{state: tc.state, missing: tc.missing, deleteErr: tc.deleteErr} - m := &manager{paths: p, instanceManager: fake, logger: slog.Default()} - err := m.reapPriorMachineBuilder(context.Background(), "job") - if tc.wantError { - require.Error(t, err) - } else { - require.NoError(t, err) - } - require.Equal(t, tc.unrecorded, fake.lookedUp == machineBuilderName("job")) - require.Equal(t, tc.wantStop, fake.stopped) - require.Equal(t, tc.wantDelete, fake.deleted) - }) - } -} diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go deleted file mode 100644 index 4985387c9..000000000 --- a/lib/builds/macos_build_manager.go +++ /dev/null @@ -1,182 +0,0 @@ -package builds - -import ( - "bytes" - "context" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/instances" -) - -// machineBuild reports whether a request selects the native macOS build path. -func machineBuild(req *CreateBuildRequest) bool { - return req != nil && req.MachineBaseImage != "" -} - -func (m *manager) skipMachineBuildRecovery(meta *buildMetadata) bool { - if !m.config.MachineBuildOnly { - return false - } - if machineBuild(meta.Request) && m.config.MachineBuild != nil { - return false - } - m.logger.Warn("leaving unsupported build pending in machine-build-only mode", "id", meta.ID) - return true -} - -// executeMachineBuild is a backend of the same queued build job. runBuild owns -// deadline/status/log completion, and CreateBuild already staged verified source. -func (m *manager) executeMachineBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { - backend := m.config.MachineBuild - if backend == nil { - return nil, fmt.Errorf("machine build backend is not configured") - } - if err := m.reapPriorMachineBuilder(ctx, id); err != nil { - return nil, machineFailure("reap_prior_builder", err) - } - base, err := m.imageManager.GetImage(ctx, req.MachineBaseImage) - if err != nil { - return nil, machineFailure("resolve_base", err) - } - sourcePath, err := filepath.Abs(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")) - if err != nil { - return nil, err - } - job := MachineBuildRequest{ - ID: id, BaseImage: req.MachineBaseImage, SourceHash: req.SourceHash, Policy: *policy, - registryBase: machineRegistryBase(backend.Publisher, req.MachineBaseImage), - onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, - onPublished: func(ctx context.Context, publication MachinePublication) error { - if err := m.imageManager.WaitForReady(ctx, "builds/"+id+"@"+publication.Digest); err != nil { - return fmt.Errorf("machine image import: %w", err) - } - return nil - }, - } - result, err := backend.runPrepared(ctx, job, base, sourcePath) - if err != nil { - return nil, err - } - ref, err := images.ParseNormalizedRef(result.Publication.Reference) - if err != nil || ref.Repository() != strings.TrimSuffix(stripRegistryScheme(m.config.RegistryURL), "/")+"/builds/"+id { - return nil, fmt.Errorf("machine publication is outside the authorized job repository") - } - return &BuildResult{Success: true, ImageDigest: result.Publication.Digest, ImageRef: result.Publication.Reference, Provenance: result.Provenance}, nil -} - -// reapPriorMachineBuilder removes the builder recorded by an interrupted attempt of -// this same job before a re-enqueued attempt starts. Nothing from that builder is -// published, so a forced stop is acceptable; a failed delete fails this attempt -// rather than leaving two builders with storage in play. -func (m *manager) reapPriorMachineBuilder(ctx context.Context, id string) error { - meta, err := readMetadata(m.paths, id) - if err != nil { - return err - } - // An attempt interrupted before it recorded its builder still named it after the build. - prior := machineBuilderName(id) - if meta.BuilderInstance != nil && *meta.BuilderInstance != "" { - prior = *meta.BuilderInstance - } - inst, err := m.instanceManager.GetInstance(ctx, prior) - if errors.Is(err, instances.ErrNotFound) { - return nil - } - if err != nil { - return err - } - if inst.State != instances.StateStopped { - if _, err := m.instanceManager.StopInstance(ctx, inst.Id); err != nil { - return fmt.Errorf("stop prior builder: %w", err) - } - } - if err := m.instanceManager.DeleteInstance(ctx, inst.Id); err != nil && !errors.Is(err, instances.ErrNotFound) { - return fmt.Errorf("delete prior builder: %w", err) - } - m.logger.Info("reaped prior machine builder", "build_id", id, "instance_id", inst.Id) - return nil -} - -// storeMachineSource stages machine source privately and records its SHA256 -// provenance. A caller-supplied source hash must match. -func (m *manager) storeMachineSource(ctx context.Context, meta *buildMetadata, data []byte) error { - sourceDir := m.paths.BuildSourceDir(meta.ID) - if err := os.MkdirAll(sourceDir, 0700); err != nil { - return fmt.Errorf("store source: %w", err) - } - hash, err := stageBuildSource(ctx, bytes.NewReader(data), filepath.Join(sourceDir, "source.tar.gz"), int64(len(data))) - if err != nil { - return fmt.Errorf("store source: %w", err) - } - if meta.Request.SourceHash != "" && meta.Request.SourceHash != hash { - return ErrSourceHashMismatch - } - meta.Request.SourceHash = hash - if err := writeMetadata(m.paths, meta); err != nil { - return fmt.Errorf("write source provenance: %w", err) - } - return nil -} - -// admitMachineBuild validates a machine build request and returns its policy. -// Machine resources inherit the installed base, not Linux builder defaults. -func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) (*BuildPolicy, error) { - if m.config.MachineBuild == nil || m.config.MachineBuild.Driver == nil || m.config.MachineBuild.Publisher == nil { - return nil, fmt.Errorf("machine build backend is not configured") - } - if req.Dockerfile != "" || req.BuilderID != "" || len(req.BuildArgs) != 0 || len(req.Secrets) != 0 || req.CacheScope != "" || req.GlobalCacheKey != "" || req.IsAdminBuild { - return nil, fmt.Errorf("machine builds do not support Linux builder/cache/secret options") - } - if len(sourceData) > maxBuildSourceBytes { - return nil, fmt.Errorf("machine build source too large") - } - ref, err := images.ParseNormalizedRef(req.MachineBaseImage) - if err != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { - return nil, fmt.Errorf("machine base must be pinned by sha256 digest") - } - policy := &BuildPolicy{} - if req.BuildPolicy != nil { - requested := *req.BuildPolicy - policy = &requested - } - if err := normalizeMachinePolicy(policy); err != nil { - return nil, err - } - req.BuildPolicy = policy - return policy, nil -} - -// machineRegistryBase returns the base as a registry base when the publisher -// writes to the registry that stores it, so its layers are reused. Any other base -// publishes all of its layers. -func machineRegistryBase(publisher MachineBuildPublisher, baseImage string) *MachineBase { - streaming, ok := publisher.(*StreamingMachinePublisher) - if !ok { - return nil - } - registry, err := streaming.registry() - if err != nil { - return nil - } - ref, err := images.ParseNormalizedRef(baseImage) - if err != nil || !ref.IsDigest() { - return nil - } - repo, ok := strings.CutPrefix(ref.Repository(), registry.Host+"/") - if !ok { - return nil - } - return &MachineBase{Repo: repo, Digest: ref.Digest()} -} - -// removeMachineSource deletes a machine build's staged source after the job is terminal. -func (m *manager) removeMachineSource(id string) { - if err := os.Remove(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")); err != nil && !errors.Is(err, os.ErrNotExist) { - m.logger.Warn("failed to remove machine build source", "id", id, "error", err) - } -} diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go deleted file mode 100644 index 22fccd29b..000000000 --- a/lib/builds/macos_build_manager_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package builds - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/instances" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/require" -) - -type machineJobImages struct { - images.Manager - fixture *machineFixture - resolveErr error -} - -func (m machineJobImages) GetImage(ctx context.Context, ref string) (*images.Image, error) { - if m.resolveErr != nil { - return nil, m.resolveErr - } - if ref == "registry/base@"+machineTestDigest { - return m.fixture.image, nil - } - return &images.Image{Status: images.StatusReady, Digest: machineTestDigest}, nil -} - -func (m machineJobImages) WaitForReady(context.Context, string) error { return nil } - -func newMachineTestManager(t *testing.T, f *machineFixture, publisher MachineBuildPublisher, imageManager images.Manager, machineOnly bool) *manager { - t.Helper() - config := DefaultConfig() - config.RegistrySecret = "synthetic-test-secret" - config.MachineBuildOnly = machineOnly - if f != nil && publisher != nil { - config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: publisher} - } - if imageManager == nil && f != nil { - imageManager = machineJobImages{fixture: f} - } - api, err := NewManager(paths.New(t.TempDir()), config, noPriorBuilder{}, nil, nil, imageManager, nil, nil, nil) - require.NoError(t, err) - return api.(*manager) -} - -type machineJobPublisher struct { - fixture *machineFixture - publishedID string - // sourcePath, when set, must still exist at publish time. - sourcePath func(id string) string -} - -func (p *machineJobPublisher) Publish(ctx context.Context, id string, storage MachineStorage, base *MachineBase) (MachinePublication, error) { - p.publishedID = id - if p.sourcePath != nil { - if _, err := os.Stat(p.sourcePath(id)); err != nil { - return MachinePublication{}, errors.New("source removed before publish") - } - } - // Reuse the fixture's storage assertions with its fixed ID. - receipt, err := p.fixture.Publish(ctx, "build-test", storage, base) - if err == nil { - receipt.Reference = "localhost:4973/builds/" + id + "@" + receipt.Digest - } - return receipt, err -} -func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { - f := newMachineFixture() - publisher := &machineJobPublisher{fixture: f} - m := newMachineTestManager(t, f, publisher, nil, false) - publisher.sourcePath = func(id string) string { return filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz") } - source := []byte("synthetic compressed-source bytes") - sum := sha256.Sum256(source) - req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: hex.EncodeToString(sum[:])} - build, err := m.CreateBuild(context.Background(), req, source) - require.NoError(t, err) - require.Eventually(t, func() bool { - b, e := m.GetBuild(context.Background(), build.ID) - return e == nil && (b.Status == StatusReady || b.Status == StatusFailed) - }, 5*time.Second, 10*time.Millisecond) - done, err := m.GetBuild(context.Background(), build.ID) - require.NoError(t, err) - require.Equal(t, StatusReady, done.Status, "failure: %v", done.Error) - require.NotNil(t, done.ImageRef) - require.Equal(t, "localhost:4973/builds/"+build.ID+"@"+machineTestDigest, *done.ImageRef) - require.Equal(t, build.ID, publisher.publishedID) - require.Equal(t, req.SourceHash, done.Provenance.SourceHash) - require.Equal(t, machineTestDigest, done.Provenance.BaseImageDigest) - require.Equal(t, f.ID(), *done.BuilderInstanceID) - meta, err := readMetadata(m.paths, build.ID) - require.NoError(t, err) - require.Equal(t, req.MachineBaseImage, meta.Request.MachineBaseImage) - require.Equal(t, "isolated", meta.Request.BuildPolicy.NetworkMode) - require.Equal(t, 0, meta.Request.BuildPolicy.CPUs, "machine defaults must not use Linux CPU defaults") - _, err = readBuildConfig(m.paths, build.ID) - require.Error(t, err, "machine builds must not persist a builder config or registry token") - require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) -} -func TestMachineBuildsOnOneBaseRunConcurrently(t *testing.T) { - fixture := newMachineFixture() - fixture.waitForCancel = true - m := newMachineTestManager(t, fixture, fixture, nil, false) - req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} - var ids []string - for range 2 { - build, err := m.CreateBuild(context.Background(), req, []byte("source")) - require.NoError(t, err) - ids = append(ids, build.ID) - } - require.Eventually(t, func() bool { return m.queue.ActiveCount() == 2 }, 5*time.Second, 10*time.Millisecond, "builds on one base must not serialize") - for _, id := range ids { - // Cancel only once the attempt has recorded its builder, so it is Building. - require.Eventually(t, func() bool { - meta, err := readMetadata(m.paths, id) - return err == nil && meta.BuilderInstance != nil - }, 5*time.Second, 10*time.Millisecond) - require.NoError(t, m.CancelBuild(context.Background(), id)) - } - require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) -} - -func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { - m := newMachineTestManager(t, nil, nil, nil, false) - _, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) - require.ErrorContains(t, err, "not configured") - f := newMachineFixture() - m.config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: f} - _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base:mutable"}, []byte("source")) - require.ErrorContains(t, err, "pinned") - _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, Secrets: []SecretRef{{ID: "unsupported"}}}, []byte("source")) - require.ErrorContains(t, err, "secret options") - _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: "wrong"}, []byte("source")) - require.True(t, errors.Is(err, ErrSourceHashMismatch)) - _, err = m.admitMachineBuild(&CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, make([]byte, maxBuildSourceBytes+1)) - require.ErrorContains(t, err, "source too large") - list, err := m.ListBuilds(context.Background()) - require.NoError(t, err) - require.Empty(t, list) -} - -func TestQueuedMachineCancellationRemovesSource(t *testing.T) { - fixture := newMachineFixture() - fixture.waitForCancel = true - m := newMachineTestManager(t, fixture, fixture, nil, false) - req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} - // Fill both concurrency slots so the next build stays queued. - for range m.config.MaxConcurrentBuilds { - first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) - require.NoError(t, err) - t.Cleanup(func() { - _ = m.CancelBuild(context.Background(), first.ID) - require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) - }) - require.Eventually(t, func() bool { - meta, err := readMetadata(m.paths, first.ID) - return err == nil && meta.BuilderInstance != nil - }, 5*time.Second, 10*time.Millisecond) - } - queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) - require.NoError(t, err) - source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") - require.FileExists(t, source) - require.NoError(t, m.CancelBuild(context.Background(), queued.ID)) - done, err := m.GetBuild(context.Background(), queued.ID) - require.NoError(t, err) - require.Equal(t, StatusCancelled, done.Status) - require.NotNil(t, done.CompletedAt) - require.NoFileExists(t, source) -} - -func TestMachineCancellationInterruptsPhasesWithoutPublishing(t *testing.T) { - fixture := newMachineFixture() - fixture.waitForCancel = true - m := newMachineTestManager(t, fixture, fixture, nil, false) - build, e := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("synthetic source")) - require.NoError(t, e) - require.Eventually(t, func() bool { - meta, e := readMetadata(m.paths, build.ID) - return e == nil && meta.BuilderInstance != nil - }, 5*time.Second, 10*time.Millisecond) - require.NoError(t, m.CancelBuild(context.Background(), build.ID)) - require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(build.ID); return !active }, 5*time.Second, 10*time.Millisecond) - done, e := m.GetBuild(context.Background(), build.ID) - require.NoError(t, e) - require.Equal(t, StatusCancelled, done.Status) - require.False(t, fixture.publisherCalled) - require.Contains(t, fixture.calls, "abort") - require.Contains(t, fixture.calls, "destroy") -} - -func TestMachineFailureUsesSharedFailedStatusWithoutPublication(t *testing.T) { - for _, tc := range []struct { - name string - phase string - resolveErr error - wantError string - }{ - {name: "base resolution", resolveErr: errors.New("sensitive-resolution-error"), wantError: "machine build resolve_base failed"}, - {name: "provision", phase: "provision", wantError: "machine build provision failed"}, - } { - t.Run(tc.name, func(t *testing.T) { - f := newMachineFixture() - f.fail = tc.phase - m := newMachineTestManager(t, f, f, machineJobImages{fixture: f, resolveErr: tc.resolveErr}, false) - build, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) - require.NoError(t, err) - require.Eventually(t, func() bool { - b, e := m.GetBuild(context.Background(), build.ID) - return e == nil && b.Status == StatusFailed - }, 5*time.Second, 10*time.Millisecond) - done, err := m.GetBuild(context.Background(), build.ID) - require.NoError(t, err) - require.Equal(t, StatusFailed, done.Status) - require.Nil(t, done.ImageDigest) - require.NotContains(t, f.calls, "publish") - require.NotContains(t, *done.Error, "sensitive-resolution-error") - require.Equal(t, tc.wantError, *done.Error) - if tc.phase == "provision" { - require.Contains(t, f.calls, "destroy") - } else { - require.Empty(t, f.calls) - } - }) - } -} - -func TestMachineFailureShowsOnlyExitCode(t *testing.T) { - err := machineFailure("provision", fmt.Errorf("guest said secret-token: %w", &machineExitError{code: 7})) - require.Equal(t, "machine build provision failed: build.sh exited with code 7", err.Error()) - require.Equal(t, "machine build provision failed", machineFailure("provision", fmt.Errorf("secret-token")).Error()) - admission := fmt.Errorf("%w: macOS template identity is already in use", instances.ErrInsufficientResources) - require.Contains(t, machineFailure("start", admission).Error(), "machine build start failed: insufficient resources") -} - -// noPriorBuilder reports that no earlier attempt left a builder instance behind. -type noPriorBuilder struct{ instances.Manager } - -func (noPriorBuilder) GetInstance(context.Context, string) (*instances.Instance, error) { - return nil, instances.ErrNotFound -} diff --git a/lib/builds/macos_build_policy_test.go b/lib/builds/macos_build_policy_test.go deleted file mode 100644 index 32c1592f6..000000000 --- a/lib/builds/macos_build_policy_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package builds - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMachinePolicyNormalization(t *testing.T) { - for _, tc := range []struct { - name string - policy BuildPolicy - err string - }{ - {name: "defaults"}, - {name: "minimum timeout", policy: BuildPolicy{TimeoutSeconds: 1}}, - {name: "maximum timeout", policy: BuildPolicy{TimeoutSeconds: 86400}}, - {name: "egress", policy: BuildPolicy{NetworkMode: "egress"}}, - {name: "negative timeout", policy: BuildPolicy{TimeoutSeconds: -1}, err: "invalid machine build timeout"}, - {name: "excessive timeout", policy: BuildPolicy{TimeoutSeconds: 86401}, err: "invalid machine build timeout"}, - {name: "unsupported mode", policy: BuildPolicy{NetworkMode: "host"}, err: "unsupported machine network policy"}, - {name: "domains", policy: BuildPolicy{NetworkMode: "egress", AllowedDomains: []string{"example.test"}}, err: "unsupported machine network policy"}, - } { - t.Run(tc.name, func(t *testing.T) { - policy := tc.policy - err := normalizeMachinePolicy(&policy) - if tc.err != "" { - require.EqualError(t, err, tc.err) - return - } - require.NoError(t, err) - want := tc.policy - if want.TimeoutSeconds == 0 { - want.TimeoutSeconds = 600 - } - if want.NetworkMode == "" { - want.NetworkMode = "isolated" - } - require.Equal(t, want, policy) - require.NoError(t, normalizeMachinePolicy(&policy)) - require.Equal(t, want, policy) - }) - } -} diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 5df457b60..5f09f21fc 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -25,6 +25,57 @@ import ( "github.com/kernel/hypeman/lib/images" ) +// MachineStorage is a closed macOS boot disk and platform to publish: the read-only +// ASIF layers below its overlay, base first, the overlay, and its auxiliary storage. +// A standalone base disk is an Overlay with no Layers. +type MachineStorage struct { + Layers []string + Overlay string + Aux string + Platform images.MacOSImage +} + +// MachinePublication is a verified publication: a digest-pinned reference. +type MachinePublication struct{ Reference, Digest string } + +// ValidMachineDigest reports whether s is a canonical sha256 digest. +func ValidMachineDigest(s string) bool { + if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { + return false + } + return strings.Trim(s[7:], "0123456789abcdef") == "" +} + +// validateMachineStorage checks storage before publication: the platform is valid, +// the files are distinct nonempty regular files, and the overlay stacks on its +// layers as ASIF. +func validateMachineStorage(storage MachineStorage) error { + if err := storage.Platform.Validate(); err != nil { + return err + } + var seen []os.FileInfo + for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + for _, other := range seen { + if os.SameFile(other, info) { + return fmt.Errorf("machine storage files must be distinct") + } + } + seen = append(seen, info) + } + disk, err := images.OpenMacOSDiskChain(append(append([]string(nil), storage.Layers...), storage.Overlay)) + if err != nil { + return fmt.Errorf("machine disk: %w", err) + } + return disk.Close() +} + // StreamingMachinePublisher targets only the configured internal registry and a // build-scoped repository. go-containerregistry hashes and compresses each layer // from a reopenable tar stream, so nothing is written to disk, skips blobs the @@ -80,7 +131,7 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor if err != nil { return MachinePublication{}, err } - if err := validateMachineStorage(storage, nil); err != nil { + if err := validateMachineStorage(storage); err != nil { return MachinePublication{}, err } repo := "builds/" + id @@ -249,7 +300,7 @@ func writeMachineTar(ctx context.Context, entries []machineTarEntry, w io.Writer err = archive.WriteHeader(&tar.Header{Name: entry.name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) } if err == nil { - _, err = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) + _, err = io.CopyN(archive, contextReader{ctx, source}, info.Size()) } closeErr := source.Close() if err != nil { @@ -261,3 +312,16 @@ func writeMachineTar(ctx context.Context, entries []machineTarEntry, w io.Writer } return archive.Close() } + +// contextReader stops a long read when ctx is done. +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} diff --git a/lib/builds/macos_build_recovery_test.go b/lib/builds/macos_build_recovery_test.go deleted file mode 100644 index fa8096645..000000000 --- a/lib/builds/macos_build_recovery_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package builds - -import ( - "context" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { - for _, status := range []string{StatusQueued, StatusBuilding, StatusPushing} { - t.Run(status, func(t *testing.T) { - f := newMachineFixture() - m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, true) - m.config.BuilderImage = "linux-builder-must-not-start" - policy := &BuildPolicy{} - require.NoError(t, normalizeMachinePolicy(policy)) - meta := &buildMetadata{ID: "build-test", Status: status, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: policy}} - require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("synthetic compressed-source bytes"))) - require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "linux-job", Status: StatusQueued, Request: &CreateBuildRequest{BuilderID: "linux-builder"}})) - t.Cleanup(func() { - require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) - }) - - // Hold the recovered build active so the second recovery sees it running. - f.startGate = make(chan struct{}) - require.NoError(t, m.Start(context.Background())) - require.Eventually(t, func() bool { return m.queue.ActiveCount() == 1 }, 5*time.Second, 10*time.Millisecond) - m.RecoverPendingBuilds() - close(f.startGate) - require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 && m.queue.PendingCount() == 0 }, 5*time.Second, 10*time.Millisecond) - build, err := m.GetBuild(context.Background(), meta.ID) - require.NoError(t, err) - require.Equal(t, StatusReady, build.Status, "failure: %v", build.Error) - require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz"), "source is removed once the image is ready") - require.False(t, m.ReadyForBuilds(), "Linux builder image must not be prepared") - var starts int - for _, call := range f.calls { - if call == "start" { - starts++ - } - } - require.Equal(t, 1, starts, "recovery must not duplicate an active build") - linux, err := readMetadata(m.paths, "linux-job") - require.NoError(t, err) - require.Equal(t, StatusQueued, linux.Status, "Linux jobs are preserved for Linux-mode recovery") - require.True(t, m.BuilderHasBuilds("linux-builder"), "unrecovered disk jobs still protect their builders") - }) - } -} - -func TestRecoveryCleansCancelledMachineBuildSource(t *testing.T) { - f := newMachineFixture() - m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, false) - meta := &buildMetadata{ - ID: "cancelled-build", - Status: StatusCancelled, - Request: &CreateBuildRequest{ - MachineBaseImage: "registry/base@" + machineTestDigest, - }, - } - require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("source"))) - meta.Status = StatusCancelled - require.NoError(t, writeMetadata(m.paths, meta)) - - m.RecoverPendingBuilds() - - require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz")) -} - -func TestMachineOnlyStartPreservesBuildsWhenBackendDisabled(t *testing.T) { - m := newMachineTestManager(t, nil, nil, nil, true) - require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "disabled", Status: StatusBuilding, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}})) - require.NoError(t, m.Start(context.Background())) - require.Equal(t, 0, m.queue.ActiveCount()) - meta, err := readMetadata(m.paths, "disabled") - require.NoError(t, err) - require.Equal(t, StatusBuilding, meta.Status) - require.False(t, m.ReadyForBuilds()) -} diff --git a/lib/builds/macos_build_source.go b/lib/builds/macos_build_source.go deleted file mode 100644 index 37b7ee2f5..000000000 --- a/lib/builds/macos_build_source.go +++ /dev/null @@ -1,119 +0,0 @@ -package builds - -import ( - "archive/tar" - "compress/gzip" - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strings" - "unicode/utf8" -) - -// machineEntrypoint is the script a machine build source must contain at its -// root. It runs as root with /bin/sh in the guest's private build directory. -const machineEntrypoint = "build.sh" -const maxMachineExtractBytes = 128 << 20 - -// extractMachineSource unpacks a build-owned tar.gz on the host into an exclusive -// directory, never with guest tar or a caller-supplied destination. Links, -// devices, duplicates and decompression bombs fail before anything is copied to -// the VM. The archive must contain the entrypoint as a regular file. -func extractMachineSource(ctx context.Context, source, destination string) error { - file, err := os.Open(source) - if err != nil { - return err - } - defer file.Close() - compressed, err := gzip.NewReader(file) - if err != nil { - return err - } - defer compressed.Close() - compressed.Multistream(false) - if err = os.Mkdir(destination, 0700); err != nil { - return err - } - complete := false - defer func() { - if !complete { - _ = os.RemoveAll(destination) - } - }() - limited := &io.LimitedReader{R: compressed, N: maxMachineExtractBytes + 1} - reader := tar.NewReader(limited) - seen := map[string]bool{} - var total int64 - for count := 0; ; count++ { - if err = ctx.Err(); err != nil { - return err - } - header, e := reader.Next() - if e == io.EOF { - break - } - if e != nil { - return e - } - if count >= 1024 { - return fmt.Errorf("too many machine source entries") - } - name := path.Clean(header.Name) - if name == "." && header.Typeflag == tar.TypeDir { - continue - } - if !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || !filepath.IsLocal(name) || seen[name] { - return fmt.Errorf("invalid or duplicate machine source path") - } - seen[name] = true - if header.Size < 0 || header.Size > 64<<20 || total+header.Size > maxMachineExtractBytes { - return fmt.Errorf("machine source expands beyond limit") - } - total += header.Size - target := filepath.Join(destination, name) - switch header.Typeflag { - case tar.TypeDir: - if header.Size != 0 { - return fmt.Errorf("directory has data") - } - if err = os.MkdirAll(target, 0700); err != nil { - return err - } - case tar.TypeReg, tar.TypeRegA: - if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { - return err - } - output, e := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600|os.FileMode(header.Mode)&0100) - if e != nil { - return e - } - _, copyErr := io.Copy(output, reader) - closeErr := output.Close() - if copyErr != nil { - return copyErr - } - if closeErr != nil { - return closeErr - } - default: - return fmt.Errorf("machine source must contain only regular files and directories") - } - } - // Consume bounded trailing tar padding to verify gzip integrity without allowing - // an unbounded post-tar decompression stream. - if _, err = io.Copy(io.Discard, limited); err != nil { - return err - } - if limited.N <= 0 { - return fmt.Errorf("machine source expands beyond limit") - } - info, err := os.Lstat(filepath.Join(destination, machineEntrypoint)) - if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { - return fmt.Errorf("machine source must contain %s", machineEntrypoint) - } - complete = true - return nil -} diff --git a/lib/builds/macos_build_source_test.go b/lib/builds/macos_build_source_test.go deleted file mode 100644 index 00f0d5f4d..000000000 --- a/lib/builds/macos_build_source_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package builds - -import ( - "archive/tar" - "compress/gzip" - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func writeMachineSource(t *testing.T, headers []*tar.Header, bodies []string) string { - t.Helper() - name := filepath.Join(t.TempDir(), "source.tar.gz") - f, e := os.Create(name) - require.NoError(t, e) - gz := gzip.NewWriter(f) - writer := tar.NewWriter(gz) - for i, h := range headers { - require.NoError(t, writer.WriteHeader(h)) - if h.Typeflag == tar.TypeReg { - _, e = writer.Write([]byte(bodies[i])) - require.NoError(t, e) - } - } - require.NoError(t, writer.Close()) - require.NoError(t, gz.Close()) - require.NoError(t, f.Close()) - return name -} - -func TestMachineSourceExtraction(t *testing.T) { - source := writeMachineSource(t, []*tar.Header{{Name: machineEntrypoint, Size: 4, Typeflag: tar.TypeReg, Mode: 0755}, {Name: "lib/data", Size: 1, Typeflag: tar.TypeReg}}, []string{"true", "x"}) - root := filepath.Join(t.TempDir(), "source") - require.NoError(t, extractMachineSource(context.Background(), source, root)) - info, e := os.Stat(filepath.Join(root, machineEntrypoint)) - require.NoError(t, e) - require.Equal(t, os.FileMode(0700), info.Mode().Perm()) - info, e = os.Stat(filepath.Join(root, "lib", "data")) - require.NoError(t, e) - require.Equal(t, os.FileMode(0600), info.Mode().Perm()) -} - -func TestMachineSourceRejectsUnsafeArchives(t *testing.T) { - entry := &tar.Header{Name: machineEntrypoint, Typeflag: tar.TypeReg, Size: 4} - for _, tc := range []struct { - name string - headers []*tar.Header - bodies []string - }{ - {"missing entrypoint", []*tar.Header{{Name: "other", Typeflag: tar.TypeReg, Size: 1}}, []string{"x"}}, - {"empty entrypoint", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeReg}}, []string{""}}, - {"entrypoint directory", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeDir}}, []string{""}}, - {"traversal", []*tar.Header{entry, {Name: "../outside", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x"}}, - {"symlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc"}}, []string{"true", ""}}, - {"hardlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeLink, Linkname: "other"}}, []string{"true", ""}}, - {"duplicate", []*tar.Header{entry, {Name: "a", Typeflag: tar.TypeReg, Size: 1}, {Name: "a", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x", "y"}}, - } { - t.Run(tc.name, func(t *testing.T) { - root := filepath.Join(t.TempDir(), "source") - require.Error(t, extractMachineSource(context.Background(), writeMachineSource(t, tc.headers, tc.bodies), root)) - require.NoDirExists(t, root) - }) - } -} diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go deleted file mode 100644 index b82b5c6bd..000000000 --- a/lib/builds/macos_build_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package builds - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "testing" - - "github.com/Code-Hex/vz/v4/diskimagekit" - "github.com/kernel/hypeman/lib/images" -) - -const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - -type machineFixture struct { - mu sync.Mutex - image *images.Image - calls []string - fail string - stop MachineStopReceipt - cancel context.CancelFunc - waitForCancel bool - mutateStorage func(*MachineStorage) error - storageDir string - publisherCalled bool - cleanupContextLive bool - startGate chan struct{} -} - -func newMachineFixture() *machineFixture { - return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} -} -func (f *machineFixture) record(phase string) error { - f.mu.Lock() - defer f.mu.Unlock() - f.calls = append(f.calls, phase) - if f.fail == phase { - return errors.New("sensitive-command-argument") - } - return nil -} -func (f *machineFixture) Start(_ context.Context, _ string, _ *images.Image, p BuildPolicy) (MachineBuildSession, error) { - if p.CPUs != 4 || p.MemoryMB != 8192 { - return nil, errors.New("wrong resource defaults") - } - if f.startGate != nil { - <-f.startGate - } - return f, f.record("start") -} -func (f *machineFixture) ID() string { return "isolated-test-instance" } -func (f *machineFixture) Provision(ctx context.Context, source string) error { - if _, err := os.ReadFile(source); err != nil { - return err - } - if f.cancel != nil { - f.cancel() - return ctx.Err() - } - if f.waitForCancel { - <-ctx.Done() - return ctx.Err() - } - return f.record("provision") -} -func (f *machineFixture) Sanitize(context.Context) error { return f.record("sanitize") } -func (f *machineFixture) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { - f.cleanupContextLive = ctx.Err() == nil - phase := "stop" - if force { - phase = "abort" - } - return f.stop, f.record(phase) -} -func (f *machineFixture) Storage(context.Context) (MachineStorage, error) { - if err := f.record("storage"); err != nil { - return MachineStorage{}, err - } - dir, err := os.MkdirTemp("", "machine-storage-") - if err != nil { - return MachineStorage{}, err - } - f.storageDir = dir - storage, err := machineStorageFixture(dir) - if err != nil { - return MachineStorage{}, err - } - storage.Platform = *f.image.MacOS - // The builder has its own identity; its storage carries it, not the base's. - storage.Platform.MachineIdentifier, storage.Platform.MAC = []byte{3}, "02:00:00:00:00:02" - if f.mutateStorage != nil { - if err := f.mutateStorage(&storage); err != nil { - return MachineStorage{}, err - } - } - return storage, nil -} - -// machineStorageFixture writes a one-layer ASIF chain, its writable overlay and an -// aux file to dir, as a stopped builder has. The platform is left to the caller. -func machineStorageFixture(dir string) (MachineStorage, error) { - storage := MachineStorage{Layers: []string{filepath.Join(dir, "disk.0.asif")}, Overlay: filepath.Join(dir, "boot.asif"), Aux: filepath.Join(dir, "mac-aux.img")} - img, err := diskimagekit.CreateASIF(storage.Layers[0], (4<<20)/512) - if err != nil { - return storage, err - } - defer img.Close() - if _, err := img.WriteAt([]byte("base"), 0); err != nil { - return storage, err - } - if err := img.CreateLayer(storage.Overlay, diskimagekit.Overlay); err != nil { - return storage, err - } - if _, err := img.WriteAt([]byte("build output"), 1<<20); err != nil { - return storage, err - } - return storage, os.WriteFile(storage.Aux, []byte("aux"), 0600) -} -func (f *machineFixture) Destroy(ctx context.Context) error { - f.cleanupContextLive = ctx.Err() == nil - if f.storageDir != "" { - os.RemoveAll(f.storageDir) - } - return f.record("destroy") -} -func (f *machineFixture) Publish(ctx context.Context, id string, storage MachineStorage, _ *MachineBase) (MachinePublication, error) { - f.publisherCalled = true - if err := ctx.Err(); err != nil { - return MachinePublication{}, err - } - if id != "build-test" { - return MachinePublication{}, errors.New("wrong job namespace") - } - for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { - if _, err := os.Stat(path); err != nil { - return MachinePublication{}, errors.New("storage missing at publish") - } - } - publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} - return publication, f.record("publish") -} -func machineTestRunner(t *testing.T, f *machineFixture) *machineRunnerFixture { - t.Helper() - return &machineRunnerFixture{MachineBuildBackend: MachineBuildBackend{Driver: f, Publisher: f}, WorkDir: t.TempDir()} -} -func machineTestRequest() MachineBuildRequest { - return MachineBuildRequest{ID: "build-test", BaseImage: "localhost/macos@" + machineTestDigest} -} - -func TestMachineBuildPublicationOrder(t *testing.T) { - f := newMachineFixture() - r := machineTestRunner(t, f) - result, err := r.Run(context.Background(), machineTestRequest(), strings.NewReader("source")) - if err != nil { - t.Fatal(err) - } - expected := []string{"start", "provision", "sanitize", "stop", "storage", "publish", "destroy"} - if !reflect.DeepEqual(f.calls, expected) { - t.Fatalf("wrong phase order: %v", f.calls) - } - hash := sha256.Sum256([]byte("source")) - if result.Provenance.SourceHash != hex.EncodeToString(hash[:]) || result.Provenance.BaseImageDigest != machineTestDigest { - t.Fatal("unverified provenance") - } - entries, err := os.ReadDir(r.WorkDir) - if err != nil || len(entries) != 0 { - t.Fatal("workspace not cleaned") - } -} - -func TestMachineBuildFailuresDoNotPublish(t *testing.T) { - for _, tc := range []struct { - name string - phase string - receipt *MachineStopReceipt - }{ - {name: "start", phase: "start"}, - {name: "provision", phase: "provision"}, - {name: "sanitize", phase: "sanitize"}, - {name: "stop", phase: "stop"}, - {name: "storage", phase: "storage"}, - {name: "unconfirmed stop", receipt: &MachineStopReceipt{}}, - {name: "forced stop", receipt: &MachineStopReceipt{VMMExited: true}}, - } { - t.Run(tc.name, func(t *testing.T) { - f := newMachineFixture() - f.fail = tc.phase - if tc.receipt != nil { - f.stop = *tc.receipt - } - _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) - if f.publisherCalled { - t.Fatal("failed build reached publication") - } - if tc.phase != "" && strings.Contains(err.Error(), "sensitive-command-argument") { - t.Fatal("raw guest error leaked") - } - for _, phase := range f.calls { - if phase == "storage" && tc.receipt != nil { - t.Fatal("unconfirmed stop reached storage") - } - if phase == "destroy" && tc.receipt != nil && !tc.receipt.VMMExited { - t.Fatal("destroyed unconfirmed live storage") - } - } - }) - } -} - -func TestMachineBuildInputAdmission(t *testing.T) { - for _, kind := range []string{"floating base", "wrong digest", "linux", "pending", "shaping", "source hash", "CPU overflow", "memory precision"} { - t.Run(kind, func(t *testing.T) { - f := newMachineFixture() - req := machineTestRequest() - switch kind { - case "floating base": - req.BaseImage = "localhost/macos:latest" - case "wrong digest": - f.image.Digest = "sha256:" + strings.Repeat("b", 64) - case "linux": - f.image.Platform = "linux/arm64" - case "pending": - f.image.Status = images.StatusPending - case "shaping": - req.Policy.MemoryMB = 4096 - case "source hash": - req.SourceHash = strings.Repeat("0", 64) - case "CPU overflow": - f.image.MacOS.CPUs = ^uint(0) - case "memory precision": - f.image.MacOS.Memory++ - } - runMachineBuildFailure(t, f, context.Background(), req, strings.NewReader("source")) - for _, phase := range f.calls { - if phase == "start" { - t.Fatal("bad input started a VM") - } - } - }) - } -} - -var _ MachineBuildDriver = (*machineFixture)(nil) -var _ MachineBuildSession = (*machineFixture)(nil) -var _ MachineBuildPublisher = (*machineFixture)(nil) diff --git a/lib/builds/manager.go b/lib/builds/manager.go index ac0e2ac3e..806ae5622 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -87,12 +87,6 @@ type Manager interface { // Config holds configuration for the build manager type Config struct { - // MachineBuild is the opt-in native macOS build backend. - MachineBuild *MachineBuildBackend - - // MachineBuildOnly disables Linux builder preparation and recovery. - MachineBuildOnly bool - // MaxConcurrentBuilds is the maximum number of concurrent builds MaxConcurrentBuilds int @@ -155,7 +149,6 @@ type manager struct { logger *slog.Logger metrics *Metrics createMu sync.Mutex - machineBuilds sync.Map // build ID -> *machineBuildControl builderReady atomic.Bool // pendingRecovered is set once RecoverPendingBuilds has re-enqueued // persisted pending builds; until then BuilderHasBuilds also scans @@ -211,10 +204,6 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { - if m.config.MachineBuildOnly { - m.RecoverPendingBuilds() - return nil - } go func() { for { m.ensureBuilderImage(ctx) @@ -455,12 +444,7 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc // Apply defaults to build policy policy := req.BuildPolicy - if machineBuild(&req) { - var err error - if policy, err = m.admitMachineBuild(&req, sourceData); err != nil { - return nil, err - } - } else if policy == nil { + if policy == nil { defaultPolicy := DefaultBuildPolicy() policy = &defaultPolicy } else { @@ -497,15 +481,6 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc CreatedAt: time.Now(), } - if machineBuild(&req) { - // Stage verified input before making the job visible to recovery. - if err := m.storeMachineSource(ctx, meta, sourceData); err != nil { - deleteBuild(m.paths, id) - return nil, err - } - return m.enqueueBuild(meta, req, policy), nil - } - // Write initial metadata if err := writeMetadata(m.paths, meta); err != nil { return nil, fmt.Errorf("write metadata: %w", err) @@ -625,23 +600,20 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc return nil, fmt.Errorf("write build config: %w", err) } - return m.enqueueBuild(meta, req, policy), nil -} - -// enqueueBuild queues a build whose inputs are stored. Builds sharing a builder -// are serialized by builder ID so a waiting same-builder build stays pending -// instead of holding a global concurrency slot while blocked on the builder's -// disk. Machine builds have no builder ID, so builds on one base run concurrently. -func (m *manager) enqueueBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { - queuePos := m.queue.EnqueueSerial(meta.ID, req, req.BuilderID, func() { - m.runBuild(context.Background(), meta.ID, req, policy) + // Enqueue the build. Builds sharing a builder are serialized by builder + // ID so a waiting same-builder build stays pending instead of holding a + // global concurrency slot while blocked on the builder's disk. + queuePos := m.queue.EnqueueSerial(id, req, req.BuilderID, func() { + m.runBuild(context.Background(), id, req, policy) }) + build := meta.toBuild() if queuePos > 0 { build.QueuePosition = &queuePos } - m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) - return build + + m.logger.Info("build created", "id", id, "queue_position", queuePos) + return build, nil } // gcBoundsForDisk computes the BuildKit GC bounds for a builder disk: @@ -706,29 +678,18 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques start := time.Now() m.logger.Info("starting build", "id", id) - // Register machine cancellation before exposing the running status. Ordinary - // Linux builder cancellation retains its existing instance-delete behavior. + // Update status to building + m.updateStatus(id, StatusBuilding, nil) + + // Create timeout context buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() - if machineBuild(&req) { - m.machineBuilds.Store(id, &machineBuildControl{cancel: cancel}) - defer m.machineBuilds.Delete(id) - defer func() { - meta, err := readMetadata(m.paths, id) - if err == nil && (meta.Status == StatusReady || meta.Status == StatusFailed || meta.Status == StatusCancelled) { - m.removeMachineSource(id) - } - }() - } - m.updateStatus(id, StatusBuilding, nil) // Mirror base images to the local registry before launching the VM. // BuildKit is configured with our registry as a mirror for docker.io, // so pre-cached images will be served locally without pulling from Docker Hub. - if !machineBuild(&req) { - if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { - m.logger.Warn("failed to mirror base images", "id", id, "error", err) - } + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) } // Run the build in a builder VM @@ -793,9 +754,6 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // If image_name is set, re-tag the image so it's accessible by that name. // The builder pushed to builds/{id} but the user wants it as image_name. imageRef := buildRepo - if machineBuild(&req) && result.ImageRef != "" { - imageRef = result.ImageRef - } if req.ImageName != "" { ref, err := images.ParseNormalizedRef(req.ImageName) if err != nil { @@ -840,9 +798,6 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // executeBuild runs the build in a builder VM func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { - if machineBuild(&req) { - return m.executeMachineBuild(ctx, id, req, policy) - } if !m.builderReady.Load() { return nil, fmt.Errorf("builder image is being prepared, please retry shortly") } @@ -1151,11 +1106,6 @@ func (m *manager) updateStatus(id string, status string, err error) { // updateBuildComplete updates the build with final results func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) { - if active, ok := m.machineBuilds.Load(id); ok { - control := active.(*machineBuildControl) - control.mu.Lock() - defer control.mu.Unlock() - } meta, readErr := readMetadata(m.paths, id) if readErr != nil { m.logger.Error("read metadata for completion", "id", id, "error", readErr) @@ -1296,18 +1246,12 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { case StatusQueued: // Remove from queue if m.queue.Cancel(id) { - if machineBuild(meta.Request) { - return m.cancelQueuedMachineBuild(meta) - } m.updateStatus(id, StatusCancelled, nil) return nil } return ErrBuildInProgress // Was already picked up case StatusBuilding, StatusPushing: - if machineBuild(meta.Request) { - return m.cancelMachineBuild(id) - } // Can't cancel a running build easily // Would need to terminate the builder instance if meta.BuilderInstance != nil { @@ -1505,36 +1449,15 @@ func (m *manager) StreamBuildEvents(ctx context.Context, id string, follow bool) // RecoverPendingBuilds recovers builds that were interrupted on restart func (m *manager) RecoverPendingBuilds() { - all, err := listAllBuilds(m.paths) + pending, err := listPendingBuilds(m.paths) if err != nil { - m.logger.Error("list builds for machine cleanup", "error", err) + m.logger.Error("list pending builds for recovery", "error", err) return } - for _, meta := range all { - if !machineBuild(meta.Request) { - continue - } - switch meta.Status { - case StatusReady, StatusFailed: - m.removeMachineSource(meta.ID) - case StatusCancelled: - if m.config.MachineBuild != nil { - if err := m.reapPriorMachineBuilder(context.Background(), meta.ID); err != nil { - m.logger.Error("failed to reap cancelled machine build", "id", meta.ID, "error", err) - continue - } - } - m.removeMachineSource(meta.ID) - } - } - pending := pendingBuilds(all) for _, meta := range pending { meta := meta // Shadow loop variable for closure capture m.logger.Info("recovering build", "id", meta.ID, "status", meta.Status) - if m.skipMachineBuildRecovery(meta) { - continue - } // Re-enqueue the build if meta.Request != nil { @@ -1562,16 +1485,11 @@ func (m *manager) RecoverPendingBuilds() { if len(pending) > 0 { m.logger.Info("recovered pending builds", "count", len(pending)) } - if !m.config.MachineBuildOnly { - m.pendingRecovered.Store(true) - } + m.pendingRecovered.Store(true) } // refreshBuildToken regenerates the registry token for a build and updates the config file func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) error { - if machineBuild(req) { - return nil // Machine builds have no builder config or registry token. - } // Read existing build config config, err := readBuildConfig(m.paths, buildID) if err != nil { diff --git a/lib/builds/source.go b/lib/builds/source.go deleted file mode 100644 index b2075607f..000000000 --- a/lib/builds/source.go +++ /dev/null @@ -1,75 +0,0 @@ -package builds - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "os" -) - -const maxBuildSourceBytes = 64 << 20 - -type buildContextReader struct { - ctx context.Context - r io.Reader -} - -func (r buildContextReader) Read(p []byte) (int, error) { - if err := r.ctx.Err(); err != nil { - return 0, err - } - return r.r.Read(p) -} - -// stageBuildSource streams compressed source once into a private, exclusive job -// file. Both backends consume this file and use its hash for input verification. -func stageBuildSource(ctx context.Context, source io.Reader, path string, limit int64) (hash string, err error) { - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - return "", err - } - defer func() { - if closeErr := file.Close(); err == nil { - err = closeErr - } - }() - h := sha256.New() - n, err := io.Copy(io.MultiWriter(file, h), io.LimitReader(buildContextReader{ctx, source}, limit+1)) - if err != nil { - return "", err - } - if n > limit { - return "", fmt.Errorf("build source too large") - } - if err = file.Sync(); err != nil { - return "", err - } - return hex.EncodeToString(h.Sum(nil)), nil -} - -// verifyBuildSource rejects missing/replaced source on recovery before a machine -// is launched. It never writes a second source copy. -func verifyBuildSource(ctx context.Context, path, expected string) error { - info, err := os.Lstat(path) - if err != nil { - return err - } - if !info.Mode().IsRegular() || info.Size() > maxBuildSourceBytes { - return fmt.Errorf("invalid staged source") - } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - h := sha256.New() - if _, err = io.Copy(h, io.LimitReader(buildContextReader{ctx, file}, maxBuildSourceBytes+1)); err != nil { - return err - } - if hex.EncodeToString(h.Sum(nil)) != expected { - return ErrSourceHashMismatch - } - return nil -} diff --git a/lib/builds/source_test.go b/lib/builds/source_test.go deleted file mode 100644 index 1bb7e8a84..000000000 --- a/lib/builds/source_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package builds - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSharedBuildSourceStagingAndRecoveryVerification(t *testing.T) { - ctx := context.Background() - path := filepath.Join(t.TempDir(), "source.tar.gz") - hash, err := stageBuildSource(ctx, strings.NewReader("source"), path, maxBuildSourceBytes) - require.NoError(t, err) - info, err := os.Stat(path) - require.NoError(t, err) - require.Equal(t, os.FileMode(0600), info.Mode().Perm()) - require.NoError(t, verifyBuildSource(ctx, path, hash)) - _, err = stageBuildSource(ctx, strings.NewReader("replacement"), path, maxBuildSourceBytes) - require.Error(t, err, "exclusive staging must preserve existing source") - require.NoError(t, verifyBuildSource(ctx, path, hash)) - canceled, cancel := context.WithCancel(ctx) - cancel() - require.ErrorIs(t, verifyBuildSource(canceled, path, hash), context.Canceled) - require.NoError(t, os.WriteFile(path, []byte("changed"), 0600)) - require.ErrorIs(t, verifyBuildSource(ctx, path, hash), ErrSourceHashMismatch) - require.NoError(t, os.Remove(path)) - require.Error(t, verifyBuildSource(ctx, path, hash)) - target := filepath.Join(filepath.Dir(path), "target") - require.NoError(t, os.WriteFile(target, []byte("source"), 0600)) - require.NoError(t, os.Symlink(target, path)) - require.ErrorContains(t, verifyBuildSource(ctx, path, hash), "invalid staged source") -} diff --git a/lib/builds/storage.go b/lib/builds/storage.go index 1add7a3c1..ce555aac9 100644 --- a/lib/builds/storage.go +++ b/lib/builds/storage.go @@ -144,10 +144,7 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { if err != nil { return nil, err } - return pendingBuilds(all), nil -} -func pendingBuilds(all []*buildMetadata) []*buildMetadata { var pending []*buildMetadata for _, meta := range all { switch meta.Status { @@ -155,10 +152,13 @@ func pendingBuilds(all []*buildMetadata) []*buildMetadata { pending = append(pending, meta) } } + + // Sort by created_at ascending (oldest first for FIFO recovery) sort.Slice(pending, func(i, j int) bool { return pending[i].CreatedAt.Before(pending[j].CreatedAt) }) - return pending + + return pending, nil } // deleteBuild removes a build's data from disk diff --git a/lib/builds/types.go b/lib/builds/types.go index 571259328..4ad6008e9 100644 --- a/lib/builds/types.go +++ b/lib/builds/types.go @@ -43,10 +43,6 @@ type CreateBuildRequest struct { // The Dockerfile specifies the runtime (e.g., FROM node:20-alpine). Dockerfile string `json:"dockerfile,omitempty"` - // MachineBaseImage selects the installed-machine backend. The HTTP multipart - // field machine_base_image maps here; it is enabled only with macos_builds. - MachineBaseImage string `json:"machine_base_image,omitempty"` - // BaseImageDigest optionally pins the base image by digest for reproducibility BaseImageDigest string `json:"base_image_digest,omitempty"` @@ -223,9 +219,6 @@ type BuildResult struct { // ImageDigest is the digest of the pushed image (only on success) ImageDigest string `json:"image_digest,omitempty"` - // ImageRef is the pullable output reference for machine builds. - ImageRef string `json:"image_ref,omitempty"` - // Error is the error message (only on failure) Error string `json:"error,omitempty"` diff --git a/lib/instances/create.go b/lib/instances/create.go index 8af4a9147..e7ecdde66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -385,11 +385,6 @@ func (m *manager) createInstance( HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } - if req.FreshMacOSIdentity { - if err := m.freshMacOSIdentity(stored); err != nil { - return nil, err - } - } // 12. Ensure directories log.DebugContext(ctx, "creating instance directories", "instance_id", id) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 5cc1ce936..d9ea69a40 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -114,7 +114,7 @@ func (m *manager) deleteInstanceWithOptions( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown = m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, false) + gracefulShutdown = m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) if gracefulShutdown { gracefulSpanEnd(nil) } else { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 6fec8d6b1..b7a10a72e 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -551,7 +551,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe inst, err := m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false) return returnWithReadiness(inst, err, false) case StateStopped: - inst, err := m.stopInstance(ctx, forkID, false) + inst, err := m.stopInstance(ctx, forkID) return returnWithReadiness(inst, err, false) } } diff --git a/lib/instances/macos_agent_startup.go b/lib/instances/macos_agent_startup.go index 74025e0cb..05451f57f 100644 --- a/lib/instances/macos_agent_startup.go +++ b/lib/instances/macos_agent_startup.go @@ -83,7 +83,7 @@ func (m *manager) stopOverdueMacOSAgentStartup(ctx context.Context, id string, l reason += ": " + last.(string) } log.WarnContext(ctx, "stopping macOS guest whose agent did not start", "instance_id", id, "reason", reason) - if _, err := m.stopInstance(ctx, id, false); err != nil { + if _, err := m.stopInstance(ctx, id); err != nil { return err } meta, err = m.loadMetadata(id) diff --git a/lib/instances/macos_build_export.go b/lib/instances/macos_build_export.go deleted file mode 100644 index af30080f1..000000000 --- a/lib/instances/macos_build_export.go +++ /dev/null @@ -1,47 +0,0 @@ -package instances - -import ( - "fmt" - "os" - - "github.com/kernel/hypeman/lib/images" -) - -// MacOSBuildExportReceipt describes one strict stop, not persisted state. -type MacOSBuildExportReceipt struct{ Graceful, VMMExited bool } - -// MacOSStorage is a stopped macOS guest's closed boot disk and platform: the -// read-only ASIF layers below its writable overlay, base first, the overlay, and its -// auxiliary storage. A standalone base disk is an Overlay with no Layers. -type MacOSStorage struct { - Layers []string - Overlay string - Aux string - Platform images.MacOSImage -} - -// Files returns the storage's paths: the overlay, the auxiliary storage, then the layers. -func (s MacOSStorage) Files() []string { - return append([]string{s.Overlay, s.Aux}, s.Layers...) -} - -// CheckFiles requires every storage file to be a distinct nonempty regular file. -func (s MacOSStorage) CheckFiles() error { - var seen []os.FileInfo - for _, path := range s.Files() { - info, err := os.Lstat(path) - if err != nil { - return err - } - if !info.Mode().IsRegular() || info.Size() == 0 { - return fmt.Errorf("machine storage must be nonempty regular files") - } - for _, other := range seen { - if os.SameFile(other, info) { - return fmt.Errorf("machine storage files must be distinct") - } - } - seen = append(seen, info) - } - return nil -} diff --git a/lib/instances/macos_build_export_darwin.go b/lib/instances/macos_build_export_darwin.go deleted file mode 100644 index 95381600f..000000000 --- a/lib/instances/macos_build_export_darwin.go +++ /dev/null @@ -1,90 +0,0 @@ -//go:build darwin - -package instances - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/kernel/hypeman/lib/images" -) - -// StopMacOSBuildInstance performs only the strict graceful stop used by build -// export. It copies no storage and never falls back to a forced hypervisor stop. -// Success requires the shutdown acknowledgement and confirmed VMM exit. -func (m *manager) StopMacOSBuildInstance(ctx context.Context, id string) (receipt MacOSBuildExportReceipt, err error) { - if err = ctx.Err(); err != nil { - return - } - if _, bounded := ctx.Deadline(); !bounded { - return receipt, fmt.Errorf("machine stop requires a deadline") - } - lock := m.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - current, err := m.currentInstanceWithoutHydration(ctx, id) - if err != nil { - return receipt, err - } - if current.MacOS == nil || !current.GuestAgentEnabled() || (current.State != StateRunning && current.State != StateInitializing) { - return receipt, fmt.Errorf("strict stop requires an active macOS guest with a system agent") - } - if err = m.markRestartManualStopLocked(ctx, id); err != nil { - return - } - stopped, err := m.stopInstance(ctx, id, true) - if err != nil { - return receipt, err - } - receipt = MacOSBuildExportReceipt{Graceful: true, VMMExited: true} - m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) - return receipt, nil -} - -// MacOSBuildStorage returns the closed storage of a Stopped macOS instance, checked -// under the instance lock. The caller publishes from these paths. A start in the -// meantime changes the overlay under the publisher, whose digest check then fails -// the upload instead of publishing it. -func (m *manager) MacOSBuildStorage(ctx context.Context, id string) (MacOSStorage, error) { - if err := ctx.Err(); err != nil { - return MacOSStorage{}, err - } - if _, bounded := ctx.Deadline(); !bounded { - return MacOSStorage{}, fmt.Errorf("machine storage requires a deadline") - } - lock := m.getInstanceLock(id) - lock.Lock() - defer lock.Unlock() - current, err := m.currentInstanceWithoutHydration(ctx, id) - if err != nil { - return MacOSStorage{}, err - } - if current.State != StateStopped || current.MacOS == nil { - return MacOSStorage{}, fmt.Errorf("machine storage requires a stopped macOS instance") - } - storage := MacOSStorage{ - Layers: macOSDiskParents(¤t.StoredMetadata), - Overlay: m.paths.InstanceMacOSBootDisk(id), - Aux: filepath.Join(current.DataDir, macOSAuxStorageFile), - Platform: *current.MacOS, - } - if err := checkClosedMacOSStorage(ctx, storage); err != nil { - return MacOSStorage{}, err - } - return storage, nil -} - -// checkClosedMacOSStorage requires every storage file to be a distinct nonempty -// regular file that no process holds open. -func checkClosedMacOSStorage(ctx context.Context, storage MacOSStorage) error { - if err := storage.CheckFiles(); err != nil { - return err - } - for _, path := range storage.Files() { - if err := images.RequireClosedFile(ctx, path); err != nil { - return err - } - } - return nil -} diff --git a/lib/instances/macos_build_export_darwin_test.go b/lib/instances/macos_build_export_darwin_test.go deleted file mode 100644 index 8f665c268..000000000 --- a/lib/instances/macos_build_export_darwin_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build darwin - -package instances - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/require" -) - -func TestMacOSBuildStorageRequiresClosedDistinctFiles(t *testing.T) { - root := t.TempDir() - storage := MacOSStorage{ - Layers: []string{filepath.Join(root, "disk.0.asif")}, - Overlay: filepath.Join(root, paths.MacOSBootDiskFile), - Aux: filepath.Join(root, "mac-aux.img"), - } - for _, path := range []string{storage.Layers[0], storage.Overlay, storage.Aux} { - require.NoError(t, os.WriteFile(path, []byte("synthetic"), 0600)) - } - require.NoError(t, checkClosedMacOSStorage(context.Background(), storage)) - - file, err := os.Open(storage.Layers[0]) - require.NoError(t, err) - require.Error(t, checkClosedMacOSStorage(context.Background(), storage), "an open layer is refused") - require.NoError(t, file.Close()) - - require.NoError(t, os.Remove(storage.Aux)) - require.NoError(t, os.Link(storage.Overlay, storage.Aux)) - require.Error(t, checkClosedMacOSStorage(context.Background(), storage), "aliased files are refused") - require.NoError(t, os.Remove(storage.Aux)) - require.NoError(t, os.WriteFile(storage.Aux, nil, 0600)) - require.Error(t, checkClosedMacOSStorage(context.Background(), storage), "empty files are refused") -} - -func TestMacOSBuildStopAndStorageRequireBoundedInputs(t *testing.T) { - m := &manager{} - _, e := m.StopMacOSBuildInstance(context.Background(), "unused") - require.Error(t, e) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, e = m.StopMacOSBuildInstance(ctx, "unused") - require.ErrorIs(t, e, context.Canceled) - _, e = m.MacOSBuildStorage(context.Background(), "unused") - require.Error(t, e) - _, e = m.MacOSBuildStorage(ctx, "unused") - require.ErrorIs(t, e, context.Canceled) -} diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 4ad65c94f..b21b56a2f 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -186,7 +186,7 @@ func (m *manager) rekeyMacOSCloneOnStart(ctx context.Context, stored *StoredMeta // Teardown runs even if saving the pending flag fails. Otherwise the clone would keep // running, networked, with its source's SSH host keys. saveErr := m.saveMetadata(&metadata{StoredMetadata: *stored}) - _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id, false) + _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id) if saveErr != nil || stopErr != nil { return fmt.Errorf("macOS clone rekey failed and teardown needs attention: %w", errors.Join(err, saveErr, stopErr)) } @@ -224,19 +224,6 @@ func (m *manager) clearMacOSStopReceipt(stored *StoredMetadata) error { return nil } -// freshMacOSIdentity gives a new macOS instance its own machine identifier and MAC -// instead of its image's, so it can run beside the image's other instances. The -// guest's SSH host keys are rekeyed on first start. -func (m *manager) freshMacOSIdentity(stored *StoredMetadata) error { - if stored.MacOS == nil { - return nil - } - if err := prepareMacOSClone(stored); err != nil { - return err - } - return m.assignMacOSMAC(stored) -} - // assignMacOSCloneMAC gives a networked clone a MAC no stored instance uses. It runs // under the fork admission lock, so concurrent forks cannot pick the same MAC. func (m *manager) assignMacOSCloneMAC(clone *StoredMetadata) error { diff --git a/lib/instances/macos_identity_test.go b/lib/instances/macos_identity_test.go index 0d3f98506..af30fb596 100644 --- a/lib/instances/macos_identity_test.go +++ b/lib/instances/macos_identity_test.go @@ -44,21 +44,6 @@ func TestPrepareMacOSCloneRebindsIdentityWithoutAliasingSource(t *testing.T) { require.False(t, source.IdentityRekeyPending) } -func TestFreshMacOSIdentityRebindsIdentifierAndMAC(t *testing.T) { - p := paths.New(t.TempDir()) - require.NoError(t, os.MkdirAll(p.GuestsDir(), 0700)) - m := &manager{paths: p} - stored := testMacOSSource() - stored.Id = "builder" - require.NoError(t, m.freshMacOSIdentity(stored)) - require.True(t, stored.IdentityRekeyPending) - require.NotEqual(t, []byte("source-identifier"), stored.MacOS.MachineIdentifier) - require.NotEqual(t, "a6:fd:95:db:a2:62", stored.MacOS.MAC) - require.Equal(t, stored.MacOS.MAC, stored.MAC) - _, err := net.ParseMAC(stored.MAC) - require.NoError(t, err) -} - func TestPrepareMacOSCloneRefusesWithoutGuestAgent(t *testing.T) { for _, disable := range []func(*StoredMetadata){ func(s *StoredMetadata) { s.MacOS.GuestAgent = false }, diff --git a/lib/instances/manager.go b/lib/instances/manager.go index aa234f69a..b64d1c1c9 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -736,7 +736,7 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - inst, err := m.stopInstance(ctx, id, false) + inst, err := m.stopInstance(ctx, id) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventStop, inst) } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 88775bf14..0bf65d7af 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -255,7 +255,7 @@ func TestGracefulShutdownWaitsForSocketOwnerInsteadOfExitedStoredPID(t *testing. }} m := &manager{} - assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), + assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), "stop and delete must fall back to the hardened kill path while the socket owner is alive") assert.True(t, ProcessExists(owner.Process.Pid)) } @@ -297,7 +297,7 @@ func TestGracefulShutdownFallbackKillsCapturedPIDAfterListenerCloses(t *testing. VsockSocket: filepath.Join(t.TempDir(), "missing-vsock.sock"), }} - assert.False(t, m.tryGuestShutdown(context.Background(), inst, 0, false)) + assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 0)) require.NoError(t, m.killHypervisor(context.Background(), inst)) assert.False(t, ProcessExists(owner.Process.Pid)) } @@ -752,7 +752,7 @@ func TestMacOSGracefulShutdownWithoutLiveVMMIsNotGraceful(t *testing.T) { sent = true return nil }} - assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), + assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), "a macOS stop with no VMM to signal must not produce a graceful receipt") assert.False(t, sent) } @@ -800,7 +800,7 @@ func TestMacOSGracefulShutdownRequiresGuestStopReceipt(t *testing.T) { return err }} - assert.Equal(t, guestStopped, m.tryGuestShutdown(context.Background(), inst, 5, false)) + assert.Equal(t, guestStopped, m.tryGracefulGuestShutdown(context.Background(), inst, 5)) assert.NoFileExists(t, receipt, "the receipt is consumed") }) } diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 056bb076b..227b6c76c 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -173,7 +173,7 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err return err } - stopped, err := m.stopInstance(ctx, id, false) + stopped, err := m.stopInstance(ctx, id) if err != nil { _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) return err diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 4e4f23340..2d8ee7a39 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -39,10 +39,9 @@ func hasVFIODevices(stored *StoredMetadata) bool { return storedVGPUDevicePath(stored) != "" || len(stored.Devices) > 0 } -// tryGuestShutdown asks guest init to shut down and waits for the hypervisor -// process to exit. Returns true if the process exited in time. With strict, only -// an acknowledged shutdown within the context deadline counts. -func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int, strict bool) bool { +// tryGracefulGuestShutdown asks guest init to shut down and waits for the +// hypervisor process to exit. Returns true if the process exited in time. +func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int) bool { log := logger.FromContext(ctx) if !inst.GuestAgentEnabled() { @@ -68,7 +67,7 @@ func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTime if pid == 0 { // Nothing was signalled. Linux treats an already-gone VMM as stopped, but // macOS only counts a stop as graceful after a shutdown attempt. - return !strict && inst.MacOS == nil + return inst.MacOS == nil } inst.HypervisorProcessIdentity.Set(pid) if inst.MacOS != nil { @@ -105,14 +104,6 @@ func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTime // If we couldn't signal the guest, don't burn the full graceful timeout. waitTimeout = shutdownFailureFallbackWait } - if strict { - if ctx.Err() != nil { - return false - } - if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < waitTimeout { - waitTimeout = max(0, time.Until(deadline)) - } - } if WaitForProcessExit(pid, waitTimeout) { // The shim exits after a forced stop too, so macOS also needs its receipt that @@ -122,7 +113,7 @@ func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTime return false } log.DebugContext(ctx, "VM shut down gracefully", "instance_id", inst.Id) - return shutdownSent || !strict + return true } log.WarnContext(ctx, "graceful shutdown timed out, falling back to hypervisor shutdown", "instance_id", inst.Id) @@ -133,9 +124,10 @@ func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTime // Flow: send Shutdown RPC -> wait for VM to power off -> // fall back to hypervisor shutdown -> final SIGKILL if still alive. // Multi-hop orchestration: Running/Initializing → Shutdown → Stopped -// stopInstance stops an instance. With strict, a failed graceful shutdown is an -// error instead of falling back to a forced hypervisor stop. -func (m *manager) stopInstance(ctx context.Context, id string, strict bool) (_ *Instance, retErr error) { +func (m *manager) stopInstance( + ctx context.Context, + id string, +) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) log.InfoContext(ctx, "stopping instance", "instance_id", id) @@ -188,7 +180,7 @@ func (m *manager) stopInstance(ctx context.Context, id string, strict bool) (_ * attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown := m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, strict) + gracefulShutdown := m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) if gracefulShutdown { gracefulSpanEnd(nil) } else { @@ -197,10 +189,6 @@ func (m *manager) stopInstance(ctx context.Context, id string, strict bool) (_ * // 5. Fallback hypervisor shutdown if guest graceful shutdown didn't work if !gracefulShutdown { - // Export callers must never turn a forced stop into a graceful receipt. - if strict { - return nil, errGracefulShutdownFailed - } log.DebugContext(ctx, "shutting down hypervisor (fallback)", "instance_id", id) shutdownCtx, shutdownSpanEnd := m.startLifecycleStep(ctx, "shutdown_hypervisor", attribute.String("instance_id", id), diff --git a/lib/instances/types.go b/lib/instances/types.go index 68226bfbf..f42ff6089 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -270,7 +270,6 @@ type CreateInstanceRequest struct { Name string // Required Image string // Required: OCI reference Platform string // Optional: target platform as os/arch[/variant]; drives image resolution and auto-pull. Empty means host platform. - FreshMacOSIdentity bool // Internal: give a macOS instance its own machine identifier and MAC instead of the image's. Size int64 // Base memory in bytes (default: 1GB) HotplugSize int64 // Hotplug memory in bytes (default: 0, set explicitly to enable) OverlaySize int64 // Overlay disk size in bytes (default: 10GB) diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 9e8a64d25..148967567 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -203,7 +203,6 @@ func extractRepoFromPath(path string) string { } // isWriteOperation returns true if the HTTP method implies a write operation - func isWriteOperation(method string) bool { return method == http.MethodPut || method == http.MethodPost || method == http.MethodPatch || method == http.MethodDelete } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index b6b825e65..4c604c755 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -274,12 +274,6 @@ const ( SnapshotTargetStateStopped SnapshotTargetState = "Stopped" ) -// Defines values for CreateBuildMultipartBodyMachineNetworkMode. -const ( - Egress CreateBuildMultipartBodyMachineNetworkMode = "egress" - Isolated CreateBuildMultipartBodyMachineNetworkMode = "isolated" -) - // Defines values for GetInstanceLogsParamsSource. const ( App GetInstanceLogsParamsSource = "app" @@ -2008,22 +2002,6 @@ type CreateBuildMultipartBody struct { // Admin builds can populate the shared global cache that all tenant builds read from. IsAdminBuild *string `json:"is_admin_build,omitempty"` - // MachineBaseImage Experimental installed darwin/arm64 base pinned by sha256 digest. - // Requires server macos_only and macos_builds. The source must contain - // build.sh at its root, which runs as root in the guest. Machine source - // is limited to 64 MiB compressed; only machine_network_mode, - // timeout_seconds, memory_mb, cpus and tags are accepted alongside - // source and this selector. CPU/memory zero inherit the base. - // Linux/cache/secrets/admin options are rejected. The builder and its - // output get their own machine identifier and MAC, so the output can - // run beside the base. - MachineBaseImage *string `json:"machine_base_image,omitempty"` - - // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables - // unrestricted VZ NAT for checksum-verified artifact downloads; it does not - // enforce domain allowlists. Linux builds do not use this selector. - MachineNetworkMode *CreateBuildMultipartBodyMachineNetworkMode `json:"machine_network_mode,omitempty"` - // MemoryMb Memory limit for builder VM in MB (default 2048) MemoryMb *int `json:"memory_mb,omitempty"` @@ -2043,9 +2021,6 @@ type CreateBuildMultipartBody struct { TimeoutSeconds *int `json:"timeout_seconds,omitempty"` } -// CreateBuildMultipartBodyMachineNetworkMode defines parameters for CreateBuild. -type CreateBuildMultipartBodyMachineNetworkMode string - // GetBuildEventsParams defines parameters for GetBuildEvents. type GetBuildEventsParams struct { // Follow Continue streaming new events after initial output @@ -19351,403 +19326,397 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV0s20qlvuPYjqMdK9a2bM+3J8yhwG6QxKgb6ABoyUzK", - "VeeveYCp84TzJKfWAtA3osmWbEn2xDVVE5mNOxYWFtblt/7oRTLNpGDC6N7RHz0dLVhK8c8nxtBo8U4m", - "ecpes99ypg38nCmZMWU4w0KpzIWZZNQs4F8x05HimeFS9I56p9QsyNWCKUYusRWiFzJPYjJlBOuxuNfv", - "sfc0zRLWO+rtpMLsxNTQXr9nlhn8pI3iYt770O8pRmMpkqXtZkbzxPSOZjTRrN/o9gSaJlQTqDLAOkV7", - "UykTRkXvA7b4W84Vi3tHv1Sn8WtRWE7/ziIDnT/JjTwzVMTT5alMeLRcnexLLvL32BuhuZEpNTwi2tYh", - "GVYiU6pZTKQgNDL8khEupjIXMXnz9JREUggWQWN6LORUM3XJYjJTMiVmwchCaoNljKLRBTF0mrDhWPT6", - "jf1gAr7Em1fprwtmFkwFBss1ca2QmVTELLgmXMDXiA2rG2ZUzlZXtt/jccImhqdM5mZ1oX6UVySRYo7T", - "8u2SNNeGLOglI78zJclvOU34bMnFvH2RpmwmFSM/LjOWUkGyhEZME24IF0b62dg1KmnsQRoiLj4XUrFJ", - "zLThgkL7k0wqo1dH/wr/oAmplMWhYXliFtR4KhfSkAvGsvpE6RW9qC/jL3t7/cej0ejXfo8bltpjRd/z", - "NE97R4cPHuw/6PdSLuy/d4vRc2HYnKneh+IXqhRdVqajZa4iNol4rNbNJEo4E4Y8PX72+oYT6O2Ohvi/", - "nUe9fm/38d5w9/AR/nv3sFed1srC10f+Yf3ROzPU5HqVB9nTNHGEMqkQyeqsf87TKVNEzkiUK8WESZYE", - "jxSLOxBdbdqj0FZEUsz4PFcsXu3bH7naci6oJlRYpjFo8IuysU7nLgImFssrMVEspVzAGq8M4rX/ROCE", - "EneIYEiRFEbJJAGmYAxLM6P9KeoDGxeEZlnCI2Q9tUN1kI50r98TeZLQ6coIy91mCZ9zLNBpabiubJKv", - "S4wkTBimihPeZWlqbLGt43K5g7tR8sVOXS5kEk9yYXiy2utb+NmtabUnrsmCJTGRs1mf8BmhBFqBny2N", - "15d9b7R3OBgdDEaHb3YfHo0eH40e/K3X782kSqnpHfViatgAdrnL3iDT1lxE4d0RTRJJ4UJSLLK7U1xY", - "tQ2cskimjEDTa0d+8PEjb71unpabCgWJK1jerIGtHjaujG7UnVBtCiaE+8XNckIDY3rDU6YNTTPgQzCG", - "ymK2cSHfYHMf/MqvXeDdj1pgwd6biVuh4HxC9MHeZyyCG1F6blIIGNCeI++CZd0JjStGtRRWTILL9Jde", - "LnSewdXN4kmWUAPt9vo9SwaTlGsNVYsfYq4tH+n3PJFPhDQTlQthCwpmrqS6qJZ0rUx41uv3FlRPLudZ", - "3uuvu7bqRI1dsIRmGttzO64mTCmpelY0Xk5mUvlN6v1aXcI1Ta2skC6u2MAK9fq92gIU7NzPxY+72NXg", - "4Po999dE2VcFNmcnszrwalurwy2Gtp6x21vECtF+m4mrrOscIOZ0LqQ2PNKd2DwKD7C9qYwDrPNZ0Rzh", - "MROGzzhTTq5mRMH9kDLiGyHQCOGC5LpxDgrRf8IumTB6cnkwMVG2uiiNh0118yqySXkjVm7lYvuLk7KB", - "SOtzDz6cLinHM/mMXfKIrcpubmsmseKXTAXYdyEAWFZoy5EtOOvAQoQUbLu2UuKSx5x2YQcxjmnCA9Rz", - "+vSY2M/k+BnZWrD39U72Hk4f9dqbFDQN0MKPeUrFAA4EDMu3j2Wrbb88CLXMZZrmk7mSebba8vGrk5O3", - "BD8SgRJutcVHeyFJNYv4hMaxYlqH5+8/Vsc2Go1GR3TvaDQajkKjvGQilqp1Se3n8JLujmK2pslOS+ra", - "X1nSn98dPzt+Qp5KlUmFb7aNB6e6PNV5Vcmmvish+v8+50m8SvVT+Jm1rBNTmmvDhCHf22JEMfuUI1cL", - "qRmJaLRgZGofLfg8x/a6kHzRsb+9QiN4xnUmkc+TdyelNIdPQ/aeRbmp9fstibk2XEROqKpMrsOIQP2U", - "MLh/Q5IFrgBxZbgUxHjR6cZCQKQY3dAdlOjU2er5zy15TVLd1rovQrggKU8SrlkkRayrfXBhDg/aJ1M5", - "xfbaXOnqOfxMUqY1nTOyhWopfG5YDk+4JjPKExZvd5Ow2ybzdzmt3Gu1M4dkMKDTaHdvP8jQUjpnk5jP", - "nV6xSYPwO8jH0I4hPG2dCEoY3eaBXSo2W+3vB7xPsBPFZkwxEX10d5mSl0xQ96T6D+y393/tlArXHadt", - "3cHFPC2Lf+j3fstZziaZ1NyOcIVNuC9ARrjUBGuEx4yf4u1OFKUNVevPB5b4BCexFDY3ro1T/YC8Recb", - "q7yBMk2GjuyoEHAqXKCVbz8HSSsgskhhmAiszks5JwkX9hnEhHF7gcLeMmPfJXK+3ftk61As/+rhh3Hf", - "gHnZH1pag2/94lWQyHl1NReMKjNltcVsuVddQ+XoWpf/tHZ8Ghco1WyynoOcciFYjDp3d7BtSZCtg28f", - "PEUX3Ewu4QYOnTkc1k/cEFeitalERhcznrDJguqFHTGNY24Vrqe1mQREyJoxg6KSwDeIog0+qs9+fLL3", - "4JC4DgJr6LS/UGB1JpXa0LwtSwxVU5okQdpoJ7fr39GrFBKmgFLh23b3FBToCdNyup7bTfd4z/XC/oW8", - "G0aFd1+v34uAvBL4e4Vw+733A2h1cEkVrjo0XxnWf/ueKr99X3Za+fW06L/y42s3lMpPP/hRVX57Wg7Q", - "rwpTq+fBPdNw0mGpUnJhBlzgFoAA7jRPU7t5xSvLaTKIFBUJj6ku7HydYPV0RaQiW69/eLq/v/94u7tw", - "xfXFRPPf2WQ+XSs3u0E7QRmqEahGuCBzPqfTpWF6SJ5SIaQBY2S0oGLOYkJnBmq5sdYe4A9Gm4wwrUIS", - "U20ikvn9gD5+9P49NY8P+ZV+/Hs6VfO/79Nem3Yx1y3rixo46nYTihJFRWATb3ZRp/Q9vPwtlYSMnyfW", - "XEWKQm4PdHMIQ1I+6Gf8Pay5Ibu1lQ4ubvj1V5iwhBSDXPDfctztLKHL1TegYTQdIEWEltfyDXuA9Obj", - "o+H82Dp+pleUGzg5hc3UTblPZBLDvTPjSpted2vYtQQDpj6ViFQ7ZWGJaZUimgvYeqkX42w9KwmfsWgZ", - "JWyVuXv2nancqVtjBncO/lkoELuycaYCnNj/eFp0Ufv5Wdlf7ffntvMP/d5TmtEpT7jnzHU+7WzyE6f4", - "27RL1cae2aqvXc0P/d6MUZMrFljLM/QPIFZPPEjYJUuIKw3Ee0SenB4TnasZmsyRWm1ZUNtLzfRYKDan", - "Kk6YRlK/WvBoUagrQfWgCRWlWwB5klzRpSaZYpoJczQW40JDrse9Phnbx5f721KJ/du6hvhCYq6Ydv8A", - "XcO414e2EjnX496Q/Ci1GcBz2clSR2Ts9DF63CNb9k+SUa3NQsl8vhiLlAo6ZykTpk/QSQPVwMCZkuU2", - "oSIm456SmhlDByzNE+T90NiTLEvYWJzxhEdSkJRGr85c3StuFuS1rVS5PHHGcFn3SeZUz5qmbCyoVYPC", - "Ti5JpuSUkYTmIlowTZiYSRWx7SE5ZWrgl9jvLaGKwWZYJS7JBRwRRsvN0L/8Spgwynk6dGcuMJPr0B8s", - "ffGI1tepeWxrfCisI9ep/LOrAqzKzTj4RlHLUpteoWd7Ixaqfn8d1SwB/bHAFcVd5VDqSpDals0SOkdS", - "qRwiS4x6LCKZpvjQc70QzS6ZokmxQ3DHSdQnwN3gbqSxiBKZx4PFMmPqkmup+nBDsAidEVSf/MbSvI+d", - "wl+DlEdKXqZEirFIgIx3aBofHmwPyROScA3E4efPLXE7EkNOgPoADqMd96jXyY97dmJcE7j4h+S5MIpb", - "miPaEtzU3qQN4uq6eRVmtXK34f5cp7EzW2PFwmF/dgTdX2GxFbopCbAg4woXDd1Zaxjwqgzu13W9McqN", - "r9itiAq3UzXqPCLcEJpljCpNuBiLkppEjFTqKmWK4XJobtzWpcyQrZQaZC+gtR0L5BDfaFLb/e0hQbnR", - "+oeVHAmqEMWSpR0QGws/5iueJKg4JDl6MCCHswcNNWMVDwo8TFQUvC8prV3O+OXsorUJ1I6I7fu/n5+8", - "JXqpDUvJlAuqltuE65LXWtJcNcyFRUa3e/hyR1GtPutYotcT1CVS1AXI5nndaEJwImhJF5so7EcZcrGk", - "KlqE3Oi0IU9P3xL4zA2LgIhr46UqPQyakqRuaQ56pSjE2vWuNYdMZ+OcJRwpHPGmyR4X90hYRCrM8avW", - "LqhZseOyhEXAq6w/jX2/MeKMy0SmyNIFW53NTusS+cb1pt412ZJ6Bya8XblZ4EyrXAzJK0FQjCA1KWIs", - "sCgXUZLHTJMKPyel+tbLFzU3qILsx+Jf/+//Z2WJmFxyWkgb5B1XBnxIfrev2JmiKQOmV7/SogWLLsai", - "KYYQaFVLQv2lUiwz19ULRfH5whAhr4ZjcWzHVAx4S8uZuaKK5Rk8M8lg4EY9cHLWtnXiwoa4abho/tLY", - "m8riXNOXsGbOKza0v0phm0j151JsqdPqnBp2RQMP4hdAeoNLrtFlDWnClSXHp0OCn9GUEC3sV+CiIMWS", - "LUsVXMzHAr84kXibOInWkpkzTQ7Jq5SbgviFLK4Xd9ONBTgYThkTaEVMLq0DckmqS9bcAPDm3AUD73A3", - "dDTmMPaJkZN5HtS7+rsu5opFhrw7GRg5eHdCjKKzGY+AjjKm3KjtBdMcdJCfg49B0rLSvqJ3oUiG8MhQ", - "PJ6DkAPL5UR/+5u9nDKmYFxvnpzCBYer/y0Z9wQ1tk7J6QdgyeExi8nPT97A1Rq9Ott2q+aeprbhHlw7", - "ISV4v6fzqWCmbfz2K/rghrcCXWk3MV+7RCtbtIm8P06igYvfNvDNGpmk5GAgnYAQW5DgkLysya+2QuG4", - "AiLoqkRftOAP8bekGO13KNCQlFGhCXWyfmVEhOux8NJHTeAASZu4FYWqKheWS9n7cFDhviiWWHkEJbLC", - "Sf8S/n9wqWV04Vw87Cuz4LSCpni6q0u3Illx00nI2agCqDxYAkvqe68JnvBG0IJmeiENvqr9P3YU00Yq", - "tg2Pc+cvBd8zmmtG/kJSlkq1HAtfvm/PWUKN9WEo6s6AMfXIVpRIwap6BOtAQMdCG5llLHaGiW/JzB1u", - "WjgRSlXoqm2hsXBd6B0rAWhSzAI3oLTAJMtykysz2UZVA86mmBbMOU/tsBfSZEk+H7h59shWwi+ZmzZM", - "kP/ObBtgRZFSDJw/JBQtdSf4iRiq5sw41bPG9pFmxr2x2MJzS/DfYFhJc8EjvM2x3DzLB1UVR49svTh9", - "u1P6LI1F5TNWAaXegMtBwlMO7G0LfiDHO6+Ighsaf+ZibkePhe10iklieTfFIXlHk9ydU6R2WN+x8NI/", - "jnRgVG4Wszw5Ipe/OyGsvh/FThZS0e5+32maEtQwtUoy706IppfMEyTWgie1NnCMaHzJlOHaaWMvyJYj", - "p4GzcyHdaSIYi4mQY1FtCw4ykikMsSnyE6uI0aSxSGNRim0NT3J/tXmTYaGK8IJPd43N+hdNi43h418t", - "13ginxWP+sZFkvF2qyroIvGg0Khcpsr7lZFXGRNQKpZRnjJhCBcz6YvWvchGw702z7SW7s+q10rRvV5I", - "ZcicG6IYiHFSOG427g1irgwcf53PZvw9viJz1AJZmQZb0n3gUONeLi6EvBLjXn2YDw/p3iye7m/cDT/s", - "fm0Ng/uATM9ppVvj9dbb0J6uNZiBqEmMhLdz9eXvKPwTmMvKkCS49NQAzkrCWVyh7b711p0zwYBzxdYH", - "1BAvobWYfAZBUfbWrUrXssS07Kj1nG3d0A1zmCdyiheen0jpdDokx/W16xOKH1Bgrq9yES1RcQwlW2w4", - "H8LtkkV8AJ6hA7o3GI0Goya1JwcD6/KeUWOYggH+P7/Qwe9PBn8bDR7/Wv45GQ5+/ct/BB/kHb1VvV3b", - "zXPLn6U+8YOturA2B7rJvfXGlrXq8NsPL6oWWnc6UgyPAU02jgE8Dp5WirfT+tPjVa83u1KxBD30kMud", - "hE8VVcsdMefi/RGIc9rU12192XUKlkBwjBWMfAFCNXEall92LqniVJhf7RjJuKocAPmN4K0B9/4zHBEZ", - "DHw7jn15z/uaAWBIfpaGUGeRhifdERZa7XeuaJpS1HZe0oTHeDicmU3zmFnlrMBv5NJKSYqZXAlyMBr5", - "L5PKoEBukNpPZdxzilMVLYhTCCmCqhAbIROxDLo0MqaNMKXKUqyeNDhmf9kp/tryfw5//cv2//6PT0rq", - "SGhraNyqM67Jzxp+3lCIbCXyiqmIakYSBrPVfRLzOTfaCm8xukahQu1bEllnD+u6KBVhwquosVydnNPl", - "gGZ84DQv1tz9kom5WfSODveDi7vl/hj8+p/+p5aFVXkSeq+9ljlqXvFzNfDaj6GT8cWvbp6g0SXl4thW", - "210VJz9uh/1E1u20fdS1bjXcMUVU0IaBrEbgg8dRGhIiLplScBQtX3t68gyeMBfMMTR4NZJxPhrtR1gA", - "/2TuFxDi4CmDv23XJR7bXEjY+aXHooVEN8gkkddRUvabTL3NS3CtjSy02iX3X/UpRCO6NYujs7srSKZK", - "XjCFJhlYYs40uWBLqy6Y1xSaTFySS6q8afANhiVgEf8JrUEMn8fwiMXgfIhlZcqxRfdms2FQjCYlt0wp", - "mLvwXakjmTFgqK4YTo2cM3F5TlKa2We9YlZpnlLDFIfnYqlYTFnMgUePBcODQTIKZz6KpAIVK+wt6mDL", - "VfhGk3P7TDzH5s+5ALI+twezoSr9o/fq7ZvvX739+dnk1enzn58cT356/j/ws63UO/rlj54F0ygcr75n", - "VDFF/uMPnO8H6zyLPn0Q6r6Qyr12vYke/VVoxocyY4LyYSTTXr/3n9V//vrhV+/uaSP3LntHwYF9CIqZ", - "TvcZ8MH3AVCaSFV6nlLEBUEW5dUOFXVDQ5VvJb9rHQl8onA5mWY6GJvS1FuUcujuaHTy/Y7zHnng/7E9", - "JM50i8MHHiSVE4/1AsinwOVAe1qSyMhFaMzaQvB9VyEGj7bWTHJhNjKnsugqjxoMyq/XYEU7Uy52NGzD", - "ILreuiPd3NhR+bm45EoKfJ6juDRNmK6flZ9fPXs+ef7zu94RXARxjnGEvX7v9NXrN72j3v5oNOqFCJS9", - "z7hiOuj4+GSqZZIbRrBQ6WM6JCe5RudOxzxmOepBUT7TJM1BsZQsCXsfJbkGPoXSgDFJKP740WBv983u", - "o6PR6Gg0+ltnh1Ug/Q3M+8XpWxuhjued0cQsJmia21TxRyz7FIt+6HvdJD7tA6tU7KnXVsLpdXXI1qIu", - "XdnJESTIcW//xff2QO2+GPcqhOhCgYtWbMON1XvxfWhdKtqoVUmv+GYj3ETsqX4VjMbq63Pt1ERVS/hA", - "UITYgZIxVTGZSqrib+u+NFgTFPnfaILaCpo4O5EVvbGOlSUl0/hW9j4+pbJV9xtLgJqf1KoSqSAMLaWX", - "HFjOIE259PXqbMkpU2o61GHFwBTQ41XchXr9HszM/cdPsNfvXf5ej6nerA50rjGd3oobHoE0ybhga16B", - "n4mwDzrlRNJ4sPuJZf2Kv1sDhsZ+qG938Tj1BL6C7jSlIr7isVlMICIehhy4Id0XUhQursn3MBOa/Osf", - "/3x3UipEdl9MM3dn7u49+Mg7s3FLQtPBWJtiInkWnsbbLDyJdyf/+sc//UzudxJWpLyReO72/7ltoQlQ", - "43C7rDd+i921kMPK+FzZYkvfgGEQunLlJVMJXVauEzem3u4IeXpjVIpbc6OrV1Ehr79coDUvrlWvF2Dz", - "M84S66NlXW/xfaG9hcjaDvG0AE/6RvsIJEamUho7AlgbRlewUcK30seppIbu5oikMJQLptx4ixvDXin2", - "xZIeHgzJ8/cZUxwEJpqQmKorLqzTi6/p3tyECgAESrhgA546C65dEjtJ5MnQMBjavMORtg5Hq1p7VFbH", - "xN2PtrJUDYXYbWqWFEPGPMkKJL11x+e1LV0++gMkGaDI76kuLLNd6LAgw929E/fnniXIKvEVh8ueNa4K", - "4rNdfQtiKDydDXaqLazdlBHoiVFtyAF5wb8nXEAof8LICf+eUHc7cUHe/c3uA/I63UmW0hc8s34eEzov", - "wlLXIf+dXfDMveexhvcysHdQnEPLeIaGY2GRfYBtIO2AR7633RoIItDWcQFuW7zPVmU00A6UN5otbs20", - "Khd9Ms0NPPmlYcQpXxzEAIwFC08ZyUsn0sYD3E1w1TECl+WCKcGSiX1g644rYysRV6l1cXCqM6odEpky", - "eVZfr2c/nZyRrWdLQVMekZ9sqycyzsEb0PKE7frq9cciUwjvAp3AWeCuXzkjMjcDORsYxZgfYoqNFXYY", - "B3Jx+eL0rYNJ0eCk9JrBwjIRO4dXL+wUHq/iG2NN4bVmq/03Fr0NCseb9zue7DNXvDza3TWS/Z4xSQjc", - "LrFsDYKIbBwFGK4K6vOhfX04gS9kicVgucGQvNXADUbg9iIVei2QqTSL6usSryVdfQGVEJplMXh/hp+Y", - "5Wu2cb4PFkG7dZTlddLd67diGl5ahwkQZ3RXznUJhSuMCwGXApxrryOjCoLNuCCf1R2z8LJVvZIdYLln", - "1NQBObqqwm3LCAa70TvUabRxSbprtCs61hXdtlcA/tExaGJ9+8fCD2StRr/UAX5EX2e2keYSubb7fmY3", - "WKXjYk0adoBPszxPdEVn2gm31co8Vp7RZOsc1KyOjkGxet4n5/9Z+wH4qZen4LlwRexqII8W8FO1fb3i", - "2bxBj3tj72aqb74fT3Qr6ha53CVGUaGtn/2CZmxIfsSLkRiWgrQINxWvuhYKefUtkfaN4quOBQxN130t", - "vTZf87kA1zN0mgeGE8dW5e80dt7/Rofgjr0ebkVit6NzoEXIy5y3PTn3qvfzOv9dVcyv6q2cpn5FYYG/", - "W0UFqnLMTpob6N6+XpxNWubGgpi5qTcEvLr6f5NjkBtLoYq8wf6fFeyiiSF9GdBYwOTkrLTXVAxHbfYZ", - "J/yFbUcXbIlb7u1EdMVSVDURhQ05zqXdiTKVqqg7tFcJ1xX7krUUOeMQOh/Xj2jQbLJpK2C9Oi9//eUf", - "jMMZ+MmWFOMe8x6f1HMhmJztD5WOmuHioybhyIZ1nfet6oOhfpEIIJbE++Y3m0e3ZHRiPHe/DF1r50WQ", - "4KfAHC+jP6pbSyo760VpbMaLYobF/bpsAOjUevOkrHvqosXsqdiV4p6ReYDLjiKvi5pJ3cPr4/RAzyuN", - "rXEI69rEapSsXd/KmB09WWBnH0Ri9wPtXzXE4ybwf9x4hlvkmHqX5zRJzsmWK7RNFPs7osK6vQIPu2L7", - "AeXWkUDhbvbupD8WlgucL4zJJvB/egKn+LzZmKvrT3gJyv5ohKLrwcF+PWbDDrjRbF1dHkTDad8a8Lu6", - "Jf+tFqW8R7/Hz8UNBIg3ZMvQORwdB3sE98T2xyrsudAsyhXbfNafoHjkoHdwTAnlYvDjmzenRLE51xhq", - "HAxqsK7xLRhtTllQuqv56RpZ19ru7R88OHz46PEwvlBDFqlhrgfwkBnsDmlKf5eCXmkQu3bSJc2yo8vd", - "jXzebkExvnaW71+27aQgU2DJ3iG5yyv5aVmlNBhecBF3beAnKLvZ6dU/4m/b7JIpNsizuaIx+8RGlxv7", - "NeFqtu/qhkwo6x2ac21AEVFCAG+t92OuL9alTAYxNfQ6bsx2uKsey+nSNmWf5KH2Wt3Dz5o+4TXcnmA+", - "hI91MvNjCW1LG6awVSSweGJkIHzXyw7Hz2DNfdlO4KkIgWzk5HLG5aYgfxhaPXQXq3v1BjQxyCLuNHXe", - "qsELrb9V1r07qfrWDMdigFLYEXlWdFA0WzRJ8YlBY2vl35KqFj+MhoApQJ+QdydD8qYY7TfaGwXsmEgR", - "OZqjQRGFooEVyaoDyDHQhptmdaeWtHjM2+hCJN23YZG1BRWghQIN9WVT3piPjevGjXJ3SRV6pqNCeB06", - "2Wt7H61DKKt5hDwYjHYHuw/e7I6u6xFyC5DTobae1HmL8/2vcp+nb4+f7Tm15/aNQck+OSh1mJU9K4MW", - "yFauWSUgV6pgqEIlIqAlFOHGEQa3FDBQwm+uK2tX4g2UvA207RAeGBbp3wAPu8kwN4KuVia3qtBZZqh+", - "KE9JRZC3O5pFPBh5DS6D3ytGLzCLwOq9jTgUE3ufhf0NESxkukRwLFRyKSnNTJMCQ6ZUfu8ePDx4tH94", - "8Gg0CuA5rxK8jPgELbKdBgAuOAldFtCGW+jGEJNpIqd1Qn+wf/jo4ejx7l7XcVjDfbd1KN7fvhbZcivy", - "F5/pzX+pDWpv7+Hh/v7+6PBw76DTqGxj3QblytZFkof7Dw92H+0ddFqF0LvuucfXbgrwoWQPT2x+JPjX", - "QGcs4jMeEcSlI1ABkQMWYMb3Vuj6mZzS2KfECN8dhvJEr/Xus525klbfmuaJ4VnC7DfckE4mCZz5M2wp", - "mGNMiCIFyfVacqjkG/2+/FyKIijOxGyaz+cO8q9YuhMXzV8KT5wl8REpnmxr+RzuZjmwX9vowM2hIzW8", - "lFcF+F6FCOzVZZEqFCMFndhNq83KhwtxkeVBkmhdyh9yZUE4sFFCpzI31oLvcsaUnVg8G4EW7VzE3dQd", - "P0h1sTG+A27iIjVOR4WBD/SfIRBAI8S/KvQVlnbrj+C+a/La1rCKwvLnLK/nBexjT06hKDw6QVzojV0z", - "XaXLsNyCOnPv1Gf7K3nnHXk0DmbWCezTvrDVnGFKKLNRYgFKsf5TZ1i8c7gYVNyoSOmw7oJd3cWiY8Dj", - "AMh2ACqU21nxdY7Tha6hLESkCzcAfE6YeVyBl2qctDOPuOGWfDgWLt6q+Mk5BUBFuw7WkUAqPuf1jut6", - "1jt0Xb4OZXriujF1Viu2+QzqwgwY4gEWcHpmPYGt92IV79btSa/fOytyZznGVF8a/2NgRcoAg1XoodO3", - "13VAzpSc8RAeEHodua/uoeZdc18ejM4Gu/9tvTuB/FBi48J6KoEdYdjIf4Tlu11EL07fnraNqUg+Raqj", - "W5nTGpSj0rOmjIopFsfFmnPtW64k6qWCGHrBQMOEJ65WzMJ3sXhInlpfG5Rr+Aw12sYXs/AqwIBs35fs", - "qAaVAz5hghWNWlNqnANVVXHeoBiIAtpbPREj7sXp2zpew+OQ+I2AK9McBjZJA/rAH+A7sQUKwIiT7+si", - "+N5BqOkw4z6tERC+9Gc0Au+AzhQS0CE2prEJkfHF6dvXLuFSG1Y1kFORlMnCVA/Jz0VKMlheXeJRDQPK", - "xQYJWtdzKDDRiQw5sPygGCP4rUyFCnSDRm+Lblw4GtAY0a2k8AkZwfffYQ3+llNFheGCxeTdD/pbMnJW", - "9nc/EBt/5HgW11W/y/rj8mFoS1vjCE8XSw26PbssNikKF1XFJnKBzi+T07KiUwEH3ifhdHye45AtSMKI", - "C3j2enD86t1OGrPLfm1M8NF66b44fbtdYcuXHsyiKFvnxpdtGia/VUV2yEmZkbJ5EBBH26k+m9QBezMi", - "UxZR79Td3LyoSOA8RXcNVJSBmdswQRR1emoqao0g6wLH+zxJho5n+iET1EzPkHVXsF/t7eaYmmIZxcdX", - "IXm9N9gpIzqPIsbihsdLc9QVejsiuUip0guaNCd3VAAsRAuqaGSYIt+8/6YXzsCErLTr3VChzuJy6kyW", - "lasoQI+Vc9d2xN/9UD+eVjqwYdVOp84FNwgTh9jgW8UgLZr5kJzQpVO9ZHjNOnopd4zOZhYstbhHFEsY", - "1Zg5XZN3P2z25TTS0KRtDm/go+NTWzAhOEUwzD7JamwAfq+ilFW7PQxeGeV8ApYp7LBp5hiSYxFme9Zh", - "CHEGLWSY1nwu4Dmonc8Q3Kd+JVf2rn53BjVJTVzGXn3hatPpBy6AEMWEbisbB7r6MlrN5Cov6mxKXmzO", - "Hmobae/3qQ9VbZjC3PPTn1uMaPXOS8RWdRzDKlQ1y6iihiVL+x4pqLOeDYKt+qFAtMA1YmWfQ/EPNgNQ", - "rtjELBTT4DNWU1Ds91ezSGuMSrhkLnGenVPFXGckSam6wFPmn98kF3YF6nGx+5vAsRbGZNeYFDhVWJ2c", - "YeqSJs2ILb3il/OMQZTWlJkrxoSfCtWEVr3imxHvuiV5GcTyMMVlfQ17+4F+z2ykBJkrGjFia/kc9AVb", - "g6PXdSl7+y3DggtH65b93V23v67qLE+67XFoWLubdthE19ngN09PfRKuIsm5X+a91VWGRBL2yLkaG7Z2", - "T69PB+e7EhZFe0VmnzLMD+dizapR7N4rEfPfCYzYq0SOV5iDrsY6u37wFNil6ttz/munN2HzuIfcbyDS", - "KJQsHj+4KL05gg1ibILK8eXFYysp2Tu5cj+7SCqQtLgAXOa633eUq6TX7w1mblZHOzvA7xPEWz3Y3320", - "s975e63Xv3NynMR8nVbIlXLOch4CwIJV4qTrJLFDs6yD3hxrbrofkD2tbIJNEA93W+WR5UNcwF7TcF58", - "TyPjU1qiIr3mKEGrxxZR0euCDDSY2ixV8Np9XD2fo6DnCrih1sl/Z4X2YWbWExloxCopG+toifz3oHQq", - "VShvoVTGhStPmX+0F/ehdwT2+N81k/roUXWWhw8e7D/YxIeQ2TSOuTt3gana0nVABlZZbnd+XQPwH12X", - "OdyX9WSF67KBpt48DZDU+gWVGRPXWs8HB/t711vPrhM59s6cDb4UwrMCACuUicrI4pQZGlND60wGNdDA", - "ZTAxFmUpBg7Ovl3PWlq8nqoAVTfN6vepfGZaEny+9nj2KRV8ho8kW7Las17QvQeHRzYNccxmBw8Oh8Ph", - "dWF7nhffum3FjnWtrSD4DPXi4/bhFtB5uszlj97pkzc/AiPLtbKX1o6ecnFU+Xfxz/ID/mH/OeUijOrT", - "JXM1n61krK5tb5Y72HQWH8FMhAvG8pJSB6++FhMSxhQglFgQI7PmW317YJgFjfN6zpNrgGKuCeUHcQU0", - "8q1rc+Mk1KLQhZpK8umqXqFDImpw/Vzn+eE1zljG9Wk1HUWO7lV/mBtlWddrE9Gu5CnMmChSzyaJ/SuS", - "AlHJQ3loa1ek/9YhjSFeIy5fYdFl9cei98qPT6sDqfzu0yBWfnIJaX+9piPbWoH0r6ty6GYu5MXRDYc5", - "rP4vboWuyb8diGYwguOe78KbOHvWe381/6/f/o8+ffj33d9evnv3P5cv/uvZz/x/3iWnrz4KoGk9fuq9", - "gqB+MtxT9HCsgZ92JaUTsMys0hM8/1pW2H2xGgcTLTBdMZmyI3DBfskNUzb5ZSNkGfI1MHwpYS0iBWYp", - "dGgW21D51PohQOU/vBj8odlG7GArlNuQAihJ59NYppSL7bEYC9cW8RPR+C6Av2IS0cxmsOQCFLPg2K0w", - "3agzDJed98kfNMs+bENKFGoIe29TAWQUMzbOCtuE8G4dblTWed0VZwXeso0jHoviti4ARK13wLDMNsJZ", - "0oz0a1mU9e8393J6NApBrWIUGmwkZtTCJ0hB2UBGRXgceTTaXn3PbXhjFDS0hvyc+4wNU4Yw4wAptoQ2", - "P2Mxj5Cv+OheW7ISWG0pzSnxMiXfL3FvXtuQ05jQ3CyYMM4xsk8iKS846+OW9gkYZ9FfC2taL5yFzAbT", - "5WAhswKGpbCFWz+W+iP7/wzcRAfvmOIz11MQTQNIJCB04pFxM8MypXZhZWJvbDIiodEo74rbrFbaJjKy", - "ARgmV8Ln2mGYcRfBhgrqYyGZ/FsSJRy1TnqBNrwFooUaaCYE+NkbFYFudBrFbNb8d90Rae/BIb5g/b/3", - "9zrHmdulW0dleRJ406ae9XXg2JZN4gCseDDxivANroNwA1pOTFBPYST+94z4hsoTVzASa+mzka/apW5J", - "dCXmdTsYnOiOgQMVmtDcdJ3d6imsxTp2aMH6FGG1pAOc0XOcB3nz8owYplKPerEVwe7gKbHYNQOude6y", - "CD55evJ8e9gLosTVTFq4VWtjIeuDDiCk2HPQ6jpV6mhoyvrk+BnGuLtrpXyLYVDSD1KRxN6K5WV0hHA+", - "dXUPtflHj585ATRZlo5KVmwZ97Z9i1nzejsir323hBZDKaKVS9ryTZaXCTbr3FZtxNRK643gdrSPueef", - "u48xPgqTs1pejEi3rfdXd52jW3H42NSQXftCqlRs139V9v5TY8J/ehl9/3oyurNCT7IF1SHqXlStmlho", - "Zd+rhuw6u1ftnn61fieaB8/WX32Kqso1ZBjcc7XqIRDkB4Pd3Te7B9dX310XzruO1FfB+CwQvbtDcd8G", - "pHUAoJqbSWtICIHPLgDE60XenUB0JSCvwceGdmR3/2EXpQT22jWYovIRRGocUsGlPPJeEQRgARAveJJY", - "AUbzuaAJeUy2zo5f/HT88uU2GZBXr06aW7GuRnB/boDsjbcArqN1jgqAr9WAPEgR8fvmzUv4EiUMg6as", - "HH5xc7zvjarFDvjffnAvTt9CjQXVE+9u3R5hTMsoffaea6NXcRc7RS18DN64rVoqxrpM0rbhnCo3gpb/", - "WEPVDqJ4bt8C2rgPOVlZznsA4r7P0N7PDwR8LWz3x2JvOz3DLUFvt15pIdjqBo7Jg7bb7eYg2rcynBoM", - "VIhtVSUcj7twY9Tqfo8HYs6fOD8+cnxapqkrjRG++cacHu8Ndw8fYabl3VEXxp7SaE3fJ0+edu98tGdv", - "mSM6PYriI3yw39Rm5QjbPkFocgWRuWP/SBz37IVZed1Wjq0t0y3mYhUc/GZY4E0xrgWWG8VZ77ikJ+na", - "REkdgoqbUIdp7kBjU54kXLNIiljXZeQF1URnFivZJgwqJPixwAH2SZG3HaUUQqNI5aXq0UnXTt7PM0f3", - "mLQ4k0Izl7XkJ7bUJOVoBC26R9dHTYrYtXgstpSPcywCGjFfcQw/YNRQn7inUB+9hjExElQYC73IDTCx", - "bQhAEZDeWDmtLJlytBhtE53bJy2OF1djCQxT85ipsYBiATTmP4rnydEhICP0e8VLDpK7jEYharpV4+fQ", - "wa5jCxgCQ4VLp4fe7ioXJBcxU+S11MwYSpglh6aL3DUNpx+Jce6rdxOvXPVSrgofzE2Q6J2EqY/GeMah", - "trzP0Tv0Bo/zBzcX0TtFCnr51cUIulqT63gwuJAMePFN4bazCjkWOzWkZi7Wo4i9eWsTB9en7lyPjQSj", - "ulqSdycnNbcHF6HRbeLIJVr2QWbX2oa9DTqSjaO5jnm5goi9GQX745Gpm5JKRUL85DjUVQujhy6wFFpT", - "bNUezuv0a/CqDMZAcWH3CYhmzQQbuaMt0EvpGFltVS/1zKGyOlHax1Q4RPoCtKU06uul3pnmeieL+I79", - "rncQUecRIuocBCEPYnY5yfPQ0wg+uX7I27cAe4O/ICK0DZCpdU/p4e6j0aPHg0fT3cPBQTzaHdDd/cPB", - "3gM6mu1HD/d39/bXBBV1iIG9eVhr8MUc8GMuvNYn3ns+5NTcFrvQkE2cP/YVF7G8ql1/QQfZau/O+XZT", - "96uu9Z2HEAwJSqg2Vn3RwslO8JJnEbSN5XycdZEnLqzoPHwz2v1Y7Q8OruWOeKNyYc2qWKA0IaSVAVc3", - "qz7Om7F8HJAPfNm0WtXOuy/a6OjB46MHH7toPnhj0xib5HSHm9vmEeaR0xvRIT5IuKJH8grKnpOJrFbf", - "Fl/2+r1cVP9GYaDhS1187hTE1XZg+2E2su5aaYE8OK69V9BTxSJnxkcgqfj3COZ8KYA1QAR6CrowUtHF", - "WSBBtMMdV94u0AyaxZyKzgYu22AMjIzk2sXzcQGMGO2P0IjDRzgiL7AsfqKpfda5QdjE3VXTG42X1l8G", - "zpfv2j6y1g/5zL2vsA48tgj8C6cNy+BUtuubsNLZEflZYp3itSdkU/dri+Mza7V4oyzZciD7Hu8GO3Oi", - "5hH5oRAvCwHVCaRbmrk/J45hlTBT2zWwD7fj4EhY2bkKcEW/Z1e01+/5hYI/A1AXb0uqXzl/VVIMOZIx", - "muBZLsP0c8MTB66PM+Ha8Ei74BHY3Daxx8VmsnhiH09tPqk29NWWIUUlL1W9OyFbiJ/6F+IU2/Cv7cJ/", - "tXbX7T0+eHz4cO/xYSeUtHKAm0Xjp4hMsDq4jXJylOUTpxtpmzok0cM1j6xWofB9AWtXSeaZksB6YOa+", - "wWrnj4ePq+BwscyBDxdDckiSFosaNyyIf1jwohY/yN94cslnM/Hb79HF3t8VT3ffH+q96W4L6LXtKKx2", - "O646F6zoqNl0YFN7hvG7kKCUboW4e800zoCcMUOQfgaERvi8KeKpHcnlurbiQcI62N/ff/TwwV4nunKj", - "qxycCSrhApeyG0FZkmBJsvX67IzsVAjOtumRYLIAzEHjnBGXoH1UF0iHu6P9EJW0XNwl1bi2L9PWJX/n", - "no9uUm7RMXi7eFqunPLgau/vjx4ePHj0oNsxdurhiXq/nsO4crY/nz6juvNbKE2+eXJKoHU1oxHksn9i", - "9cGj1oyKuWbk5ydvLLSTkCRjalDeiL6lepir8zW71vzMteaHSWRs8ocNU6x5qjw8hOit3W6ojyH3FYdn", - "Wjv6dS4YOL4B8grsa2ApVpl4v+3eCQlmllRfsyihPH0S+WCNxj1mkzxMlC1WbkKXK8a95VeuwA51O6nJ", - "CuWXDfXBOkQqkositdBws3H3k9ho2/m/vWg23w+hAB8Bq+fgyWy6xhssZabYJZe5/gQNSWOjb2eJlOpa", - "dduePq+ZBgueB8V6d/KNJo7WiDYsq7/GHDWuAXG74eSudZ5rJBIm8rbF6rQbXbZ+3YT7Lae2vw6ao8YN", - "WqETY+BcudjsRvqUJlGOybRosZ8wK7woEFwgy5KljRJIEikFiRZUoLlFudR9Yk4oARSGYdCnFr5MZkH/", - "DHlFEmlB3yHZj8szZQcB1UAYAs3SVjWboyWlRvrnB6llMi6TUJ0aH6ThTLVhdKYiqB7WkxpZwSO3VWo6", - "1UTONT4vDcY/DJtpMDKqbFgDFTZv2mVqX6UB1+3AEBvMPHSj4kc4mPap7IQXDFm3K0kjJXUFWOzdiR52", - "jZ4r4qE3u4bWB9uBdK1pNHCV4RWnO6dXDN2Pgcigj7khkYYx+nCN06VXc6ZU5Jh5qkLITqU/7OxYCXrt", - "SQGYdc3BajPBdDKgCixsYEX8fqFZ8mWC96JnbTdZLufBfKPaK1QVbqptgO08Nbii4dXqFzQYIuNVhLq1", - "oHglVGATFus6YJVl/hGusVVewSAkWxg9U2FLFSzJ7S7uNuHHL/Sz8u51mYjB4tEVo3E9JOMpNYtjMZOr", - "nOA6Nliv03Z+rxlTHiYxZoKz2L9KC2OsU5px7fLQxTlzK4fd1oD00FmC4svJadsgWKjO65sddlE02zGs", - "zzaD/bqCHXaS63Bo7huV41ohAQHxlEG6ndxJuZ6EDWOrDSs2zxOqEDqw45D1Mk24uOjSul6mU5nwiECF", - "poV9JiGEbAKf9Hc4l+1Os4MKk7bUZGd2cC7C0G5Io99yCt/BLLcb8c2o09mx9Xegfic/tKDP+Q88YQ7l", - "8q3g7yuEXk/HcLA3aou7b2m0FnG/CvN6Xc7tSDZ44nMdiFJcK+X4pGosdjkysDQmh8Oj1uFW8iDM3ph4", - "M9tQPQTl40BOnlp+3YA4IVOGEUR+aivNdWGLXaYSTC0D2fb+Lqd11WpXB+JAwsItUYJtKDYLRgrgjq5V", - "bdsSK2tS2d3roFkgW4WJYqVrgkRsSu1Yemq18ZPXK1kOF8wtGfdztBkPO2Tw8Z4kBRCC67U7IkIzH2XA", - "8xnxbrRZYmZnTKe1rCRZhTRlSiECPUg4UvjZIICLVHatPYIWxHUviWIp5WIsuCjUrQjLxohgl0xV4m2l", - "gkfWHII9/lp54sHvLM3M0iWDQDX8N5qAMc6PcSyqg4TGQY0+JE+E1SyqPDO1dLEsdq8+IBSfJF8xozB/", - "KjcLMlNML6pzD+XMBRnvSqq4NRnZkvgiMA2K1lpi5AUTVVZWNBN8GtqGJrbWqj+gTXiNX937k9RyUJNm", - "jun1/eWaqbCQWEypKNLJCaZyVCqPEzyQMACXW9T9ZVl8gafSAT2lbP6/fZPlT6dF4/XfGsUqCCkeM/yJ", - "VdsGVbBRlq+uTsMmt9HpBQPq1uHirFolyJZ3xvYJmvR2S6KkTvdkt5i+ZtiBH82OZlG994NHDx4edsxU", - "9VFmP4sD9qmNfJfpGuNey06ddLEgPXrw6PHj/YMHj/euZWHxESot+9MWpVLdH4ARMXBYk3/945/vTurU", - "svcAvblH1xpUnrUP6W3WYUDvTv71j3/6Ud14QCFGswr33+IB0Orvk1R30rsc1I2B3Yxka973T2pKAlqw", - "GbLFENQb3Hfsug3KwTQARbpJwTSjETfLACOnVz7BhSvSQO3u0HpjsCGR17bt8FWBc+l8WpQAHxZX4D+t", - "lblBC486J7zT+bTNov2q2SuWI6XVouot0cFZwVJEWMFdzOeK6ppreMys3aFfxKqtxu3YEuvxe5tBFTAU", - "hOSttRWFshY05ElXqbr9je2s2C1rap3miv+65hy2H8FraX0DN3JA6RtledeGSrkdLsCb1ZpMq6ko1+b6", - "rOWtLG/d6/fbIQ55NelJcYNdv79K6OV1KjZozdKjG4Nb8rLtfo0kWqipElUTUKDJhA0qaRCwNNG5tQjC", - "mXeA9YFY0OhCzmZ11NwH7SjrCCCEYWO+F2oMvEz6oPV3OosmRLdFCxr3Huhxj0gFIZnpuNcwWwUDMVP6", - "fuI6qKPEjNbBnpeB9I1Baj+DaSKjC5tlEZP3D8mIpIwKTXKBh79hVdsdrbcO9XtZZW/8OuLjbmUp3WaS", - "KVvQS46ZWZxNZV5z6WTvudHoeortHJFYWtyoWoppN0MoZsMkj8pJ46VDxdI1DA1COSm8b2xZFhV8M0xs", - "LX5nSvYd7AFw7FevTvrWgQGdGO3Aap6SfqJ2BL1+r+yikaih/D3siQzGChx3E/g/XV3HanQ7WlYV08xo", - "hwRekkODCCB8SZhmRoC020OuHqC2eiXlAt0GnfsHLKjv3RIIiVmEJ1KvnsU6od+AuBsRCG6lQyEI+yES", - "xkOxJpPPa2cQbg7AKhsq2eFtO1UPcWslnGgjs179VE/Y+4ixuAkdGi7S1eve1Qx63b+kDm2oSNzuSqPn", - "9OrshrcXKoZjbVvtanSAkGIAp67YUns4LXKdR72pE1p19avgGJMQUGuoQIfxC/Z+/Vr/zN4bRFqP84TF", - "q2tbHEnLqtxltGnFbxwj2XagpWIb03PeQtpK67l+o8SVtiq7j9yVrtSt5Ktc2Z0zZnzZM0dGrTtUTxlT", - "M2m5JogvUvexsaTUJ+6CJ7vpdoMEDxZhrYiD9+0Y7Qn6FjC+z/j7NcRjC9iHcR0gpTxInhqIRSrdSul7", - "cvAQk4jpxtgFny9Msqw74BwEUJk+KqmrYoYJryjssvPlbvqKq95ubjurrYeE47MKyNBKchQnkk7WIXA/", - "LYp57XxGl6jFaTUSPtw/GI3290Y3guD2w7rGcj0tq7gUqPV22oLzKvUIq1rMLAeqtlCEa6/m1baJ78rV", - "JNooRtMjDOHJaMRIwmYIt1eExm62LDa7Xj94J1C5eNyC/v1GuX3zNvh68p2iKzGv/MuqPnCudTSj6vcN", - "BtEWNhOtgPMFovf2B6PDN7v7Rw8Oj3Z3bwM2u1iktriRh7/vXj1M9ujsIHm0fPjb7uLhfC/dD77DLriI", - "u9LqT1C21cumvCTrqEg1lka23BwyppoJ0+taI0BzTrhgA13EWm0OeFzDC6z9feP5v56e385grexwVp9k", - "VYSghuggZd0PkpebzFrbRXM2x8/Wz+JGsUzNgYTprTkUKNVxMJjrYrf3kRgPueh4Db2tFOx8Ea2Nr9t0", - "FYUs7HjSg7vcsuIh8m5APFRmve4CX73kArrTuVTcLNL1t0VRrAAkR7/p37WJ68hRQ3I8F1Kx+s+Fm1z1", - "EQWVe/1e8vtB/cy437tjiDks44IA3VZXpYIObmQJu2TJ+lXAIuXDQ1lPdqoYLsR3u4Pdx+iHkPx+8N1o", - "8LjucdC3q1Vdvl1fuvbrqMsaVpMJ+iRUu4+v5XHt13MdBf3EQ6nwynvZoRw7Gi9z0/urw4fu1ja4/Lyy", - "xw1MoFYB9GMlPXe5TapCUwwa2RDKfUVRqxuvxyqRkSmbc6G76G33R4Xi9gEobskTBzVuE8LLop9q80Ar", - "VTrhacpiTg2OSpk1EQx7HXVxzbfE9bKc+FoBaW0YFtcebwZb2BRwtemaHH5EZO9HvX67vXjX4YCgXs0/", - "URGNDAv2CZ+BDrrevk97bR8dGBiJ/mpHHvKtJFnHA3QpB3o9SZ/MpSFlMH5HfVsu2vWCxfghR2e+Fn3D", - "EsTeJ4FWKaDI+Dr2dfyMZErGeVTGjyY46BI7ROVi2FXI3zi0W9VvYIg3KDk26zfaFBrd9JNt+93QTQLB", - "tm/17mjzVt+KUqTfy7N4Mw+zhbpxsGthwG8IQQyoaOrL3pAEK5P5tQNHf11dwdU3r9UlRyAS5Zk3sABN", - "rVJSwNyCJoaQX+8zljDDQo0QmcRllATXJRfdzFJ3Dx8t2kycaJFaHchPjGWEGoukhP2lYM0LDcwnMC3u", - "kq2Rxw3X1uA1sImP3GrVB/dwoyTWulVVFW5bMgbL5Rs6b4u8clFRf/vZ1ESzVSwVz/BrQtrr9mQCrqZ3", - "e2vVH9+GWu4+hbRXzvTQAID18MQFzrrvv/QFBrGuTrwHIfN8iCze0Dlm3dtgR2kllVCGSjRyUGKoS5OF", - "qTrhnwTzkWoP/eIFBKzuI3pd2kLCDaFz2sxB0Zbg0lqiN/sQtaeMeUPna9F9m+GwVfX7k8HfrLqdTIZH", - "O9/95f8e/PqfQbV7Q4GgmRrEbIYeVxdsObD5nEBZMaxj22IyiZ42dO7ODKMpKtMQN95xpep4H4wK7rn8", - "maYrU0BXtUoypt2NE/rLf7Q7elWW8S1eGBvP7kfnWrmNnLRG+nt5K2Vq7p3qfUTd9nAsMEjvgi01qaR4", - "my5rJ/YbXVSpuOKTc0vuQyYuz8mUY85MPRbwvKdRxMCePXRZgziORUlkw4rRpNqOSzXnz4uzzFrHCgif", - "XgFGfvX2zfev3v78bPLq9PnPT44nPz3/n95RT7Crge0hHgDtHTw4dOneqyu5G0w5cv3MGUNy4uIVnM/D", - "LMeXPUKfgY0UQtCSJWHvoyTX4DaBS29McvMcGatRyzfPOfGRoMrGJCH3DAf+DYDeMGZ7rzrvIq49MXKN", - "efKdlocLsiq6WMIZ9/BKMdJLEsEMJbAV4dUuN7a+6IeLbneJhZsNHHYM/YXDFtCivuQagTu8E0ylMNnC", - "EBqfzNdHEG9fD/72SdFg0AXzE+dsGj3+FHlV365NpHopkwG881qSTwTV6nYtgiEE2JQNzei1WV/m08Bj", - "xum453xOAwaXkGHlk+Q/9QPaGDq2sv+tieDCAR3Pmpk57LG0S9XIJNHQlmgzaI/3SEG8b8EyRrBmG8TL", - "K06GteZ3UmF2XJ7iEHBILBEffl3YdnnKPODkACttjkZe+8CszKwykva9OZF5aFvWLRDE4wOPVKyyEViB", - "xTdcMheg1AGOBumtjk7mKqN4jnZ3TbYKFZhfgiLsetUusD6jxQl9X/QAJeA10TTE4jzKjFq7L77H7AOv", - "3S4B+3RN4DAar9ww1n6ditatiaeq1c2oUtXqvG354MFzvGoN92s7Ww3iLPuokWaIHv9KuflBKnwXt4O/", - "3DpkP17+MVMIq9cE5O+EZs9TFk9kbtaffygG0qO98otMs2WmYq8DoEjEUS2uuY0XeHiScgyrKw3LwaJc", - "cbMEpZF7mUwxHtSnB8aFxI7w57JjTMn64QNqz2eByJkXTDDFI0x4a6EGBb6YALavzHtoU2CuQOCiCPTq", - "6bHTu3gUZXxHc4Ok5x1Qn5wegzGeKat76I2G+8MRHuaMCZrx3lFvf7g7HPXwVbXAKe5MIWG/CyR3L+ri", - "BX8cO0noe18IaiqaMoM1fglAIqADpiuOTxA6rzwiM8qVe0VmiYwLguFQGzM4+Av1yN7KfbvsnZXHGG+N", - "YUAse+U291cUlPHs4DT3RiOHFW/c9YtBTDZyYufvzo227LeTVOeWKJDQYOWZ52XLYuk/9HsHo91rjWnd", - "UPDshjp+K6iLZmb4On8wGt1+p8fCxie6aHPnF1Y9cUhI1bP2y6+wZzpPU6qWfsGqq5VJ3SYYM9BrubL2", - "HWc0iYBVYNolALRh9juoRKl14Va5wGzVviJQaP0U2Lb9JhdoTd/LePnJlrDWh9dRfKizMzguH1bo+dPR", - "TkHGqxvpPnnQcku1d0BA39Mi1fq9nZSD0ePb7xR0vwmPIFrVE7BzzOYafZ8ShGD3IExSkd9yaSgp4hq+", - "oCPtZNZpQW798ira+YPHH+zxTljIHnDKVEqFjRKxZTYc+pXjbG0z5XFee6t5wj9+1nM3lUcjshcVCnL1", - "I1q9tprC4Op1dNA7auvTTi++R8I/uIMT7iZbZNu9zyOHOU7BrvwlHSdna5yWQkhQlnvBzOdC86O7vLJc", - "XoY/4Sn6Ugj4BSskvHK3Vi6FnUzlwj6AgxLg6zJy09X7pi78vSm/VNyF0K4BTWOGEGMNzDReDolfU/vo", - "B6WvthD96tK6m9XP1ykM73M5YXt3ccJwxoWl6Os19fWaWnfKLbX4KeDBrJzyDjqIa2kg/nz6h2trH77q", - "HrrrHjppHgS7sqUBD3JInGtuJGNG9ALynYKx2QI/eSccQ9Vw/juhKlrwS4bofpj3Lk8Mz6hCF5uUgJXA", - "2tBbFRNr1RJFczvQ3MA7ZJYL3AT00GyCLjCTNiDO0hWTCwHKWqq910yJq7iSod2e/aCCvWiwvBrJ1UJq", - "VgAbClO5zTHOW9vXMTY7HIs3DvEWFhC9yj2v0SxB3N41+h8pCB0LV+Fbz0K8R5ymacm5qELwRG4hOu22", - "rEwVRzrRkQyBDr1hggoz0BmL+IxHbloXbOkcW4MNdkplBQP244S8MN5Ra287DFyHrkthlOJnxTfiKKlu", - "vxHSuCTfpZHLYylRBbkPghlK5omc0mRi1+eCBWyCL7CEW5TS4FJak4SMmc3Kny3NQgr7dz7Nhcnt31Ml", - "rzRTYHMaC4xIcWsN3gqFgEiuMDdemkllrJ8D9rljh7jzxwVbfhiOxZM45cLW0LYKTbQEMwXUM9KDh1ju", - "1UIP9jSF7eBPc21kWoWA9XRnhylzk+XGhdZoZvoh+NOxMJL84UEuP+z8Ufb4AY3FjMZAJ5UidkooW7eN", - "Wk8ozH6CRQPmdoYLMO7BRWrdPOaKCoNjKlE6yby6pVtFmghMQttc4YgKksnMpthAolpQILlaGwhaQZOE", - "GDxKvi4I7riTLfNJabTggk1KBhcwfr3PmOIpExDEiSbVBNxkY6quuNixydKR23nOt4QR7j04dIxvOBav", - "i3ADe/OkNJJ6giwJ9FP2n3bI9lXizg3CMsBhQ6BXy9ZgHQ2qtJSUxienQp5F7W81C/OQnNg5ujbHAmjT", - "ohHBThwekBP+fSUO+1vLKv3KeHjCVMasPxbOJueTPPVdoNkknfYJcCScD8hKpOozRihA6Wges7FwU8Ny", - "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1Lf6C89pAsfCHh8C", - "BkSHcnEl/Hyr/AUqnDx52ifauoK6ehEVY6FycKOFGVWHt47EqgsZ4NvWy8qPwpEv+szWsrRwLeEcxEPy", - "/D0ISNwQhmk8XYicHotcKAbdwwqQd3/DnGXAP2xGYQB5ZQo5H6HK8BmNbP5NtCN+Szj8i2ng5mPBxEzC", - "ZsUypdxmoLtKuDZ6SHAn/DgdiFSuWWNPa1ghfuzwE445mDWjIKhWTFCk3ubNxgU5+b5yv40OHoWvOEc2", - "q63/19mrnwkKysAWbbHSg9KuvgDCAsc2TOfjxIzngFxmbceYMnXc4/G4V3hYxNs4VlgcVOAPBmjm/w6G", - "9p3tps/j74ZDaMp6EByRX/6wrRzB9ZalFqN43PvQJ5UPc24W+bT49muYANugFc9qdzPZspLntuc3MMOK", - "EG6lVjgS0glmwLxIKRRUfcmmXFC1XBfkHFh6t4JyZp2LK4vxxxidice9o7F3Jx73+uMeE5f4m/M5Hvc+", - "hFegwbRa9OuFv0FBRIej0fZmlH63vgE3gg62uk+slmlVVBTJhWEHHUT0n8tk92+tEvq+UKII6LyETbM3", - "sLeH3fdTG3q/g83/WRqQ4hOWMnTyGlTTJMJNgek9B1bOgev4JsbKyuu+qrUKmCsbT3QqIpb4J/pmpfDd", - "GxqBiiIGQu7dnpz7OjeFKb1Ib/JF2c5xs8rzvdbUd88UN7qr265m4rsf+v3ibG0BS5uzs7FLHxYRTu6E", - "wF1O7UZsYXiLnuGYBmdMGPIcfx26/3o9EQLRnidyfn5k1XyQp4QkXLjYoUpQA8gtbi2xksXuKurZfxaZ", - "NbesiPOvf/wTB8XF/F//+KdLiPGvf/wTj/uOxaTEFP/nC0aVmTJqzo8IRMgOaALvMTcZTMLNLplakv2R", - "sw/iJxiSYFERgCFzeGDCg9/kSlRirGySS+0adGZFmA8XOdMO+wwK8pnLwGX9oAM6Xn+W7VLe6YnuBzBk", - "cQaVCcCt6GkAATi54BgTZp/PvbB5xc65ZmBpunSvOPZu5i+GvTeWegd2gNdkMLjEoXOHH9ykydbZ2fPt", - "IcFnoKUKzLKGj5qyGfe+GX7lSZt5kuUodYaCq2x5U0QzOuUJ9+aJlhRR9gg6lUoZi1AkaPBNHPmRAo8B", - "t2znNN3HomPx6mwHzTGGRSZXrO84gXKwzGUOSeli4rAH5F/coCfpwJUdixmj0AY5fmaZQCVzQRFEXTQs", - "EP0I/eG5qaWr7I+Fhd92WiU4eCDJJlgJ+59Tw67osk8qkq9NKZVQAy913YfCY2HDkt0aDBDfiVSGOUR+", - "Zoc08F7/Lr5TsRkoenAiNkOf7XsLFB4uGqJfxqD77mxkuh0WkU4ch/nN8YkqrfoSW3p15ndjG/VwUcKR", - "GlAJN0enQY94LkVtV4vg0wVV8SCScAlUEfAgWiFh8byNxz6ttNW7RUmm1k/gOP3YJNcvTbhYrE4ADrFF", - "9Vxv5X/mynQz87sW/0x2frtA1zH0W2sPs/zGru5Xo38Ho3943bwDQMgK/8zD1t5edIDt4p6CAzztra65", - "/VJZsvtQNZItjweGFlSpyOnTY0LjWDGtt/+9FZEwU0ulpfwH9yOw4vtwU3Njkcruhde31AnkS2EHr92o", - "CfXzaiYlr95vO7WMZa03XZG8rLzybv/2aHR6nWukFHpLWvt6k2x0zOc6gjDnKrUMUDRKWCG+FOe0SkWb", - "tMrW5b+4ctaKS7YU+FO5A3l3+mXXdS6ad8MdMMVnDYZ4j4ywDstAqCjQG74kan5b7KKb1zr18+dFmqO7", - "k4LuWhUdIvMv6bkYN5YNuKAFRWq9QF8wY6GQbvOd7noITBz03O5U24Eu7ayLadmq1gHHTgg1Mevfvse2", - "SLenr23vz/TyxeW5jsTilvyriNLhsVuu1boH7rHLm39779sa6uMd+9M4AgssMnxwummEUINUtFQvRbT9", - "1aXmk1M0LnblEav8vFlcaLLxe/nOuiu57onwGvUc3MZ18a7lgswSQJS2RoCYz9CJ0FQQEi0j2ruDUb52", - "7t1EUcOc7+SXiBFwmifWuhFJcclUBdW1eqXu/IEO7pufSp55rb1d375+OWAiknFhPGmXSd2XT/xgsvRf", - "i/u/+1P3Bca+cy8etAmMH7H/NvCEFDC//2vvBwf0+7/2fqBJxgX7X/tPEmqYNtu3Riyju7rp7voB8wUT", - "H7xfeH3RVljTjqHzasx6nTI91PV1KLMGUe2V3UXQ3Sejvk8v5DVxvTtJeHdG9/CSumt0o2OfwsRuqVQe", - "6a4BX/7vLfSd1Qj6zvV1te45hsJYBIgvEFJJKo/QTB3EPffE7XiTwFCcTcqIolRHfYQv/6dSSdhJX0sp", - "UazrV71EF71EdbnWqiZswdtVTtg+7sn6XhBbaLXx01dovjsw6DiKrEDz1SzcJTjfQmqDnz4PnJZ/p0gc", - "XhyF6n3W0WRacoq195o/U8fP+rjDfdjT42clPsMdGVD9OO5cI+D6vXtp7Ek65fNc5roamp1SEy2YdrAo", - "CavfDF+arqKUG1q1FZ8xlY7u8k67c2XEV7q/JTVJc0Mt87ZeEJseI75U18eIK4/QHAgZ76BjXIK3vk/+", - "ud0SneJB47uScQ1b/0O/y7hCjyRImKNJ+Y4h+LQ5Gov/DQ+jXwyj6a/f+YD4fDTaO8Tfmbj89TsfEy9O", - "PKkwYRR3uZqf/PwMXU/mGF2A6ZxLRJzmOCwMCZKez9v0b/dyK71vuj/dPBV+fbp1erpVlmv9080WvOW3", - "Wz33250/3jy9hRbcfvuTPt/+5Dbl2tNS57MZjzgTmEEPtZJ6xYnaPjG/mpNvph+lwjlxVDwwa5JI52dk", - "wbU2SOhFWvZP6uLYb02kIwk1hqWZASS2iEFCbxs6qBe5AeAnn1cHJ+hTNPJyPqHr3Tc18Y2EI3dr+X/W", - "Be4GX7pFbtm7fuq6jr/Q0Flpg+Vj/7gsRZv21+X9Eu/tvik7XLV3/6r8kknMPt9Wly6DF0IgT6S1P6W5", - "jTMuapYQs0Py5s1LH1NsAfxc1lEjfapRn45+LKqpRhEGz+dwtQXqEIAOgwDq++SdMaNxwgXDIAymQ+G/", - "9QTB93osPr0EHM5+fNe29y7H0m7oPUrA98YK7kTWLJwNjLMUF0ezkhi5OC1e3sRT80XxK8eAAownJOvt", - "0NzIgUMp2FlIC3MbRvo+TWiEQN9QzGLQOmAYCzpdbQrRXpRMEqYstnCWGy9uAcy3GxwXRhbZ851kdg7N", - "T3JheHLetz6QUDbRhIqlQ/Mbi1pnTuZD8AYEJsERKpbZETdSgcOgOagaoRTiLFS7JDS5oks9Fg7uwVZP", - "JUZFRxaGO0mG5EeJSDs2JX6F8dp81N/osTjnccImDijnnHBN9EIqwwSLSSovma73y6hKOFM4iacUVk6T", - "lC5LnFG3PjJjFq6yBscj4d9UxBwzG0PPxZSPxoKSvdGIpIwK7cA1NJ3hhePaIDiI2oC+JZQcjB67Wo19", - "wwwMfvm34DQpxS5lRKeQxBqoGIbIzTZuYOoyjSumjVSwfTOutN2vQr/pUsjWNpZrnzg77pNclPAhqOuv", - "WNdgu0yuBM7TmScZV8U16BCTpiyisJ5C1vtBXGsZRbkKXZCw1ZWU9/+OgmNleme4VGFwjgRVBhHEAEhE", - "flzgmZZ4lLa/baGqkqj+HBdN8JBIRSip0HWp0WBRjqxxC0Fnz8v8zWLiyp9vf+vPDhxfxwj88RfzL+l+", - "QiKSs1ntAG6+muwBXhcUt0rCf9Zz+tQxvRqLizmdC6kNjzwz9DD71Ufz1wdhpwfh+pUNUvNMqot2T+gf", - "pLro+gLzHptf1kOsOsPP0BABw0Mnnfu3R6A23D5WgGju/JHWpK/ilKLQxY0ufJUJJDpgqqqVv3OzQfVV", - "t2WRNuEyVdbYXeCiwSNk4n60uf8r/uZoYohcq39aZzUkNtjs0mdturRPjxJJ7nq8Ek5VFfDBvgW19T/o", - "e3EI6cpv2BZK76vbFWSqADS6Eam16NzDkgagWsfirbbJLc6t6emcFDwYBFqb48GlHwHYVvgN27eorjTL", - "zgso/e0j8gIPcmXbbOdbNnMS0JqWCbNorJdpen60mv3+3ckJVsIyC5vn/vyI+Iz3xf2hoVQVhhVmkVBt", - "yM8OXHareIzjjp6DDrMyv20H0FqmOhiLEFgrYJ3aBvmMnFdwW89bQAU9v30p5/qzMRWVOaPsXIwk7umI", - "tMlE3Gtz8uBJ2PCzOxqF8iZ0hI+1w7hl9NiVwbyU8yJfVY2UaZZ1JV83TKTiyzRdQ8Nkq+QgRJtY5uYv", - "2sRMKazsqLuNuMkWjew/DL0AQnXIo/5gb49Fy1LZGYaXCphqJV+L/ddlmvb6PTceqHdlsjSQueXj4Xib", - "DX7oh3aogrn7JxbP7+1KdLnNiqPikwVZice9oADKFc8LbulNoH/rN1QF+7dx3TkdQPsz4rUt8Kc3cXpF", - "452J7+5F0xCfgvRxb6cYxayGLvxzkuQr463rzYQsUJVxZI/vYmQIvUyiRGpWs5N9OcCNTl/YEL3b9W1+", - "jQcwvDj3WTG7OAKduapnvuZnoMLY5HLjx0z8dO/c92Z1BF8yCIVemQ2wvwba3yannM+ekD7dlqxMtQuF", - "fKXN6ytrOxEmPLdWWYSrELuMo7mRKTU8wnSA0UJKXSH7Aprf5tJ1OviCMlFDZZUFLhDjHEj13Gnzz52I", - "eeQ0j4RWP7k+hljdhW+Ea/hPZY0fKsqVguP3/QsKM9NoSJmmOJuRjOaagZyZg0ZsGSXM539kYFePaGZy", - "xTDbNCMpFzzN02rOBdixS4r4UOe76XmfTHNDEtCyqeKj91mKZJoyEVs5aSwWjF5ypmBREmqYiJYDzYTm", - "6BoFaR8wZydoarKYosEMAykVAwrEBBYpMzSmhqKgcw4nfmJjwc5JphgSkdWOCPa+pIYY85t+a7PpQLPn", - "fqDnhGG6CK4XRYLziMZMRME0CmefNxv79Cr9M2aaE70nB6sb8dL79Liqqq79cD4PZ6wvFvKjA5tfI/Tq", - "9kd1PYjGk9G/55G2c/VzvCc7XbHE607x52GgK4juszHS3b8VTioS57a7yqlEMv+z6hELhlJDhIAAXbuN", - "N7WvFdlZi2W+Fs/b+cP/eXwD7eJnwgn7rQ/7tjyA5aQ/B5brVvVGPPee1KpOl1TVyd0fC3aDuj/xSaoK", - "l/tSlK015L2Cb1e5k1EUX19SfGXbTbbt/EZuyra9bnbFM6LCyLkYoKttmIM7NW4rq3aqg3/ToJ7G7Cos", - "895ZZGm5uHMYTc8aM7oEVcmfwdd6jf0okkpZFBFq2JcF/V3RGlajLFA3VyYY7fug13cnJ9ttXEKZtTxC", - "mS+YQ1Qim6BaGq8qcF9dMqV47OFNn548K22yKhdD8irlhhhJLhjLysAgDM4E1M4CT6Ux7CZwSr/HhFHL", - "THJhNo6iLHo7gyl/kA5W5jMUJV0uia/m8M7mcNTsf3nsDLkMocUE1r9MDTUbc1xzMZMqtXIZncocWgce", - "BMsE+2mdLWY8YXqpDUutcydAQcBxw7xELve8q2d3uY+uzXBybNRhxlTKteZS6LFwITcZU9A3VIf2K35q", - "QYOAoQV/PbVM8vPwgYTBWLc/atpWDZGvMKd176i3Q7NsJ6aGtvjZueF9xJB+QKdGopfpFAIowSvyQpOt", - "hF/Y5wm51CSBP7bXekVOsN6nzqx/85MFK30sZjKYt9TSbEHMf6rgOMfWvGHys2Br9/TO/GgvwhesenA9", - "L5zJVharN/JYxWgywLz8HouJ5IYn/HfLdqERrg2PbBQZLfbx3UnB4IdjccKM4pG190bgf44gFfjS3cmU", - "jHYAx28/yjgC+u0zHBwy3/bPKfb49PQtlktZKtWyPxZUuIbfPDm1luYZjVhzoIIZsCST451XG3zWz3CZ", - "/o39J+0E1wJSBDf8q3ny+rAxrWdItxxRma17tsnsT+/g66TJrzqOL1PHATRczmarwGrzAGthfcalTPIU", - "/mH/ON4EVWdotHiHRT8bydsOZ2M3foJfxKF0c4qZzfF8LwYYu2Bfqv8sLJyfAgoxNc/E4C3wxPwZqfvT", - "mxKq6/gZml7divr86Z/N2brrm8+NwYOmVNfjSznmltL8TIxcrwm7orxdE/Z9IqML7dB1qipMohhFyHz4", - "sYQ4d+ZKFBMw2Jc4VCqLfcZ0fywaylAL4qQJJYaplAua7OCcbSMI1u41avRScoy5jzBmZqBB1Z3JJEFE", - "dkQ0hNlAiZlvoGJd1i7DZLVM1TBqJMA2yZR5APvt0NPtr5SbH6Sqo9F/LnzxTWX9EeWRctT9bwDgb+/x", - "owD5T+h7dNuOc2fc9iPaeiHLH61aqk9wb8a9/ZEe9/pk3NtLxz3YgacU1bnUkAck5SI3gGz5zOraMKr6", - "cEQ0i6SItcfR99rE/ZFumbkjy5aA3UOsd5dij6MqXMrXrpMQe4ByBOpjABHZqh44dybjPh66mMjcWNOD", - "O1euVMwMqke279xuXDkjX9/2XTj5X93xrfEo3GVgl5Wtt5w9y/WCtavcXtqkWbmZwjoUSbb1gvxdTnUf", - "og6sZh4Q81b4HtQ+tR3cRe4I6Oo6eSPc3L8mjeiQNKJcqzD+pnX2hCvZU4cF4WTvM6kM/O5wCxwN4UsC", - "wUAwFearp8djEQErsmiRiqUSuZODuLe38JO/npHnT1/3yTNM+Ex+zKfbQ/IKIm2wVe3sRWNhJTHLvCIK", - "wOp2BnHoerZjR+q5Tcd16OBaUvzep7P04MkIWHn8XnmH9X5vwWiMEskfvZfSdhYAkn79Eg4Q7J+rWWx7", - "b63wATp8tRw8mRmmVps9cTFbooBBcZe0RxV0ght2jB1qD6ZX9mllA4t2sr/XC4CffPiax+P2E4XfjcUO", - "+3WpHSGa70vNEYwHsWCOIRZYva6LTBhtEcuOl619YGCXx88+yaPiVq3U63hXLV3Av+vpwpl+sYamrLZP", - "QMRFBp2Nll4fqLywCNe2GoloRiNuln1Ck8TdUe4mKLxjBoX4O1WMXoDefggoZK5nH1wMttq+N9SSmOsL", - "24KzxQ4JuMfpfFoMjuBBs1ZjXHMI0TWSRDSJ8oQaRthsxiKMC8aUPLrFllsMpXeLZ6fsJAgLU/Gwz7+4", - "tIVhmsDdK8miSXE7dqt3FIsSytN2PHknqKHzI7oaTKFRKcB3IbEURyMltSauqQFL+JxDWg30lIBMGqDm", - "oikbiyyhQjBFcm099GHog0wxrXPFfANwdzmK6pMSKzJT0jjXhERKpcfCU/i7E6INy9aQ2Wvb8gnO+ZZk", - "W9u46+melNSNMbSrQlwRAhtiKcUuONARqIbuwS3eDui+pcQv5eC/UXwOvq+ZktQyWeuXZI+1X0576GvR", - "060pTM+KUt1SmBatViIkK9GDa7H+JiV8ety7ngdioPML3goH6T5dL6L5J6jUse965Gx4EO7TR84ylI31", - "3zHx6VklYLGrAquk8C9NnVQZee2o1oJ+N0N8dY7yvc2o285YXvcG4fUlI3fRWihv24P38yOE0d0iTtx1", - "3rwvm7ZqyFu1t2kL/MDmBAmfBQXeTmaEe0ZcuUFmhM8KAwBo5x6xWIIH9b5i+mu2Z6m+Jje41VB+m+EA", - "oeHaQvkt13POq2sfSu9cmW7PJNfin0mCtwt0HfndL/vXV3+HJ0NlsTaZoIHgWZqZpavkbZWl05nmv7Nh", - "iyG48Fu9PVPwDVw6Px15eDptdei8R/SyezTk3IvPqMsGyTW4+ZVZ93xg9xeGd1g9c7WLZQdunQFV0YJf", - "snale/0EuyXKFBtkMkPjSmwXzK2Hv8sMVcP578Q17/Bf3b8wnSgmMmAxiblikQFvEWEkcgTbxzeaKCmN", - "+y7Vst1LxB6RH5RMn7jZbLgP3ZlyyrDSzzBdDmJq6ODSc5s1KrSP8O70/pTA8AgX5MX3ZIu9N8omLSEz", - "ePlAtLlfUvY+YizWSJPb1QHvjsJDhIYn82mXUa5JP/MK/6AJiXJtZOr3/vgZ2cJ0dnMmYC9A1J+hJJsp", - "ecljFtfGCM77dlV3Wxb0unpXECqKXIT+cWEHdy8yTJcLaf47z+psoXCJmXJBcXAbE7zUz5QFFID+KBfe", - "AcftkR/F1yvMvfy2/GMHKBFTq7pFNFJauOntr9fcl3zNVYOh/J1Wu+28e8565XW3+KiOYUu3kYSiiJ27", - "W7X1u88npIfrLzKax6nOL4sHaZva/PMiwdHd3Q93rS5/9wWHgIKy/LKxbLYBdRkmmJfo0x2zS5bILMUU", - "91i21+/lKukd9RbGZEc7O+j7vZDaHB08frjf+/Drh/9/ABXpBjflAQIA", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", + "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", + "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", + "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", + "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", + "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", + "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", + "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", + "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", + "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", + "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", + "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", + "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", + "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", + "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", + "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", + "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", + "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", + "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", + "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", + "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", + "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", + "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", + "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", + "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", + "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", + "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", + "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", + "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", + "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", + "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", + "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", + "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", + "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", + "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", + "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", + "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", + "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", + "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", + "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", + "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", + "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", + "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", + "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", + "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", + "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", + "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", + "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", + "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", + "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", + "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", + "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", + "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", + "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", + "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", + "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", + "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", + "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", + "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", + "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", + "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", + "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", + "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", + "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", + "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", + "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", + "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", + "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", + "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", + "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", + "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", + "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", + "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", + "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", + "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", + "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", + "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", + "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", + "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", + "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", + "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", + "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", + "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", + "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", + "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", + "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", + "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", + "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", + "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", + "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", + "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", + "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", + "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", + "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", + "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", + "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", + "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", + "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", + "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", + "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", + "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", + "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", + "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", + "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", + "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", + "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", + "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", + "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", + "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", + "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", + "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", + "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", + "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", + "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", + "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", + "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", + "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", + "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", + "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", + "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", + "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", + "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", + "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", + "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", + "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", + "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", + "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", + "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", + "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", + "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", + "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", + "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", + "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", + "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", + "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", + "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", + "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", + "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", + "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", + "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", + "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", + "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", + "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", + "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", + "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", + "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", + "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", + "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", + "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", + "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", + "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", + "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", + "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", + "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", + "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", + "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", + "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", + "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", + "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", + "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", + "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", + "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", + "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", + "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", + "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", + "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", + "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", + "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", + "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", + "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", + "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", + "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", + "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", + "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", + "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", + "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", + "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", + "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", + "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", + "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", + "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", + "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", + "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", + "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", + "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", + "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", + "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", + "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", + "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", + "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", + "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", + "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", + "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", + "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", + "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", + "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", + "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", + "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", + "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", + "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", + "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", + "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", + "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", + "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", + "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", + "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", + "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", + "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", + "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", + "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", + "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", + "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", + "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", + "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", + "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", + "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", + "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", + "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", + "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", + "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", + "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", + "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", + "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", + "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", + "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", + "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", + "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", + "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", + "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", + "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", + "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", + "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", + "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", + "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", + "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", + "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", + "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", + "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", + "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", + "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", + "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", + "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", + "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", + "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", + "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", + "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", + "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", + "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", + "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", + "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", + "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", + "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", + "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", + "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", + "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", + "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", + "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", + "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", + "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", + "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", + "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", + "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", + "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", + "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", + "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", + "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", + "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", + "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", + "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", + "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", + "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", + "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", + "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", + "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", + "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", + "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", + "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", + "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", + "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", + "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", + "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", + "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", + "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", + "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", + "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", + "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", + "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", + "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", + "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", + "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", + "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", + "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", + "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", + "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", + "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", + "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", + "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", + "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", + "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", + "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", + "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", + "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", + "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", + "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", + "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", + "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", + "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", + "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", + "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", + "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", + "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", + "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", + "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", + "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", + "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", + "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", + "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", + "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", + "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", + "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", + "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", + "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", + "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", + "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", + "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", + "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", + "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", + "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", + "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", + "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", + "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", + "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", + "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", + "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", + "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", + "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", + "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", + "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", + "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", + "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", + "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", + "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", + "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", + "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", + "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", + "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", + "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", + "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", + "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", + "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", + "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", + "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", + "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", + "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", + "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", + "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", + "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", + "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", + "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", + "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", + "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", + "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", + "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", + "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", + "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", + "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", + "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", + "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", + "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", + "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", + "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", + "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", + "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", + "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", + "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", + "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", + "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", + "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", + "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", + "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", + "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", + "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", + "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", + "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", + "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", + "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", + "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", + "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", + "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", + "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", + "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index ff2334317..8de7c33d3 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -6,7 +6,6 @@ import ( "log/slog" "net/netip" "os" - "path/filepath" "strconv" "strings" "time" @@ -426,20 +425,15 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } - registryURL := cfg.Registry.URL - if !cfg.MacOSBuilds { - var err error - registryURL, err = builderRegistryURL(registryURL, networkManager) - if err != nil { - return nil, err - } + registryURL, err := builderRegistryURL(cfg.Registry.URL, networkManager) + if err != nil { + return nil, err } if registryURL != cfg.Registry.URL { log.Info("resolved registry URL for builder VMs", "original", cfg.Registry.URL, "resolved", registryURL) } buildConfig := builds.Config{ - MachineBuildOnly: cfg.MacOSOnly, MaxConcurrentBuilds: cfg.Build.MaxConcurrentSourceBuilds, BuilderImage: cfg.Build.BuilderImage, RegistryURL: registryURL, @@ -450,17 +444,6 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins DockerSocket: cfg.Build.DockerSocket, } - if cfg.MacOSBuilds { - workDir := filepath.Join(p.BuildsDir(), "machine-work") - if err := os.MkdirAll(workDir, 0700); err != nil { - return nil, err - } - buildConfig.MachineBuild = &builds.MachineBuildBackend{ - Driver: &builds.GuestMachineBuildDriver{Instances: instanceManager, WorkDir: workDir}, - Publisher: &builds.StreamingMachinePublisher{RegistryURL: registryURL, Tokens: builds.NewRegistryTokenGenerator(cfg.JwtSecret)}, - } - } - // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics) var secretProvider builds.SecretProvider if cfg.Build.SecretsDir != "" { diff --git a/openapi.yaml b/openapi.yaml index a51553ab5..73aa90cde 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4683,25 +4683,6 @@ paths: dockerfile: type: string description: Dockerfile content. Required if not included in the source tarball. - machine_base_image: - type: string - description: | - Experimental installed darwin/arm64 base pinned by sha256 digest. - Requires server macos_only and macos_builds. The source must contain - build.sh at its root, which runs as root in the guest. Machine source - is limited to 64 MiB compressed; only machine_network_mode, - timeout_seconds, memory_mb, cpus and tags are accepted alongside - source and this selector. CPU/memory zero inherit the base. - Linux/cache/secrets/admin options are rejected. The builder and its - output get their own machine identifier and MAC, so the output can - run beside the base. - machine_network_mode: - type: string - enum: [isolated, egress] - description: | - Native machine builds only. Defaults to isolated. Explicit egress enables - unrestricted VZ NAT for checksum-verified artifact downloads; it does not - enforce domain allowlists. Linux builds do not use this selector. base_image_digest: type: string description: Optional pinned base image digest From 975b48a5132a92fda8bf9b2b18aacc592372fbbd Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:11:11 +0000 Subject: [PATCH 15/33] Add import-tart: publish a stopped Tart VM as a machine image A standalone Tart VM with an ASIF disk becomes a one-layer image. A stacked VM's read-only layers come from Tart's content store, in the order its manifest lists them, under its writable overlay. With --base, the image's layers are the base's, so only the overlay is uploaded. Each image gets a fresh machine identifier and MAC, since Tart keeps its source's. --- cmd/import-tart/main.go | 79 ++++++++++++++++++ lib/builds/macos_build_publisher.go | 19 +++++ lib/builds/tart.go | 119 ++++++++++++++++++++++++++++ lib/builds/tart_test.go | 115 +++++++++++++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 cmd/import-tart/main.go create mode 100644 lib/builds/tart.go create mode 100644 lib/builds/tart_test.go diff --git a/cmd/import-tart/main.go b/cmd/import-tart/main.go new file mode 100644 index 000000000..05f8137e6 --- /dev/null +++ b/cmd/import-tart/main.go @@ -0,0 +1,79 @@ +//go:build darwin && arm64 + +// import-tart publishes a stopped Tart macOS VM to hypeman's registry as a machine +// image. A stacked VM built on an image already in the registry passes that image +// as --base, so only its overlay is uploaded. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/Code-Hex/vz/v4" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" +) + +func main() { + vm := flag.String("vm", "", "stopped Tart VM directory, e.g. ~/.tart/vms/NAME") + registry := flag.String("registry", "", "hypeman registry URL, e.g. https://hypeman.example or http://localhost:4973") + name := flag.String("name", "", "image name; the image is published as builds/NAME") + base := flag.String("base", "", "digest-pinned registry image the VM's stacked disk is built on") + guestAgent := flag.Bool("guest-agent", false, "the image runs hypeman's guest agent") + timeout := flag.Duration("timeout", 2*time.Hour, "publication timeout") + flag.Parse() + secret := os.Getenv("HYPEMAN_JWT_SECRET") + if *vm == "" || *registry == "" || *name == "" || secret == "" { + fmt.Fprintln(os.Stderr, "--vm, --registry, --name and HYPEMAN_JWT_SECRET are required") + os.Exit(2) + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + ctx, cancelTimeout := context.WithTimeout(ctx, *timeout) + defer cancelTimeout() + publication, err := run(ctx, *vm, *registry, *name, *base, *guestAgent, secret) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := json.NewEncoder(os.Stdout).Encode(publication); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(ctx context.Context, vm, registry, name, baseRef string, guestAgent bool, secret string) (builds.MachinePublication, error) { + storage, err := builds.ReadTartVM(vm, guestAgent) + if err != nil { + return builds.MachinePublication{}, err + } + for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { + if err := images.RequireClosedFile(ctx, path); err != nil { + return builds.MachinePublication{}, fmt.Errorf("%s: %w", path, err) + } + } + var base *builds.MachineBase + if baseRef != "" { + if base, err = builds.ParseMachineBase(registry, baseRef); err != nil { + return builds.MachinePublication{}, err + } + } + // Tart keeps its source's identity, so each image gets its own. + id, err := vz.NewMacMachineIdentifier() + if err != nil { + return builds.MachinePublication{}, err + } + storage.Platform.MachineIdentifier = id.DataRepresentation() + if storage.Platform.MAC, err = network.NewUniqueMAC(nil); err != nil { + return builds.MachinePublication{}, err + } + publisher := &builds.StreamingMachinePublisher{RegistryURL: registry, Tokens: builds.NewRegistryTokenGenerator(secret)} + return publisher.Publish(ctx, name, storage, base) +} diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go index 5f09f21fc..8e9384d61 100644 --- a/lib/builds/macos_build_publisher.go +++ b/lib/builds/macos_build_publisher.go @@ -182,6 +182,25 @@ func (t refuseRedirects) RoundTrip(req *http.Request) (*http.Response, error) { return response, err } +// ParseMachineBase returns ref, a digest-pinned image in the registry at +// registryURL, as a publication base. +func ParseMachineBase(registryURL, ref string) (*MachineBase, error) { + registry, err := (&StreamingMachinePublisher{RegistryURL: registryURL}).registry() + if err != nil { + return nil, err + } + parsed, err := images.ParseNormalizedRef(ref) + if err != nil || !parsed.IsDigest() { + return nil, fmt.Errorf("machine base must be pinned by digest") + } + repo, ok := strings.CutPrefix(parsed.Repository(), registry.Host+"/") + if !ok { + return nil, fmt.Errorf("machine base %s is not in the registry %s", ref, registry.Host) + } + base := &MachineBase{Repo: repo, Digest: parsed.Digest()} + return base, base.validate() +} + // registry parses the configured registry address, defaulting to HTTPS. Plaintext // is allowed only on loopback. func (p *StreamingMachinePublisher) registry() (*url.URL, error) { diff --git a/lib/builds/tart.go b/lib/builds/tart.go new file mode 100644 index 000000000..46a454708 --- /dev/null +++ b/lib/builds/tart.go @@ -0,0 +1,119 @@ +package builds + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/images" +) + +// Tart's OCI media types and annotations for stacked images. +const ( + tartDiskMediaType = "application/vnd.cirruslabs.tart.disk.v2" + tartOverlayMediaType = "application/vnd.cirruslabs.tart.disk.asif.overlay.v1" + tartFileDigestAnnotation = "org.cirruslabs.tart.disk-file-content-digest" +) + +type tartConfig struct { + OS string `json:"os"` + Arch string `json:"arch"` + ECID []byte `json:"ecid"` + HardwareModel []byte `json:"hardwareModel"` + CPUCount uint `json:"cpuCount"` + MemorySize uint64 `json:"memorySize"` + MACAddress string `json:"macAddress"` + DiskFormat string `json:"diskFormat"` +} + +type tartManifest struct { + Layers []struct { + MediaType string `json:"mediaType"` + Annotations map[string]string `json:"annotations"` + } `json:"layers"` +} + +// ReadTartVM returns the storage of a stopped Tart macOS VM directory, for +// publishing as a machine image. A standalone VM's disk.img must be ASIF and +// becomes a one-layer image. A stacked VM's read-only layers, base first, come +// from Tart's content store, read from its manifest.json, with its writable +// overlay.asif on top. The platform keeps Tart's identity; callers that publish +// a new image give it a fresh one. +func ReadTartVM(dir string, guestAgent bool) (MachineStorage, error) { + var config tartConfig + data, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + return MachineStorage{}, err + } + if err := json.Unmarshal(data, &config); err != nil { + return MachineStorage{}, fmt.Errorf("tart config: %w", err) + } + if config.OS != "darwin" || config.Arch != "arm64" { + return MachineStorage{}, fmt.Errorf("tart VM is %s/%s, want darwin/arm64", config.OS, config.Arch) + } + if _, err := os.Lstat(filepath.Join(dir, "state.vzvmsave")); err == nil { + return MachineStorage{}, fmt.Errorf("tart VM is suspended; shut it down from the guest first") + } + storage := MachineStorage{ + Aux: filepath.Join(dir, "nvram.bin"), + Platform: images.MacOSImage{ + HardwareModel: config.HardwareModel, + MachineIdentifier: config.ECID, + MAC: config.MACAddress, + CPUs: config.CPUCount, + Memory: config.MemorySize, + GuestAgent: guestAgent, + }, + } + manifest := filepath.Join(dir, "manifest.json") + overlay := filepath.Join(dir, "overlay.asif") + if _, err := os.Lstat(overlay); errors.Is(err, os.ErrNotExist) { + if config.DiskFormat != "asif" { + return MachineStorage{}, fmt.Errorf("tart VM disk format is %q, want asif", config.DiskFormat) + } + storage.Overlay = filepath.Join(dir, "disk.img") + return storage, nil + } + storage.Overlay = overlay + storage.Layers, err = tartStackLayers(manifest, filepath.Join(filepath.Dir(filepath.Dir(dir)), "cache", "content")) + return storage, err +} + +// tartStackLayers returns the content-store paths of a stacked VM's read-only +// layers, base first: the whole-file digest on the first chunk of the base and of +// each overlay. +func tartStackLayers(path, contentDir string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var manifest tartManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("tart manifest: %w", err) + } + var layers []string + for i, layer := range manifest.Layers { + if layer.MediaType != tartDiskMediaType && layer.MediaType != tartOverlayMediaType { + continue + } + digest, ok := layer.Annotations[tartFileDigestAnnotation] + if !ok { + continue + } + hex, found := strings.CutPrefix(digest, "sha256:") + if !found || !ValidMachineDigest(digest) { + return nil, fmt.Errorf("tart manifest layer %d has an invalid disk digest", i) + } + if len(layers) == 0 && layer.MediaType != tartDiskMediaType { + return nil, fmt.Errorf("tart manifest disk layers must start with the base") + } + layers = append(layers, filepath.Join(contentDir, "sha256", hex)) + } + if len(layers) == 0 { + return nil, fmt.Errorf("tart manifest names no stacked disk files") + } + return layers, nil +} diff --git a/lib/builds/tart_test.go b/lib/builds/tart_test.go new file mode 100644 index 000000000..fa3d3c6f6 --- /dev/null +++ b/lib/builds/tart_test.go @@ -0,0 +1,115 @@ +package builds + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Code-Hex/vz/v4/diskimagekit" + "github.com/stretchr/testify/require" +) + +// tartTestVM writes a Tart VM directory under home/vms/name with config.json and +// nvram.bin, and returns its path. +func tartTestVM(t *testing.T, home, name string, config map[string]any) string { + t.Helper() + dir := filepath.Join(home, "vms", name) + require.NoError(t, os.MkdirAll(dir, 0700)) + base := map[string]any{"version": 1, "os": "darwin", "arch": "arm64", "ecid": []byte{2}, "hardwareModel": []byte{1}, + "cpuCountMin": 4, "cpuCount": 4, "memorySizeMin": 8 << 30, "memorySize": 8 << 30, "macAddress": "02:00:00:00:00:01", "diskFormat": "asif"} + for k, v := range config { + base[k] = v + } + data, err := json.Marshal(base) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), data, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "nvram.bin"), []byte("nvram"), 0600)) + return dir +} + +func TestReadTartVMStandalone(t *testing.T) { + dir := tartTestVM(t, t.TempDir(), "base", nil) + img, err := diskimagekit.CreateASIF(filepath.Join(dir, "disk.img"), (4<<20)/512) + require.NoError(t, err) + require.NoError(t, img.Close()) + + storage, err := ReadTartVM(dir, true) + require.NoError(t, err) + require.Empty(t, storage.Layers) + require.Equal(t, filepath.Join(dir, "disk.img"), storage.Overlay) + require.Equal(t, filepath.Join(dir, "nvram.bin"), storage.Aux) + require.Equal(t, []byte{1}, storage.Platform.HardwareModel) + require.Equal(t, []byte{2}, storage.Platform.MachineIdentifier) + require.Equal(t, uint(4), storage.Platform.CPUs) + require.True(t, storage.Platform.GuestAgent) + require.NoError(t, validateMachineStorage(storage)) + + raw := tartTestVM(t, t.TempDir(), "raw", map[string]any{"diskFormat": "raw"}) + _, err = ReadTartVM(raw, false) + require.ErrorContains(t, err, "want asif") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "state.vzvmsave"), []byte("x"), 0600)) + _, err = ReadTartVM(dir, false) + require.ErrorContains(t, err, "suspended") + + linux := tartTestVM(t, t.TempDir(), "linux", map[string]any{"os": "linux"}) + _, err = ReadTartVM(linux, false) + require.ErrorContains(t, err, "want darwin/arm64") +} + +func TestReadTartVMStacked(t *testing.T) { + home := t.TempDir() + content := filepath.Join(home, "cache", "content", "sha256") + require.NoError(t, os.MkdirAll(content, 0700)) + // A base and one immutable overlay in the content store, named by digest. + digest := func(name string) string { + sum := sha256.Sum256([]byte(name)) + return hex.EncodeToString(sum[:]) + } + basePath, layerPath := filepath.Join(content, digest("base")), filepath.Join(content, digest("layer")) + img, err := diskimagekit.CreateASIF(basePath, (4<<20)/512) + require.NoError(t, err) + require.NoError(t, img.CreateLayer(layerPath, diskimagekit.Overlay)) + + dir := tartTestVM(t, home, "build", nil) + require.NoError(t, img.CreateLayer(filepath.Join(dir, "overlay.asif"), diskimagekit.Overlay)) + require.NoError(t, img.Close()) + disk := func(media, digest string) map[string]any { + layer := map[string]any{"mediaType": media} + if digest != "" { + layer["annotations"] = map[string]string{tartFileDigestAnnotation: "sha256:" + digest} + } + return layer + } + writeManifest := func(layers ...map[string]any) { + all := append([]map[string]any{{"mediaType": "application/vnd.cirruslabs.tart.config.v1"}}, layers...) + all = append(all, map[string]any{"mediaType": "application/vnd.cirruslabs.tart.nvram.v1"}) + data, err := json.Marshal(map[string]any{"layers": all}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0600)) + } + writeManifest(disk(tartDiskMediaType, digest("base")), disk(tartDiskMediaType, ""), disk(tartOverlayMediaType, digest("layer"))) + + storage, err := ReadTartVM(dir, false) + require.NoError(t, err) + require.Equal(t, []string{basePath, layerPath}, storage.Layers) + require.Equal(t, filepath.Join(dir, "overlay.asif"), storage.Overlay) + require.NoError(t, validateMachineStorage(storage)) + + writeManifest(disk(tartOverlayMediaType, digest("layer"))) + _, err = ReadTartVM(dir, false) + require.ErrorContains(t, err, "must start with the base") + + writeManifest(disk(tartDiskMediaType, "")) + _, err = ReadTartVM(dir, false) + require.ErrorContains(t, err, "names no stacked disk files") + + // A layer missing from the content store fails validation. + writeManifest(disk(tartDiskMediaType, digest("missing"))) + storage, err = ReadTartVM(dir, false) + require.NoError(t, err) + require.Error(t, validateMachineStorage(storage)) +} From e196b4cf6f6fcf4255acfc599c6aadafd2e3d5dd Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:42:33 +0000 Subject: [PATCH 16/33] Document building macOS images with Packer and Tart --- docs/macos-experimental.md | 42 +++++++++++++++++++ docs/macos-support.md | 2 +- ...uild_layer_test.go => macos_layer_test.go} | 0 ..._build_publisher.go => macos_publisher.go} | 0 ...lisher_test.go => macos_publisher_test.go} | 0 5 files changed, 43 insertions(+), 1 deletion(-) rename lib/builds/{macos_build_layer_test.go => macos_layer_test.go} (100%) rename lib/builds/{macos_build_publisher.go => macos_publisher.go} (100%) rename lib/builds/{macos_build_publisher_test.go => macos_publisher_test.go} (100%) diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 081d946ec..1d23fc1fe 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -66,6 +66,48 @@ This schema is experimental pending reconciliation with other machine-image platforms. There is no identity rebinding on pull or concurrent-template guarantee. Ordinary OCI transport does not imply sparse distribution efficiency, boot portability, or container execution semantics. +## Building images with Packer and Tart + +hypeman does not build macOS images itself. Build them with Packer's Tart builder +on a macOS 27 host, as stacked Tart VMs, then publish the result with +`import-tart`. A stacked VM shares a read-only base disk and writes only to its own +`overlay.asif`, so a build on a base already in the registry uploads only that +overlay. + +1. Publish the base once from a standalone Tart VM whose `disk.img` is ASIF + (`diskFormat: asif`). Every layer is uploaded. +2. Get the base as a Tart image (`tart push` to any OCI registry) and create the + build VM with `tart clone --stacked`. The Packer Tart plugin cannot clone + stacked itself, so point its `vm_base_name` at that local clone. +3. Build. The template must set `recovery_partition = "keep"` (the default deletes + the recovery partition and grows APFS), regenerate the guest's SSH host keys + (`rm -f /etc/ssh/ssh_host_* && ssh-keygen -A`), and end by shutting the guest + down from inside, with `skip_clean = true` on that step. `tart stop` is a hard + power-off. +4. Publish the stopped VM over the base: + +```sh +HYPEMAN_JWT_SECRET=... go run ./cmd/import-tart \ + --vm ~/.tart/vms/my-build \ + --registry https://hypeman.example \ + --name my-build \ + --base hypeman.example/builds/my-base@sha256:... \ + --guest-agent +``` + +The image is published as `builds/`. `import-tart` reads the VM's layers from +Tart's content store in the order its `manifest.json` lists them, refuses a +suspended or running VM, and checks that the overlay stacks on those layers. With +`--base`, the VM's layers must be that image's layers, and only the config and +overlay are uploaded. Each image gets a fresh machine identifier and MAC, since +Tart keeps its source's; the SSH host keys are the template's job. The registry +imports a pushed image on its host, as `builds/@sha256:...`. Other hosts pull +it by digest with registry credentials. + +Building on a build needs that build as a Tart image too: `tart clone --stacked` +only stacks on images pulled from a registry, and a local clone copies the +overlay instead of stacking on it. + ## API Use the usual JWT-authenticated API with a VZ-capable, signed build: diff --git a/docs/macos-support.md b/docs/macos-support.md index 8646a1146..704b3d40b 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -14,7 +14,7 @@ owns the behavior. | Stats | Partial | CPU and memory from the VM process. Network counters are 0 under NAT. | PR 1 | | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 | -| Builds | Unsupported | Linux Dockerfile builds unsupported. | PR 6 | +| Builds | Unsupported | Build with Packer and Tart, then publish with `import-tart`. See [macos-experimental.md](macos-experimental.md#building-images-with-packer-and-tart). Linux Dockerfile builds unsupported. | PR 6 | | Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | | Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | | Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_layer_test.go similarity index 100% rename from lib/builds/macos_build_layer_test.go rename to lib/builds/macos_layer_test.go diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_publisher.go similarity index 100% rename from lib/builds/macos_build_publisher.go rename to lib/builds/macos_publisher.go diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_publisher_test.go similarity index 100% rename from lib/builds/macos_build_publisher_test.go rename to lib/builds/macos_publisher_test.go From 8a3906df16ec239158390ed49a03f8f0e21f4ac5 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:54:15 +0000 Subject: [PATCH 17/33] Attach the guest agent to macOS guests on a read-only disk hypeman builds an APFS disk holding its macOS guest agent and attaches a copy to each macOS guest that declares the agent, second after the boot disk. A cold start takes the host's current disk; saved state keeps the copy it was saved with. A launcher installed once in the base runs the agent from the disk. Also rewrite the aux path and ASIF parents when forking a snapshot. --- .gitignore | 1 + Makefile | 11 ++- docs/macos-experimental.md | 5 +- docs/macos-guest-agent.md | 26 +++++- lib/hypervisor/vz/fork.go | 8 ++ lib/hypervisor/vz/fork_test.go | 9 +- lib/instances/create.go | 2 +- lib/instances/macos.go | 50 ++++++++++- lib/instances/macos_agent_disk_test.go | 87 +++++++++++++++++++ lib/instances/macos_memory_snapshot_test.go | 7 +- lib/instances/snapshot_macos.go | 7 +- lib/instances/start.go | 6 ++ lib/instances/types.go | 4 + lib/paths/paths.go | 14 +++ .../guest_agent/macos/install-launcher.sh | 42 +++++++++ lib/system/macos_agent_disk.go | 8 ++ lib/system/macos_agent_disk_darwin.go | 69 +++++++++++++++ lib/system/macos_agent_disk_darwin_test.go | 23 +++++ lib/system/macos_agent_disk_other.go | 15 ++++ 19 files changed, 384 insertions(+), 10 deletions(-) create mode 100644 lib/instances/macos_agent_disk_test.go create mode 100755 lib/system/guest_agent/macos/install-launcher.sh create mode 100644 lib/system/macos_agent_disk.go create mode 100644 lib/system/macos_agent_disk_darwin.go create mode 100644 lib/system/macos_agent_disk_darwin_test.go create mode 100644 lib/system/macos_agent_disk_other.go diff --git a/.gitignore b/.gitignore index 614d5996b..936c7cadd 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ cloud-hypervisor cloud-hypervisor/** lib/system/exec_agent/exec-agent lib/system/guest_agent/guest-agent +lib/system/guest_agent/guest-agent-darwin lib/system/init/init lib/hypervisor/vz/vz-shim/vz-shim diff --git a/Makefile b/Makefile index 3f0732b07..c42f760c9 100644 --- a/Makefile +++ b/Makefile @@ -246,6 +246,12 @@ lib/system/guest_agent/guest-agent: lib/system/guest_agent/*.go @echo "Building guest-agent for Linux..." cd lib/system/guest_agent && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o guest-agent . +# Build the macOS guest-agent, carried on the read-only agent disk of macOS guests. +# Darwin AF_VSOCK needs cgo, so it builds only on macOS, the only host that embeds it. +lib/system/guest_agent/guest-agent-darwin: lib/system/guest_agent/*.go + @echo "Building guest-agent for macOS..." + cd lib/system/guest_agent && CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -o guest-agent-darwin . + # Build init binary (runs as PID 1 in guest VM) for embedding # Cross-compile for Linux since it runs inside the VM lib/system/init/init: lib/system/init/*.go @@ -254,6 +260,9 @@ lib/system/init/init: lib/system/init/*.go build-embedded: @$(MAKE) -B lib/system/guest_agent/guest-agent +ifeq ($(shell uname -s),Darwin) + @$(MAKE) -B lib/system/guest_agent/guest-agent-darwin +endif @$(MAKE) -B lib/system/init/init # Build the binary @@ -391,7 +400,7 @@ clean: rm -rf lib/vmm/binaries/cloud-hypervisor/ rm -rf lib/hypervisor/firecracker/binaries/firecracker/ rm -rf lib/ingress/binaries/ - rm -f lib/system/guest_agent/guest-agent + rm -f lib/system/guest_agent/guest-agent lib/system/guest_agent/guest-agent-darwin rm -f lib/system/init/init rm -f lib/hypervisor/vz/vz-shim/vz-shim diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 1d23fc1fe..010c624a2 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -83,7 +83,10 @@ overlay. the recovery partition and grows APFS), regenerate the guest's SSH host keys (`rm -f /etc/ssh/ssh_host_* && ssh-keygen -A`), and end by shutting the guest down from inside, with `skip_clean = true` on that step. `tart stop` is a hard - power-off. + power-off. For the guest agent, install the launcher once in the base with + `lib/system/guest_agent/macos/install-launcher.sh` and publish with + `--guest-agent`; hypeman supplies the agent on its own disk (see + [macos-guest-agent.md](macos-guest-agent.md#agent-disk)). 4. Publish the stopped VM over the base: ```sh diff --git a/docs/macos-guest-agent.md b/docs/macos-guest-agent.md index fda435897..bbb85c405 100644 --- a/docs/macos-guest-agent.md +++ b/docs/macos-guest-agent.md @@ -9,7 +9,9 @@ is unchanged. ## Build -Build on a macOS development machine with the macOS SDK and cgo enabled: +`make build-embedded` on macOS builds the agent that hypeman embeds for the agent +disk. To build it by hand on a macOS development machine with the macOS SDK and +cgo enabled: ```sh CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build \ @@ -23,6 +25,28 @@ listener and cgo-disabled build path. The Darwin listener accepts only host CID 2, marks descriptors close-on-exec, and provides net.Conn deadlines for gRPC. Do not start the executable on the development host to test guest services. +## Agent disk + +hypeman carries the macOS agent and attaches it to each macOS guest whose image +declares `guest_agent: true`, on a read-only disk second after the boot disk. The +disk is an APFS volume named `HypemanAgent` holding `hypeman-guest-agent` and a +`VERSION` file, built on the host with `hdiutil` the first time a guest needs it, +under `system/macos-agent/`. It is named by the agent's hash, so a new hypeman +builds a new disk. + +Each guest keeps its own copy (`agent.img`). Creating a guest and every cold start +copy the host's current disk, so an updated agent takes effect at the next cold +start. Standby, restore and snapshots keep the copy the guest was saved with, so +saved state restores against the same disk. Guests created before this existed +keep booting without the disk. + +The image runs the agent from the disk with a small launcher, installed once in the +base by `lib/system/guest_agent/macos/install-launcher.sh` (as root, for example +from a Packer provisioner). It removes an agent installed in the image itself and +adds a LaunchDaemon that waits for the volume, mounts it read-only if macOS has +not, and runs the agent from it. An image with its own agent and no launcher still +works; the disk is attached and unused. + ## Provisioning boundary Install the binary **inside a stopped-template provisioning guest**, not on the diff --git a/lib/hypervisor/vz/fork.go b/lib/hypervisor/vz/fork.go index 87c4acede..0ee47810c 100644 --- a/lib/hypervisor/vz/fork.go +++ b/lib/hypervisor/vz/fork.go @@ -94,9 +94,17 @@ func rewriteShimConfigPaths(cfg shimconfig.ShimConfig, sourceDir, targetDir stri cfg.VsockSocket = replace(cfg.VsockSocket) cfg.LogPath = replace(cfg.LogPath) cfg.RestoreMachineStatePath = replace(cfg.RestoreMachineStatePath) + cfg.MacAuxStoragePath = replace(cfg.MacAuxStoragePath) for i := range cfg.Disks { cfg.Disks[i].Path = replace(cfg.Disks[i].Path) + parents := make([]string, len(cfg.Disks[i].Parents)) + for j, parent := range cfg.Disks[i].Parents { + parents[j] = replace(parent) + } + if len(parents) > 0 { + cfg.Disks[i].Parents = parents + } } return cfg diff --git a/lib/hypervisor/vz/fork_test.go b/lib/hypervisor/vz/fork_test.go index 240ddb1f9..fa683ae8e 100644 --- a/lib/hypervisor/vz/fork_test.go +++ b/lib/hypervisor/vz/fork_test.go @@ -42,7 +42,10 @@ func TestPrepareFork_RewritesSnapshotManifest(t *testing.T) { Disks: []shimconfig.DiskConfig{ {Path: sourceDir + "/overlay.raw"}, {Path: "/volumes/shared.raw"}, + {Path: sourceDir + "/boot.asif", Parents: []string{sourceDir + "/disk.0.asif", "/images/shared.asif"}}, + {Path: sourceDir + "/agent.img", Readonly: true}, }, + MacAuxStoragePath: sourceDir + "/mac-aux.img", Networks: []shimconfig.NetworkConfig{ {MAC: "02:00:00:00:00:01"}, }, @@ -77,9 +80,13 @@ func TestPrepareFork_RewritesSnapshotManifest(t *testing.T) { assert.Equal(t, targetDir+"/kernel/initrd", updated.ShimConfig.InitrdPath) assert.Equal(t, targetDir+"/vz.sock", updated.ShimConfig.ControlSocket) assert.Equal(t, targetDir+"/logs/vz-shim.log", updated.ShimConfig.LogPath) - require.Len(t, updated.ShimConfig.Disks, 2) + require.Len(t, updated.ShimConfig.Disks, 4) assert.Equal(t, targetDir+"/overlay.raw", updated.ShimConfig.Disks[0].Path) assert.Equal(t, "/volumes/shared.raw", updated.ShimConfig.Disks[1].Path) + assert.Equal(t, targetDir+"/boot.asif", updated.ShimConfig.Disks[2].Path) + assert.Equal(t, []string{targetDir + "/disk.0.asif", "/images/shared.asif"}, updated.ShimConfig.Disks[2].Parents) + assert.Equal(t, shimconfig.DiskConfig{Path: targetDir + "/agent.img", Readonly: true}, updated.ShimConfig.Disks[3]) + assert.Equal(t, targetDir+"/mac-aux.img", updated.ShimConfig.MacAuxStoragePath) require.Len(t, updated.ShimConfig.Networks, 1) assert.Equal(t, "02:00:00:00:00:01", updated.ShimConfig.Networks[0].MAC) } diff --git a/lib/instances/create.go b/lib/instances/create.go index e7ecdde66..f044416d4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -424,7 +424,7 @@ func (m *manager) createInstance( // 13. Create overlay disk with specified size log.DebugContext(ctx, "creating overlay disk", "instance_id", id, "size_bytes", stored.OverlaySize) - if err := m.createBootDisk(stored, imageInfo); err != nil { + if err := m.createBootDisk(ctx, stored, imageInfo); err != nil { log.ErrorContext(ctx, "failed to create overlay disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create overlay disk: %w", err) } diff --git a/lib/instances/macos.go b/lib/instances/macos.go index b64cd6a41..b3712b47b 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -5,7 +5,10 @@ import ( "context" "errors" "fmt" + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/system" "net" + "os" "path/filepath" "strings" "syscall" @@ -162,7 +165,7 @@ func (m *manager) checkMacOSStart(stored *StoredMetadata, req StartInstanceReque // createBootDisk creates the Linux overlay, or the macOS boot disk: the image's // ASIF chain and auxiliary storage cloned into the instance, with an empty // writable overlay on top. -func (m *manager) createBootDisk(stored *StoredMetadata, img *images.Image) error { +func (m *manager) createBootDisk(ctx context.Context, stored *StoredMetadata, img *images.Image) error { if stored.MacOS == nil { return m.createOverlayDisk(stored.Id, stored.OverlaySize) } @@ -175,7 +178,38 @@ func (m *manager) createBootDisk(stored *StoredMetadata, img *images.Image) erro return err } stored.MacOSDiskLayers = len(chain) - return nil + if !stored.GuestAgentEnabled() { + return nil + } + stored.MacOSAgentDisk = true + return m.installMacOSAgentDisk(ctx, stored) +} + +// macOSAgentDiskSource returns the host's current guest agent disk. Tests replace it. +var macOSAgentDiskSource = system.MacOSAgentDisk + +// installMacOSAgentDisk gives a macOS guest a copy of the host's current guest agent +// disk. Each guest keeps its own copy, so saved state restores with the disk it was +// saved with; a cold start takes the host's current agent. +func (m *manager) installMacOSAgentDisk(ctx context.Context, stored *StoredMetadata) error { + if !stored.MacOSAgentDisk { + return nil + } + source, err := macOSAgentDiskSource(ctx, m.paths) + if err != nil { + return err + } + disk := m.paths.InstanceMacOSAgentDisk(stored.Id) + staged := disk + ".tmp" + _ = os.Remove(staged) + if err := forkvm.CopyRegularFile(source, staged); err != nil { + return fmt.Errorf("copy macOS agent disk: %w", err) + } + if err := os.Chmod(staged, 0400); err != nil { + _ = os.Remove(staged) + return err + } + return os.Rename(staged, disk) } // macOSDiskLayer is the ith read-only layer of a macOS guest's boot disk chain. @@ -183,6 +217,16 @@ func macOSDiskLayer(dataDir string, i int) string { return filepath.Join(dataDir, fmt.Sprintf("disk.%d.asif", i)) } +// macOSDisks returns a macOS guest's disks: its boot disk, then its read-only guest +// agent disk. The boot disk must come first; VZ does not boot with it second. +func macOSDisks(p *paths.Paths, stored *StoredMetadata) []hypervisor.DiskConfig { + disks := []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk(stored.Id), Parents: macOSDiskParents(stored)}} + if stored.MacOSAgentDisk { + disks = append(disks, hypervisor.DiskConfig{Path: p.InstanceMacOSAgentDisk(stored.Id), Readonly: true}) + } + return disks +} + // macOSDiskParents returns the read-only layers below a macOS guest's boot overlay. func macOSDiskParents(stored *StoredMetadata) []string { parents := make([]string, stored.MacOSDiskLayers) @@ -233,7 +277,7 @@ func (m *manager) admitMacOSBoot(ctx context.Context, stored *StoredMetadata) (f func (m *manager) macOSVMConfig(inst *Instance) hypervisor.VMConfig { c := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: inst.Vcpus, MemoryBytes: inst.Size, VsockCID: inst.VsockCID, VsockSocket: inst.VsockSocket, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: inst.MacOS.HardwareModel, MachineIdentifierData: inst.MacOS.MachineIdentifier, AuxStoragePath: filepath.Join(inst.DataDir, macOSAuxStorageFile)}, - Disks: []hypervisor.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(inst.Id), Parents: macOSDiskParents(&inst.StoredMetadata)}}} + Disks: macOSDisks(m.paths, &inst.StoredMetadata)} if inst.NetworkEnabled { c.Networks = []hypervisor.NetworkConfig{{MAC: inst.MAC, IP: inst.IP, Netmask: macOSGuestNetmask}} } diff --git a/lib/instances/macos_agent_disk_test.go b/lib/instances/macos_agent_disk_test.go new file mode 100644 index 000000000..568d99128 --- /dev/null +++ b/lib/instances/macos_agent_disk_test.go @@ -0,0 +1,87 @@ +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// useMacOSAgentDisk makes the host's agent disk a file holding content. +func useMacOSAgentDisk(t *testing.T, content string) { + t.Helper() + disk := filepath.Join(t.TempDir(), "agent.img") + require.NoError(t, os.WriteFile(disk, []byte(content), 0644)) + prev := macOSAgentDiskSource + macOSAgentDiskSource = func(context.Context, *paths.Paths) (string, error) { return disk, nil } + t.Cleanup(func() { macOSAgentDiskSource = prev }) +} + +func TestMacOSAgentDiskInstallAndRefresh(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + stored := &StoredMetadata{Id: "mac", MacOS: macOSGuestMetadata(2).MacOS} + require.NoError(t, os.MkdirAll(m.paths.InstanceDir(stored.Id), 0700)) + + useMacOSAgentDisk(t, "agent v1") + require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored)) + require.NoFileExists(t, m.paths.InstanceMacOSAgentDisk(stored.Id), "a guest without the flag gets no agent disk") + + stored.MacOSAgentDisk = true + require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored)) + data, err := os.ReadFile(m.paths.InstanceMacOSAgentDisk(stored.Id)) + require.NoError(t, err) + require.Equal(t, "agent v1", string(data)) + info, err := os.Stat(m.paths.InstanceMacOSAgentDisk(stored.Id)) + require.NoError(t, err) + require.Equal(t, os.FileMode(0400), info.Mode().Perm()) + + useMacOSAgentDisk(t, "agent v2") + require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored), "a cold start replaces the guest's copy") + data, err = os.ReadFile(m.paths.InstanceMacOSAgentDisk(stored.Id)) + require.NoError(t, err) + require.Equal(t, "agent v2", string(data)) +} + +func TestMacOSDisksAttachAgentDiskSecond(t *testing.T) { + p := paths.New(t.TempDir()) + stored := &StoredMetadata{Id: "mac", DataDir: p.InstanceDir("mac"), MacOSDiskLayers: 1} + require.Equal(t, []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk("mac"), Parents: []string{macOSDiskLayer(stored.DataDir, 0)}}}, macOSDisks(p, stored)) + stored.MacOSAgentDisk = true + disks := macOSDisks(p, stored) + require.Len(t, disks, 2) + require.Equal(t, p.InstanceMacOSBootDisk("mac"), disks[0].Path, "the boot disk stays first") + require.Equal(t, hypervisor.DiskConfig{Path: p.InstanceMacOSAgentDisk("mac"), Readonly: true}, disks[1]) +} + +func TestMacOSMemorySnapshotRestoresAgentDiskItWasSavedWith(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + meta.MacOSAgentDisk = true + require.NoError(t, m.saveMetadata(meta)) + agentDisk := m.paths.InstanceMacOSAgentDisk(meta.Id) + require.NoError(t, os.WriteFile(agentDisk, []byte("agent-at-checkpoint"), 0400)) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + ctx := context.Background() + snapshot, err := m.CreateSnapshot(ctx, meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.NoError(t, err) + + require.NoError(t, os.Remove(agentDisk)) + require.NoError(t, os.WriteFile(agentDisk, []byte("agent-after"), 0400)) + _, err = m.RestoreSnapshot(ctx, meta.Id, snapshot.Id, RestoreSnapshotRequest{TargetState: StateStandby}) + require.NoError(t, err) + data, err := os.ReadFile(agentDisk) + require.NoError(t, err) + require.Equal(t, "agent-at-checkpoint", string(data), "saved state restores with the agent disk it was saved with") +} + +func TestMacOSMemorySnapshotRequiresItsAgentDisk(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + meta.MacOSAgentDisk = true + require.NoError(t, m.saveMetadata(meta)) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + _, err := m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.ErrorIs(t, err, ErrInvalidRequest, "a payload without its agent disk is refused") +} diff --git a/lib/instances/macos_memory_snapshot_test.go b/lib/instances/macos_memory_snapshot_test.go index df3152b28..ef205e947 100644 --- a/lib/instances/macos_memory_snapshot_test.go +++ b/lib/instances/macos_memory_snapshot_test.go @@ -51,8 +51,10 @@ func writeMacOSMemorySnapshotPayload(t *testing.T, m *manager, guestDir string, VCPUs: stored.Vcpus, MemoryBytes: stored.Size, MacHardwareModelData: base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel), MacMachineIdentifierData: base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier), MacAuxStoragePath: filepath.Join(m.paths.InstanceDir(stored.Id), macOSAuxStorageFile), - Disks: []shimconfig.DiskConfig{{Path: m.paths.InstanceMacOSBootDisk(stored.Id), Parents: macOSDiskParents(stored)}}, }} + for _, disk := range macOSDisks(m.paths, stored) { + manifest.ShimConfig.Disks = append(manifest.ShimConfig.Disks, shimconfig.DiskConfig{Path: disk.Path, Readonly: disk.Readonly, Parents: disk.Parents}) + } if stored.NetworkEnabled { manifest.ShimConfig.Networks = []shimconfig.NetworkConfig{{MAC: stored.MAC}} if stored.IP != "" { @@ -144,6 +146,9 @@ func TestMacOSMemorySnapshotRejectsInvalidPayloadBeforeStorageSwap(t *testing.T) {"hardware", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacHardwareModelData = "different" }}, {"disk", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Disks[0].Path = "/wrong-disk" }}, {"aux", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacAuxStoragePath = "/wrong-aux" }}, + {"extra-disk", func(m *shimconfig.SnapshotManifest) { + m.ShimConfig.Disks = append(m.ShimConfig.Disks, shimconfig.DiskConfig{Path: "/agent.img", Readonly: true}) + }}, {"network", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Networks[0].MAC = "02:00:00:00:00:02" }}, {"network-address", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Networks[0].IPv4Address, m.ShimConfig.Networks[0].IPv4Subnet = "192.168.128.2", "192.168.128.0/29" diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go index 17eea7f72..f2d473146 100644 --- a/lib/instances/snapshot_macos.go +++ b/lib/instances/snapshot_macos.go @@ -163,7 +163,9 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St if cfg.VCPUs != stored.Vcpus || cfg.MemoryBytes != stored.Size || cfg.MacHardwareModelData != base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel) || cfg.MacMachineIdentifierData != base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier) { return fmt.Errorf("%w: macOS snapshot configuration does not match its metadata", ErrInvalidRequest) } - if len(cfg.Disks) != 1 || cfg.Disks[0].Readonly || cfg.Disks[0].Path != m.paths.InstanceMacOSBootDisk(stored.Id) || !slices.Equal(cfg.Disks[0].Parents, macOSDiskParents(stored)) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { + if !slices.EqualFunc(cfg.Disks, macOSDisks(m.paths, stored), func(saved shimconfig.DiskConfig, want hypervisor.DiskConfig) bool { + return saved.Path == want.Path && saved.Readonly == want.Readonly && slices.Equal(saved.Parents, want.Parents) + }) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { return fmt.Errorf("%w: macOS snapshot storage configuration does not match its instance", ErrInvalidRequest) } if cfg.KernelPath != "" || cfg.InitrdPath != "" || cfg.EnableMemoryBalloon || cfg.RequireMemoryBalloon || cfg.EnableRosetta { @@ -190,6 +192,9 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St for i := range stored.MacOSDiskLayers { payload = append(payload, macOSDiskLayer(guestDir, i)) } + if stored.MacOSAgentDisk { + payload = append(payload, filepath.Join(guestDir, paths.MacOSAgentDiskFile)) + } for _, path := range payload { info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { diff --git a/lib/instances/start.go b/lib/instances/start.go index 898d6a166..af1f5f3c1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -97,6 +97,12 @@ func (m *manager) startInstance( }() } + // A cold start takes the host's current guest agent. + if err := m.installMacOSAgentDisk(ctx, stored); err != nil { + log.ErrorContext(ctx, "failed to refresh macOS agent disk", "instance_id", id, "error", err) + return nil, fmt.Errorf("refresh macOS agent disk: %w", err) + } + // 3. Get image info (needed for buildHypervisorConfig). imageInfo, err := m.resolveBootImage(ctx, stored) if err != nil { diff --git a/lib/instances/types.go b/lib/instances/types.go index f42ff6089..9dbce1f7f 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -97,6 +97,10 @@ type StoredMetadata struct { // MacOSDiskLayers is the number of read-only image layers below a macOS guest's // boot overlay, cloned into its data directory. MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` + // MacOSAgentDisk marks a macOS guest that boots with its copy of the read-only + // guest agent disk. Guests created before the disk existed keep booting without + // it, so their saved state still restores. + MacOSAgentDisk bool `json:"macos_agent_disk,omitempty"` // IdentityRekeyPending marks a macOS clone whose SSH host keys still come from its // source. The next start rekeys them and clears the flag. IdentityRekeyPending bool `json:"identity_rekey_pending,omitempty"` diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 1940e4a99..67ba0275d 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -260,6 +260,20 @@ func (p *Paths) InstanceMacOSBootDisk(id string) string { return filepath.Join(p.InstanceDir(id), MacOSBootDiskFile) } +// MacOSAgentDiskFile is a macOS guest's copy of the read-only guest agent disk. +const MacOSAgentDiskFile = "agent.img" + +// InstanceMacOSAgentDisk returns the path to a macOS guest's guest agent disk. +func (p *Paths) InstanceMacOSAgentDisk(id string) string { + return filepath.Join(p.InstanceDir(id), MacOSAgentDiskFile) +} + +// SystemMacOSAgentDisk returns the path to the guest agent disk built for an agent +// version, from which macOS guests take their copies. +func (p *Paths) SystemMacOSAgentDisk(version string) string { + return filepath.Join(p.DataDir(), "system", "macos-agent", version+".img") +} + // RestoreStagingDir holds a macOS restore in progress inside its instance directory, // so deleting the instance also deletes any staging it left behind. const RestoreStagingDir = "restore-staging" diff --git a/lib/system/guest_agent/macos/install-launcher.sh b/lib/system/guest_agent/macos/install-launcher.sh new file mode 100755 index 000000000..3a9259350 --- /dev/null +++ b/lib/system/guest_agent/macos/install-launcher.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Install the launcher that runs hypeman's guest agent from the read-only agent +# disk hypeman attaches to macOS guests. Run as root in the base image, for +# example from a Packer provisioner. A guest agent installed in the image itself +# is removed, so the agent always comes from the disk. +set -eu +launchctl bootout system/com.kernel.hypeman.guest-agent 2>/dev/null || true +rm -f /Library/LaunchDaemons/com.kernel.hypeman.guest-agent.plist /usr/local/sbin/hypeman-guest-agent +install -d -o root -g wheel -m 0755 /usr/local/libexec +cat > /usr/local/libexec/hypeman-agent-launcher <<'SH' +#!/bin/sh +# Run hypeman's guest agent from its read-only agent disk, mounting it if macOS +# has not yet. +install -d -o root -g wheel -m 0755 /var/run/hypeman +for i in $(seq 1 120); do + [ -x /Volumes/HypemanAgent/hypeman-guest-agent ] && exec /Volumes/HypemanAgent/hypeman-guest-agent + /usr/sbin/diskutil mount readOnly HypemanAgent >/dev/null 2>&1 + sleep 1 +done +echo "hypeman agent disk not found" >&2 +exit 1 +SH +chown root:wheel /usr/local/libexec/hypeman-agent-launcher +chmod 0755 /usr/local/libexec/hypeman-agent-launcher +cat > /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist <<'PLIST' + + + + + Labelcom.kernel.hypeman.agent-launcher + ProgramArguments/usr/local/libexec/hypeman-agent-launcher + RunAtLoad + KeepAlive + UserNameroot + GroupNamewheel + StandardOutPath/var/log/hypeman-guest-agent.log + StandardErrorPath/var/log/hypeman-guest-agent.log + + +PLIST +chown root:wheel /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist +chmod 0644 /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist diff --git a/lib/system/macos_agent_disk.go b/lib/system/macos_agent_disk.go new file mode 100644 index 000000000..8f6695db6 --- /dev/null +++ b/lib/system/macos_agent_disk.go @@ -0,0 +1,8 @@ +package system + +// MacOSAgentVolume is the volume name of the read-only guest agent disk attached +// to macOS guests. The launcher in a macOS image runs MacOSAgentBinary from it. +const ( + MacOSAgentVolume = "HypemanAgent" + MacOSAgentBinary = "hypeman-guest-agent" +) diff --git a/lib/system/macos_agent_disk_darwin.go b/lib/system/macos_agent_disk_darwin.go new file mode 100644 index 000000000..b4ba94010 --- /dev/null +++ b/lib/system/macos_agent_disk_darwin.go @@ -0,0 +1,69 @@ +//go:build darwin + +package system + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + + "github.com/kernel/hypeman/lib/paths" +) + +// DarwinGuestAgentBinary is the macOS guest agent, built by the Makefile. +// +//go:embed guest_agent/guest-agent-darwin +var DarwinGuestAgentBinary []byte + +var macOSAgentDiskMu sync.Mutex + +// MacOSAgentDisk returns the read-only guest agent disk for the embedded macOS +// agent, building it on first use. Disks are named by the agent's hash, so a new +// agent gets a new disk. +func MacOSAgentDisk(ctx context.Context, p *paths.Paths) (string, error) { + sum := sha256.Sum256(DarwinGuestAgentBinary) + version := hex.EncodeToString(sum[:6]) + path := p.SystemMacOSAgentDisk(version) + macOSAgentDiskMu.Lock() + defer macOSAgentDiskMu.Unlock() + if _, err := os.Stat(path); err == nil { + return path, nil + } + return path, buildMacOSAgentDisk(ctx, DarwinGuestAgentBinary, version, path) +} + +// buildMacOSAgentDisk writes a raw APFS disk image at path holding the agent and a +// VERSION file. +func buildMacOSAgentDisk(ctx context.Context, binary []byte, version, path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + work, err := os.MkdirTemp(filepath.Dir(path), ".build-") + if err != nil { + return err + } + defer os.RemoveAll(work) + src := filepath.Join(work, "src") + if err := os.Mkdir(src, 0755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(src, MacOSAgentBinary), binary, 0755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(src, "VERSION"), []byte(version+"\n"), 0644); err != nil { + return err + } + // UDTO is a raw image, which Virtualization attaches as is. hdiutil appends .cdr. + out := filepath.Join(work, "agent") + cmd := exec.CommandContext(ctx, "/usr/bin/hdiutil", "create", "-quiet", "-srcfolder", src, "-volname", MacOSAgentVolume, "-fs", "APFS", "-format", "UDTO", out) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("build macOS agent disk: %w: %s", err, output) + } + return os.Rename(out+".cdr", path) +} diff --git a/lib/system/macos_agent_disk_darwin_test.go b/lib/system/macos_agent_disk_darwin_test.go new file mode 100644 index 000000000..dcc45ae84 --- /dev/null +++ b/lib/system/macos_agent_disk_darwin_test.go @@ -0,0 +1,23 @@ +//go:build darwin + +package system + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildMacOSAgentDisk(t *testing.T) { + path := filepath.Join(t.TempDir(), "macos-agent", "v1.img") + require.NoError(t, buildMacOSAgentDisk(context.Background(), []byte("#!/bin/sh\n"), "v1", path)) + info, err := os.Stat(path) + require.NoError(t, err) + require.Zero(t, info.Size()%512, "a raw disk is whole sectors") + entries, err := os.ReadDir(filepath.Dir(path)) + require.NoError(t, err) + require.Len(t, entries, 1, "the build directory is removed") +} diff --git a/lib/system/macos_agent_disk_other.go b/lib/system/macos_agent_disk_other.go new file mode 100644 index 000000000..f2f504f29 --- /dev/null +++ b/lib/system/macos_agent_disk_other.go @@ -0,0 +1,15 @@ +//go:build !darwin + +package system + +import ( + "context" + "errors" + + "github.com/kernel/hypeman/lib/paths" +) + +// MacOSAgentDisk is only available on macOS hosts. +func MacOSAgentDisk(context.Context, *paths.Paths) (string, error) { + return "", errors.New("the macOS guest agent disk requires a macOS host") +} From 2fa2a9c1c8f95b1d48279b87407b446e96b3052a Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:57:59 +0000 Subject: [PATCH 18/33] Let macOS guests boot with read-only disks after the boot disk --- lib/hypervisor/vz/macos_config_test.go | 9 +++++++++ lib/hypervisor/vz/starter.go | 11 +++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/hypervisor/vz/macos_config_test.go b/lib/hypervisor/vz/macos_config_test.go index 38fbac0b0..858c2d874 100644 --- a/lib/hypervisor/vz/macos_config_test.go +++ b/lib/hypervisor/vz/macos_config_test.go @@ -40,6 +40,15 @@ func TestMacOSStarterConfig(t *testing.T) { bad = cfg bad.Disks = []hypervisor.DiskConfig{{Readonly: true}} require.Error(t, NewStarter().ValidateConfig(bad)) + agent := cfg + agent.Disks = []hypervisor.DiskConfig{{Path: "/vm/root.raw"}, {Path: "/vm/agent.img", Readonly: true}} + require.NoError(t, NewStarter().ValidateConfig(agent), "a read-only disk may follow the boot disk") + bad = cfg + bad.Disks = []hypervisor.DiskConfig{{Path: "/vm/agent.img", Readonly: true}, {Path: "/vm/root.raw"}} + require.Error(t, NewStarter().ValidateConfig(bad), "the boot disk comes first") + bad = cfg + bad.Disks = []hypervisor.DiskConfig{{Path: "/vm/root.raw"}, {Path: "/vm/data.img"}} + require.Error(t, NewStarter().ValidateConfig(bad), "later disks are read-only") bad = cfg bad.MacOS = nil require.Error(t, NewStarter().ValidateConfig(bad)) diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index 70e09f137..12ced7afe 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -119,8 +119,15 @@ func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { if config.HotplugBytes != 0 || config.EnableRosetta || config.GuestMemory.EnableBalloon || config.GuestMemory.RequireBalloon || len(config.PCIDevices) != 0 || config.VGPUDevicePath != "" { return fmt.Errorf("macOS does not support hotplug, ballooning, Rosetta shares or PCI passthrough") } - if len(config.Disks) != 1 || config.Disks[0].Readonly || config.Disks[0].EffectiveFormat() != hypervisor.DiskFormatRaw { - return fmt.Errorf("macOS requires one writable raw boot disk") + // The boot disk comes first; VZ does not boot macOS from a later disk. Any other + // disk, such as the guest agent disk, is read-only. + if len(config.Disks) == 0 || config.Disks[0].Readonly || config.Disks[0].EffectiveFormat() != hypervisor.DiskFormatRaw { + return fmt.Errorf("macOS requires a writable raw boot disk first") + } + for _, disk := range config.Disks[1:] { + if !disk.Readonly || len(disk.Parents) != 0 || disk.EffectiveFormat() != hypervisor.DiskFormatRaw { + return fmt.Errorf("macOS disks after the boot disk must be read-only raw disks") + } } if config.VCPUs < 2 || uint(config.VCPUs) > vz.VirtualMachineConfigurationMaximumAllowedCPUCount() { return fmt.Errorf("macOS CPU count is outside the host's supported range") From 53a30255c89d217b64cb3ca9ed89425314b2adca Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:02:23 +0000 Subject: [PATCH 19/33] Drop imports left unused by the removed fresh-identity test --- lib/instances/macos_identity_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/instances/macos_identity_test.go b/lib/instances/macos_identity_test.go index af30fb596..2978e7da1 100644 --- a/lib/instances/macos_identity_test.go +++ b/lib/instances/macos_identity_test.go @@ -4,12 +4,10 @@ package instances import ( "net" - "os" "testing" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" - "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) From f1f405b7f147da5ebb4c574e2f9d560217b732f1 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:08:01 +0000 Subject: [PATCH 20/33] Add a Packer template for macOS browser images It builds on a stacked Tart clone of the base: installs Chrome, checked by its code signature, runs any extra scripts, installs hypeman's agent launcher, checks the image with a smoke test, regenerates the SSH host keys, and shuts the guest down from inside. --- images/macos/browser.pkr.hcl | 88 ++++++++++++++++++++++++++++++++++ images/macos/install-chrome.sh | 18 +++++++ images/macos/smoke-test.sh | 24 ++++++++++ 3 files changed, 130 insertions(+) create mode 100644 images/macos/browser.pkr.hcl create mode 100755 images/macos/install-chrome.sh create mode 100755 images/macos/smoke-test.sh diff --git a/images/macos/browser.pkr.hcl b/images/macos/browser.pkr.hcl new file mode 100644 index 000000000..06e0c120d --- /dev/null +++ b/images/macos/browser.pkr.hcl @@ -0,0 +1,88 @@ +# Builds a macOS image for hypeman on a stacked Tart VM. Clone the base with +# `tart clone --stacked` first and pass that VM as vm_base_name: the build writes +# only to the clone's overlay, which import-tart publishes over the base. +# See docs/macos-experimental.md. +packer { + required_plugins { + tart = { + version = "= 1.21.0" + source = "github.com/cirruslabs/tart" + } + } +} + +variable "vm_base_name" { + type = string + description = "Stopped Tart VM to build on, made with tart clone --stacked." +} + +variable "vm_name" { + type = string + description = "Tart VM the build writes, which import-tart publishes." +} + +variable "ssh_username" { + type = string +} + +variable "ssh_password" { + type = string + sensitive = true +} + +variable "extra_scripts" { + type = list(string) + default = [] + description = "Scripts run as root, in order, after Chrome is installed." +} + +locals { + sudo = "echo '${var.ssh_password}' | sudo -S -p '' sh -eux '{{ .Path }}'" +} + +source "tart-cli" "macos" { + vm_base_name = var.vm_base_name + vm_name = var.vm_name + headless = true + # The default, "delete", removes the recovery partition and grows APFS, which + # rewrites the whole disk instead of the build's changes. + recovery_partition = "keep" + ssh_username = var.ssh_username + ssh_password = var.ssh_password + ssh_timeout = "300s" +} + +build { + sources = ["source.tart-cli.macos"] + + provisioner "shell" { + execute_command = local.sudo + scripts = concat(["${path.root}/install-chrome.sh"], var.extra_scripts) + } + + # hypeman supplies its guest agent on a read-only disk; the image only carries + # the launcher that runs it. + provisioner "shell" { + execute_command = local.sudo + script = "${path.root}/../../lib/system/guest_agent/macos/install-launcher.sh" + } + + provisioner "shell" { + script = "${path.root}/smoke-test.sh" + } + + # Every image gets its own SSH host keys; Tart keeps its source's. The guest + # shuts itself down: tart stop is a hard power-off. + provisioner "shell" { + execute_command = local.sudo + inline = [ + "rm -f /etc/ssh/ssh_host_*", + "ssh-keygen -A", + "shutdown -h now", + ] + expect_disconnect = true + # Without this, Packer retries removing its script over the dead connection + # for five minutes. + skip_clean = true + } +} diff --git a/images/macos/install-chrome.sh b/images/macos/install-chrome.sh new file mode 100755 index 000000000..eba3b2ace --- /dev/null +++ b/images/macos/install-chrome.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Install Google Chrome from Google's stable DMG. The download URL is not +# versioned, so the app is checked by its code signature: it must be signed by +# Google's Developer ID team and pass Gatekeeper. +set -eu +team=EQHXZ8M8AV +work=$(mktemp -d) +mount="$work/mount" +trap 'hdiutil detach -quiet "$mount" 2>/dev/null || true; rm -rf "$work"' EXIT +curl -fsSL -o "$work/chrome.dmg" https://dl.google.com/chrome/mac/universal/stable/GGRO/googlechrome.dmg +hdiutil attach -quiet -nobrowse -readonly -mountpoint "$mount" "$work/chrome.dmg" +app="$mount/Google Chrome.app" +codesign --verify --deep --strict "$app" +codesign -dv "$app" 2>&1 | grep -qx "TeamIdentifier=$team" +spctl --assess --type execute "$app" +rm -rf "/Applications/Google Chrome.app" +ditto "$app" "/Applications/Google Chrome.app" +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --version diff --git a/images/macos/smoke-test.sh b/images/macos/smoke-test.sh new file mode 100755 index 000000000..a54909440 --- /dev/null +++ b/images/macos/smoke-test.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Checks the image before it is published: hypeman's agent launcher is installed, +# no agent is baked into the image, and Chrome starts headless and answers CDP. +# Runs as the build user, leaving no browser profile behind. +set -eu +test -x /usr/local/libexec/hypeman-agent-launcher +test -f /Library/LaunchDaemons/com.kernel.hypeman.agent-launcher.plist +test ! -e /usr/local/sbin/hypeman-guest-agent + +chrome="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" +test -x "$chrome" +profile=$(mktemp -d) +"$chrome" --headless=new --user-data-dir="$profile" --remote-debugging-port=9222 --no-first-run about:blank >/dev/null 2>&1 & +pid=$! +trap 'kill $pid 2>/dev/null; wait $pid 2>/dev/null; rm -rf "$profile"' EXIT +for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9222/json/version; then + echo + exit 0 + fi + sleep 1 +done +echo "Chrome did not answer on its debugging port" >&2 +exit 1 From 6b5866f48e5194c8a38dae477b158733b6219e4b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:16:42 +0000 Subject: [PATCH 21/33] Add a workflow that builds and publishes macOS images It clones the base stacked, builds and checks the image with the Packer template, and publishes it over the base with import-tart. It runs by hand on a self-hosted macOS 27 runner. Also stop install-chrome.sh using codesign --strict, which rejects Chrome's own Alerts helper. --- .github/workflows/macos-image.yml | 95 +++++++++++++++++++++++++++++++ docs/macos-experimental.md | 30 +++++++--- images/macos/install-chrome.sh | 12 ++-- 3 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/macos-image.yml diff --git a/.github/workflows/macos-image.yml b/.github/workflows/macos-image.yml new file mode 100644 index 000000000..d3bf9b4e8 --- /dev/null +++ b/.github/workflows/macos-image.yml @@ -0,0 +1,95 @@ +name: macOS image + +# Builds a macOS browser image with Packer and Tart on a stacked clone of a base, +# checks it, and publishes it to hypeman's registry over that base. See +# docs/macos-experimental.md. +on: + workflow_dispatch: + inputs: + tart_base: + description: The base as a Tart image (OCI reference) to clone stacked + required: true + base: + description: The same base in hypeman's registry, pinned by digest + required: true + name: + description: Image name; published as builds/NAME + required: true + +permissions: + contents: read + +# A Mac runs at most two VMs; builds on one runner go one at a time. +concurrency: + group: macos-image-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + # Stacked Tart VMs and the Swift vz-shim need macOS 27 and the macOS 27 SDK. + runs-on: [self-hosted, macos, arm64, tart] + timeout-minutes: 90 + env: + VM: hypeman-image-${{ github.run_id }} + TART_VERSION: 2.41.1 + TART_SHA256: e680a734febccfce7d4e910c1f7f13fc3b00f19a181c945e2d746de3214c32c2 + PACKER_VERSION: 1.16.2 + PACKER_SHA256: 3d7e4df8e7e8c21bdf495c4f63ad9189e22540dc8126dee6db791aa8519e8b3a + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + + - name: Install Tart and Packer + run: | + bin="$RUNNER_TEMP/bin" + mkdir -p "$bin" + curl -fsSL -o "$RUNNER_TEMP/tart.tar.gz" "https://github.com/cirruslabs/tart/releases/download/$TART_VERSION/tart.tar.gz" + echo "$TART_SHA256 $RUNNER_TEMP/tart.tar.gz" | shasum -a 256 -c + tar -xzf "$RUNNER_TEMP/tart.tar.gz" -C "$RUNNER_TEMP" + ln -sf "$RUNNER_TEMP/tart.app/Contents/MacOS/tart" "$bin/tart" + curl -fsSL -o "$RUNNER_TEMP/packer.zip" "https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_${PACKER_VERSION}_darwin_arm64.zip" + echo "$PACKER_SHA256 $RUNNER_TEMP/packer.zip" | shasum -a 256 -c + unzip -oq "$RUNNER_TEMP/packer.zip" -d "$bin" + echo "$bin" >> "$GITHUB_PATH" + + - name: Build import-tart + run: | + make build-embedded build-vz-shim + go build -o "$RUNNER_TEMP/bin/import-tart" ./cmd/import-tart + + - name: Clone the base stacked + env: + TART_BASE: ${{ inputs.tart_base }} + TART_REGISTRY_USERNAME: ${{ secrets.TART_REGISTRY_USERNAME }} + TART_REGISTRY_PASSWORD: ${{ secrets.TART_REGISTRY_PASSWORD }} + run: tart clone --stacked "$TART_BASE" "$VM-base" + + - name: Build and check the image + working-directory: images/macos + env: + PKR_VAR_ssh_username: ${{ vars.MACOS_IMAGE_SSH_USERNAME }} + PKR_VAR_ssh_password: ${{ secrets.MACOS_IMAGE_SSH_PASSWORD }} + run: | + packer init browser.pkr.hcl + packer build -var vm_base_name="$VM-base" -var vm_name="$VM" browser.pkr.hcl + + - name: Publish + env: + BASE: ${{ inputs.base }} + NAME: ${{ inputs.name }} + REGISTRY: ${{ vars.HYPEMAN_REGISTRY_URL }} + HYPEMAN_JWT_SECRET: ${{ secrets.HYPEMAN_JWT_SECRET }} + run: | + import-tart --vm "$HOME/.tart/vms/$VM" --registry "$REGISTRY" --name "$NAME" --base "$BASE" --guest-agent | tee "$RUNNER_TEMP/publication.json" + echo "Published \`$(jq -r .Reference "$RUNNER_TEMP/publication.json")\`" >> "$GITHUB_STEP_SUMMARY" + + - name: Delete the build VMs + if: always() + run: | + tart delete "$VM" || true + tart delete "$VM-base" || true diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 010c624a2..bffc82a24 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -79,14 +79,22 @@ overlay. 2. Get the base as a Tart image (`tart push` to any OCI registry) and create the build VM with `tart clone --stacked`. The Packer Tart plugin cannot clone stacked itself, so point its `vm_base_name` at that local clone. -3. Build. The template must set `recovery_partition = "keep"` (the default deletes - the recovery partition and grows APFS), regenerate the guest's SSH host keys - (`rm -f /etc/ssh/ssh_host_* && ssh-keygen -A`), and end by shutting the guest - down from inside, with `skip_clean = true` on that step. `tart stop` is a hard - power-off. For the guest agent, install the launcher once in the base with - `lib/system/guest_agent/macos/install-launcher.sh` and publish with - `--guest-agent`; hypeman supplies the agent on its own disk (see - [macos-guest-agent.md](macos-guest-agent.md#agent-disk)). +3. Build with `images/macos/browser.pkr.hcl`. It installs Chrome (from Google's + stable DMG, checked by its code signature and Gatekeeper), runs any + `extra_scripts`, installs hypeman's agent launcher (hypeman supplies the agent + on its own disk; see [macos-guest-agent.md](macos-guest-agent.md#agent-disk)), + checks the image with `images/macos/smoke-test.sh` (launcher present, no baked + agent, headless Chrome answers CDP), regenerates the SSH host keys and shuts the + guest down from inside. It keeps the recovery partition; Packer's default + deletes it and rewrites the whole disk. The base must have an SSH user with sudo + for Packer. + + ```sh + cd images/macos && packer init browser.pkr.hcl + packer build -var vm_base_name=base-clone -var vm_name=my-build \ + -var ssh_username=admin -var ssh_password=... browser.pkr.hcl + ``` + 4. Publish the stopped VM over the base: ```sh @@ -98,6 +106,12 @@ HYPEMAN_JWT_SECRET=... go run ./cmd/import-tart \ --guest-agent ``` +The `macOS image` workflow (`.github/workflows/macos-image.yml`, run by hand) does +steps 2 to 4 on a self-hosted macOS 27 runner labelled `tart`. It needs the +`HYPEMAN_REGISTRY_URL` and `MACOS_IMAGE_SSH_USERNAME` variables and the +`HYPEMAN_JWT_SECRET`, `MACOS_IMAGE_SSH_PASSWORD`, `TART_REGISTRY_USERNAME` and +`TART_REGISTRY_PASSWORD` secrets. + The image is published as `builds/`. `import-tart` reads the VM's layers from Tart's content store in the order its `manifest.json` lists them, refuses a suspended or running VM, and checks that the overlay stacks on those layers. With diff --git a/images/macos/install-chrome.sh b/images/macos/install-chrome.sh index eba3b2ace..f01f3dcb6 100755 --- a/images/macos/install-chrome.sh +++ b/images/macos/install-chrome.sh @@ -1,7 +1,8 @@ #!/bin/sh # Install Google Chrome from Google's stable DMG. The download URL is not -# versioned, so the app is checked by its code signature: it must be signed by -# Google's Developer ID team and pass Gatekeeper. +# versioned, so the copy to be installed is checked by its code signature: it must +# be signed by Google's Developer ID team and pass Gatekeeper. (codesign --strict +# rejects Chrome's own Alerts helper, so the deep check is not strict.) set -eu team=EQHXZ8M8AV work=$(mktemp -d) @@ -9,10 +10,11 @@ mount="$work/mount" trap 'hdiutil detach -quiet "$mount" 2>/dev/null || true; rm -rf "$work"' EXIT curl -fsSL -o "$work/chrome.dmg" https://dl.google.com/chrome/mac/universal/stable/GGRO/googlechrome.dmg hdiutil attach -quiet -nobrowse -readonly -mountpoint "$mount" "$work/chrome.dmg" -app="$mount/Google Chrome.app" -codesign --verify --deep --strict "$app" +app="$work/Google Chrome.app" +ditto "$mount/Google Chrome.app" "$app" +codesign --verify --deep "$app" codesign -dv "$app" 2>&1 | grep -qx "TeamIdentifier=$team" spctl --assess --type execute "$app" rm -rf "/Applications/Google Chrome.app" -ditto "$app" "/Applications/Google Chrome.app" +mv "$app" "/Applications/Google Chrome.app" "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --version From 356062decbe9c390dbe3d712de83c73027ac54de Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:50:34 +0000 Subject: [PATCH 22/33] Tighten the macOS disk contract and share its names macOS boot disks are typed ASIF and must stack on their image's layers; other macOS disks must be read-only raw. Image labels, file names and the distinct-file check live in lib/images, and layer reuse requires the same layers beneath. Share the macOS-only platform default and the image resolve tail, count a guest's agent disk and aux storage in disk utilization, and let a wait for the agent disk build honor its context. --- cmd/api/api/instances.go | 5 +- lib/diskutilization/diskutilization.go | 5 +- lib/hypervisor/config.go | 2 + lib/hypervisor/config_validation.go | 4 ++ lib/hypervisor/vz/macos_config_test.go | 9 +-- lib/hypervisor/vz/starter.go | 9 +-- lib/images/macos_asif.go | 11 ++-- lib/images/macos_bundle.go | 33 +++++++--- lib/images/macos_import_darwin.go | 14 ++--- lib/images/macos_machine.go | 69 ++++++++++++++++----- lib/images/macos_machine_test.go | 14 +++-- lib/images/macos_unpack.go | 29 ++++----- lib/images/manager.go | 2 +- lib/images/oci.go | 5 +- lib/images/storage.go | 2 +- lib/instances/create.go | 4 +- lib/instances/create_image.go | 29 ++++----- lib/instances/macos.go | 31 ++++----- lib/instances/macos_agent_disk_test.go | 3 +- lib/instances/macos_memory_snapshot_test.go | 3 +- lib/instances/macos_storage.go | 2 +- lib/instances/snapshot_macos.go | 3 +- lib/instances/snapshot_macos_darwin_test.go | 4 +- lib/paths/paths.go | 3 + lib/system/macos_agent_disk_darwin.go | 12 ++-- 25 files changed, 192 insertions(+), 115 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index ced753723..3b9dc5e3b 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -63,10 +63,7 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - platform := derefString(request.Body.Platform) - if s.Config != nil && s.Config.MacOSOnly && platform == "" { - platform = "darwin/arm64" - } + platform := instances.DefaultPlatform(s.Config != nil && s.Config.MacOSOnly, derefString(request.Body.Platform)) macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform) if err != nil { log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image) diff --git a/lib/diskutilization/diskutilization.go b/lib/diskutilization/diskutilization.go index e209c96db..3da7e4667 100644 --- a/lib/diskutilization/diskutilization.go +++ b/lib/diskutilization/diskutilization.go @@ -89,7 +89,10 @@ func Collect(p *paths.Paths) (Breakdown, error) { instanceID := guest.Name() - breakdown.RootfsOverlays += allocatedBytesForPath(p.InstanceOverlay(instanceID)) + allocatedBytesForPath(p.InstanceMacOSBootDisk(instanceID)) + // A macOS guest's own writes and files. Its disk.N.asif layers are clones of + // its image's, which Images already counts. + breakdown.RootfsOverlays += allocatedBytesForPath(p.InstanceOverlay(instanceID)) + allocatedBytesForPath(p.InstanceMacOSBootDisk(instanceID)) + + allocatedBytesForPath(p.InstanceMacOSAgentDisk(instanceID)) + allocatedBytesForPath(filepath.Join(p.InstanceDir(instanceID), paths.MacOSAuxStorageFile)) volumeOverlays, err := sumMatchingFilesAllocatedBytes(p.InstanceVolumeOverlaysDir(instanceID), func(path string, entry fs.DirEntry) bool { return !entry.IsDir() && filepath.Ext(entry.Name()) == ".raw" diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index b0f457fc1..4aadd5f89 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -104,6 +104,8 @@ type DiskFormat string const ( DiskFormatRaw DiskFormat = "raw" DiskFormatQCOW2 DiskFormat = "qcow2" + // DiskFormatASIF is an ASIF overlay on its Parents, only for macOS boot disks. + DiskFormatASIF DiskFormat = "asif" ) // EffectiveFormat preserves raw disks for existing callers. diff --git a/lib/hypervisor/config_validation.go b/lib/hypervisor/config_validation.go index 26a23065f..7de9bbd68 100644 --- a/lib/hypervisor/config_validation.go +++ b/lib/hypervisor/config_validation.go @@ -42,6 +42,10 @@ func ValidateBootConfig(cfg VMConfig) error { for i, disk := range cfg.Disks { switch disk.EffectiveFormat() { case DiskFormatRaw, DiskFormatQCOW2: + case DiskFormatASIF: + if cfg.EffectiveBootMode() != BootModeMacOS { + return fmt.Errorf("disk %d: ASIF disks are only for macOS boot", i) + } default: return fmt.Errorf("disk %d has unsupported format %q", i, disk.Format) } diff --git a/lib/hypervisor/vz/macos_config_test.go b/lib/hypervisor/vz/macos_config_test.go index 858c2d874..ecc276b11 100644 --- a/lib/hypervisor/vz/macos_config_test.go +++ b/lib/hypervisor/vz/macos_config_test.go @@ -11,7 +11,8 @@ import ( ) func TestMacOSStarterConfig(t *testing.T) { - cfg := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: 2, MemoryBytes: 4 << 30, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: []byte{1}, MachineIdentifierData: []byte{2}, AuxStoragePath: "/vm/aux.img"}, Disks: []hypervisor.DiskConfig{{Path: "/vm/root.raw"}}} + bootDisk := hypervisor.DiskConfig{Path: "/vm/boot.asif", Format: hypervisor.DiskFormatASIF, Parents: []string{"/vm/disk.0.asif"}} + cfg := hypervisor.VMConfig{BootMode: hypervisor.BootModeMacOS, VCPUs: 2, MemoryBytes: 4 << 30, MacOS: &hypervisor.MacOSPlatform{HardwareModelData: []byte{1}, MachineIdentifierData: []byte{2}, AuxStoragePath: "/vm/aux.img"}, Disks: []hypervisor.DiskConfig{bootDisk}} require.NoError(t, NewStarter().ValidateConfig(cfg)) shim := buildShimConfigFromVMConfig(cfg, "/vm/vz.sock") require.Equal(t, base64.StdEncoding.EncodeToString([]byte{1}), shim.MacHardwareModelData) @@ -41,13 +42,13 @@ func TestMacOSStarterConfig(t *testing.T) { bad.Disks = []hypervisor.DiskConfig{{Readonly: true}} require.Error(t, NewStarter().ValidateConfig(bad)) agent := cfg - agent.Disks = []hypervisor.DiskConfig{{Path: "/vm/root.raw"}, {Path: "/vm/agent.img", Readonly: true}} + agent.Disks = []hypervisor.DiskConfig{bootDisk, {Path: "/vm/agent.img", Readonly: true}} require.NoError(t, NewStarter().ValidateConfig(agent), "a read-only disk may follow the boot disk") bad = cfg - bad.Disks = []hypervisor.DiskConfig{{Path: "/vm/agent.img", Readonly: true}, {Path: "/vm/root.raw"}} + bad.Disks = []hypervisor.DiskConfig{{Path: "/vm/agent.img", Readonly: true}, bootDisk} require.Error(t, NewStarter().ValidateConfig(bad), "the boot disk comes first") bad = cfg - bad.Disks = []hypervisor.DiskConfig{{Path: "/vm/root.raw"}, {Path: "/vm/data.img"}} + bad.Disks = []hypervisor.DiskConfig{bootDisk, {Path: "/vm/data.img"}} require.Error(t, NewStarter().ValidateConfig(bad), "later disks are read-only") bad = cfg bad.MacOS = nil diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go index 12ced7afe..15479b941 100644 --- a/lib/hypervisor/vz/starter.go +++ b/lib/hypervisor/vz/starter.go @@ -119,10 +119,11 @@ func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { if config.HotplugBytes != 0 || config.EnableRosetta || config.GuestMemory.EnableBalloon || config.GuestMemory.RequireBalloon || len(config.PCIDevices) != 0 || config.VGPUDevicePath != "" { return fmt.Errorf("macOS does not support hotplug, ballooning, Rosetta shares or PCI passthrough") } - // The boot disk comes first; VZ does not boot macOS from a later disk. Any other - // disk, such as the guest agent disk, is read-only. - if len(config.Disks) == 0 || config.Disks[0].Readonly || config.Disks[0].EffectiveFormat() != hypervisor.DiskFormatRaw { - return fmt.Errorf("macOS requires a writable raw boot disk first") + // The boot disk comes first; VZ does not boot macOS from a later disk. It is a + // writable ASIF overlay on its image's layers. Any other disk, such as the guest + // agent disk, is read-only and raw. + if boot := config.Disks; len(boot) == 0 || boot[0].Readonly || boot[0].EffectiveFormat() != hypervisor.DiskFormatASIF || len(boot[0].Parents) == 0 { + return fmt.Errorf("macOS requires a writable ASIF boot disk with its image layers first") } for _, disk := range config.Disks[1:] { if !disk.Readonly || len(disk.Parents) != 0 || disk.EffectiveFormat() != hypervisor.DiskFormatRaw { diff --git a/lib/images/macos_asif.go b/lib/images/macos_asif.go index 6aa28ac2f..5098a7f6a 100644 --- a/lib/images/macos_asif.go +++ b/lib/images/macos_asif.go @@ -13,10 +13,11 @@ import ( "github.com/kernel/hypeman/lib/paths" ) -// A macOS machine image's boot disk is an ASIF chain: disk.0.asif is the base and -// each later file is an overlay on the one before it. A pulled image has one file -// per OCI layer; an imported image has one. -func macOSDiskLayerFile(i int) string { +// MacOSDiskLayerFile names the ith file of a macOS boot disk chain, in an image or +// an instance. The chain is ASIF: disk.0.asif is the base and each later file is an +// overlay on the one before it. A pulled image has one file per OCI layer; an +// imported image has one. +func MacOSDiskLayerFile(i int) string { return fmt.Sprintf("disk.%d.asif", i) } @@ -36,7 +37,7 @@ func MacOSDiskChain(p *paths.Paths, imageName, digest string) ([]string, error) } chain := make([]string, meta.MacOSDiskLayers) for i := range chain { - chain[i] = filepath.Join(layout.dir, macOSDiskLayerFile(i)) + chain[i] = filepath.Join(layout.dir, MacOSDiskLayerFile(i)) } return chain, nil } diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 55663beb0..5a6f3fde6 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -50,9 +50,8 @@ func readMacOSMachineBundle(root, diskName, auxName, configName string) (*macOSM if err != nil { return nil, err } - // Compare inodes: different hardlink names do not establish distinctness. - if sameFile(disk, aux) || sameFile(disk, config) || sameFile(aux, config) { - return nil, fmt.Errorf("machine bundle files must be distinct") + if err := CheckMachineFiles(disk, aux, config); err != nil { + return nil, err } platform, err := readMacOSPlatform(config) if err != nil { @@ -93,6 +92,9 @@ func readMacOSMachineLayers(root, diskName, auxName, configName string) (*macOSM if err != nil { return nil, fmt.Errorf("machine image top layer: %w", err) } + if err := CheckMachineFiles(append([]string{aux, config}, payload.Disks...)...); err != nil { + return nil, err + } platform, err := readMacOSPlatform(config) if err != nil { return nil, err @@ -154,8 +156,25 @@ func machineBundleFile(root, relative string) (string, error) { return path, nil } -func sameFile(a, b string) bool { - ai, errA := os.Stat(a) - bi, errB := os.Stat(b) - return errA == nil && errB == nil && os.SameFile(ai, bi) +// CheckMachineFiles requires each path to be a nonempty regular file, not a +// symlink, and no two to be the same file. Inodes are compared: different hardlink +// names do not establish distinctness. +func CheckMachineFiles(paths ...string) error { + seen := make([]os.FileInfo, 0, len(paths)) + for _, path := range paths { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + for _, other := range seen { + if os.SameFile(other, info) { + return fmt.Errorf("machine storage files must be distinct") + } + } + seen = append(seen, info) + } + return nil } diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index 2f3a4585b..2b1a8b16d 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -29,16 +29,16 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if ref.IsDigest() { return nil, fmt.Errorf("%w: import requires a tagged name", ErrInvalidName) } - diskName := "disk.asif" + diskName := MacOSMachineDiskFile if _, err := os.Lstat(filepath.Join(source, diskName)); os.IsNotExist(err) { diskName = "disk.img" } - bundle, err := readMacOSMachineBundle(source, diskName, "aux.img", "config.json") + bundle, err := readMacOSMachineBundle(source, diskName, MacOSMachineAuxFile, MacOSMachinePlatformFile) if err != nil { return nil, err } mac := bundle.Platform - for _, file := range []string{diskName, "aux.img"} { + for _, file := range []string{diskName, MacOSMachineAuxFile} { file = filepath.Join(source, file) info, e := os.Lstat(file) if e != nil { @@ -62,8 +62,8 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) return nil, err } defer os.RemoveAll(stage) - clones := map[string]string{bundle.Aux: "aux.img"} - if diskName == "disk.asif" { + clones := map[string]string{bundle.Aux: MacOSMachineAuxFile} + if diskName == MacOSMachineDiskFile { clones[bundle.Disks[0]] = machineDiskFile } else if err = ConvertRawToASIF(ctx, bundle.Disks[0], filepath.Join(stage, machineDiskFile)); err != nil { return nil, fmt.Errorf("convert boot disk to ASIF: %w", err) @@ -73,7 +73,7 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) return nil, fmt.Errorf("clone bundle: %w", err) } } - for _, name := range []string{machineDiskFile, "aux.img"} { + for _, name := range []string{machineDiskFile, MacOSMachineAuxFile} { if err = os.Chmod(filepath.Join(stage, name), 0600); err != nil { return nil, err } @@ -87,7 +87,7 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) h := sha256.New() canonical, _ := json.Marshal(mac) h.Write(canonical) - for _, name := range []string{machineDiskFile, "aux.img"} { + for _, name := range []string{machineDiskFile, MacOSMachineAuxFile} { f, e := os.Open(filepath.Join(stage, name)) if e != nil { return nil, e diff --git a/lib/images/macos_machine.go b/lib/images/macos_machine.go index d7661870a..5396b2136 100644 --- a/lib/images/macos_machine.go +++ b/lib/images/macos_machine.go @@ -3,10 +3,10 @@ package images import ( "context" "fmt" + "log/slog" "os" "path/filepath" "runtime" - "slices" "time" gcr "github.com/google/go-containerregistry/pkg/v1" @@ -21,8 +21,31 @@ const ( MacOSMachineFormatLabel = "io.hypeman.machine-image.disk-format" MacOSMachineAuxLabel = "io.hypeman.machine-image.aux-path" MacOSMachinePlatformLabel = "io.hypeman.machine-image.platform-path" + // MacOSMachineDiskDigestsLabel lists the sha256 of each layer's disk file, base + // first, comma-separated. A publication over a base checks its layers against it. + MacOSMachineDiskDigestsLabel = "io.hypeman.machine-image.disk-digests" ) +// The fixed label values of a macOS machine image, and the file names a published +// image uses in each layer. +const ( + MacOSMachineVersion = "2" + MacOSMachineKind = "macos-image" + MacOSMachineFormat = "asif" + MacOSMachineDiskFile = "disk.asif" + MacOSMachineAuxFile = "aux.img" + MacOSMachinePlatformFile = "config.json" +) + +// MacOSMachineLabels returns the labels of a macOS machine image that uses the +// standard file names. +func MacOSMachineLabels() map[string]string { + return map[string]string{ + MacOSMachineVersionLabel: MacOSMachineVersion, MacOSMachineKindLabel: MacOSMachineKind, MacOSMachineFormatLabel: MacOSMachineFormat, + MacOSMachineDiskLabel: MacOSMachineDiskFile, MacOSMachineAuxLabel: MacOSMachineAuxFile, MacOSMachinePlatformLabel: MacOSMachinePlatformFile, + } +} + // stagedMacOS is the part of a staged macOS machine image beyond its base disk. type stagedMacOS struct { layers []string // staged overlays above the base, in order, beside their final paths @@ -52,13 +75,13 @@ func installMacOSMachine(layout imageLayout, staged stagedImageFiles, meta *imag } var installed []string for i, layer := range staged.macos.layers { - path := filepath.Join(layout.dir, macOSDiskLayerFile(i+1)) + path := filepath.Join(layout.dir, MacOSDiskLayerFile(i+1)) if err := os.Rename(layer, path); err != nil { return installed, err } installed = append(installed, path) } - auxPath := filepath.Join(layout.dir, "aux.img") + auxPath := filepath.Join(layout.dir, MacOSMachineAuxFile) if err := os.Rename(staged.macos.aux, auxPath); err != nil { return installed, err } @@ -97,26 +120,40 @@ func checkCachedMachinePlatform(p *paths.Paths, ref *ResolvedRef, requested Plat return nil } -// readyMacOSLayer returns the disk of a ready macOS image's chain layer that has -// diffID, or "" if no ready image has one. -func (m *manager) readyMacOSLayer(diffID string) string { +// reusableMacOSLayers returns, for each lower layer of a chain with diffIDs, the +// disk of a ready image whose chain has the same layers up to and including it, or +// "". Matching the layers beneath keeps each reused overlay on the chain it was +// built on. The top layer is never reused, since it carries the image's own files. +func (m *manager) reusableMacOSLayers(diffIDs []string) []string { + reuse := make([]string, len(diffIDs)) metas, err := listAllMetadata(m.paths) if err != nil { - return "" + slog.Warn("list images for macOS layer reuse", "error", err) + return reuse } for _, meta := range metas { if meta.Status != StatusReady || meta.MacOS == nil || len(meta.MacOSDiskDiffIDs) != meta.MacOSDiskLayers { continue } - i := slices.Index(meta.MacOSDiskDiffIDs, diffID) - if i < 0 { + shared := 0 + for shared < len(meta.MacOSDiskDiffIDs) && shared < len(diffIDs)-1 && meta.MacOSDiskDiffIDs[shared] == diffIDs[shared] { + shared++ + } + if shared == 0 { continue } - if chain, err := MacOSDiskChain(m.paths, meta.Name, meta.Digest); err == nil { - return chain[i] + chain, err := MacOSDiskChain(m.paths, meta.Name, meta.Digest) + if err != nil { + slog.Warn("read macOS image chain for layer reuse", "image", meta.Name, "error", err) + continue + } + for i := range shared { + if reuse[i] == "" { + reuse[i] = chain[i] + } } } - return "" + return reuse } // stageMacOSOutputs stages a macOS machine image's boot disk chain and auxiliary @@ -155,7 +192,7 @@ func stageMacOSMachine(staged *stagedImageFiles, root string, meta *containerMet chain := []string{staged.disk} for i, disk := range payload.Disks { if i > 0 { - layer := filepath.Join(layoutDir, macOSDiskLayerFile(i)+".tmp-"+buildID) + layer := filepath.Join(layoutDir, MacOSDiskLayerFile(i)+".tmp-"+buildID) staged.macos.layers = append(staged.macos.layers, layer) chain = append(chain, layer) } @@ -230,9 +267,9 @@ func validateMacOSMachine(meta *containerMetadata) error { return fmt.Errorf("macOS machine requires darwin/arm64, not %s", normalized) } labels := meta.Labels - for _, want := range [][2]string{{MacOSMachineVersionLabel, "2"}, {MacOSMachineKindLabel, "macos-image"}, {MacOSMachineFormatLabel, "asif"}} { - if label := want[0]; labels[label] != want[1] { - return fmt.Errorf("darwin artifact requires %s=%q, got %q; ordinary container images are not macOS machines", label, want[1], labels[label]) + for label, want := range map[string]string{MacOSMachineVersionLabel: MacOSMachineVersion, MacOSMachineKindLabel: MacOSMachineKind, MacOSMachineFormatLabel: MacOSMachineFormat} { + if labels[label] != want { + return fmt.Errorf("darwin artifact requires %s=%q, got %q; ordinary container images are not macOS machines", label, want, labels[label]) } } pathLabels := []string{MacOSMachineDiskLabel, MacOSMachineAuxLabel, MacOSMachinePlatformLabel} diff --git a/lib/images/macos_machine_test.go b/lib/images/macos_machine_test.go index 2bae089c5..7f393797e 100644 --- a/lib/images/macos_machine_test.go +++ b/lib/images/macos_machine_test.go @@ -260,14 +260,16 @@ func TestMacOSMachinePullReusesReadyLayers(t *testing.T) { mgr, err := NewManager(p, 1, nil) require.NoError(t, err) client := mgr.(*manager).ociClient - lookup := client.readyMachineLayer + lookup := client.reusableMachineLayers var reused []string - client.readyMachineLayer = func(diffID string) string { - path := lookup(diffID) - if path != "" { - reused = append(reused, path) + client.reusableMachineLayers = func(diffIDs []string) []string { + paths := lookup(diffIDs) + for _, path := range paths { + if path != "" { + reused = append(reused, path) + } } - return path + return paths } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index 6cd7bafbb..53ab5a80a 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -35,8 +35,8 @@ var maxMachineUnpackedBytes int64 = 1 << 40 // the rootfs unpack, it writes files sparsely: zero chunks become holes, so a // mostly empty boot disk occupies only its data. Machine bundles hold only regular // files and directories; any other entry is rejected. Each layer is checked -// against its config diff ID. A layer below the top whose disk a ready image -// already has is cloned from that image instead of unpacked. +// against its config diff ID. A layer below the top that a ready image already has, +// on the same layers beneath it, is cloned from that image instead of unpacked. func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDir string) error { path, err := layout.FromPath(c.cacheDir) if err != nil { @@ -64,10 +64,18 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi if err := os.MkdirAll(targetDir, 0700); err != nil { return fmt.Errorf("create target dir: %w", err) } + diffIDs := make([]string, len(configFile.RootFS.DiffIDs)) + for i, diffID := range configFile.RootFS.DiffIDs { + diffIDs[i] = diffID.String() + } + reuse := make([]string, len(layers)) + if c.reusableMachineLayers != nil { + reuse = c.reusableMachineLayers(diffIDs) + } remaining := maxMachineUnpackedBytes for i, l := range layers { dir := filepath.Join(targetDir, strconv.Itoa(i)) - if i < len(layers)-1 && c.reuseMachineLayer(configFile.RootFS.DiffIDs[i], dir, configFile.Config.Labels[MacOSMachineDiskLabel]) { + if reuse[i] != "" && cloneMachineLayer(reuse[i], dir, configFile.Config.Labels[MacOSMachineDiskLabel]) { continue } if err := unpackMachineLayer(ctx, l, configFile.RootFS.DiffIDs[i], dir, &remaining); err != nil { @@ -77,17 +85,10 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi return nil } -// reuseMachineLayer clones a ready image's disk with diffID into dir, where -// unpacking the layer would put it, and reports whether it did. A failed clone, as -// when that image is being deleted, leaves dir absent for an unpack. -func (c *ociClient) reuseMachineLayer(diffID gcr.Hash, dir, diskName string) bool { - if c.readyMachineLayer == nil { - return false - } - source := c.readyMachineLayer(diffID.String()) - if source == "" { - return false - } +// cloneMachineLayer clones a ready image's disk into dir, where unpacking the layer +// would put it, and reports whether it did. A failed clone, as when that image is +// being deleted, leaves dir absent for an unpack. +func cloneMachineLayer(source, dir, diskName string) bool { target := filepath.Join(dir, diskName) if err := os.MkdirAll(filepath.Dir(target), 0700); err == nil { if err = forkvm.CopyRegularFile(source, target); err == nil { diff --git a/lib/images/manager.go b/lib/images/manager.go index 5bfcd4453..8b1f733c3 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -110,7 +110,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string), } - ociClient.readyMachineLayer = m.readyMacOSLayer + ociClient.reusableMachineLayers = m.reusableMacOSLayers // Initialize metrics if meter is provided if meter != nil { diff --git a/lib/images/oci.go b/lib/images/oci.go index 0ebb57636..abb744f6a 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -25,8 +25,9 @@ import ( // ociClient handles OCI image operations without requiring Docker daemon type ociClient struct { cacheDir string - // readyMachineLayer returns a ready image's disk for a macOS layer diff ID, or "". - readyMachineLayer func(diffID string) string + // reusableMachineLayers returns, for each layer of a macOS chain with diffIDs, a + // ready image's disk to clone instead of unpacking it, or "". + reusableMachineLayers func(diffIDs []string) []string } type credentialedManifestInspector struct { diff --git a/lib/images/storage.go b/lib/images/storage.go index b707d1b72..ab78dac8d 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -158,7 +158,7 @@ func legacyLayout(p *paths.Paths, repository, digestHex string) imageLayout { // machineDiskFile is the base of a macOS machine image's ASIF boot disk chain, // whether imported locally or pulled. Rootfs images use erofs or ext4 instead. -var machineDiskFile = macOSDiskLayerFile(0) +const machineDiskFile = "disk.0.asif" // MacOSDiskLayerFile(0) // withMachineDisk points layout at a machine image's base disk when the directory // has one. Rootfs layouts never do, so a stat is enough to pick the file. diff --git a/lib/instances/create.go b/lib/instances/create.go index f044416d4..415344620 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,9 +99,7 @@ func (m *manager) createInstance( if err := validateExpiresAt(req.ExpiresAt, m.nowUTC()); err != nil { return nil, err } - if m.macOSOnly && strings.TrimSpace(req.Platform) == "" { - req.Platform = "darwin/arm64" - } + req.Platform = DefaultPlatform(m.macOSOnly, req.Platform) if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index 825196d8b..62508c947 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -24,26 +24,15 @@ func resolveImageForCreate(ctx context.Context, imageManager createImageResolver return nil, fmt.Errorf("get macOS image: %w", err) } if err != nil || img.MacOS == nil { - img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}) - if err != nil { + if img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}); err != nil { return nil, fmt.Errorf("pull macOS image: %w", err) } } - pinned, err := pinnedImageName(imageName, img.Digest) - if err != nil { - return nil, err + img, err = awaitResolvedImage(ctx, imageManager, img, imageName, platform, log) + if err == nil && img.MacOS == nil { + err = fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) } - img, err = awaitReady(ctx, imageManager, img, pinned, log) - if err != nil { - return nil, err - } - if img.MacOS == nil { - return nil, fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) - } - if err := validateResolvedImagePlatform(img, platform); err != nil { - return nil, err - } - return img, nil + return img, err } // An empty platform means the host platform: a no-platform run must boot a // host-native guest and never silently emulate, so we resolve the host variant @@ -72,6 +61,12 @@ func resolveImageForCreate(ctx context.Context, imageManager createImageResolver if err != nil { return nil, fmt.Errorf("resolve image %s for platform %s: %w", imageName, resolvePlatform, err) } + return awaitResolvedImage(ctx, imageManager, img, imageName, resolvePlatform, log) +} + +// awaitResolvedImage pins img by digest, waits for it to be ready and checks it +// is for platform. +func awaitResolvedImage(ctx context.Context, imageManager createImageResolver, img *images.Image, imageName, platform string, log *slog.Logger) (*images.Image, error) { pinned, err := pinnedImageName(imageName, img.Digest) if err != nil { return nil, err @@ -80,7 +75,7 @@ func resolveImageForCreate(ctx context.Context, imageManager createImageResolver if err != nil { return nil, err } - if err := validateResolvedImagePlatform(img, resolvePlatform); err != nil { + if err := validateResolvedImagePlatform(img, platform); err != nil { return nil, err } return img, nil diff --git a/lib/instances/macos.go b/lib/instances/macos.go index b3712b47b..e08073dde 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -5,8 +5,6 @@ import ( "context" "errors" "fmt" - "github.com/kernel/hypeman/lib/forkvm" - "github.com/kernel/hypeman/lib/system" "net" "os" "path/filepath" @@ -14,6 +12,7 @@ import ( "syscall" "time" + "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" @@ -22,10 +21,10 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + "github.com/kernel/hypeman/lib/system" ) -// macOSAuxStorageFile is the instance's copy of the image's auxiliary storage. -const macOSAuxStorageFile = "mac-aux.img" +const macOSAuxStorageFile = paths.MacOSAuxStorageFile // requireMacOSImageName checks lookup errors before resolving the reference as // darwin/arm64. The resolved image is validated by checkMacOSCreate. @@ -60,8 +59,8 @@ func (m *manager) checkMacOSCreate(req *CreateInstanceRequest, img *images.Image } config := hypervisor.VMConfig{ BootMode: hypervisor.BootModeMacOS, VCPUs: prepared.Vcpus, MemoryBytes: prepared.Size, - MacOS: &hypervisor.MacOSPlatform{HardwareModelData: img.MacOS.HardwareModel, MachineIdentifierData: img.MacOS.MachineIdentifier, AuxStoragePath: "mac-aux.img"}, - Disks: []hypervisor.DiskConfig{{Path: paths.MacOSBootDiskFile}}, + MacOS: &hypervisor.MacOSPlatform{HardwareModelData: img.MacOS.HardwareModel, MachineIdentifierData: img.MacOS.MachineIdentifier, AuxStoragePath: macOSAuxStorageFile}, + Disks: []hypervisor.DiskConfig{{Path: paths.MacOSBootDiskFile, Format: hypervisor.DiskFormatASIF, Parents: []string{images.MacOSDiskLayerFile(0)}}}, } if err := validatePlannedVMConfig(starter, hvType, config); err != nil { return err @@ -162,6 +161,15 @@ func (m *manager) checkMacOSStart(stored *StoredMetadata, req StartInstanceReque return nil } +// DefaultPlatform returns the create platform: a macOS-only server defaults an +// unset one to darwin/arm64. +func DefaultPlatform(macOSOnly bool, platform string) string { + if macOSOnly && strings.TrimSpace(platform) == "" { + return "darwin/arm64" + } + return platform +} + // createBootDisk creates the Linux overlay, or the macOS boot disk: the image's // ASIF chain and auxiliary storage cloned into the instance, with an empty // writable overlay on top. @@ -173,7 +181,7 @@ func (m *manager) createBootDisk(ctx context.Context, stored *StoredMetadata, im if err != nil { return err } - aux := filepath.Join(filepath.Dir(chain[0]), "aux.img") + aux := filepath.Join(filepath.Dir(chain[0]), images.MacOSMachineAuxFile) if err := createMacOSStorage(chain, aux, stored.DataDir); err != nil { return err } @@ -212,15 +220,10 @@ func (m *manager) installMacOSAgentDisk(ctx context.Context, stored *StoredMetad return os.Rename(staged, disk) } -// macOSDiskLayer is the ith read-only layer of a macOS guest's boot disk chain. -func macOSDiskLayer(dataDir string, i int) string { - return filepath.Join(dataDir, fmt.Sprintf("disk.%d.asif", i)) -} - // macOSDisks returns a macOS guest's disks: its boot disk, then its read-only guest // agent disk. The boot disk must come first; VZ does not boot with it second. func macOSDisks(p *paths.Paths, stored *StoredMetadata) []hypervisor.DiskConfig { - disks := []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk(stored.Id), Parents: macOSDiskParents(stored)}} + disks := []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk(stored.Id), Format: hypervisor.DiskFormatASIF, Parents: macOSDiskParents(stored)}} if stored.MacOSAgentDisk { disks = append(disks, hypervisor.DiskConfig{Path: p.InstanceMacOSAgentDisk(stored.Id), Readonly: true}) } @@ -231,7 +234,7 @@ func macOSDisks(p *paths.Paths, stored *StoredMetadata) []hypervisor.DiskConfig func macOSDiskParents(stored *StoredMetadata) []string { parents := make([]string, stored.MacOSDiskLayers) for i := range parents { - parents[i] = macOSDiskLayer(stored.DataDir, i) + parents[i] = filepath.Join(stored.DataDir, images.MacOSDiskLayerFile(i)) } return parents } diff --git a/lib/instances/macos_agent_disk_test.go b/lib/instances/macos_agent_disk_test.go index 568d99128..3d9cb5ddf 100644 --- a/lib/instances/macos_agent_disk_test.go +++ b/lib/instances/macos_agent_disk_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) @@ -49,7 +50,7 @@ func TestMacOSAgentDiskInstallAndRefresh(t *testing.T) { func TestMacOSDisksAttachAgentDiskSecond(t *testing.T) { p := paths.New(t.TempDir()) stored := &StoredMetadata{Id: "mac", DataDir: p.InstanceDir("mac"), MacOSDiskLayers: 1} - require.Equal(t, []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk("mac"), Parents: []string{macOSDiskLayer(stored.DataDir, 0)}}}, macOSDisks(p, stored)) + require.Equal(t, []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk("mac"), Format: hypervisor.DiskFormatASIF, Parents: []string{filepath.Join(stored.DataDir, images.MacOSDiskLayerFile(0))}}}, macOSDisks(p, stored)) stored.MacOSAgentDisk = true disks := macOSDisks(p, stored) require.Len(t, disks, 2) diff --git a/lib/instances/macos_memory_snapshot_test.go b/lib/instances/macos_memory_snapshot_test.go index ef205e947..de9e9397a 100644 --- a/lib/instances/macos_memory_snapshot_test.go +++ b/lib/instances/macos_memory_snapshot_test.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -37,7 +38,7 @@ func macOSMemorySnapshotFixture(t *testing.T) (*manager, *metadata) { meta.MacOSDiskLayers = 1 require.NoError(t, m.saveMetadata(meta)) require.NoError(t, os.Rename(m.paths.InstanceOverlay(id), m.paths.InstanceMacOSBootDisk(id))) - require.NoError(t, os.WriteFile(macOSDiskLayer(meta.DataDir, 0), []byte("image layer"), 0400)) + require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, images.MacOSDiskLayerFile(0)), []byte("image layer"), 0400)) require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, macOSAuxStorageFile), []byte("aux-at-checkpoint"), 0600)) return m, meta } diff --git a/lib/instances/macos_storage.go b/lib/instances/macos_storage.go index 6d4ed400b..21d2b6fa8 100644 --- a/lib/instances/macos_storage.go +++ b/lib/instances/macos_storage.go @@ -26,7 +26,7 @@ func createMacOSStorage(chain []string, aux, dataDir string) (err error) { }() copies := make([]string, len(chain)) for i, layer := range chain { - copies[i] = macOSDiskLayer(dataDir, i) + copies[i] = filepath.Join(dataDir, images.MacOSDiskLayerFile(i)) if err = copyMacOSStorageFile(layer, copies[i], 0400, &created); err != nil { return err } diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go index f2d473146..3f9d6bd5e 100644 --- a/lib/instances/snapshot_macos.go +++ b/lib/instances/snapshot_macos.go @@ -16,6 +16,7 @@ import ( "github.com/kernel/hypeman/lib/forkvm" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -190,7 +191,7 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St } payload := []string{filepath.Join(guestDir, paths.MacOSBootDiskFile), filepath.Join(guestDir, macOSAuxStorageFile), filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile)} for i := range stored.MacOSDiskLayers { - payload = append(payload, macOSDiskLayer(guestDir, i)) + payload = append(payload, filepath.Join(guestDir, images.MacOSDiskLayerFile(i))) } if stored.MacOSAgentDisk { payload = append(payload, filepath.Join(guestDir, paths.MacOSAgentDiskFile)) diff --git a/lib/instances/snapshot_macos_darwin_test.go b/lib/instances/snapshot_macos_darwin_test.go index 84c452589..8b3275783 100644 --- a/lib/instances/snapshot_macos_darwin_test.go +++ b/lib/instances/snapshot_macos_darwin_test.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/vz" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" @@ -30,13 +31,14 @@ func TestMacOSStoppedSnapshotForkRebindsIdentity(t *testing.T) { source.MAC = source.MacOS.MAC source.MacOS.GuestAgent = true source.StopGraceful = true + source.MacOSDiskLayers = 1 now := time.Now() source.GuestAgentReadyAt = &now source.Phases.Record(phasetracking.PhaseRunning, now.Add(-time.Hour)) source.Phases.Record(phasetracking.PhaseStopped, now) require.NoError(t, vz.NewStarter().ValidateConfig(m.plannedStoredVMConfig(SnapshotKindStopped, source))) require.NoError(t, m.saveSnapshotRecord(&snapshotRecord{Snapshot: Snapshot{Id: snapshotID, Kind: SnapshotKindStopped, SourceInstanceID: source.Id, SourceName: source.Name, SourceHypervisor: source.HypervisorType, CreatedAt: now}, StoredMetadata: source})) - writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{paths.MacOSBootDiskFile: "snapshot-disk", macOSAuxStorageFile: "snapshot-aux"}) + writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{paths.MacOSBootDiskFile: "snapshot-disk", images.MacOSDiskLayerFile(0): "snapshot-layer", macOSAuxStorageFile: "snapshot-aux"}) first, err := m.ForkSnapshot(context.Background(), snapshotID, ForkSnapshotRequest{Name: "mac-snapshot-clone-1"}) require.NoError(t, err) diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 67ba0275d..00d76cb4c 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -260,6 +260,9 @@ func (p *Paths) InstanceMacOSBootDisk(id string) string { return filepath.Join(p.InstanceDir(id), MacOSBootDiskFile) } +// MacOSAuxStorageFile is a macOS guest's copy of its image's auxiliary storage. +const MacOSAuxStorageFile = "mac-aux.img" + // MacOSAgentDiskFile is a macOS guest's copy of the read-only guest agent disk. const MacOSAgentDiskFile = "agent.img" diff --git a/lib/system/macos_agent_disk_darwin.go b/lib/system/macos_agent_disk_darwin.go index b4ba94010..4a7dab48f 100644 --- a/lib/system/macos_agent_disk_darwin.go +++ b/lib/system/macos_agent_disk_darwin.go @@ -11,7 +11,6 @@ import ( "os" "os/exec" "path/filepath" - "sync" "github.com/kernel/hypeman/lib/paths" ) @@ -21,7 +20,8 @@ import ( //go:embed guest_agent/guest-agent-darwin var DarwinGuestAgentBinary []byte -var macOSAgentDiskMu sync.Mutex +// macOSAgentDiskBuild serializes disk builds; waiting for one honors ctx. +var macOSAgentDiskBuild = make(chan struct{}, 1) // MacOSAgentDisk returns the read-only guest agent disk for the embedded macOS // agent, building it on first use. Disks are named by the agent's hash, so a new @@ -30,8 +30,12 @@ func MacOSAgentDisk(ctx context.Context, p *paths.Paths) (string, error) { sum := sha256.Sum256(DarwinGuestAgentBinary) version := hex.EncodeToString(sum[:6]) path := p.SystemMacOSAgentDisk(version) - macOSAgentDiskMu.Lock() - defer macOSAgentDiskMu.Unlock() + select { + case macOSAgentDiskBuild <- struct{}{}: + case <-ctx.Done(): + return "", ctx.Err() + } + defer func() { <-macOSAgentDiskBuild }() if _, err := os.Stat(path); err == nil { return path, nil } From 5bf81e61744c80fd1eb8c3fe5d3cd72d7a1aeda4 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:50:34 +0000 Subject: [PATCH 23/33] Move registry tokens into lib/registrytoken Image publication needs tokens without the rest of lib/builds, which links the API server. lib/builds keeps its names as aliases. --- lib/builds/registry_token.go | 223 +----------------- lib/registrytoken/token.go | 221 +++++++++++++++++ .../token_test.go} | 24 +- 3 files changed, 241 insertions(+), 227 deletions(-) create mode 100644 lib/registrytoken/token.go rename lib/{builds/registry_token_test.go => registrytoken/token_test.go} (93%) diff --git a/lib/builds/registry_token.go b/lib/builds/registry_token.go index 3a2f8abfe..0aaa34d03 100644 --- a/lib/builds/registry_token.go +++ b/lib/builds/registry_token.go @@ -1,220 +1,13 @@ -// Package builds implements registry token generation for secure builder VM authentication. package builds -import ( - "fmt" - "time" +import "github.com/kernel/hypeman/lib/registrytoken" - "github.com/golang-jwt/jwt/v5" +// Registry tokens are minted by lib/registrytoken, which image publication uses +// without the rest of this package. +type ( + RepoPermission = registrytoken.RepoPermission + RegistryTokenClaims = registrytoken.Claims + RegistryTokenGenerator = registrytoken.Generator ) -// RepoPermission defines access permissions for a specific repository. -// This enables fine-grained control where different repos can have different scopes. -type RepoPermission struct { - // Repo is the repository path (e.g., "builds/abc123", "cache/tenant-x", "cache/global/node") - Repo string `json:"repo"` - - // Scope is the access scope for this repo: "pull" for read-only, "push" for read+write - Scope string `json:"scope"` -} - -// RegistryTokenClaims contains the claims for a scoped registry access token. -// These tokens are issued to builder VMs to grant limited access to specific repositories. -type RegistryTokenClaims struct { - jwt.RegisteredClaims - - // BuildID is the build job identifier for audit purposes - BuildID string `json:"build_id"` - - // RepoAccess defines per-repository access permissions (new two-tier format) - // If present, this takes precedence over the legacy Repositories/Scope fields - RepoAccess []RepoPermission `json:"repo_access,omitempty"` - - // Repositories is the list of allowed repository paths (legacy format, kept for backward compat) - // Deprecated: Use RepoAccess for new tokens - Repositories []string `json:"repos,omitempty"` - - // Scope is the access scope (legacy format, kept for backward compat) - // Deprecated: Use RepoAccess for new tokens - Scope string `json:"scope,omitempty"` -} - -// RegistryTokenGenerator creates scoped registry access tokens -type RegistryTokenGenerator struct { - secret []byte -} - -// NewRegistryTokenGenerator creates a new token generator with the given secret -func NewRegistryTokenGenerator(secret string) *RegistryTokenGenerator { - return &RegistryTokenGenerator{ - secret: []byte(secret), - } -} - -// GeneratePushToken creates a short-lived token granting push access to specific repositories. -// The token expires after the specified duration (typically matching the build timeout). -// Deprecated: Use GenerateToken for new code that needs per-repo scopes. -func (g *RegistryTokenGenerator) GeneratePushToken(buildID string, repos []string, ttl time.Duration) (string, error) { - if buildID == "" { - return "", fmt.Errorf("build ID is required") - } - if len(repos) == 0 { - return "", fmt.Errorf("at least one repository is required") - } - - now := time.Now() - claims := RegistryTokenClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Subject: "builder-" + buildID, - IssuedAt: jwt.NewNumericDate(now), - ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), - Issuer: "hypeman", - }, - BuildID: buildID, - Repositories: repos, - Scope: "push", - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - return token.SignedString(g.secret) -} - -// GenerateToken creates a short-lived token with per-repository access permissions. -// This supports the two-tier cache model where different repos can have different scopes. -// For example: pull on cache/global/*, push on cache/{tenant} -func (g *RegistryTokenGenerator) GenerateToken(buildID string, repoAccess []RepoPermission, ttl time.Duration) (string, error) { - if buildID == "" { - return "", fmt.Errorf("build ID is required") - } - if len(repoAccess) == 0 { - return "", fmt.Errorf("at least one repository permission is required") - } - - now := time.Now() - claims := RegistryTokenClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Subject: "builder-" + buildID, - IssuedAt: jwt.NewNumericDate(now), - ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), - Issuer: "hypeman", - }, - BuildID: buildID, - RepoAccess: repoAccess, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - return token.SignedString(g.secret) -} - -// ValidateToken parses and validates a registry token, returning the claims if valid. -func (g *RegistryTokenGenerator) ValidateToken(tokenString string) (*RegistryTokenClaims, error) { - token, err := jwt.ParseWithClaims(tokenString, &RegistryTokenClaims{}, func(token *jwt.Token) (interface{}, error) { - // Validate signing method - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) - } - return g.secret, nil - }) - - if err != nil { - return nil, fmt.Errorf("parse token: %w", err) - } - - claims, ok := token.Claims.(*RegistryTokenClaims) - if !ok || !token.Valid { - return nil, fmt.Errorf("invalid token") - } - - return claims, nil -} - -// IsRepositoryAllowed checks if the given repository path is allowed by the token claims. -func (c *RegistryTokenClaims) IsRepositoryAllowed(repo string) bool { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Repo == repo { - return true - } - } - return false - } - - // Fall back to legacy format - for _, allowed := range c.Repositories { - if allowed == repo { - return true - } - } - return false -} - -// GetRepoScope returns the scope for a specific repository. -// Returns empty string if the repository is not allowed. -func (c *RegistryTokenClaims) GetRepoScope(repo string) string { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Repo == repo { - return perm.Scope - } - } - return "" - } - - // Fall back to legacy format - all repos have the same scope - for _, allowed := range c.Repositories { - if allowed == repo { - return c.Scope - } - } - return "" -} - -// IsPushAllowedForRepo returns true if the token grants push (write) access to the given repo. -func (c *RegistryTokenClaims) IsPushAllowedForRepo(repo string) bool { - scope := c.GetRepoScope(repo) - return scope == "push" -} - -// IsPullAllowedForRepo returns true if the token grants pull (read) access to the given repo. -// Push scope also implicitly grants pull access. -func (c *RegistryTokenClaims) IsPullAllowedForRepo(repo string) bool { - scope := c.GetRepoScope(repo) - return scope == "push" || scope == "pull" -} - -// IsPushAllowed returns true if the token grants push (write) access to any repo. -// Deprecated: Use IsPushAllowedForRepo for per-repo scope checking. -func (c *RegistryTokenClaims) IsPushAllowed() bool { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Scope == "push" { - return true - } - } - return false - } - - // Fall back to legacy format - return c.Scope == "push" -} - -// IsPullAllowed returns true if the token grants pull (read) access. -// Push tokens also implicitly grant pull access. -// Deprecated: Use IsPullAllowedForRepo for per-repo scope checking. -func (c *RegistryTokenClaims) IsPullAllowed() bool { - // Check new per-repo access format first - if len(c.RepoAccess) > 0 { - for _, perm := range c.RepoAccess { - if perm.Scope == "push" || perm.Scope == "pull" { - return true - } - } - return false - } - - // Fall back to legacy format - return c.Scope == "push" || c.Scope == "pull" -} +var NewRegistryTokenGenerator = registrytoken.NewGenerator diff --git a/lib/registrytoken/token.go b/lib/registrytoken/token.go new file mode 100644 index 000000000..df69a9790 --- /dev/null +++ b/lib/registrytoken/token.go @@ -0,0 +1,221 @@ +// Package registrytoken mints and checks the JWTs that scope access to hypeman's +// registry, for builder VMs and image publication. +package registrytoken + +import ( + "fmt" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// RepoPermission defines access permissions for a specific repository. +// This enables fine-grained control where different repos can have different scopes. +type RepoPermission struct { + // Repo is the repository path (e.g., "builds/abc123", "cache/tenant-x", "cache/global/node") + Repo string `json:"repo"` + + // Scope is the access scope for this repo: "pull" for read-only, "push" for read+write + Scope string `json:"scope"` +} + +// Claims contains the claims for a scoped registry access token. +// These tokens are issued to builder VMs to grant limited access to specific repositories. +type Claims struct { + jwt.RegisteredClaims + + // BuildID is the build job identifier for audit purposes + BuildID string `json:"build_id"` + + // RepoAccess defines per-repository access permissions (new two-tier format) + // If present, this takes precedence over the legacy Repositories/Scope fields + RepoAccess []RepoPermission `json:"repo_access,omitempty"` + + // Repositories is the list of allowed repository paths (legacy format, kept for backward compat) + // Deprecated: Use RepoAccess for new tokens + Repositories []string `json:"repos,omitempty"` + + // Scope is the access scope (legacy format, kept for backward compat) + // Deprecated: Use RepoAccess for new tokens + Scope string `json:"scope,omitempty"` +} + +// Generator creates scoped registry access tokens +type Generator struct { + secret []byte +} + +// NewGenerator creates a new token generator with the given secret +func NewGenerator(secret string) *Generator { + return &Generator{ + secret: []byte(secret), + } +} + +// GeneratePushToken creates a short-lived token granting push access to specific repositories. +// The token expires after the specified duration (typically matching the build timeout). +// Deprecated: Use GenerateToken for new code that needs per-repo scopes. +func (g *Generator) GeneratePushToken(buildID string, repos []string, ttl time.Duration) (string, error) { + if buildID == "" { + return "", fmt.Errorf("build ID is required") + } + if len(repos) == 0 { + return "", fmt.Errorf("at least one repository is required") + } + + now := time.Now() + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Subject: "builder-" + buildID, + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), + Issuer: "hypeman", + }, + BuildID: buildID, + Repositories: repos, + Scope: "push", + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + return token.SignedString(g.secret) +} + +// GenerateToken creates a short-lived token with per-repository access permissions. +// This supports the two-tier cache model where different repos can have different scopes. +// For example: pull on cache/global/*, push on cache/{tenant} +func (g *Generator) GenerateToken(buildID string, repoAccess []RepoPermission, ttl time.Duration) (string, error) { + if buildID == "" { + return "", fmt.Errorf("build ID is required") + } + if len(repoAccess) == 0 { + return "", fmt.Errorf("at least one repository permission is required") + } + + now := time.Now() + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Subject: "builder-" + buildID, + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(ttl)), + Issuer: "hypeman", + }, + BuildID: buildID, + RepoAccess: repoAccess, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + return token.SignedString(g.secret) +} + +// ValidateToken parses and validates a registry token, returning the claims if valid. +func (g *Generator) ValidateToken(tokenString string) (*Claims, error) { + token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) { + // Validate signing method + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return g.secret, nil + }) + + if err != nil { + return nil, fmt.Errorf("parse token: %w", err) + } + + claims, ok := token.Claims.(*Claims) + if !ok || !token.Valid { + return nil, fmt.Errorf("invalid token") + } + + return claims, nil +} + +// IsRepositoryAllowed checks if the given repository path is allowed by the token claims. +func (c *Claims) IsRepositoryAllowed(repo string) bool { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Repo == repo { + return true + } + } + return false + } + + // Fall back to legacy format + for _, allowed := range c.Repositories { + if allowed == repo { + return true + } + } + return false +} + +// GetRepoScope returns the scope for a specific repository. +// Returns empty string if the repository is not allowed. +func (c *Claims) GetRepoScope(repo string) string { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Repo == repo { + return perm.Scope + } + } + return "" + } + + // Fall back to legacy format - all repos have the same scope + for _, allowed := range c.Repositories { + if allowed == repo { + return c.Scope + } + } + return "" +} + +// IsPushAllowedForRepo returns true if the token grants push (write) access to the given repo. +func (c *Claims) IsPushAllowedForRepo(repo string) bool { + scope := c.GetRepoScope(repo) + return scope == "push" +} + +// IsPullAllowedForRepo returns true if the token grants pull (read) access to the given repo. +// Push scope also implicitly grants pull access. +func (c *Claims) IsPullAllowedForRepo(repo string) bool { + scope := c.GetRepoScope(repo) + return scope == "push" || scope == "pull" +} + +// IsPushAllowed returns true if the token grants push (write) access to any repo. +// Deprecated: Use IsPushAllowedForRepo for per-repo scope checking. +func (c *Claims) IsPushAllowed() bool { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Scope == "push" { + return true + } + } + return false + } + + // Fall back to legacy format + return c.Scope == "push" +} + +// IsPullAllowed returns true if the token grants pull (read) access. +// Push tokens also implicitly grant pull access. +// Deprecated: Use IsPullAllowedForRepo for per-repo scope checking. +func (c *Claims) IsPullAllowed() bool { + // Check new per-repo access format first + if len(c.RepoAccess) > 0 { + for _, perm := range c.RepoAccess { + if perm.Scope == "push" || perm.Scope == "pull" { + return true + } + } + return false + } + + // Fall back to legacy format + return c.Scope == "push" || c.Scope == "pull" +} diff --git a/lib/builds/registry_token_test.go b/lib/registrytoken/token_test.go similarity index 93% rename from lib/builds/registry_token_test.go rename to lib/registrytoken/token_test.go index e8eaf1037..898155ee5 100644 --- a/lib/builds/registry_token_test.go +++ b/lib/registrytoken/token_test.go @@ -1,4 +1,4 @@ -package builds +package registrytoken import ( "testing" @@ -9,7 +9,7 @@ import ( ) func TestRegistryTokenGenerator_GeneratePushToken(t *testing.T) { - generator := NewRegistryTokenGenerator("test-secret-key") + generator := NewGenerator("test-secret-key") t.Run("valid token generation", func(t *testing.T) { token, err := generator.GeneratePushToken("build-123", []string{"builds/build-123", "cache/tenant-x"}, 30*time.Minute) @@ -40,7 +40,7 @@ func TestRegistryTokenGenerator_GeneratePushToken(t *testing.T) { } func TestRegistryTokenGenerator_ValidateToken(t *testing.T) { - generator := NewRegistryTokenGenerator("test-secret-key") + generator := NewGenerator("test-secret-key") t.Run("valid token", func(t *testing.T) { token, err := generator.GeneratePushToken("build-abc", []string{"builds/build-abc"}, time.Hour) @@ -63,12 +63,12 @@ func TestRegistryTokenGenerator_ValidateToken(t *testing.T) { t.Run("invalid signature", func(t *testing.T) { // Generate with one secret - gen1 := NewRegistryTokenGenerator("secret-1") + gen1 := NewGenerator("secret-1") token, err := gen1.GeneratePushToken("build-123", []string{"builds/build-123"}, time.Hour) require.NoError(t, err) // Validate with different secret - gen2 := NewRegistryTokenGenerator("secret-2") + gen2 := NewGenerator("secret-2") _, err = gen2.ValidateToken(token) require.Error(t, err) assert.Contains(t, err.Error(), "signature is invalid") @@ -81,7 +81,7 @@ func TestRegistryTokenGenerator_ValidateToken(t *testing.T) { } func TestRegistryTokenClaims_IsRepositoryAllowed(t *testing.T) { - claims := &RegistryTokenClaims{ + claims := &Claims{ Repositories: []string{"builds/abc123", "cache/tenant-x"}, } @@ -98,20 +98,20 @@ func TestRegistryTokenClaims_IsRepositoryAllowed(t *testing.T) { func TestRegistryTokenClaims_IsPushAllowed(t *testing.T) { t.Run("push scope", func(t *testing.T) { - claims := &RegistryTokenClaims{Scope: "push"} + claims := &Claims{Scope: "push"} assert.True(t, claims.IsPushAllowed()) assert.True(t, claims.IsPullAllowed()) // push implies pull }) t.Run("pull scope", func(t *testing.T) { - claims := &RegistryTokenClaims{Scope: "pull"} + claims := &Claims{Scope: "pull"} assert.False(t, claims.IsPushAllowed()) assert.True(t, claims.IsPullAllowed()) }) } func TestRegistryTokenGenerator_GenerateToken(t *testing.T) { - generator := NewRegistryTokenGenerator("test-secret-key") + generator := NewGenerator("test-secret-key") t.Run("valid token with per-repo permissions", func(t *testing.T) { repoAccess := []RepoPermission{ @@ -147,7 +147,7 @@ func TestRegistryTokenGenerator_GenerateToken(t *testing.T) { func TestRegistryTokenClaims_RepoAccess(t *testing.T) { // Test claims with new per-repo access format - claims := &RegistryTokenClaims{ + claims := &Claims{ RepoAccess: []RepoPermission{ {Repo: "builds/abc123", Scope: "push"}, {Repo: "cache/global/node", Scope: "pull"}, @@ -195,7 +195,7 @@ func TestRegistryTokenClaims_RepoAccess(t *testing.T) { func TestRegistryTokenClaims_RepoAccessPullOnly(t *testing.T) { // Test claims with only pull access - claims := &RegistryTokenClaims{ + claims := &Claims{ RepoAccess: []RepoPermission{ {Repo: "cache/global/node", Scope: "pull"}, }, @@ -212,7 +212,7 @@ func TestRegistryTokenClaims_RepoAccessPullOnly(t *testing.T) { func TestRegistryTokenClaims_LegacyFallback(t *testing.T) { // Test that legacy format still works when RepoAccess is empty - claims := &RegistryTokenClaims{ + claims := &Claims{ Repositories: []string{"builds/abc123", "cache/tenant-x"}, Scope: "push", } From a97a4107d1859d588882edcb286ec5b4b004756c Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:50:35 +0000 Subject: [PATCH 24/33] Move machine image publication into lib/machineimage The publisher and Tart reader no longer live in lib/builds, so import-tart stops linking the API server and its embedded binaries, and the workflow builds it directly. Published images record each layer's disk digest; with --base the VM's layers must match them, where before only the layer count was checked. The publisher now gives every image a fresh machine identifier and MAC itself. Tart's content store follows TART_HOME. The workflow's publish step runs with pipefail. --- .github/workflows/macos-image.yml | 6 +- cmd/import-tart/main.go | 33 ++- .../layer_test.go} | 2 +- .../publisher.go} | 223 +++++++++++------- .../publisher_test.go} | 31 ++- lib/{builds => machineimage}/tart.go | 57 +++-- lib/{builds => machineimage}/tart_test.go | 8 +- 7 files changed, 214 insertions(+), 146 deletions(-) rename lib/{builds/macos_layer_test.go => machineimage/layer_test.go} (99%) rename lib/{builds/macos_publisher.go => machineimage/publisher.go} (54%) rename lib/{builds/macos_publisher_test.go => machineimage/publisher_test.go} (73%) rename lib/{builds => machineimage}/tart.go (60%) rename lib/{builds => machineimage}/tart_test.go (96%) diff --git a/.github/workflows/macos-image.yml b/.github/workflows/macos-image.yml index d3bf9b4e8..de247aa57 100644 --- a/.github/workflows/macos-image.yml +++ b/.github/workflows/macos-image.yml @@ -58,9 +58,7 @@ jobs: echo "$bin" >> "$GITHUB_PATH" - name: Build import-tart - run: | - make build-embedded build-vz-shim - go build -o "$RUNNER_TEMP/bin/import-tart" ./cmd/import-tart + run: go build -o "$RUNNER_TEMP/bin/import-tart" ./cmd/import-tart - name: Clone the base stacked env: @@ -79,6 +77,8 @@ jobs: packer build -var vm_base_name="$VM-base" -var vm_name="$VM" browser.pkr.hcl - name: Publish + # pipefail, so a failed publish is not hidden by tee. + shell: bash env: BASE: ${{ inputs.base }} NAME: ${{ inputs.name }} diff --git a/cmd/import-tart/main.go b/cmd/import-tart/main.go index 05f8137e6..cc8e37aa7 100644 --- a/cmd/import-tart/main.go +++ b/cmd/import-tart/main.go @@ -16,9 +16,9 @@ import ( "time" "github.com/Code-Hex/vz/v4" - "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/machineimage" + "github.com/kernel/hypeman/lib/registrytoken" ) func main() { @@ -49,31 +49,30 @@ func main() { } } -func run(ctx context.Context, vm, registry, name, baseRef string, guestAgent bool, secret string) (builds.MachinePublication, error) { - storage, err := builds.ReadTartVM(vm, guestAgent) +func run(ctx context.Context, vm, registry, name, baseRef string, guestAgent bool, secret string) (machineimage.Publication, error) { + storage, err := machineimage.ReadTartVM(vm, guestAgent) if err != nil { - return builds.MachinePublication{}, err + return machineimage.Publication{}, err } for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { if err := images.RequireClosedFile(ctx, path); err != nil { - return builds.MachinePublication{}, fmt.Errorf("%s: %w", path, err) + return machineimage.Publication{}, fmt.Errorf("%s: %w", path, err) } } - var base *builds.MachineBase + var base *machineimage.Base if baseRef != "" { - if base, err = builds.ParseMachineBase(registry, baseRef); err != nil { - return builds.MachinePublication{}, err + if base, err = machineimage.ParseBase(registry, baseRef); err != nil { + return machineimage.Publication{}, err } } - // Tart keeps its source's identity, so each image gets its own. + publisher := &machineimage.Publisher{RegistryURL: registry, Tokens: registrytoken.NewGenerator(secret), NewMachineIdentifier: newMachineIdentifier} + return publisher.Publish(ctx, name, storage, base) +} + +func newMachineIdentifier() ([]byte, error) { id, err := vz.NewMacMachineIdentifier() if err != nil { - return builds.MachinePublication{}, err + return nil, err } - storage.Platform.MachineIdentifier = id.DataRepresentation() - if storage.Platform.MAC, err = network.NewUniqueMAC(nil); err != nil { - return builds.MachinePublication{}, err - } - publisher := &builds.StreamingMachinePublisher{RegistryURL: registry, Tokens: builds.NewRegistryTokenGenerator(secret)} - return publisher.Publish(ctx, name, storage, base) + return id.DataRepresentation(), nil } diff --git a/lib/builds/macos_layer_test.go b/lib/machineimage/layer_test.go similarity index 99% rename from lib/builds/macos_layer_test.go rename to lib/machineimage/layer_test.go index 54b59f8ef..5ad9f65b1 100644 --- a/lib/builds/macos_layer_test.go +++ b/lib/machineimage/layer_test.go @@ -1,4 +1,4 @@ -package builds +package machineimage import ( "context" diff --git a/lib/builds/macos_publisher.go b/lib/machineimage/publisher.go similarity index 54% rename from lib/builds/macos_publisher.go rename to lib/machineimage/publisher.go index 8e9384d61..14704d3f0 100644 --- a/lib/builds/macos_publisher.go +++ b/lib/machineimage/publisher.go @@ -1,9 +1,14 @@ -package builds +// Package machineimage publishes macOS machine images to hypeman's registry, from +// stopped Tart VMs or other ASIF disk chains. +package machineimage import ( "archive/tar" "compress/gzip" "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" @@ -11,6 +16,7 @@ import ( "net/http" "net/url" "os" + "slices" "strings" "time" @@ -23,127 +29,125 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/registrytoken" ) -// MachineStorage is a closed macOS boot disk and platform to publish: the read-only -// ASIF layers below its overlay, base first, the overlay, and its auxiliary storage. -// A standalone base disk is an Overlay with no Layers. -type MachineStorage struct { - Layers []string - Overlay string - Aux string - Platform images.MacOSImage +// Storage is a closed macOS boot disk and platform to publish: the read-only ASIF +// layers below its overlay, base first, the overlay, and its auxiliary storage. A +// standalone base disk is an Overlay with no Layers. LayerDigests, when known, are +// the sha256 of each layer file; otherwise the publisher hashes them. +type Storage struct { + Layers []string + LayerDigests []string + Overlay string + Aux string + Platform images.MacOSImage } -// MachinePublication is a verified publication: a digest-pinned reference. -type MachinePublication struct{ Reference, Digest string } +// Publication is a verified publication: a digest-pinned reference. +type Publication struct{ Reference, Digest string } -// ValidMachineDigest reports whether s is a canonical sha256 digest. -func ValidMachineDigest(s string) bool { +// validDigest reports whether s is a canonical sha256 digest. +func validDigest(s string) bool { if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { return false } return strings.Trim(s[7:], "0123456789abcdef") == "" } -// validateMachineStorage checks storage before publication: the platform is valid, -// the files are distinct nonempty regular files, and the overlay stacks on its -// layers as ASIF. -func validateMachineStorage(storage MachineStorage) error { - if err := storage.Platform.Validate(); err != nil { +// validate checks storage before publication: the platform is valid, the files +// are distinct nonempty regular files, and the overlay stacks on its layers as ASIF. +func (s Storage) validate() error { + if err := s.Platform.Validate(); err != nil { return err } - var seen []os.FileInfo - for _, path := range append([]string{storage.Overlay, storage.Aux}, storage.Layers...) { - info, err := os.Lstat(path) - if err != nil { - return err - } - if !info.Mode().IsRegular() || info.Size() == 0 { - return fmt.Errorf("machine storage must be nonempty regular files") - } - for _, other := range seen { - if os.SameFile(other, info) { - return fmt.Errorf("machine storage files must be distinct") - } - } - seen = append(seen, info) + if len(s.LayerDigests) != 0 && len(s.LayerDigests) != len(s.Layers) { + return fmt.Errorf("machine storage has %d layer digests for %d layers", len(s.LayerDigests), len(s.Layers)) + } + if err := images.CheckMachineFiles(append([]string{s.Overlay, s.Aux}, s.Layers...)...); err != nil { + return err } - disk, err := images.OpenMacOSDiskChain(append(append([]string(nil), storage.Layers...), storage.Overlay)) + disk, err := images.OpenMacOSDiskChain(append(append([]string(nil), s.Layers...), s.Overlay)) if err != nil { return fmt.Errorf("machine disk: %w", err) } return disk.Close() } -// StreamingMachinePublisher targets only the configured internal registry and a -// build-scoped repository. go-containerregistry hashes and compresses each layer -// from a reopenable tar stream, so nothing is written to disk, skips blobs the -// registry already holds, and commits the manifest last. The registry stores each -// upload on disk and commits it only if it matches its digest. +// Publisher targets only the configured internal registry and a build-scoped +// repository. go-containerregistry hashes and compresses each layer from a +// reopenable tar stream, so nothing is written to disk, skips blobs the registry +// already holds, and commits the manifest last. The registry stores each upload on +// disk and commits it only if it matches its digest. // No credentials, redirect destinations or arbitrary output repositories come from the source. -type StreamingMachinePublisher struct { +type Publisher struct { RegistryURL string - Tokens *RegistryTokenGenerator + Tokens *registrytoken.Generator + // NewMachineIdentifier returns a fresh VZ machine identifier. Every published + // image gets its own identifier and MAC, so it never runs with its source's. + NewMachineIdentifier func() ([]byte, error) } -// MachineBase is a pinned base image in the publisher's registry. A machine -// published over it reuses its layers, so only the new overlay is uploaded. -type MachineBase struct { +// Base is a pinned base image in the publisher's registry. A machine published +// over it reuses its layers, so only the new overlay is uploaded. +type Base struct { Repo string // builds/{id} repository holding the base manifest and layers Digest string // canonical sha256 digest of the base manifest } -func (b MachineBase) validate() error { +func (b Base) validate() error { if !strings.HasPrefix(b.Repo, "builds/") || strings.Contains(b.Repo, "..") || len(b.Repo) > 256 { return fmt.Errorf("machine base repository must be under builds/") } - if !ValidMachineDigest(b.Digest) { + if !validDigest(b.Digest) { return fmt.Errorf("machine base digest must be a canonical sha256 digest") } return nil } -// machineLabels describe a published machine image: one ASIF disk per layer, with -// the auxiliary storage and platform in the top layer. -var machineLabels = map[string]string{ - images.MacOSMachineVersionLabel: "2", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "asif", - images.MacOSMachineDiskLabel: "disk.asif", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", -} - -// Publish commits storage as a machine image in builds/{id}. With a base, the -// storage's layers must be the base's and the base's registry layers are reused; +// Publish commits storage as a machine image in builds/{id}, with a fresh machine +// identifier and MAC. With a base, the storage's layers must be the base's, by the +// disk digests the base records, and the base's registry layers are reused; // without one, every layer is uploaded. -func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, storage MachineStorage, base *MachineBase) (MachinePublication, error) { +func (p *Publisher) Publish(ctx context.Context, id string, storage Storage, base *Base) (Publication, error) { deadline, ok := ctx.Deadline() if !ok || ctx.Err() != nil { - return MachinePublication{}, fmt.Errorf("bounded publication required") + return Publication{}, fmt.Errorf("bounded publication required") } - if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil { - return MachinePublication{}, fmt.Errorf("scoped machine publisher required") + if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil || p.NewMachineIdentifier == nil { + return Publication{}, fmt.Errorf("scoped machine publisher required") } if base != nil { if err := base.validate(); err != nil { - return MachinePublication{}, err + return Publication{}, err } } - registry, err := p.registry() + registry, err := parseRegistryURL(p.RegistryURL) if err != nil { - return MachinePublication{}, err + return Publication{}, err + } + if err := storage.validate(); err != nil { + return Publication{}, err + } + if storage.Platform.MachineIdentifier, err = p.NewMachineIdentifier(); err != nil { + return Publication{}, fmt.Errorf("new machine identifier: %w", err) } - if err := validateMachineStorage(storage); err != nil { - return MachinePublication{}, err + if storage.Platform.MAC, err = randomMAC(); err != nil { + return Publication{}, err } repo := "builds/" + id - access := []RepoPermission{{Repo: repo, Scope: "push"}} + access := []registrytoken.RepoPermission{{Repo: repo, Scope: "push"}} if base != nil { - access = append(access, RepoPermission{Repo: base.Repo, Scope: "pull"}) + access = append(access, registrytoken.RepoPermission{Repo: base.Repo, Scope: "pull"}) } token, err := p.Tokens.GenerateToken(id, access, time.Until(deadline)) if err != nil { - return MachinePublication{}, err + return Publication{}, err + } + transport := &http.Transport{} + if defaults, ok := http.DefaultTransport.(*http.Transport); ok { + transport = defaults.Clone() } - transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil defer transport.CloseIdleConnections() options := []remote.Option{remote.WithContext(ctx), remote.WithAuth(&authn.Bearer{Token: token}), remote.WithTransport(refuseRedirects{transport})} @@ -153,20 +157,30 @@ func (p *StreamingMachinePublisher) Publish(ctx context.Context, id string, stor } img, err := machineImage(ctx, registry.Host, storage, base, nameOptions, options) if err != nil { - return MachinePublication{}, err + return Publication{}, err } target, err := name.NewTag(registry.Host+"/"+repo+":latest", nameOptions...) if err != nil { - return MachinePublication{}, err + return Publication{}, err } if err = remote.Write(target, img, options...); err != nil { - return MachinePublication{}, fmt.Errorf("write machine image: %w", err) + return Publication{}, fmt.Errorf("write machine image: %w", err) } digest, err := img.Digest() if err != nil { - return MachinePublication{}, err + return Publication{}, err } - return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest.String(), Digest: digest.String()}, nil + return Publication{Reference: registry.Host + "/" + repo + "@" + digest.String(), Digest: digest.String()}, nil +} + +// randomMAC returns a random locally administered unicast MAC address. +func randomMAC() (string, error) { + mac := make([]byte, 6) + if _, err := rand.Read(mac); err != nil { + return "", err + } + mac[0] = mac[0]&^0x01 | 0x02 + return net.HardwareAddr(mac).String(), nil } // refuseRedirects fails a redirected registry request rather than follow it, so @@ -182,10 +196,10 @@ func (t refuseRedirects) RoundTrip(req *http.Request) (*http.Response, error) { return response, err } -// ParseMachineBase returns ref, a digest-pinned image in the registry at -// registryURL, as a publication base. -func ParseMachineBase(registryURL, ref string) (*MachineBase, error) { - registry, err := (&StreamingMachinePublisher{RegistryURL: registryURL}).registry() +// ParseBase returns ref, a digest-pinned image in the registry at registryURL, as a +// publication base. +func ParseBase(registryURL, ref string) (*Base, error) { + registry, err := parseRegistryURL(registryURL) if err != nil { return nil, err } @@ -197,14 +211,13 @@ func ParseMachineBase(registryURL, ref string) (*MachineBase, error) { if !ok { return nil, fmt.Errorf("machine base %s is not in the registry %s", ref, registry.Host) } - base := &MachineBase{Repo: repo, Digest: parsed.Digest()} + base := &Base{Repo: repo, Digest: parsed.Digest()} return base, base.validate() } -// registry parses the configured registry address, defaulting to HTTPS. Plaintext -// is allowed only on loopback. -func (p *StreamingMachinePublisher) registry() (*url.URL, error) { - address := p.RegistryURL +// parseRegistryURL parses a registry address, defaulting to HTTPS. Plaintext is +// allowed only on loopback. +func parseRegistryURL(address string) (*url.URL, error) { if !strings.Contains(address, "://") { address = "https://" + address } @@ -220,8 +233,18 @@ func (p *StreamingMachinePublisher) registry() (*url.URL, error) { // machineImage returns the machine image for storage: the base's layers, from the // registry or the storage's own layer files, then a top layer with the overlay, -// auxiliary storage and platform. -func machineImage(ctx context.Context, host string, storage MachineStorage, base *MachineBase, nameOptions []name.Option, options []remote.Option) (v1.Image, error) { +// auxiliary storage and platform. Its labels record every layer's disk digest. +func machineImage(ctx context.Context, host string, storage Storage, base *Base, nameOptions []name.Option, options []remote.Option) (v1.Image, error) { + layerDigests := storage.LayerDigests + if len(layerDigests) == 0 { + for _, path := range storage.Layers { + digest, err := fileDigest(ctx, path) + if err != nil { + return nil, err + } + layerDigests = append(layerDigests, digest) + } + } img := empty.Image if base != nil { ref, err := name.NewDigest(host+"/"+base.Repo+"@"+base.Digest, nameOptions...) @@ -238,7 +261,7 @@ func machineImage(ctx context.Context, host string, storage MachineStorage, base if config.OS != "darwin" || config.Architecture != "arm64" { return nil, fmt.Errorf("base is not a darwin/arm64 machine image") } - for label, want := range machineLabels { + for label, want := range images.MacOSMachineLabels() { if config.Config.Labels[label] != want { return nil, fmt.Errorf("base is not an ASIF machine image: %s=%q", label, config.Config.Labels[label]) } @@ -246,9 +269,13 @@ func machineImage(ctx context.Context, host string, storage MachineStorage, base if layers := len(config.RootFS.DiffIDs); layers != len(storage.Layers) { return nil, fmt.Errorf("machine base has %d layers but the disk stacks on %d", layers, len(storage.Layers)) } + baseDigests := strings.Split(config.Config.Labels[images.MacOSMachineDiskDigestsLabel], ",") + if !slices.Equal(baseDigests, layerDigests) { + return nil, fmt.Errorf("machine base's disks are not the layers the disk stacks on: base records %q", config.Config.Labels[images.MacOSMachineDiskDigestsLabel]) + } } else { for _, path := range storage.Layers { - layer, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: path}}) + layer, err := machineLayer(ctx, []machineTarEntry{{name: images.MacOSMachineDiskFile, path: path}}) if err != nil { return nil, err } @@ -257,11 +284,15 @@ func machineImage(ctx context.Context, host string, storage MachineStorage, base } } } + overlayDigest, err := fileDigest(ctx, storage.Overlay) + if err != nil { + return nil, err + } platform, err := json.Marshal(storage.Platform) if err != nil { return nil, err } - top, err := machineLayer(ctx, []machineTarEntry{{name: "disk.asif", path: storage.Overlay}, {name: "aux.img", path: storage.Aux}, {name: "config.json", data: platform}}) + top, err := machineLayer(ctx, []machineTarEntry{{name: images.MacOSMachineDiskFile, path: storage.Overlay}, {name: images.MacOSMachineAuxFile, path: storage.Aux}, {name: images.MacOSMachinePlatformFile, data: platform}}) if err != nil { return nil, err } @@ -272,13 +303,29 @@ func machineImage(ctx context.Context, host string, storage MachineStorage, base if err != nil { return nil, err } - config.OS, config.Architecture, config.Config.Labels = "darwin", "arm64", machineLabels + labels := images.MacOSMachineLabels() + labels[images.MacOSMachineDiskDigestsLabel] = strings.Join(append(slices.Clone(layerDigests), overlayDigest), ",") + config.OS, config.Architecture, config.Config.Labels = "darwin", "arm64", labels if img, err = mutate.ConfigFile(img, config); err != nil { return nil, err } return mutate.ConfigMediaType(mutate.MediaType(img, types.OCIManifestSchema1), types.OCIConfigJSON), nil } +// fileDigest returns the sha256 digest of a file. +func fileDigest(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, contextReader{ctx, file}); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil +} + // machineTarEntry is one file in a machine layer, from a path or in memory. type machineTarEntry struct { name string diff --git a/lib/builds/macos_publisher_test.go b/lib/machineimage/publisher_test.go similarity index 73% rename from lib/builds/macos_publisher_test.go rename to lib/machineimage/publisher_test.go index 958ae0808..c1252270f 100644 --- a/lib/builds/macos_publisher_test.go +++ b/lib/machineimage/publisher_test.go @@ -1,4 +1,4 @@ -package builds_test +package machineimage_test import ( "context" @@ -12,11 +12,12 @@ import ( "time" "github.com/Code-Hex/vz/v4/diskimagekit" - "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/machineimage" "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/registry" + "github.com/kernel/hypeman/lib/registrytoken" "github.com/stretchr/testify/require" ) @@ -30,16 +31,18 @@ func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, return &images.Image{Status: images.StatusReady, Digest: digest}, nil } +func testMachineIdentifier() ([]byte, error) { return []byte{9}, nil } + func machinePublisherPlatform() images.MacOSImage { return images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} } // machinePublisherStorage writes a standalone ASIF disk and aux file, as a base // bundle has. -func machinePublisherStorage(t *testing.T) builds.MachineStorage { +func machinePublisherStorage(t *testing.T) machineimage.Storage { t.Helper() root := t.TempDir() - storage := builds.MachineStorage{Overlay: filepath.Join(root, "disk.asif"), Aux: filepath.Join(root, "aux.img"), Platform: machinePublisherPlatform()} + storage := machineimage.Storage{Overlay: filepath.Join(root, "disk.asif"), Aux: filepath.Join(root, "aux.img"), Platform: machinePublisherPlatform()} img, e := diskimagekit.CreateASIF(storage.Overlay, (4<<20)/512) require.NoError(t, e) _, e = img.WriteAt([]byte("synthetic disk"), 0) @@ -66,7 +69,7 @@ func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { })) server := httptest.NewServer(handler) defer server.Close() - publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + publisher := &machineimage.Publisher{RegistryURL: server.URL, Tokens: registrytoken.NewGenerator("synthetic registry secret"), NewMachineIdentifier: testMachineIdentifier} ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() result, e := publisher.Publish(ctx, "qa-build", machinePublisherStorage(t), nil) @@ -88,7 +91,7 @@ func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing. http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) })) defer server.Close() - publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic secret")} + publisher := &machineimage.Publisher{RegistryURL: server.URL, Tokens: registrytoken.NewGenerator("synthetic secret"), NewMachineIdentifier: testMachineIdentifier} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _, e := publisher.Publish(ctx, "qa-build", machinePublisherStorage(t), nil) @@ -108,7 +111,7 @@ func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { reg.Handler().ServeHTTP(w, r) }))) defer server.Close() - publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + publisher := &machineimage.Publisher{RegistryURL: server.URL, Tokens: registrytoken.NewGenerator("synthetic registry secret"), NewMachineIdentifier: testMachineIdentifier} ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() @@ -120,7 +123,7 @@ func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { // A build stacks a writable overlay on the base disk. img, e := diskimagekit.Open(baseStorage.Overlay) require.NoError(t, e) - derived := builds.MachineStorage{Layers: []string{baseStorage.Overlay}, Overlay: filepath.Join(t.TempDir(), "boot.asif"), Aux: filepath.Join(t.TempDir(), "aux.img"), Platform: machinePublisherPlatform()} + derived := machineimage.Storage{Layers: []string{baseStorage.Overlay}, Overlay: filepath.Join(t.TempDir(), "boot.asif"), Aux: filepath.Join(t.TempDir(), "aux.img"), Platform: machinePublisherPlatform()} require.NoError(t, img.CreateLayer(derived.Overlay, diskimagekit.Overlay)) _, e = img.WriteAt([]byte("build output"), 1<<20) require.NoError(t, e) @@ -129,7 +132,7 @@ func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { derived.Platform.MachineIdentifier = []byte{3} uploads.Store(0) - machineBase := &builds.MachineBase{Repo: "builds/qa-base", Digest: base.Digest} + machineBase := &machineimage.Base{Repo: "builds/qa-base", Digest: base.Digest} result, e := publisher.Publish(ctx, "qa-derived", derived, machineBase) require.NoError(t, e) require.Equal(t, int32(2), uploads.Load(), "only the config and the overlay layer are uploaded") @@ -138,4 +141,14 @@ func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { derived.Layers = nil _, e = publisher.Publish(ctx, "qa-derived", derived, machineBase) require.ErrorContains(t, e, "machine base has 1 layers but the disk stacks on 0") + + // The same number of layers, but stacked on a different base disk. + other := machinePublisherStorage(t) + img, e = diskimagekit.Open(other.Overlay) + require.NoError(t, e) + mismatched := machineimage.Storage{Layers: []string{other.Overlay}, Overlay: filepath.Join(t.TempDir(), "boot.asif"), Aux: derived.Aux, Platform: machinePublisherPlatform()} + require.NoError(t, img.CreateLayer(mismatched.Overlay, diskimagekit.Overlay)) + require.NoError(t, img.Close()) + _, e = publisher.Publish(ctx, "qa-mismatched", mismatched, machineBase) + require.ErrorContains(t, e, "not the layers the disk stacks on") } diff --git a/lib/builds/tart.go b/lib/machineimage/tart.go similarity index 60% rename from lib/builds/tart.go rename to lib/machineimage/tart.go index 46a454708..4deabe484 100644 --- a/lib/builds/tart.go +++ b/lib/machineimage/tart.go @@ -1,4 +1,4 @@ -package builds +package machineimage import ( "encoding/json" @@ -40,24 +40,25 @@ type tartManifest struct { // publishing as a machine image. A standalone VM's disk.img must be ASIF and // becomes a one-layer image. A stacked VM's read-only layers, base first, come // from Tart's content store, read from its manifest.json, with its writable -// overlay.asif on top. The platform keeps Tart's identity; callers that publish -// a new image give it a fresh one. -func ReadTartVM(dir string, guestAgent bool) (MachineStorage, error) { +// overlay.asif on top; the content store is under TART_HOME, or two levels above +// the VM directory by default. The platform keeps Tart's identity; the publisher +// gives each image a fresh one. +func ReadTartVM(dir string, guestAgent bool) (Storage, error) { var config tartConfig data, err := os.ReadFile(filepath.Join(dir, "config.json")) if err != nil { - return MachineStorage{}, err + return Storage{}, err } if err := json.Unmarshal(data, &config); err != nil { - return MachineStorage{}, fmt.Errorf("tart config: %w", err) + return Storage{}, fmt.Errorf("tart config: %w", err) } if config.OS != "darwin" || config.Arch != "arm64" { - return MachineStorage{}, fmt.Errorf("tart VM is %s/%s, want darwin/arm64", config.OS, config.Arch) + return Storage{}, fmt.Errorf("tart VM is %s/%s, want darwin/arm64", config.OS, config.Arch) } if _, err := os.Lstat(filepath.Join(dir, "state.vzvmsave")); err == nil { - return MachineStorage{}, fmt.Errorf("tart VM is suspended; shut it down from the guest first") + return Storage{}, fmt.Errorf("tart VM is suspended; shut it down from the guest first") } - storage := MachineStorage{ + storage := Storage{ Aux: filepath.Join(dir, "nvram.bin"), Platform: images.MacOSImage{ HardwareModel: config.HardwareModel, @@ -70,31 +71,38 @@ func ReadTartVM(dir string, guestAgent bool) (MachineStorage, error) { } manifest := filepath.Join(dir, "manifest.json") overlay := filepath.Join(dir, "overlay.asif") - if _, err := os.Lstat(overlay); errors.Is(err, os.ErrNotExist) { + if _, err := os.Lstat(overlay); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return Storage{}, err + } if config.DiskFormat != "asif" { - return MachineStorage{}, fmt.Errorf("tart VM disk format is %q, want asif", config.DiskFormat) + return Storage{}, fmt.Errorf("tart VM disk format is %q, want asif", config.DiskFormat) } storage.Overlay = filepath.Join(dir, "disk.img") return storage, nil } storage.Overlay = overlay - storage.Layers, err = tartStackLayers(manifest, filepath.Join(filepath.Dir(filepath.Dir(dir)), "cache", "content")) + home := os.Getenv("TART_HOME") + if home == "" { + home = filepath.Dir(filepath.Dir(dir)) + } + storage.Layers, storage.LayerDigests, err = tartStackLayers(manifest, filepath.Join(home, "cache", "content")) return storage, err } -// tartStackLayers returns the content-store paths of a stacked VM's read-only -// layers, base first: the whole-file digest on the first chunk of the base and of -// each overlay. -func tartStackLayers(path, contentDir string) ([]string, error) { +// tartStackLayers returns the content-store paths and file digests of a stacked +// VM's read-only layers, base first: the whole-file digest on the first chunk of +// the base and of each overlay. Tart names content files by that digest. +func tartStackLayers(path, contentDir string) ([]string, []string, error) { data, err := os.ReadFile(path) if err != nil { - return nil, err + return nil, nil, err } var manifest tartManifest if err := json.Unmarshal(data, &manifest); err != nil { - return nil, fmt.Errorf("tart manifest: %w", err) + return nil, nil, fmt.Errorf("tart manifest: %w", err) } - var layers []string + var layers, digests []string for i, layer := range manifest.Layers { if layer.MediaType != tartDiskMediaType && layer.MediaType != tartOverlayMediaType { continue @@ -104,16 +112,17 @@ func tartStackLayers(path, contentDir string) ([]string, error) { continue } hex, found := strings.CutPrefix(digest, "sha256:") - if !found || !ValidMachineDigest(digest) { - return nil, fmt.Errorf("tart manifest layer %d has an invalid disk digest", i) + if !found || !validDigest(digest) { + return nil, nil, fmt.Errorf("tart manifest layer %d has an invalid disk digest", i) } if len(layers) == 0 && layer.MediaType != tartDiskMediaType { - return nil, fmt.Errorf("tart manifest disk layers must start with the base") + return nil, nil, fmt.Errorf("tart manifest disk layers must start with the base") } layers = append(layers, filepath.Join(contentDir, "sha256", hex)) + digests = append(digests, digest) } if len(layers) == 0 { - return nil, fmt.Errorf("tart manifest names no stacked disk files") + return nil, nil, fmt.Errorf("tart manifest names no stacked disk files") } - return layers, nil + return layers, digests, nil } diff --git a/lib/builds/tart_test.go b/lib/machineimage/tart_test.go similarity index 96% rename from lib/builds/tart_test.go rename to lib/machineimage/tart_test.go index fa3d3c6f6..40a10e55d 100644 --- a/lib/builds/tart_test.go +++ b/lib/machineimage/tart_test.go @@ -1,4 +1,4 @@ -package builds +package machineimage import ( "crypto/sha256" @@ -45,7 +45,7 @@ func TestReadTartVMStandalone(t *testing.T) { require.Equal(t, []byte{2}, storage.Platform.MachineIdentifier) require.Equal(t, uint(4), storage.Platform.CPUs) require.True(t, storage.Platform.GuestAgent) - require.NoError(t, validateMachineStorage(storage)) + require.NoError(t, storage.validate()) raw := tartTestVM(t, t.TempDir(), "raw", map[string]any{"diskFormat": "raw"}) _, err = ReadTartVM(raw, false) @@ -97,7 +97,7 @@ func TestReadTartVMStacked(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{basePath, layerPath}, storage.Layers) require.Equal(t, filepath.Join(dir, "overlay.asif"), storage.Overlay) - require.NoError(t, validateMachineStorage(storage)) + require.NoError(t, storage.validate()) writeManifest(disk(tartOverlayMediaType, digest("layer"))) _, err = ReadTartVM(dir, false) @@ -111,5 +111,5 @@ func TestReadTartVMStacked(t *testing.T) { writeManifest(disk(tartDiskMediaType, digest("missing"))) storage, err = ReadTartVM(dir, false) require.NoError(t, err) - require.Error(t, validateMachineStorage(storage)) + require.Error(t, storage.validate()) } From 0a0c949e6485cbea8de9b807565d21c22f95c8bf Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:50:35 +0000 Subject: [PATCH 25/33] Bound chunked registry uploads and prefer the upstream for manifests Cap each upload's size, open uploads' count and bytes, and end uploads idle for an hour. Re-check that an upload is still open once its lock is held, omit an empty Range, and sync blobs before installing them. Manifests by digest come from the blob store only when the upstream registry does not know them. --- lib/registry/registry.go | 39 ++++++++++-- lib/registry/uploads.go | 134 ++++++++++++++++++++++++++++++++------- 2 files changed, 146 insertions(+), 27 deletions(-) diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 12d72dc35..6c0c60008 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -121,7 +121,7 @@ func isCacheRepo(repo string) bool { // This wraps the underlying registry to intercept manifest PUTs and trigger conversion. func (r *Registry) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if r.uploads.serve(w, req) || r.serveStoredManifest(w, req) { + if r.uploads.serve(w, req) || r.serveManifestByDigest(w, req) { return } // Intercept manifest PUT requests to store in blob store and trigger conversion @@ -174,10 +174,12 @@ func (r *Registry) Handler() http.Handler { }) } -// serveStoredManifest answers a manifest request by digest from the blob store, -// where every pushed manifest is kept. The upstream registry holds manifests in -// memory, so without this a manifest pushed before a restart is unknown by digest. -func (r *Registry) serveStoredManifest(w http.ResponseWriter, req *http.Request) bool { +// serveManifestByDigest answers a manifest read by digest, from the upstream +// registry or, when it does not know the manifest, from the blob store, where every +// pushed manifest is kept. The upstream registry holds manifests in memory, so a +// manifest pushed before a restart is otherwise unknown by digest. Other requests +// are not handled. +func (r *Registry) serveManifestByDigest(w http.ResponseWriter, req *http.Request) bool { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) if matches == nil || (req.Method != http.MethodGet && req.Method != http.MethodHead) { return false @@ -186,6 +188,33 @@ func (r *Registry) serveStoredManifest(w http.ResponseWriter, req *http.Request) if err != nil { return false } + // Manifests are small, so the upstream answer is buffered to fall back on a 404. + upstream := &bufferedResponse{header: make(http.Header), status: http.StatusOK} + r.handler.ServeHTTP(upstream, req) + if upstream.status != http.StatusNotFound || !r.serveStoredManifest(w, req, h) { + for key, values := range upstream.header { + w.Header()[key] = values + } + w.WriteHeader(upstream.status) + w.Write(upstream.body.Bytes()) + } + return true +} + +// bufferedResponse records a response for serveManifestByDigest. +type bufferedResponse struct { + header http.Header + status int + body bytes.Buffer +} + +func (b *bufferedResponse) Header() http.Header { return b.header } +func (b *bufferedResponse) WriteHeader(status int) { b.status = status } +func (b *bufferedResponse) Write(data []byte) (int, error) { return b.body.Write(data) } + +// serveStoredManifest writes the manifest h from the blob store and reports +// whether it had one. +func (r *Registry) serveStoredManifest(w http.ResponseWriter, req *http.Request, h v1.Hash) bool { path := r.blobStore.blobPath(h.String()) if info, err := os.Stat(path); err != nil || info.Size() > maxStoredManifestBytes { return false diff --git a/lib/registry/uploads.go b/lib/registry/uploads.go index 07b2ccf6e..ff8b76715 100644 --- a/lib/registry/uploads.go +++ b/lib/registry/uploads.go @@ -13,6 +13,8 @@ import ( "regexp" "strconv" "sync" + "sync/atomic" + "time" v1 "github.com/google/go-containerregistry/pkg/v1" ) @@ -20,22 +22,33 @@ import ( // uploadPattern matches /v2/{name}/blobs/uploads/ and /v2/{name}/blobs/uploads/{id}. var uploadPattern = regexp.MustCompile(`^/v2/(.+)/blobs/uploads/([0-9a-f]*)$`) +// Bounds on chunked uploads, which otherwise hold host disk until they end or the +// process restarts. One upload fits the largest machine image layer. +const ( + maxUploadBytes = 128 << 30 + maxOpenUploadBytes = 256 << 30 + maxOpenUploads = 64 + uploadIdleTimeout = time.Hour +) + // chunkedUploads stores chunked blob uploads (POST, PATCH..., PUT ?digest=) in // files beside the blob store, hashing as they arrive. The upstream registry keeps // them in memory, which a multi-gigabyte layer cannot afford. Monolithic uploads // (POST ?digest=) already stream and are left to it. type chunkedUploads struct { - dir string - store *BlobStore - mu sync.Mutex - open map[string]*chunkedUpload + dir string + store *BlobStore + mu sync.Mutex + open map[string]*chunkedUpload + openBytes atomic.Int64 } type chunkedUpload struct { - mu sync.Mutex - file *os.File - hash hash.Hash - size int64 + mu sync.Mutex + file *os.File + hash hash.Hash + size int64 + lastUsed time.Time } // newChunkedUploads removes uploads left by an earlier process: their sessions @@ -59,17 +72,16 @@ func (u *chunkedUploads) serve(w http.ResponseWriter, req *http.Request) bool { repo, id := match[1], match[2] switch { case req.Method == http.MethodPost && id == "" && req.URL.Query().Get("digest") == "": + u.removeIdle() u.start(w, repo) case id != "" && (req.Method == http.MethodPatch || req.Method == http.MethodPut || req.Method == http.MethodDelete): - u.mu.Lock() - upload := u.open[id] - u.mu.Unlock() + upload := u.lock(id) if upload == nil { http.Error(w, "blob upload unknown", http.StatusNotFound) return true } - upload.mu.Lock() defer upload.mu.Unlock() + upload.lastUsed = time.Now() switch req.Method { case http.MethodPatch: u.append(w, req, repo, id, upload) @@ -85,7 +97,55 @@ func (u *chunkedUploads) serve(w http.ResponseWriter, req *http.Request) bool { return true } +// lock returns the open upload id, locked, or nil. An upload ended while this +// request waited for it is not returned. +func (u *chunkedUploads) lock(id string) *chunkedUpload { + u.mu.Lock() + upload := u.open[id] + u.mu.Unlock() + if upload == nil { + return nil + } + upload.mu.Lock() + u.mu.Lock() + current := u.open[id] == upload + u.mu.Unlock() + if !current { + upload.mu.Unlock() + return nil + } + return upload +} + +// removeIdle ends uploads unused for uploadIdleTimeout. An upload busy with a +// request is not idle. +func (u *chunkedUploads) removeIdle() { + u.mu.Lock() + idle := make(map[string]*chunkedUpload) + for id, upload := range u.open { + if upload.mu.TryLock() { + if time.Since(upload.lastUsed) > uploadIdleTimeout { + idle[id] = upload + } else { + upload.mu.Unlock() + } + } + } + u.mu.Unlock() + for id, upload := range idle { + u.remove(id, upload) + upload.mu.Unlock() + } +} + func (u *chunkedUploads) start(w http.ResponseWriter, repo string) { + u.mu.Lock() + full := len(u.open) >= maxOpenUploads + u.mu.Unlock() + if full { + http.Error(w, "too many open blob uploads", http.StatusTooManyRequests) + return + } var raw [16]byte rand.Read(raw[:]) id := hex.EncodeToString(raw[:]) @@ -95,13 +155,33 @@ func (u *chunkedUploads) start(w http.ResponseWriter, repo string) { return } u.mu.Lock() - u.open[id] = &chunkedUpload{file: file, hash: sha256.New()} + u.open[id] = &chunkedUpload{file: file, hash: sha256.New(), lastUsed: time.Now()} u.mu.Unlock() w.Header().Set("Location", "/v2/"+repo+"/blobs/uploads/"+id) w.Header().Set("Range", "0-0") w.WriteHeader(http.StatusAccepted) } +// write appends body to upload, within the size bounds. It reports whether the +// body fit; an upload that does not fit is ended. +func (u *chunkedUploads) write(w http.ResponseWriter, id string, upload *chunkedUpload, body io.Reader) bool { + allowed := min(maxUploadBytes-upload.size, maxOpenUploadBytes-u.openBytes.Load()) + n, err := io.Copy(io.MultiWriter(upload.file, upload.hash), io.LimitReader(body, max(allowed, 0)+1)) + upload.size += n + u.openBytes.Add(n) + switch { + case err != nil: + u.remove(id, upload) + http.Error(w, "write upload failed", http.StatusInternalServerError) + return false + case n > allowed: + u.remove(id, upload) + http.Error(w, "blob upload too large", http.StatusRequestEntityTooLarge) + return false + } + return true +} + // append writes a chunk. A Content-Range must start where the upload ends. func (u *chunkedUploads) append(w http.ResponseWriter, req *http.Request, repo, id string, upload *chunkedUpload) { if r := req.Header.Get("Content-Range"); r != "" { @@ -111,28 +191,30 @@ func (u *chunkedUploads) append(w http.ResponseWriter, req *http.Request, repo, return } } - n, err := io.Copy(io.MultiWriter(upload.file, upload.hash), req.Body) - upload.size += n - if err != nil { - u.remove(id, upload) - http.Error(w, "write upload failed", http.StatusInternalServerError) + if !u.write(w, id, upload, req.Body) { return } w.Header().Set("Location", "/v2/"+repo+"/blobs/uploads/"+id) - w.Header().Set("Range", "0-"+strconv.FormatInt(upload.size-1, 10)) + if upload.size > 0 { + w.Header().Set("Range", "0-"+strconv.FormatInt(upload.size-1, 10)) + } w.WriteHeader(http.StatusAccepted) } // commit writes the final chunk and installs the blob if it matches the digest. // The upload ends either way. func (u *chunkedUploads) commit(w http.ResponseWriter, req *http.Request, repo, id string, upload *chunkedUpload) { - defer u.remove(id, upload) want, err := v1.NewHash(req.URL.Query().Get("digest")) if err != nil { + u.remove(id, upload) http.Error(w, "invalid digest", http.StatusBadRequest) return } - if _, err = io.Copy(io.MultiWriter(upload.file, upload.hash), req.Body); err == nil { + if !u.write(w, id, upload, req.Body) { + return + } + defer u.remove(id, upload) + if err = upload.file.Sync(); err == nil { err = upload.file.Close() } if err != nil { @@ -152,10 +234,18 @@ func (u *chunkedUploads) commit(w http.ResponseWriter, req *http.Request, repo, w.WriteHeader(http.StatusCreated) } +// remove ends an upload and deletes its file. Ending an ended upload does nothing. func (u *chunkedUploads) remove(id string, upload *chunkedUpload) { u.mu.Lock() - delete(u.open, id) + current := u.open[id] == upload + if current { + delete(u.open, id) + } u.mu.Unlock() + if !current { + return + } + u.openBytes.Add(-upload.size) upload.file.Close() os.Remove(upload.file.Name()) } From e9e52df049b1e5e13b210323297ca25c449abdb5 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:50:35 +0000 Subject: [PATCH 26/33] Type the vz-shim control API and drain it before exiting Routes are an enum and responses Codable. finish waits up to five seconds for control requests in flight, as the Go shim's server shutdown did. The snapshot manifest is replaced with rename(2), every image in a stack must be ASIF, and VM.swift is split into storage, network and platform files. --- cmd/vz-shim/Identity.swift | 28 ++++++ cmd/vz-shim/Network.swift | 99 ++++++++++++++++++++ cmd/vz-shim/Platform.swift | 31 +++++++ cmd/vz-shim/Shim.swift | 126 ++++++++++++++++---------- cmd/vz-shim/Socket.swift | 16 +++- cmd/vz-shim/Storage.swift | 56 ++++++++++++ cmd/vz-shim/VM.swift | 179 ------------------------------------- cmd/vz-shim/main.swift | 27 ------ 8 files changed, 308 insertions(+), 254 deletions(-) create mode 100644 cmd/vz-shim/Identity.swift create mode 100644 cmd/vz-shim/Network.swift create mode 100644 cmd/vz-shim/Platform.swift create mode 100644 cmd/vz-shim/Storage.swift diff --git a/cmd/vz-shim/Identity.swift b/cmd/vz-shim/Identity.swift new file mode 100644 index 000000000..4611b84b7 --- /dev/null +++ b/cmd/vz-shim/Identity.swift @@ -0,0 +1,28 @@ +import CryptoKit +import Foundation + +// lockMacOSIdentity holds an exclusive lock on the Mac machine identity until the +// process exits, including while paused. The lock file is never unlinked: another +// waiter may already hold its inode open. +func lockMacOSIdentity(_ config: ShimConfig) throws -> Int32? { + guard let text = config.macMachineIdentifierData, !text.isEmpty else { return nil } + guard let data = decodeBase64(text) else { throw ShimError("decode Mac identity for admission") } + let hash = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + // Fixed directory, not TMPDIR: servers with different environments must contend + // on the same lock. The name is predictable, so refuse symlinks and files owned + // by anyone else. + let path = "/tmp/hypeman-mac-identity-\(getuid())-\(hash).lock" + let fd = open(path, O_CREAT | O_RDWR | O_NOFOLLOW, 0o600) + guard fd >= 0 else { throw ShimError("open \(path): \(errnoString())") } + var info = stat() + guard fstat(fd, &info) == 0, info.st_mode & S_IFMT == S_IFREG, info.st_uid == getuid() else { + close(fd) + throw ShimError("Mac identity lock path is not an owned regular file") + } + guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { + let message = errnoString() + close(fd) + throw ShimError("Mac machine identity already in use: \(message)") + } + return fd +} diff --git a/cmd/vz-shim/Network.swift b/cmd/vz-shim/Network.swift new file mode 100644 index 000000000..81944eb8b --- /dev/null +++ b/cmd/vz-shim/Network.swift @@ -0,0 +1,99 @@ +import Foundation +import Virtualization +import vmnet + +func makeNetworkDevices(_ networks: [NetworkConfig]) throws -> [VZNetworkDeviceConfiguration] { + // No networks configured (NetworkEnabled=false): no NIC is attached. + try networks.map { n in + let device = VZVirtioNetworkDeviceConfiguration() + if let address = n.ipv4Address, !address.isEmpty { + device.attachment = try vmnetAttachment(n) + } else { + device.attachment = VZNATNetworkDeviceAttachment() + } + device.macAddress = macAddress(n.mac) + return device + } +} + +func macAddress(_ text: String?) -> VZMACAddress { + if let text, !text.isEmpty { + if let mac = VZMACAddress(string: text) { + Log.info("using specified MAC address", ["mac": text]) + return mac + } + Log.warn("failed to parse MAC address, generating random", ["mac": text]) + return VZMACAddress.randomLocallyAdministered() + } + let mac = VZMACAddress.randomLocallyAdministered() + Log.info("generated random MAC address", ["mac": mac.string]) + return mac +} + +// vmnetAttachment attaches the guest to its own shared-mode vmnet network with the +// given subnet and a DHCP reservation for the guest's MAC, so the guest's address is +// the one hypeman assigned. +func vmnetAttachment(_ n: NetworkConfig) throws -> VZNetworkDeviceAttachment { + guard #available(macOS 26.0, *) else { + throw ShimError("configure network: vmnet networks require macOS 26 or newer") + } + guard let macText = n.mac, let mac = VZMACAddress(string: macText) else { + throw ShimError("configure network: vmnet network needs the guest MAC") + } + guard let (network, bits) = parseIPv4Prefix(n.ipv4Subnet ?? ""), bits <= 30 else { + throw ShimError("configure network: parse guest subnet \(n.ipv4Subnet ?? "")") + } + guard let address = parseIPv4(n.ipv4Address ?? "") else { + throw ShimError("configure network: parse guest address \(n.ipv4Address ?? "")") + } + let mask: UInt32 = bits == 0 ? 0 : ~UInt32(0) << UInt32(32 - bits) + let subnetText = "\(formatIPv4(network & mask))/\(bits)" + + var status = vmnet_return_t.VMNET_SUCCESS + guard let config = vmnet_network_configuration_create(.VMNET_SHARED_MODE, &status), status == .VMNET_SUCCESS else { + throw ShimError("configure network: configure vmnet network \(subnetText): status \(status.rawValue)") + } + // vmnet needs the host address, the first in the subnet, to start DHCP-backed + // attachments; the network address alone leaves the guest without a lease. + var subnetAddr = in_addr(s_addr: ((network & mask) + 1).bigEndian) + var maskAddr = in_addr(s_addr: mask.bigEndian) + status = vmnet_network_configuration_set_ipv4_subnet(config, &subnetAddr, &maskAddr) + guard status == .VMNET_SUCCESS else { + throw ShimError("configure network: set vmnet subnet \(subnetText): status \(status.rawValue)") + } + var client = ether_addr_t() + withUnsafeMutableBytes(of: &client.octet) { $0.copyBytes(from: mac.ethernetAddress) } + var reserved = in_addr(s_addr: address.bigEndian) + status = vmnet_network_configuration_add_dhcp_reservation(config, &client, &reserved) + guard status == .VMNET_SUCCESS else { + throw ShimError("configure network: reserve \(formatIPv4(address)) on vmnet network \(subnetText): status \(status.rawValue)") + } + guard let network = vmnet_network_create(config, &status), status == .VMNET_SUCCESS else { + throw ShimError("configure network: create vmnet network \(subnetText): status \(status.rawValue)") + } + Log.info("attached guest vmnet network", ["subnet": subnetText, "address": formatIPv4(address), "mac": macText]) + return VZVmnetNetworkDeviceAttachment(network: network) +} + +extension VZMACAddress { + var ethernetAddress: [UInt8] { + string.split(separator: ":").compactMap { UInt8($0, radix: 16) } + } +} + +// parseIPv4 returns an address in host byte order. +func parseIPv4(_ text: String) -> UInt32? { + var addr = in_addr() + guard inet_pton(AF_INET, text, &addr) == 1 else { return nil } + return UInt32(bigEndian: addr.s_addr) +} + +func parseIPv4Prefix(_ text: String) -> (UInt32, Int)? { + let parts = text.split(separator: "/") + guard parts.count == 2, let address = parseIPv4(String(parts[0])), let bits = Int(parts[1]), (0...32).contains(bits) else { return nil } + return (address, bits) +} + +func formatIPv4(_ address: UInt32) -> String { + "\(address >> 24 & 0xff).\(address >> 16 & 0xff).\(address >> 8 & 0xff).\(address & 0xff)" +} diff --git a/cmd/vz-shim/Platform.swift b/cmd/vz-shim/Platform.swift new file mode 100644 index 000000000..ae267df72 --- /dev/null +++ b/cmd/vz-shim/Platform.swift @@ -0,0 +1,31 @@ +import Foundation +import Virtualization + +func makeGenericPlatform(_ c: inout ShimConfig) throws -> VZGenericPlatformConfiguration { + let platform = VZGenericPlatformConfiguration() + if let text = c.machineIdentifierData, !text.isEmpty { + guard let data = decodeBase64(text), let identifier = VZGenericMachineIdentifier(dataRepresentation: data) else { + throw ShimError("configure platform: recreate machine identifier") + } + platform.machineIdentifier = identifier + } else { + let identifier = VZGenericMachineIdentifier() + platform.machineIdentifier = identifier + c.machineIdentifierData = identifier.dataRepresentation.base64EncodedString() + } + return platform +} + +func cpuCount(_ requested: Int) -> Int { + var count = requested + if count == 0 { + count = max(1, ProcessInfo.processInfo.processorCount - 1) + } + return min(max(count, VZVirtualMachineConfiguration.minimumAllowedCPUCount), VZVirtualMachineConfiguration.maximumAllowedCPUCount) +} + +func memorySize(_ requested: UInt64) -> UInt64 { + // 2 GiB safety default; the caller normally provides this. + let size = requested == 0 ? 2 << 30 : requested + return min(max(size, VZVirtualMachineConfiguration.minimumAllowedMemorySize), VZVirtualMachineConfiguration.maximumAllowedMemorySize) +} diff --git a/cmd/vz-shim/Shim.swift b/cmd/vz-shim/Shim.swift index 4613a1757..108ab98d2 100644 --- a/cmd/vz-shim/Shim.swift +++ b/cmd/vz-shim/Shim.swift @@ -14,6 +14,9 @@ final class Shim: NSObject, VZVirtualMachineDelegate { private var forcedStop = false private var listeners: [UnixListener] = [] private var stateObservation: NSKeyValueObservation? + // Control requests still being served, which finish waits for. + private let inFlight = DispatchGroup() + private var finishing = false init(vm: VZVirtualMachine, vmConfig: VZVirtualMachineConfiguration, config: ShimConfig) { self.vm = vm @@ -32,7 +35,11 @@ final class Shim: NSObject, VZVirtualMachineDelegate { let control = try UnixListener(path: config.controlSocket ?? "") let vsock = try UnixListener(path: config.vsockSocket ?? "") listeners = [control, vsock] - control.serve { [self] fd in serveHTTP(fd, route) } + control.serve { [self] fd in + inFlight.enter() + defer { inFlight.leave() } + serveHTTP(fd, route) + } vsock.serve { [self] fd in proxyVsock(fd) } Log.info("control API listening", ["socket": control.path]) Log.info("vsock proxy listening", ["socket": vsock.path]) @@ -62,12 +69,21 @@ final class Shim: NSObject, VZVirtualMachineDelegate { finish() } + // finish runs on the main queue. It stops accepting requests and exits once the + // control requests in flight are done, waiting at most five seconds, as the Go + // shim's server shutdown did. The wait is off the main queue, which those + // requests may need. private func finish() { + guard !finishing else { return } + finishing = true for listener in listeners { listener.close() } - Log.info("vz-shim shutdown complete") - exit(0) + DispatchQueue.global().async { [self] in + _ = inFlight.wait(timeout: .now() + 5) + Log.info("vz-shim shutdown complete") + exit(0) + } } // writeStopReceipt records that the guest stopped the VM itself. The host counts @@ -109,45 +125,69 @@ final class Shim: NSObject, VZVirtualMachineDelegate { // MARK: Control API - private static let routes: [String: Set] = [ - "/api/v1/vm.info": ["GET"], "/api/v1/vm.pause": ["PUT"], "/api/v1/vm.resume": ["PUT"], - "/api/v1/vm.shutdown": ["PUT"], "/api/v1/vm.snapshot": ["PUT"], "/api/v1/vm.snapshot-support": ["GET"], - "/api/v1/vm.balloon": ["GET", "PUT"], - "/api/v1/vmm.ping": ["GET"], "/api/v1/vmm.shutdown": ["PUT"], - ] + // Endpoint is a control API route: a method and a path. + private enum Endpoint: String, CaseIterable { + case ping = "GET /api/v1/vmm.ping" + case vmmShutdown = "PUT /api/v1/vmm.shutdown" + case info = "GET /api/v1/vm.info" + case pause = "PUT /api/v1/vm.pause" + case resume = "PUT /api/v1/vm.resume" + case shutdown = "PUT /api/v1/vm.shutdown" + case snapshotSupport = "GET /api/v1/vm.snapshot-support" + case snapshot = "PUT /api/v1/vm.snapshot" + case getBalloon = "GET /api/v1/vm.balloon" + case setBalloon = "PUT /api/v1/vm.balloon" + + var path: Substring { rawValue.split(separator: " ")[1] } + } + + private static let paths = Set(Endpoint.allCases.map(\.path)) private func route(_ request: HTTPRequest) -> HTTPResponse { - guard let methods = Shim.routes[request.path] else { return .text(404, "404 page not found") } - guard methods.contains(request.method) else { return .text(405, "Method Not Allowed") } - switch (request.method, request.path) { - case ("GET", "/api/v1/vmm.ping"): - return HTTPResponse(status: 200, body: Data("OK".utf8)) - case ("PUT", "/api/v1/vmm.shutdown"): - return vmmShutdown() - default: - break + guard let endpoint = Endpoint(rawValue: "\(request.method) \(request.path)") else { + return Shim.paths.contains(Substring(request.path)) ? .text(405, "Method Not Allowed") : .text(404, "404 page not found") + } + switch endpoint { + case .ping: return HTTPResponse(status: 200, body: Data("OK".utf8)) + case .vmmShutdown: return vmmShutdown() + default: break } ops.lock() defer { ops.unlock() } - switch (request.method, request.path) { - case ("GET", "/api/v1/vm.info"): return info() - case ("PUT", "/api/v1/vm.pause"): return pause() - case ("PUT", "/api/v1/vm.resume"): return resume() - case ("PUT", "/api/v1/vm.shutdown"): return requestStop() - case ("GET", "/api/v1/vm.snapshot-support"): return snapshotSupport() - case ("PUT", "/api/v1/vm.snapshot"): return snapshot(request.body) - case ("GET", "/api/v1/vm.balloon"): return getBalloon() - default: return setBalloon(request.body) + switch endpoint { + case .ping, .vmmShutdown: fatalError("served above") + case .info: return info() + case .pause: return pause() + case .resume: return resume() + case .shutdown: return requestStop() + case .snapshotSupport: return snapshotSupport() + case .snapshot: return snapshot(request.body) + case .getBalloon: return getBalloon() + case .setBalloon: return setBalloon(request.body) } } + private struct VMInfo: Encodable { + let state: String + let memoryBalloonDevices: Int? + } + + private struct SnapshotSupport: Encodable { + let supported: Bool + let reason: String? + } + + private struct SnapshotRequest: Decodable { + let destinationPath: String + } + + private struct Balloon: Codable { + let targetGuestMemoryBytes: Int64 + } + private func info() -> HTTPResponse { let (state, balloons) = onMain { (stateName(vm.state), vm.memoryBalloonDevices.count) } - var response: [String: Any] = ["state": state] - if balloons > 0 { - response["memory_balloon_devices"] = balloons - } - return .json(response) + return .json(VMInfo(state: state, memoryBalloonDevices: balloons > 0 ? balloons : nil)) } private func pause() -> HTTPResponse { @@ -184,12 +224,8 @@ final class Shim: NSObject, VZVirtualMachineDelegate { } private func snapshotSupport() -> HTTPResponse { - var response: [String: Any] = ["supported": true] - if let reason = saveRestoreUnsupportedReason() { - response["supported"] = false - response["reason"] = reason - } - return .json(response) + let reason = saveRestoreUnsupportedReason() + return .json(SnapshotSupport(supported: reason == nil, reason: reason)) } func saveRestoreUnsupportedReason() -> String? { @@ -204,8 +240,7 @@ final class Shim: NSObject, VZVirtualMachineDelegate { private func snapshot(_ body: Data) -> HTTPResponse { let destination: String do { - let object = try JSONSerialization.jsonObject(with: body) as? [String: Any] - destination = object?["destination_path"] as? String ?? "" + destination = try snakeCaseDecoder.decode(SnapshotRequest.self, from: body).destinationPath } catch { return .text(400, "invalid snapshot request: \(error.localizedDescription)") } @@ -236,11 +271,13 @@ final class Shim: NSObject, VZVirtualMachineDelegate { do { let manifest = SnapshotManifest(hypervisor: "vz", machineStateFile: snapshotMachineStateFile, shimConfig: shimConfig) try JSONEncoder().encode(manifest).write(to: URL(fileURLWithPath: manifestPath + ".tmp")) - try? fm.removeItem(atPath: manifestPath) - try fm.moveItem(atPath: manifestPath + ".tmp", toPath: manifestPath) } catch { return .text(500, "write manifest failed: \(error.localizedDescription)") } + // rename(2) replaces the manifest atomically, as the Go shim's os.Rename did. + guard rename(manifestPath + ".tmp", manifestPath) == 0 else { + return .text(500, "write manifest failed: \(errnoString())") + } complete = true Log.info("VM snapshot saved", ["destination": destination, "machine_state": statePath]) return .noContent() @@ -254,13 +291,12 @@ final class Shim: NSObject, VZVirtualMachineDelegate { guard let target = onMain({ balloon()?.targetVirtualMachineMemorySize }) else { return .text(404, "no memory balloon device configured") } - return .json(["target_guest_memory_bytes": Int64(target)]) + return .json(Balloon(targetGuestMemoryBytes: Int64(target))) } private func setBalloon(_ body: Data) -> HTTPResponse { guard onMain({ balloon() != nil }) else { return .text(404, "no memory balloon device configured") } - let object = try? JSONSerialization.jsonObject(with: body) as? [String: Any] - guard let target = (object?["target_guest_memory_bytes"] as? NSNumber)?.int64Value else { + guard let target = try? snakeCaseDecoder.decode(Balloon.self, from: body).targetGuestMemoryBytes else { return .text(400, "invalid balloon request: target_guest_memory_bytes is required") } guard target >= 0 else { return .text(400, "target_guest_memory_bytes must be non-negative") } diff --git a/cmd/vz-shim/Socket.swift b/cmd/vz-shim/Socket.swift index e7f950cf2..5c6b84e9f 100644 --- a/cmd/vz-shim/Socket.swift +++ b/cmd/vz-shim/Socket.swift @@ -118,13 +118,23 @@ struct HTTPResponse { HTTPResponse(status: status, body: (message + "\n").data(using: .utf8)!) } - static func json(_ object: Any) -> HTTPResponse { - let data = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])) ?? Data("{}".utf8) + static func json(_ value: T) -> HTTPResponse { + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + encoder.outputFormatting = .sortedKeys + let data = (try? encoder.encode(value)) ?? Data("{}".utf8) return HTTPResponse(status: 200, contentType: "application/json", body: data + Data([0x0a])) } } -private let reasons = [200: "OK", 204: "No Content", 400: "Bad Request", 404: "Not Found", 405: "Method Not Allowed", 413: "Payload Too Large", 500: "Internal Server Error"] +// snakeCaseDecoder decodes control API request bodies, which use snake_case keys. +let snakeCaseDecoder: JSONDecoder = { + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + return decoder +}() + +private let reasons = [200: "OK", 204: "No Content", 400: "Bad Request", 404: "Not Found", 405: "Method Not Allowed", 500: "Internal Server Error"] func serveHTTP(_ fd: Int32, _ handle: (HTTPRequest) -> HTTPResponse) { defer { close(fd) } diff --git a/cmd/vz-shim/Storage.swift b/cmd/vz-shim/Storage.swift new file mode 100644 index 000000000..81a79ac04 --- /dev/null +++ b/cmd/vz-shim/Storage.swift @@ -0,0 +1,56 @@ +import DiskImageKit +import Foundation +import Virtualization + +func makeStorageDevices(_ disks: [DiskConfig]) throws -> [VZStorageDeviceConfiguration] { + try disks.map { disk in + guard FileManager.default.fileExists(atPath: disk.path) else { + throw ShimError("configure storage: disk image not found: \(disk.path)") + } + if disk.path.hasSuffix(".qcow2") { + throw ShimError("configure storage: qcow2 not supported by vz, use raw format: \(disk.path)") + } + let readOnly = disk.readonly ?? false + let attachment: VZStorageDeviceAttachment + do { + if let parents = disk.parents, !parents.isEmpty { + attachment = try stackedAttachment(parents: parents, overlay: disk.path, readOnly: readOnly) + } else { + attachment = try VZDiskImageStorageDeviceAttachment(url: URL(fileURLWithPath: disk.path), readOnly: readOnly) + } + } catch let error as ShimError { + throw error + } catch { + throw ShimError("configure storage: create disk attachment for \(disk.path): \(error.localizedDescription)") + } + return VZVirtioBlockDeviceConfiguration(attachment: attachment) + } +} + +// stackedAttachment attaches an ASIF overlay above its read-only parents, base +// first. Each image must name the one below it as its parent; hypeman creates them. +func stackedAttachment(parents: [String], overlay: String, readOnly: Bool) throws -> VZStorageDeviceAttachment { + guard #available(macOS 27.0, *) else { + throw ShimError("configure storage: stacked disks require macOS 27 or newer") + } + for path in parents where !FileManager.default.fileExists(atPath: path) { + throw ShimError("configure storage: disk parent not found: \(path)") + } + var stacked = try openASIF(parents[0], mode: .readOnly) + for path in parents.dropFirst() { + stacked = try stacked.appending(try openASIF(path, mode: .readOnly)) + } + stacked = try stacked.appending(try openASIF(overlay, mode: readOnly ? .readOnly : .readWrite)) + Log.info("attached stacked disk", ["base": parents[0], "overlay": overlay, "layers": parents.count + 1]) + return try VZDiskImageStorageDeviceAttachment(diskImage: stacked, cachingMode: .automatic, synchronizationMode: .full) +} + +// openASIF opens one image of a stack, which must be ASIF: the base and every overlay. +@available(macOS 27.0, *) +func openASIF(_ path: String, mode: OpenConfiguration.Mode) throws -> DiskImage { + let image = try DiskImage(opening: .open(url: URL(fileURLWithPath: path), mode: mode)) + guard image.format == .asif else { + throw ShimError("configure storage: stacked disk images must use ASIF format: \(path)") + } + return image +} diff --git a/cmd/vz-shim/VM.swift b/cmd/vz-shim/VM.swift index 3c46f35dd..4429f9a1f 100644 --- a/cmd/vz-shim/VM.swift +++ b/cmd/vz-shim/VM.swift @@ -1,7 +1,5 @@ import Foundation import Virtualization -import DiskImageKit -import vmnet // makeConfiguration builds a validated VM configuration. A Linux guest without a // machine identifier gets a new one, recorded in config so snapshots keep it. @@ -126,21 +124,6 @@ func makeLinuxConfiguration(_ c: inout ShimConfig) throws -> VZVirtualMachineCon return vc } -func makeGenericPlatform(_ c: inout ShimConfig) throws -> VZGenericPlatformConfiguration { - let platform = VZGenericPlatformConfiguration() - if let text = c.machineIdentifierData, !text.isEmpty { - guard let data = decodeBase64(text), let identifier = VZGenericMachineIdentifier(dataRepresentation: data) else { - throw ShimError("configure platform: recreate machine identifier") - } - platform.machineIdentifier = identifier - } else { - let identifier = VZGenericMachineIdentifier() - platform.machineIdentifier = identifier - c.machineIdentifierData = identifier.dataRepresentation.base64EncodedString() - } - return platform -} - func makeSerialPort(_ logPath: String?) throws -> VZSerialPortConfiguration { guard let reader = FileHandle(forReadingAtPath: "/dev/null") else { throw ShimError("configure serial: open /dev/null for reading") @@ -186,165 +169,3 @@ func makeRosettaShare(_ c: ShimConfig) throws -> [VZDirectorySharingDeviceConfig throw ShimError("configure directory sharing: rosetta requested but not supported on this host (requires Apple silicon + macOS 13+)") #endif } - -func makeStorageDevices(_ disks: [DiskConfig]) throws -> [VZStorageDeviceConfiguration] { - try disks.map { disk in - guard FileManager.default.fileExists(atPath: disk.path) else { - throw ShimError("configure storage: disk image not found: \(disk.path)") - } - if disk.path.hasSuffix(".qcow2") { - throw ShimError("configure storage: qcow2 not supported by vz, use raw format: \(disk.path)") - } - let readOnly = disk.readonly ?? false - let attachment: VZStorageDeviceAttachment - do { - if let parents = disk.parents, !parents.isEmpty { - attachment = try stackedAttachment(parents: parents, overlay: disk.path, readOnly: readOnly) - } else { - attachment = try VZDiskImageStorageDeviceAttachment(url: URL(fileURLWithPath: disk.path), readOnly: readOnly) - } - } catch let error as ShimError { - throw error - } catch { - throw ShimError("configure storage: create disk attachment for \(disk.path): \(error.localizedDescription)") - } - return VZVirtioBlockDeviceConfiguration(attachment: attachment) - } -} - -// stackedAttachment attaches an ASIF overlay above its read-only parents, base -// first. Each image must name the one below it as its parent; hypeman creates them. -func stackedAttachment(parents: [String], overlay: String, readOnly: Bool) throws -> VZStorageDeviceAttachment { - guard #available(macOS 27.0, *) else { - throw ShimError("configure storage: stacked disks require macOS 27 or newer") - } - for path in parents where !FileManager.default.fileExists(atPath: path) { - throw ShimError("configure storage: disk parent not found: \(path)") - } - var stacked = try DiskImage(opening: .open(url: URL(fileURLWithPath: parents[0]), mode: .readOnly)) - for path in parents.dropFirst() { - stacked = try stacked.appending(try openOverlay(path, mode: .readOnly)) - } - stacked = try stacked.appending(try openOverlay(overlay, mode: readOnly ? .readOnly : .readWrite)) - Log.info("attached stacked disk", ["base": parents[0], "overlay": overlay, "layers": (stacked as? any StackedImage)?.layers.count ?? 1]) - return try VZDiskImageStorageDeviceAttachment(diskImage: stacked, cachingMode: .automatic, synchronizationMode: .full) -} - -@available(macOS 27.0, *) -func openOverlay(_ path: String, mode: OpenConfiguration.Mode) throws -> DiskImage { - let image = try DiskImage(opening: .open(url: URL(fileURLWithPath: path), mode: mode)) - guard image.format == .asif else { - throw ShimError("configure storage: overlay must use ASIF format: \(path)") - } - return image -} - -func makeNetworkDevices(_ networks: [NetworkConfig]) throws -> [VZNetworkDeviceConfiguration] { - // No networks configured (NetworkEnabled=false): no NIC is attached. - try networks.map { n in - let device = VZVirtioNetworkDeviceConfiguration() - if let address = n.ipv4Address, !address.isEmpty { - device.attachment = try vmnetAttachment(n) - } else { - device.attachment = VZNATNetworkDeviceAttachment() - } - device.macAddress = macAddress(n.mac) - return device - } -} - -func macAddress(_ text: String?) -> VZMACAddress { - if let text, !text.isEmpty { - if let mac = VZMACAddress(string: text) { - Log.info("using specified MAC address", ["mac": text]) - return mac - } - Log.warn("failed to parse MAC address, generating random", ["mac": text]) - return VZMACAddress.randomLocallyAdministered() - } - let mac = VZMACAddress.randomLocallyAdministered() - Log.info("generated random MAC address", ["mac": mac.string]) - return mac -} - -// vmnetAttachment attaches the guest to its own shared-mode vmnet network with the -// given subnet and a DHCP reservation for the guest's MAC, so the guest's address is -// the one hypeman assigned. -func vmnetAttachment(_ n: NetworkConfig) throws -> VZNetworkDeviceAttachment { - guard #available(macOS 26.0, *) else { - throw ShimError("configure network: vmnet networks require macOS 26 or newer") - } - guard let macText = n.mac, let mac = VZMACAddress(string: macText) else { - throw ShimError("configure network: vmnet network needs the guest MAC") - } - guard let (network, bits) = parseIPv4Prefix(n.ipv4Subnet ?? ""), bits <= 30 else { - throw ShimError("configure network: parse guest subnet \(n.ipv4Subnet ?? "")") - } - guard let address = parseIPv4(n.ipv4Address ?? "") else { - throw ShimError("configure network: parse guest address \(n.ipv4Address ?? "")") - } - let mask: UInt32 = bits == 0 ? 0 : ~UInt32(0) << UInt32(32 - bits) - let subnetText = "\(formatIPv4(network & mask))/\(bits)" - - var status = vmnet_return_t.VMNET_SUCCESS - guard let config = vmnet_network_configuration_create(.VMNET_SHARED_MODE, &status), status == .VMNET_SUCCESS else { - throw ShimError("configure network: configure vmnet network \(subnetText): status \(status.rawValue)") - } - // vmnet needs the host address, the first in the subnet, to start DHCP-backed - // attachments; the network address alone leaves the guest without a lease. - var subnetAddr = in_addr(s_addr: ((network & mask) + 1).bigEndian) - var maskAddr = in_addr(s_addr: mask.bigEndian) - status = vmnet_network_configuration_set_ipv4_subnet(config, &subnetAddr, &maskAddr) - guard status == .VMNET_SUCCESS else { - throw ShimError("configure network: set vmnet subnet \(subnetText): status \(status.rawValue)") - } - var client = ether_addr_t() - withUnsafeMutableBytes(of: &client.octet) { $0.copyBytes(from: mac.ethernetAddress) } - var reserved = in_addr(s_addr: address.bigEndian) - status = vmnet_network_configuration_add_dhcp_reservation(config, &client, &reserved) - guard status == .VMNET_SUCCESS else { - throw ShimError("configure network: reserve \(formatIPv4(address)) on vmnet network \(subnetText): status \(status.rawValue)") - } - guard let network = vmnet_network_create(config, &status), status == .VMNET_SUCCESS else { - throw ShimError("configure network: create vmnet network \(subnetText): status \(status.rawValue)") - } - Log.info("attached guest vmnet network", ["subnet": subnetText, "address": formatIPv4(address), "mac": macText]) - return VZVmnetNetworkDeviceAttachment(network: network) -} - -extension VZMACAddress { - var ethernetAddress: [UInt8] { - string.split(separator: ":").compactMap { UInt8($0, radix: 16) } - } -} - -// parseIPv4 returns an address in host byte order. -func parseIPv4(_ text: String) -> UInt32? { - var addr = in_addr() - guard inet_pton(AF_INET, text, &addr) == 1 else { return nil } - return UInt32(bigEndian: addr.s_addr) -} - -func parseIPv4Prefix(_ text: String) -> (UInt32, Int)? { - let parts = text.split(separator: "/") - guard parts.count == 2, let address = parseIPv4(String(parts[0])), let bits = Int(parts[1]), (0...32).contains(bits) else { return nil } - return (address, bits) -} - -func formatIPv4(_ address: UInt32) -> String { - "\(address >> 24 & 0xff).\(address >> 16 & 0xff).\(address >> 8 & 0xff).\(address & 0xff)" -} - -func cpuCount(_ requested: Int) -> Int { - var count = requested - if count == 0 { - count = max(1, ProcessInfo.processInfo.processorCount - 1) - } - return min(max(count, VZVirtualMachineConfiguration.minimumAllowedCPUCount), VZVirtualMachineConfiguration.maximumAllowedCPUCount) -} - -func memorySize(_ requested: UInt64) -> UInt64 { - // 2 GiB safety default; the caller normally provides this. - let size = requested == 0 ? 2 << 30 : requested - return min(max(size, VZVirtualMachineConfiguration.minimumAllowedMemorySize), VZVirtualMachineConfiguration.maximumAllowedMemorySize) -} diff --git a/cmd/vz-shim/main.swift b/cmd/vz-shim/main.swift index e5a3c0486..a14ef4500 100644 --- a/cmd/vz-shim/main.swift +++ b/cmd/vz-shim/main.swift @@ -1,7 +1,6 @@ // hypeman-vz-shim is a subprocess that hosts one vz VM, so VMs survive hypeman // restarts. Hypeman starts it with -config and talks to it over // the control and vsock sockets named in that config. -import CryptoKit import Foundation import Virtualization @@ -19,32 +18,6 @@ func configArgument() -> String? { return nil } -// lockMacOSIdentity holds an exclusive lock on the Mac machine identity until the -// process exits, including while paused. The lock file is never unlinked: another -// waiter may already hold its inode open. -func lockMacOSIdentity(_ config: ShimConfig) throws -> Int32? { - guard let text = config.macMachineIdentifierData, !text.isEmpty else { return nil } - guard let data = decodeBase64(text) else { throw ShimError("decode Mac identity for admission") } - let hash = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() - // Fixed directory, not TMPDIR: servers with different environments must contend - // on the same lock. The name is predictable, so refuse symlinks and files owned - // by anyone else. - let path = "/tmp/hypeman-mac-identity-\(getuid())-\(hash).lock" - let fd = open(path, O_CREAT | O_RDWR | O_NOFOLLOW, 0o600) - guard fd >= 0 else { throw ShimError("open \(path): \(errnoString())") } - var info = stat() - guard fstat(fd, &info) == 0, info.st_mode & S_IFMT == S_IFREG, info.st_uid == getuid() else { - close(fd) - throw ShimError("Mac identity lock path is not an owned regular file") - } - guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { - let message = errnoString() - close(fd) - throw ShimError("Mac machine identity already in use: \(message)") - } - return fd -} - signal(SIGPIPE, SIG_IGN) guard let configJSON = configArgument(), !configJSON.isEmpty else { From d8ebf39ac66fa91a252b415e7a8cef3305421a7b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:50:35 +0000 Subject: [PATCH 27/33] Trust only the attached agent disk, and keep the sudo password off command lines The launcher runs the agent only from a fixed internal device, not a disk image a guest user attached under the same volume name. The Packer template reads the sudo password from a file it uploads. Document the disk-digest label. --- docs/macos-experimental.md | 11 +++++++---- images/macos/browser.pkr.hcl | 11 ++++++++++- lib/system/guest_agent/macos/install-launcher.sh | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index bffc82a24..01bf91cd3 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -43,6 +43,7 @@ labels describe a complete installed machine: | `io.hypeman.machine-image.disk-path` | Relative ASIF disk path, in every layer | | `io.hypeman.machine-image.aux-path` | Relative auxiliary storage path | | `io.hypeman.machine-image.platform-path` | Relative macOS platform configuration JSON path | +| `io.hypeman.machine-image.disk-digests` | Optional: sha256 of each layer's disk file, base first, comma-separated | Each layer carries one ASIF disk, and each layer's disk is an overlay on the previous layer's, so the layers form the boot disk's chain, base first. The @@ -113,10 +114,12 @@ steps 2 to 4 on a self-hosted macOS 27 runner labelled `tart`. It needs the `TART_REGISTRY_PASSWORD` secrets. The image is published as `builds/`. `import-tart` reads the VM's layers from -Tart's content store in the order its `manifest.json` lists them, refuses a -suspended or running VM, and checks that the overlay stacks on those layers. With -`--base`, the VM's layers must be that image's layers, and only the config and -overlay are uploaded. Each image gets a fresh machine identifier and MAC, since +Tart's content store (under `TART_HOME`, or `~/.tart`) in the order its +`manifest.json` lists them, refuses a suspended or running VM, and checks that the +overlay stacks on those layers. Each published image records its layers' disk +digests. With `--base`, the VM's layer digests must equal the ones that image +records, and only the config and overlay are uploaded; a base without them is +refused. Each image gets a fresh machine identifier and MAC, since Tart keeps its source's; the SSH host keys are the template's job. The registry imports a pushed image on its host, as `builds/@sha256:...`. Other hosts pull it by digest with registry credentials. diff --git a/images/macos/browser.pkr.hcl b/images/macos/browser.pkr.hcl index 06e0c120d..aa3545ea6 100644 --- a/images/macos/browser.pkr.hcl +++ b/images/macos/browser.pkr.hcl @@ -37,7 +37,10 @@ variable "extra_scripts" { } locals { - sudo = "echo '${var.ssh_password}' | sudo -S -p '' sh -eux '{{ .Path }}'" + # The sudo password is read from a file uploaded first, so it is never quoted + # into a command line. The build user's home is not readable by other users. + sudo_password = "/Users/${var.ssh_username}/.packer-sudo-password" + sudo = "sudo -S -p '' sh -eux '{{ .Path }}' < ${local.sudo_password}" } source "tart-cli" "macos" { @@ -55,6 +58,11 @@ source "tart-cli" "macos" { build { sources = ["source.tart-cli.macos"] + provisioner "file" { + content = var.ssh_password + destination = local.sudo_password + } + provisioner "shell" { execute_command = local.sudo scripts = concat(["${path.root}/install-chrome.sh"], var.extra_scripts) @@ -76,6 +84,7 @@ build { provisioner "shell" { execute_command = local.sudo inline = [ + "rm -f ${local.sudo_password}", "rm -f /etc/ssh/ssh_host_*", "ssh-keygen -A", "shutdown -h now", diff --git a/lib/system/guest_agent/macos/install-launcher.sh b/lib/system/guest_agent/macos/install-launcher.sh index 3a9259350..c74b0d879 100755 --- a/lib/system/guest_agent/macos/install-launcher.sh +++ b/lib/system/guest_agent/macos/install-launcher.sh @@ -10,10 +10,21 @@ install -d -o root -g wheel -m 0755 /usr/local/libexec cat > /usr/local/libexec/hypeman-agent-launcher <<'SH' #!/bin/sh # Run hypeman's guest agent from its read-only agent disk, mounting it if macOS -# has not yet. +# has not yet. Only the disk hypeman attaches is trusted: a fixed internal device, +# not a disk image a guest user attached under the same name. +volume=/Volumes/HypemanAgent install -d -o root -g wheel -m 0755 /var/run/hypeman for i in $(seq 1 120); do - [ -x /Volumes/HypemanAgent/hypeman-guest-agent ] && exec /Volumes/HypemanAgent/hypeman-guest-agent + if [ -x "$volume/hypeman-guest-agent" ]; then + info=$(/usr/sbin/diskutil info "$volume" 2>/dev/null) + if echo "$info" | grep -Eq '^ *Device Location: +Internal$' && + echo "$info" | grep -Eq '^ *Removable Media: +Fixed$' && + ! echo "$info" | grep -Eq '^ *Protocol: +Disk Image$'; then + exec "$volume/hypeman-guest-agent" + fi + echo "refusing $volume: it is not hypeman's attached agent disk" >&2 + exit 1 + fi /usr/sbin/diskutil mount readOnly HypemanAgent >/dev/null 2>&1 sleep 1 done From cb0c1940f673aedb2c756d81504d25f297f4bb43 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:51:13 +0000 Subject: [PATCH 28/33] Name the cause when a base predates disk digests --- lib/machineimage/publisher.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/machineimage/publisher.go b/lib/machineimage/publisher.go index 14704d3f0..f4db24f42 100644 --- a/lib/machineimage/publisher.go +++ b/lib/machineimage/publisher.go @@ -269,9 +269,12 @@ func machineImage(ctx context.Context, host string, storage Storage, base *Base, if layers := len(config.RootFS.DiffIDs); layers != len(storage.Layers) { return nil, fmt.Errorf("machine base has %d layers but the disk stacks on %d", layers, len(storage.Layers)) } - baseDigests := strings.Split(config.Config.Labels[images.MacOSMachineDiskDigestsLabel], ",") - if !slices.Equal(baseDigests, layerDigests) { - return nil, fmt.Errorf("machine base's disks are not the layers the disk stacks on: base records %q", config.Config.Labels[images.MacOSMachineDiskDigestsLabel]) + recorded := config.Config.Labels[images.MacOSMachineDiskDigestsLabel] + if recorded == "" { + return nil, fmt.Errorf("machine base records no disk digests; publish it again without a base first") + } + if !slices.Equal(strings.Split(recorded, ","), layerDigests) { + return nil, fmt.Errorf("machine base's disks are not the layers the disk stacks on: base records %q", recorded) } } else { for _, path := range storage.Layers { From 1470ec516687b8d4df56b5af6c82a05bf9723c67 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:48:18 +0000 Subject: [PATCH 29/33] Admit macOS guests by the disk they use Admission counted each macOS image at its logical disk size and reserved that size again for every instance, so a host with a few 64 GB images refused creates with tens of gigabytes free. Count a macOS image's allocated blocks, with a layer shared by images (one diff ID, or one inode) counted once, and reserve a 10 GiB allowance for each instance's writes, or what its boot overlay holds once it has written more. --- lib/images/disk_usage.go | 44 ++++++++++++++++++++++++++ lib/images/disk_usage_macos_test.go | 43 +++++++++++++++++++++++++ lib/instances/admission.go | 29 +++++++++++++++++ lib/instances/admission_allocations.go | 2 +- lib/instances/admission_macos_test.go | 23 ++++++++++++++ lib/instances/create.go | 2 +- lib/instances/macos.go | 2 +- lib/instances/macos_test.go | 2 +- 8 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 lib/images/disk_usage_macos_test.go create mode 100644 lib/instances/admission_macos_test.go diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index ca72c1994..494365288 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -26,6 +26,7 @@ func walkWithContext(ctx context.Context, root string, walkFn filepath.WalkFunc) func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir string) (int64, error) { var total int64 seenRootfs := make(map[rootfsIdentity]struct{}) + seenMacOSLayers := make(map[string]struct{}) layersDir := filepath.Join(imagesDir, "layers") err := walkWithContext(ctx, imagesDir, func(path string, info os.FileInfo, err error) error { @@ -70,6 +71,10 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir } return fmt.Errorf("unmarshal image metadata %s: %w", path, err) } + if meta.Status == StatusReady && meta.MacOS != nil { + total += macOSImageAllocatedBytes(filepath.Dir(path), &meta, seenMacOSLayers) + return nil + } if meta.Status == StatusReady { rootfsPaths, globErr := filepath.Glob(filepath.Join(filepath.Dir(path), "rootfs.*")) if globErr != nil { @@ -273,6 +278,45 @@ func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir strin return total, nil } +// macOSImageAllocatedBytes returns the disk a macOS image holds: its chain files' +// allocated blocks, not their logical size, and its auxiliary storage. A layer seen +// in an earlier image is not counted again: images that share a base clone its +// disk, keyed by diff ID, or by inode for a local import, which has none. +func macOSImageAllocatedBytes(dir string, meta *imageMetadata, seen map[string]struct{}) int64 { + var total int64 + for i := range meta.MacOSDiskLayers { + path := filepath.Join(dir, MacOSDiskLayerFile(i)) + info, err := os.Stat(path) + if err != nil { + continue + } + key := path + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + key = fmt.Sprintf("inode:%d:%d", stat.Dev, stat.Ino) + } + if i < len(meta.MacOSDiskDiffIDs) { + key = meta.MacOSDiskDiffIDs[i] + } + if _, counted := seen[key]; counted { + continue + } + seen[key] = struct{}{} + total += fileAllocatedBytes(info) + } + if info, err := os.Stat(filepath.Join(dir, MacOSMachineAuxFile)); err == nil { + total += fileAllocatedBytes(info) + } + return total +} + +// fileAllocatedBytes is a file's allocated blocks, or its size where those are unknown. +func fileAllocatedBytes(info os.FileInfo) int64 { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + return stat.Blocks * 512 + } + return info.Size() +} + type rootfsIdentity struct { dev uint64 ino uint64 diff --git a/lib/images/disk_usage_macos_test.go b/lib/images/disk_usage_macos_test.go new file mode 100644 index 000000000..23e60b72c --- /dev/null +++ b/lib/images/disk_usage_macos_test.go @@ -0,0 +1,43 @@ +package images + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// writeMacOSImageForUsage writes a ready macOS image whose chain files hold 1 MiB +// of data in a 64 MiB logical disk each, with diffIDs for its layers. +func writeMacOSImageForUsage(t *testing.T, imagesDir, digest string, diffIDs []string) { + t.Helper() + dir := filepath.Join(imagesDir, "content", digest) + require.NoError(t, os.MkdirAll(dir, 0700)) + for i := range diffIDs { + file, err := os.Create(filepath.Join(dir, MacOSDiskLayerFile(i))) + require.NoError(t, err) + _, err = file.Write(make([]byte, 1<<20)) + require.NoError(t, err) + require.NoError(t, file.Truncate(64<<20)) + require.NoError(t, file.Close()) + } + require.NoError(t, os.WriteFile(filepath.Join(dir, MacOSMachineAuxFile), []byte("aux"), 0600)) + data, err := json.Marshal(imageMetadata{Name: "builds/" + digest, Digest: "sha256:" + digest, Status: StatusReady, MacOS: &MacOSImage{}, MacOSDiskLayers: len(diffIDs), MacOSDiskDiffIDs: diffIDs, SizeBytes: 64 << 20}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0600)) +} + +func TestMacOSImageDiskUsageCountsAllocatedBytesAndSharedLayersOnce(t *testing.T) { + imagesDir := t.TempDir() + writeMacOSImageForUsage(t, imagesDir, "aaaa", []string{"sha256:base"}) + writeMacOSImageForUsage(t, imagesDir, "bbbb", []string{"sha256:base", "sha256:build"}) + + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) + require.NoError(t, err) + // Two distinct layers of about 1 MiB each, not three 64 MiB logical disks. + require.Greater(t, total, int64(2<<20)-1) + require.Less(t, total, int64(4<<20)) +} diff --git a/lib/instances/admission.go b/lib/instances/admission.go index cc22a099f..b735f0c83 100644 --- a/lib/instances/admission.go +++ b/lib/instances/admission.go @@ -1,5 +1,13 @@ package instances +import ( + "os" + "path/filepath" + "syscall" + + "github.com/kernel/hypeman/lib/paths" +) + func volumeOverlayReservationBytes(volumes []VolumeAttachment) int64 { var total int64 for _, vol := range volumes { @@ -18,5 +26,26 @@ func storedDiskReservationBytes(stored *StoredMetadata) int64 { if stored == nil { return 0 } + if stored.MacOS != nil { + return macOSBootDiskReservationBytes(stored) + } return requestedDiskReservationBytes(stored.OverlaySize, stored.Volumes) } + +// macOSBootDiskAllowance is the disk reserved for a macOS guest's writes. Its boot +// disk starts as an empty ASIF overlay on layers its image already holds, and grows +// with the guest's writes, with no fixed cap. The allowance matches the Linux +// default overlay size. +const macOSBootDiskAllowance = 10 << 30 + +// macOSBootDiskReservationBytes reserves the allowance, or what the guest's boot +// overlay already holds once it has written more. +func macOSBootDiskReservationBytes(stored *StoredMetadata) int64 { + var allocated int64 + if info, err := os.Stat(filepath.Join(stored.DataDir, paths.MacOSBootDiskFile)); err == nil { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + allocated = stat.Blocks * 512 + } + } + return max(macOSBootDiskAllowance, allocated) +} diff --git a/lib/instances/admission_allocations.go b/lib/instances/admission_allocations.go index 1f2699ac8..9087bff25 100644 --- a/lib/instances/admission_allocations.go +++ b/lib/instances/admission_allocations.go @@ -145,7 +145,7 @@ func (m *manager) allocationFromStoredMetadata(stored *StoredMetadata, active bo Name: stored.Name, Vcpus: stored.Vcpus, MemoryBytes: stored.Size + stored.HotplugSize, - OverlayBytes: stored.OverlaySize, + OverlayBytes: storedDiskReservationBytes(stored) - volumeOverlayBytes, VolumeOverlayBytes: volumeOverlayBytes, NetworkDownloadBps: stored.NetworkBandwidthDownload, NetworkUploadBps: stored.NetworkBandwidthUpload, diff --git a/lib/instances/admission_macos_test.go b/lib/instances/admission_macos_test.go new file mode 100644 index 000000000..a3a8ab1a5 --- /dev/null +++ b/lib/instances/admission_macos_test.go @@ -0,0 +1,23 @@ +package instances + +import ( + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSDiskReservationIsTheAllowanceNotTheImageSize(t *testing.T) { + dir := t.TempDir() + stored := &StoredMetadata{DataDir: dir, OverlaySize: 64 << 30, MacOS: &images.MacOSImage{}} + require.Equal(t, int64(macOSBootDiskAllowance), storedDiskReservationBytes(stored), "a stored image-sized OverlaySize is not reserved") + + require.NoError(t, os.WriteFile(filepath.Join(dir, paths.MacOSBootDiskFile), make([]byte, 1<<20), 0600)) + require.Equal(t, int64(macOSBootDiskAllowance), storedDiskReservationBytes(stored), "a boot overlay within the allowance reserves the allowance") + + linux := &StoredMetadata{OverlaySize: 64 << 30} + require.Equal(t, int64(64<<30), storedDiskReservationBytes(linux)) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 415344620..2bae403f4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -214,7 +214,7 @@ func (m *manager) createInstance( if overlaySize == 0 { overlaySize = 10 * 1024 * 1024 * 1024 // 10GB default } - // Validate overlay size against max. A macOS boot disk is sized by its image. + // Validate overlay size against max. A macOS guest reserves a fixed allowance instead. if imageInfo.MacOS == nil && overlaySize > m.limits.MaxOverlaySize { return nil, fmt.Errorf("overlay size %d exceeds maximum allowed size %d", overlaySize, m.limits.MaxOverlaySize) } diff --git a/lib/instances/macos.go b/lib/instances/macos.go index e08073dde..7c315f674 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -101,7 +101,7 @@ func prepareMacOSCreate(req *CreateInstanceRequest, img *images.Image, caps hype return fmt.Errorf("%w: macOS requires at least 4 GiB in whole MiB and 2 vCPUs", ErrInvalidRequest) } req.Size, req.Vcpus = size, vcpus - req.OverlaySize = *img.SizeBytes // Reserve the writable boot disk, not a Linux overlay. + req.OverlaySize = macOSBootDiskAllowance // Reserve room for the guest's writes, not the whole disk. req.SkipGuestAgent = req.SkipGuestAgent || !img.MacOS.GuestAgent req.SkipKernelHeaders = true return nil diff --git a/lib/instances/macos_test.go b/lib/instances/macos_test.go index 6bf99636d..9260db061 100644 --- a/lib/instances/macos_test.go +++ b/lib/instances/macos_test.go @@ -28,7 +28,7 @@ func TestMacOSRequestDefaultsAndRejections(t *testing.T) { require.NoError(t, prepareMacOSCreate(&req, testMacImage(), caps)) require.Equal(t, int64(8<<30), req.Size) require.Equal(t, 4, req.Vcpus) - require.Equal(t, int64(64<<30), req.OverlaySize) + require.Equal(t, int64(macOSBootDiskAllowance), req.OverlaySize) require.True(t, req.SkipGuestAgent) for _, r := range []CreateInstanceRequest{ {Env: map[string]string{"X": "Y"}}, {Cmd: []string{"sh"}}, {HotplugSize: 1}, {OverlaySize: 1}, {DiskIOBps: 1}, {Size: 1 << 30}, {Size: (4 << 30) + 1}, {Vcpus: 1}, {Volumes: []VolumeAttachment{{VolumeID: "v"}}}, From 97281941297665f47fbdefd1b1ac30948961b722 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:12:01 +0000 Subject: [PATCH 30/33] Derive the macOS agent disk from the guest agent setting No guests with ASIF disks predate the agent disk, so the stored flag only repeated GuestAgentEnabled. --- docs/macos-guest-agent.md | 3 +-- lib/instances/macos.go | 20 +++++++++----------- lib/instances/macos_agent_disk_test.go | 12 ++++++------ lib/instances/snapshot_macos.go | 2 +- lib/instances/types.go | 4 ---- 5 files changed, 17 insertions(+), 24 deletions(-) diff --git a/docs/macos-guest-agent.md b/docs/macos-guest-agent.md index bbb85c405..63c40f53e 100644 --- a/docs/macos-guest-agent.md +++ b/docs/macos-guest-agent.md @@ -37,8 +37,7 @@ builds a new disk. Each guest keeps its own copy (`agent.img`). Creating a guest and every cold start copy the host's current disk, so an updated agent takes effect at the next cold start. Standby, restore and snapshots keep the copy the guest was saved with, so -saved state restores against the same disk. Guests created before this existed -keep booting without the disk. +saved state restores against the same disk. The image runs the agent from the disk with a small launcher, installed once in the base by `lib/system/guest_agent/macos/install-launcher.sh` (as root, for example diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 7c315f674..8c49444a2 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -186,21 +186,18 @@ func (m *manager) createBootDisk(ctx context.Context, stored *StoredMetadata, im return err } stored.MacOSDiskLayers = len(chain) - if !stored.GuestAgentEnabled() { - return nil - } - stored.MacOSAgentDisk = true return m.installMacOSAgentDisk(ctx, stored) } // macOSAgentDiskSource returns the host's current guest agent disk. Tests replace it. var macOSAgentDiskSource = system.MacOSAgentDisk -// installMacOSAgentDisk gives a macOS guest a copy of the host's current guest agent -// disk. Each guest keeps its own copy, so saved state restores with the disk it was -// saved with; a cold start takes the host's current agent. +// installMacOSAgentDisk gives a macOS guest that runs the guest agent a copy of the +// host's current guest agent disk. Each guest keeps its own copy, so saved state +// restores with the disk it was saved with; a cold start takes the host's current +// agent. func (m *manager) installMacOSAgentDisk(ctx context.Context, stored *StoredMetadata) error { - if !stored.MacOSAgentDisk { + if stored.MacOS == nil || !stored.GuestAgentEnabled() { return nil } source, err := macOSAgentDiskSource(ctx, m.paths) @@ -220,11 +217,12 @@ func (m *manager) installMacOSAgentDisk(ctx context.Context, stored *StoredMetad return os.Rename(staged, disk) } -// macOSDisks returns a macOS guest's disks: its boot disk, then its read-only guest -// agent disk. The boot disk must come first; VZ does not boot with it second. +// macOSDisks returns a macOS guest's disks: its boot disk, then, when it runs the +// guest agent, its read-only guest agent disk. The boot disk must come first; VZ +// does not boot with it second. func macOSDisks(p *paths.Paths, stored *StoredMetadata) []hypervisor.DiskConfig { disks := []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk(stored.Id), Format: hypervisor.DiskFormatASIF, Parents: macOSDiskParents(stored)}} - if stored.MacOSAgentDisk { + if stored.GuestAgentEnabled() { disks = append(disks, hypervisor.DiskConfig{Path: p.InstanceMacOSAgentDisk(stored.Id), Readonly: true}) } return disks diff --git a/lib/instances/macos_agent_disk_test.go b/lib/instances/macos_agent_disk_test.go index 3d9cb5ddf..5b7795ff5 100644 --- a/lib/instances/macos_agent_disk_test.go +++ b/lib/instances/macos_agent_disk_test.go @@ -29,9 +29,9 @@ func TestMacOSAgentDiskInstallAndRefresh(t *testing.T) { useMacOSAgentDisk(t, "agent v1") require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored)) - require.NoFileExists(t, m.paths.InstanceMacOSAgentDisk(stored.Id), "a guest without the flag gets no agent disk") + require.NoFileExists(t, m.paths.InstanceMacOSAgentDisk(stored.Id), "a guest without the agent gets no agent disk") - stored.MacOSAgentDisk = true + stored.MacOS.GuestAgent = true require.NoError(t, m.installMacOSAgentDisk(context.Background(), stored)) data, err := os.ReadFile(m.paths.InstanceMacOSAgentDisk(stored.Id)) require.NoError(t, err) @@ -49,9 +49,9 @@ func TestMacOSAgentDiskInstallAndRefresh(t *testing.T) { func TestMacOSDisksAttachAgentDiskSecond(t *testing.T) { p := paths.New(t.TempDir()) - stored := &StoredMetadata{Id: "mac", DataDir: p.InstanceDir("mac"), MacOSDiskLayers: 1} + stored := &StoredMetadata{Id: "mac", DataDir: p.InstanceDir("mac"), MacOSDiskLayers: 1, MacOS: &images.MacOSImage{}} require.Equal(t, []hypervisor.DiskConfig{{Path: p.InstanceMacOSBootDisk("mac"), Format: hypervisor.DiskFormatASIF, Parents: []string{filepath.Join(stored.DataDir, images.MacOSDiskLayerFile(0))}}}, macOSDisks(p, stored)) - stored.MacOSAgentDisk = true + stored.MacOS.GuestAgent = true disks := macOSDisks(p, stored) require.Len(t, disks, 2) require.Equal(t, p.InstanceMacOSBootDisk("mac"), disks[0].Path, "the boot disk stays first") @@ -60,7 +60,7 @@ func TestMacOSDisksAttachAgentDiskSecond(t *testing.T) { func TestMacOSMemorySnapshotRestoresAgentDiskItWasSavedWith(t *testing.T) { m, meta := macOSMemorySnapshotFixture(t) - meta.MacOSAgentDisk = true + meta.MacOS.GuestAgent, meta.SkipGuestAgent = true, false require.NoError(t, m.saveMetadata(meta)) agentDisk := m.paths.InstanceMacOSAgentDisk(meta.Id) require.NoError(t, os.WriteFile(agentDisk, []byte("agent-at-checkpoint"), 0400)) @@ -80,7 +80,7 @@ func TestMacOSMemorySnapshotRestoresAgentDiskItWasSavedWith(t *testing.T) { func TestMacOSMemorySnapshotRequiresItsAgentDisk(t *testing.T) { m, meta := macOSMemorySnapshotFixture(t) - meta.MacOSAgentDisk = true + meta.MacOS.GuestAgent, meta.SkipGuestAgent = true, false require.NoError(t, m.saveMetadata(meta)) writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) _, err := m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go index 3f9d6bd5e..a8f4c494b 100644 --- a/lib/instances/snapshot_macos.go +++ b/lib/instances/snapshot_macos.go @@ -193,7 +193,7 @@ func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *St for i := range stored.MacOSDiskLayers { payload = append(payload, filepath.Join(guestDir, images.MacOSDiskLayerFile(i))) } - if stored.MacOSAgentDisk { + if stored.GuestAgentEnabled() { payload = append(payload, filepath.Join(guestDir, paths.MacOSAgentDiskFile)) } for _, path := range payload { diff --git a/lib/instances/types.go b/lib/instances/types.go index 9dbce1f7f..f42ff6089 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -97,10 +97,6 @@ type StoredMetadata struct { // MacOSDiskLayers is the number of read-only image layers below a macOS guest's // boot overlay, cloned into its data directory. MacOSDiskLayers int `json:"macos_disk_layers,omitempty"` - // MacOSAgentDisk marks a macOS guest that boots with its copy of the read-only - // guest agent disk. Guests created before the disk existed keep booting without - // it, so their saved state still restores. - MacOSAgentDisk bool `json:"macos_agent_disk,omitempty"` // IdentityRekeyPending marks a macOS clone whose SSH host keys still come from its // source. The next start rekeys them and clears the flag. IdentityRekeyPending bool `json:"identity_rekey_pending,omitempty"` From 394ce0ed1f50d8bf4f8ad3d6ff9a6cd4c3b586a1 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:12:02 +0000 Subject: [PATCH 31/33] Serve a stored manifest only to a repository it was pushed to A token for one repository could otherwise read another's manifest by digest through the fallback. Pushes now record their repository beside the blob store. --- lib/registry/registry.go | 53 ++++++++++++++++++++++++++++++++---- lib/registry/uploads_test.go | 5 ++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 6c0c60008..e17183cee 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strconv" "strings" "sync" @@ -42,7 +43,9 @@ type Registry struct { // the manifest blobs the registry is still serving on tag/digest pulls. // Cleared with the process; tags do not survive restart. pushedTagsMu sync.RWMutex - pushedTags map[string]string + // manifestReposMu serializes recording which repositories a manifest was pushed to. + manifestReposMu sync.Mutex + pushedTags map[string]string } // manifestPutPattern matches PUT requests to /v2/{name}/manifests/{reference} @@ -155,6 +158,9 @@ func (r *Registry) Handler() http.Handler { r.handler.ServeHTTP(wrapper, req) if wrapper.statusCode == http.StatusCreated { + if err := r.recordManifestRepo(pathRepo, digest); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to record manifest repository: %v\n", err) + } // Root every pushed manifest: the Docker manifest blob is // served by the registry but not reachable from the OCI // layout (which stores the converted manifest), so GC @@ -176,9 +182,9 @@ func (r *Registry) Handler() http.Handler { // serveManifestByDigest answers a manifest read by digest, from the upstream // registry or, when it does not know the manifest, from the blob store, where every -// pushed manifest is kept. The upstream registry holds manifests in memory, so a -// manifest pushed before a restart is otherwise unknown by digest. Other requests -// are not handled. +// pushed manifest is kept, if it was pushed to the requested repository. The +// upstream registry holds manifests in memory, so a manifest pushed before a +// restart is otherwise unknown by digest. Other requests are not handled. func (r *Registry) serveManifestByDigest(w http.ResponseWriter, req *http.Request) bool { matches := manifestPutPattern.FindStringSubmatch(req.URL.Path) if matches == nil || (req.Method != http.MethodGet && req.Method != http.MethodHead) { @@ -191,7 +197,7 @@ func (r *Registry) serveManifestByDigest(w http.ResponseWriter, req *http.Reques // Manifests are small, so the upstream answer is buffered to fall back on a 404. upstream := &bufferedResponse{header: make(http.Header), status: http.StatusOK} r.handler.ServeHTTP(upstream, req) - if upstream.status != http.StatusNotFound || !r.serveStoredManifest(w, req, h) { + if upstream.status != http.StatusNotFound || !r.manifestPushedTo(matches[1], h) || !r.serveStoredManifest(w, req, h) { for key, values := range upstream.header { w.Header()[key] = values } @@ -236,6 +242,43 @@ func (r *Registry) serveStoredManifest(w http.ResponseWriter, req *http.Request, return true } +// manifestReposPath lists, one per line, the repositories a manifest was pushed to. +func (r *Registry) manifestReposPath(h v1.Hash) string { + return filepath.Join(r.paths.SystemOCICache(), "manifest-repos", h.Hex) +} + +// recordManifestRepo records that the manifest digest was pushed to repo. +func (r *Registry) recordManifestRepo(repo, digest string) error { + h, err := v1.NewHash(digest) + if err != nil { + return err + } + r.manifestReposMu.Lock() + defer r.manifestReposMu.Unlock() + if r.manifestPushedTo(repo, h) { + return nil + } + path := r.manifestReposPath(h) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return err + } + _, err = file.WriteString(repo + "\n") + if closeErr := file.Close(); err == nil { + err = closeErr + } + return err +} + +// manifestPushedTo reports whether the manifest h was pushed to repo. +func (r *Registry) manifestPushedTo(repo string, h v1.Hash) bool { + data, err := os.ReadFile(r.manifestReposPath(h)) + return err == nil && slices.Contains(strings.Split(string(data), "\n"), repo) +} + // maxStoredManifestBytes bounds a stored blob served as a manifest, so a layer // requested as one is not read into memory. const maxStoredManifestBytes = 4 << 20 diff --git a/lib/registry/uploads_test.go b/lib/registry/uploads_test.go index afa1674fb..3b31ae739 100644 --- a/lib/registry/uploads_test.go +++ b/lib/registry/uploads_test.go @@ -167,4 +167,9 @@ func TestManifestByDigestSurvivesRestart(t *testing.T) { got, err := pulled.Digest() require.NoError(t, err) require.Equal(t, digest, got) + + otherRepo, err := name.NewDigest(strings.TrimPrefix(second.URL, "http://")+"/cache/other@"+digest.String(), name.Insecure) + require.NoError(t, err) + _, err = remote.Image(otherRepo) + require.Error(t, err, "a stored manifest is served only from the repository it was pushed to") } From 20a388ce50eb299564c2cc071801315822e33b7f Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:12:02 +0000 Subject: [PATCH 32/33] Remove agent disks for other agents Every guest has its own copy, so only the current agent's disk is needed on the host. --- lib/system/macos_agent_disk_darwin.go | 29 +++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/system/macos_agent_disk_darwin.go b/lib/system/macos_agent_disk_darwin.go index 4a7dab48f..ee38b2513 100644 --- a/lib/system/macos_agent_disk_darwin.go +++ b/lib/system/macos_agent_disk_darwin.go @@ -8,6 +8,7 @@ import ( _ "embed" "encoding/hex" "fmt" + "log/slog" "os" "os/exec" "path/filepath" @@ -25,7 +26,8 @@ var macOSAgentDiskBuild = make(chan struct{}, 1) // MacOSAgentDisk returns the read-only guest agent disk for the embedded macOS // agent, building it on first use. Disks are named by the agent's hash, so a new -// agent gets a new disk. +// agent gets a new disk; disks for other agents are removed, since every guest +// has its own copy. func MacOSAgentDisk(ctx context.Context, p *paths.Paths) (string, error) { sum := sha256.Sum256(DarwinGuestAgentBinary) version := hex.EncodeToString(sum[:6]) @@ -36,10 +38,29 @@ func MacOSAgentDisk(ctx context.Context, p *paths.Paths) (string, error) { return "", ctx.Err() } defer func() { <-macOSAgentDiskBuild }() - if _, err := os.Stat(path); err == nil { - return path, nil + if _, err := os.Stat(path); err != nil { + if err := buildMacOSAgentDisk(ctx, DarwinGuestAgentBinary, version, path); err != nil { + return "", err + } + } + removeOtherMacOSAgentDisks(path) + return path, nil +} + +// removeOtherMacOSAgentDisks deletes the disks and build directories beside path, +// left by other agents or interrupted builds. +func removeOtherMacOSAgentDisks(path string) { + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + return + } + for _, entry := range entries { + if entry.Name() != filepath.Base(path) { + if err := os.RemoveAll(filepath.Join(filepath.Dir(path), entry.Name())); err != nil { + slog.Warn("remove old macOS agent disk", "path", entry.Name(), "error", err) + } + } } - return path, buildMacOSAgentDisk(ctx, DarwinGuestAgentBinary, version, path) } // buildMacOSAgentDisk writes a raw APFS disk image at path holding the agent and a From 5d617518276119507a0cb338232018a48eeefd3d Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:12:02 +0000 Subject: [PATCH 33/33] Let import-tart publish with a minted, scoped token import-tart --mint-token prints a registry token that can push only builds/ and pull only the base, for a set lifetime. With it in HYPEMAN_REGISTRY_TOKEN, the workflow no longer holds the registry's JWT secret. --- .github/workflows/macos-image.yml | 6 ++-- cmd/import-tart/main.go | 46 ++++++++++++++++++++--------- docs/macos-experimental.md | 17 +++++++++-- lib/machineimage/publisher.go | 47 ++++++++++++++++++++++++------ lib/machineimage/publisher_test.go | 18 ++++++++++++ 5 files changed, 108 insertions(+), 26 deletions(-) diff --git a/.github/workflows/macos-image.yml b/.github/workflows/macos-image.yml index de247aa57..e93560f1b 100644 --- a/.github/workflows/macos-image.yml +++ b/.github/workflows/macos-image.yml @@ -13,7 +13,7 @@ on: description: The same base in hypeman's registry, pinned by digest required: true name: - description: Image name; published as builds/NAME + description: Image name; published as builds/NAME. The registry token must be minted for it. required: true permissions: @@ -83,7 +83,9 @@ jobs: BASE: ${{ inputs.base }} NAME: ${{ inputs.name }} REGISTRY: ${{ vars.HYPEMAN_REGISTRY_URL }} - HYPEMAN_JWT_SECRET: ${{ secrets.HYPEMAN_JWT_SECRET }} + # Minted by import-tart --mint-token for this name and base; it can publish + # nothing else. + HYPEMAN_REGISTRY_TOKEN: ${{ secrets.HYPEMAN_REGISTRY_TOKEN }} run: | import-tart --vm "$HOME/.tart/vms/$VM" --registry "$REGISTRY" --name "$NAME" --base "$BASE" --guest-agent | tee "$RUNNER_TEMP/publication.json" echo "Published \`$(jq -r .Reference "$RUNNER_TEMP/publication.json")\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/cmd/import-tart/main.go b/cmd/import-tart/main.go index cc8e37aa7..3a0b1fa75 100644 --- a/cmd/import-tart/main.go +++ b/cmd/import-tart/main.go @@ -3,6 +3,10 @@ // import-tart publishes a stopped Tart macOS VM to hypeman's registry as a machine // image. A stacked VM built on an image already in the registry passes that image // as --base, so only its overlay is uploaded. +// +// It authenticates with HYPEMAN_REGISTRY_TOKEN, a token from --mint-token that can +// publish only that image over that base, or with the registry's HYPEMAN_JWT_SECRET. +// --mint-token needs the secret and prints the token; CI then holds the token only. package main import ( @@ -25,20 +29,43 @@ func main() { vm := flag.String("vm", "", "stopped Tart VM directory, e.g. ~/.tart/vms/NAME") registry := flag.String("registry", "", "hypeman registry URL, e.g. https://hypeman.example or http://localhost:4973") name := flag.String("name", "", "image name; the image is published as builds/NAME") - base := flag.String("base", "", "digest-pinned registry image the VM's stacked disk is built on") + baseRef := flag.String("base", "", "digest-pinned registry image the VM's stacked disk is built on") guestAgent := flag.Bool("guest-agent", false, "the image runs hypeman's guest agent") timeout := flag.Duration("timeout", 2*time.Hour, "publication timeout") + mint := flag.Bool("mint-token", false, "print a registry token that can publish --name over --base, instead of publishing") + tokenTTL := flag.Duration("token-ttl", 30*24*time.Hour, "lifetime of a token from --mint-token") flag.Parse() - secret := os.Getenv("HYPEMAN_JWT_SECRET") - if *vm == "" || *registry == "" || *name == "" || secret == "" { - fmt.Fprintln(os.Stderr, "--vm, --registry, --name and HYPEMAN_JWT_SECRET are required") + token, secret := os.Getenv("HYPEMAN_REGISTRY_TOKEN"), os.Getenv("HYPEMAN_JWT_SECRET") + if *registry == "" || *name == "" || (*mint && secret == "") || (!*mint && (*vm == "" || token == "" && secret == "")) { + fmt.Fprintln(os.Stderr, "--registry and --name are required, with --vm and HYPEMAN_REGISTRY_TOKEN or HYPEMAN_JWT_SECRET to publish, or --mint-token and HYPEMAN_JWT_SECRET") os.Exit(2) } + var base *machineimage.Base + if *baseRef != "" { + var err error + if base, err = machineimage.ParseBase(*registry, *baseRef); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } + if *mint { + minted, err := machineimage.MintToken(registrytoken.NewGenerator(secret), *name, base, *tokenTTL) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(minted) + return + } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() ctx, cancelTimeout := context.WithTimeout(ctx, *timeout) defer cancelTimeout() - publication, err := run(ctx, *vm, *registry, *name, *base, *guestAgent, secret) + publisher := &machineimage.Publisher{RegistryURL: *registry, Token: token, NewMachineIdentifier: newMachineIdentifier} + if token == "" { + publisher.Tokens = registrytoken.NewGenerator(secret) + } + publication, err := publish(ctx, publisher, *vm, *name, base, *guestAgent) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -49,7 +76,7 @@ func main() { } } -func run(ctx context.Context, vm, registry, name, baseRef string, guestAgent bool, secret string) (machineimage.Publication, error) { +func publish(ctx context.Context, publisher *machineimage.Publisher, vm, name string, base *machineimage.Base, guestAgent bool) (machineimage.Publication, error) { storage, err := machineimage.ReadTartVM(vm, guestAgent) if err != nil { return machineimage.Publication{}, err @@ -59,13 +86,6 @@ func run(ctx context.Context, vm, registry, name, baseRef string, guestAgent boo return machineimage.Publication{}, fmt.Errorf("%s: %w", path, err) } } - var base *machineimage.Base - if baseRef != "" { - if base, err = machineimage.ParseBase(registry, baseRef); err != nil { - return machineimage.Publication{}, err - } - } - publisher := &machineimage.Publisher{RegistryURL: registry, Tokens: registrytoken.NewGenerator(secret), NewMachineIdentifier: newMachineIdentifier} return publisher.Publish(ctx, name, storage, base) } diff --git a/docs/macos-experimental.md b/docs/macos-experimental.md index 01bf91cd3..00606cfc5 100644 --- a/docs/macos-experimental.md +++ b/docs/macos-experimental.md @@ -99,7 +99,7 @@ overlay. 4. Publish the stopped VM over the base: ```sh -HYPEMAN_JWT_SECRET=... go run ./cmd/import-tart \ +HYPEMAN_REGISTRY_TOKEN=... go run ./cmd/import-tart \ --vm ~/.tart/vms/my-build \ --registry https://hypeman.example \ --name my-build \ @@ -110,9 +110,22 @@ HYPEMAN_JWT_SECRET=... go run ./cmd/import-tart \ The `macOS image` workflow (`.github/workflows/macos-image.yml`, run by hand) does steps 2 to 4 on a self-hosted macOS 27 runner labelled `tart`. It needs the `HYPEMAN_REGISTRY_URL` and `MACOS_IMAGE_SSH_USERNAME` variables and the -`HYPEMAN_JWT_SECRET`, `MACOS_IMAGE_SSH_PASSWORD`, `TART_REGISTRY_USERNAME` and +`HYPEMAN_REGISTRY_TOKEN`, `MACOS_IMAGE_SSH_PASSWORD`, `TART_REGISTRY_USERNAME` and `TART_REGISTRY_PASSWORD` secrets. +`HYPEMAN_REGISTRY_TOKEN` can push only `builds/` and pull only the base. +Someone with the registry's JWT secret mints it once, and mints a new one when the +name, the base or its expiry changes: + +```sh +HYPEMAN_JWT_SECRET=... go run ./cmd/import-tart --mint-token \ + --registry https://hypeman.example --name my-build \ + --base hypeman.example/builds/my-base@sha256:... --token-ttl 720h +``` + +`import-tart` also accepts `HYPEMAN_JWT_SECRET` in place of a token, minting one for +each publication. + The image is published as `builds/`. `import-tart` reads the VM's layers from Tart's content store (under `TART_HOME`, or `~/.tart`) in the order its `manifest.json` lists them, refuses a suspended or running VM, and checks that the diff --git a/lib/machineimage/publisher.go b/lib/machineimage/publisher.go index f4db24f42..c8bbd6081 100644 --- a/lib/machineimage/publisher.go +++ b/lib/machineimage/publisher.go @@ -82,7 +82,10 @@ func (s Storage) validate() error { // No credentials, redirect destinations or arbitrary output repositories come from the source. type Publisher struct { RegistryURL string - Tokens *registrytoken.Generator + // Token is a registry token from MintToken. Without one, Tokens mints a token + // for each publication. + Token string + Tokens *registrytoken.Generator // NewMachineIdentifier returns a fresh VZ machine identifier. Every published // image gets its own identifier and MAC, so it never runs with its source's. NewMachineIdentifier func() ([]byte, error) @@ -114,7 +117,7 @@ func (p *Publisher) Publish(ctx context.Context, id string, storage Storage, bas if !ok || ctx.Err() != nil { return Publication{}, fmt.Errorf("bounded publication required") } - if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil || p.NewMachineIdentifier == nil { + if !validName(id) || (p.Token == "" && p.Tokens == nil) || p.NewMachineIdentifier == nil { return Publication{}, fmt.Errorf("scoped machine publisher required") } if base != nil { @@ -136,13 +139,11 @@ func (p *Publisher) Publish(ctx context.Context, id string, storage Storage, bas return Publication{}, err } repo := "builds/" + id - access := []registrytoken.RepoPermission{{Repo: repo, Scope: "push"}} - if base != nil { - access = append(access, registrytoken.RepoPermission{Repo: base.Repo, Scope: "pull"}) - } - token, err := p.Tokens.GenerateToken(id, access, time.Until(deadline)) - if err != nil { - return Publication{}, err + token := p.Token + if token == "" { + if token, err = p.Tokens.GenerateToken(id, publishAccess(id, base), time.Until(deadline)); err != nil { + return Publication{}, err + } } transport := &http.Transport{} if defaults, ok := http.DefaultTransport.(*http.Transport); ok { @@ -173,6 +174,34 @@ func (p *Publisher) Publish(ctx context.Context, id string, storage Storage, bas return Publication{Reference: registry.Host + "/" + repo + "@" + digest.String(), Digest: digest.String()}, nil } +// MintToken returns a registry token that can publish the image name, over base +// when it is set, for ttl. A publisher given it needs no registry secret. +func MintToken(tokens *registrytoken.Generator, name string, base *Base, ttl time.Duration) (string, error) { + if !validName(name) { + return "", fmt.Errorf("invalid machine image name %q", name) + } + if base != nil { + if err := base.validate(); err != nil { + return "", err + } + } + return tokens.GenerateToken(name, publishAccess(name, base), ttl) +} + +// publishAccess is the registry access a publication needs: push on its own +// builds/{name} repository and pull on its base's. +func publishAccess(name string, base *Base) []registrytoken.RepoPermission { + access := []registrytoken.RepoPermission{{Repo: "builds/" + name, Scope: "push"}} + if base != nil { + access = append(access, registrytoken.RepoPermission{Repo: base.Repo, Scope: "pull"}) + } + return access +} + +func validName(name string) bool { + return name != "" && len(name) <= 128 && strings.Trim(name, "abcdefghijklmnopqrstuvwxyz0123456789-") == "" +} + // randomMAC returns a random locally administered unicast MAC address. func randomMAC() (string, error) { mac := make([]byte, 6) diff --git a/lib/machineimage/publisher_test.go b/lib/machineimage/publisher_test.go index c1252270f..1526f4462 100644 --- a/lib/machineimage/publisher_test.go +++ b/lib/machineimage/publisher_test.go @@ -152,3 +152,21 @@ func TestStreamingMachinePublisherReusesBaseLayers(t *testing.T) { _, e = publisher.Publish(ctx, "qa-mismatched", mismatched, machineBase) require.ErrorContains(t, e, "not the layers the disk stacks on") } + +func TestMintedTokenPublishesOnlyItsImage(t *testing.T) { + imports := &machineImportStub{imported: make(chan string, 2)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + server := httptest.NewServer(middleware.JwtAuth("synthetic registry secret")(reg.Handler())) + defer server.Close() + token, e := machineimage.MintToken(registrytoken.NewGenerator("synthetic registry secret"), "qa-minted", nil, time.Hour) + require.NoError(t, e) + publisher := &machineimage.Publisher{RegistryURL: server.URL, Token: token, NewMachineIdentifier: testMachineIdentifier} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + _, e = publisher.Publish(ctx, "qa-minted", machinePublisherStorage(t), nil) + require.NoError(t, e) + _, e = publisher.Publish(ctx, "qa-other", machinePublisherStorage(t), nil) + require.Error(t, e, "the token cannot publish another image") +}