diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c99517e68..e8d35e96e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,22 @@ permissions: contents: read jobs: + test-macos-pipeline-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ env.TEST_SOURCE_REPO }} + ref: ${{ env.TEST_SOURCE_REF }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Check macOS pipeline contracts + run: | + python3 -m unittest discover -s scripts/macos -p 'test_*.py' -v + sh -n scripts/macos/example/provision.sh + test: runs-on: [self-hosted, linux, x64, kvm] steps: diff --git a/cmd/api/api/builds.go b/cmd/api/api/builds.go index ae8e74c27..169149d47 100644 --- a/cmd/api/api/builds.go +++ b/cmd/api/api/builds.go @@ -44,9 +44,15 @@ func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequ // CreateBuild creates a new build job func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) { if s.Config != nil && s.Config.MacOSOnly { - return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + if !s.Config.MacOSBuilds { + return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + } + req, source, err := parseMachineBuildMultipart(request.Body) + if err != nil { + return oapi.CreateBuild400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil + } + return s.submitBuild(ctx, req, source) } - log := logger.FromContext(ctx) // Parse multipart form fields var sourceData []byte @@ -69,6 +75,8 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } switch part.FormName() { + case "machine_base_image", "machine_network_mode": + return oapi.CreateBuild400JSONResponse{Code: "unsupported", Message: "machine builds require macos_only and macos_builds"}, nil case "source": sourceData, err = io.ReadAll(part) if err != nil { @@ -256,6 +264,11 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } } + return s.submitBuild(ctx, domainReq, sourceData) +} + +func (s *ApiService) submitBuild(ctx context.Context, domainReq builds.CreateBuildRequest, sourceData []byte) (oapi.CreateBuildResponseObject, error) { + log := logger.FromContext(ctx) build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData) if err != nil { switch { @@ -430,6 +443,9 @@ func buildToOAPI(b *builds.Build) oapi.Build { BuildkitVersion: &b.Provenance.BuildkitVersion, Timestamp: &b.Provenance.Timestamp, } + if b.Provenance.RecipeHash != "" { + oapiBuild.Provenance.RecipeHash = &b.Provenance.RecipeHash + } if len(b.Provenance.LockfileHashes) > 0 { oapiBuild.Provenance.LockfileHashes = &b.Provenance.LockfileHashes } diff --git a/cmd/api/api/capabilities.go b/cmd/api/api/capabilities.go index e19ca8035..ee242a91c 100644 --- a/cmd/api/api/capabilities.go +++ b/cmd/api/api/capabilities.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" @@ -26,6 +25,8 @@ const ( featureLogs = "logs" featureDevices = "devices" featureRosettaEmulation = "rosetta-emulation" + featureMacOSImages = "macos-machine-images" + featureMacOSBuilds = "macos-machine-builds-v1" ) // apiVersion is the API contract version from the embedded OpenAPI document. @@ -133,13 +134,11 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities }, Runtimes: runtimes, Network: *networkCaps, - Images: oapi.CapabilitiesImages{ - Platforms: imagePlatforms(runtime.GOARCH, emulation), - DefaultPlatform: images.HostPlatformString(), - }, + Images: hostImageCapabilities(runtime.GOOS, runtime.GOARCH, emulation, s.Config.MacOSOnly), Features: serverFeatures(runtime.GOOS, emulation), } + resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH, s.Config.MacOSOnly && s.Config.MacOSBuilds)...) return oapi.GetCapabilities200JSONResponse(resp), nil } @@ -210,6 +209,33 @@ func imagePlatforms(goarch string, emulation bool) []string { return platforms } +func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi.CapabilitiesImages { + caps := oapi.CapabilitiesImages{ + Platforms: imagePlatforms(goarch, emulation), + DefaultPlatform: "linux/" + goarch, + } + if goos == "darwin" && goarch == "arm64" { + if macOSOnly { + caps.Platforms = []string{"darwin/arm64"} + caps.DefaultPlatform = "darwin/arm64" + } else { + caps.Platforms = append(caps.Platforms, "darwin/arm64") + } + } + return caps +} + +func machineImageFeatures(goos, goarch string, builds bool) []string { + if goos != "darwin" || goarch != "arm64" { + return nil + } + features := []string{featureMacOSImages} + if builds { + features = append(features, featureMacOSBuilds) + } + return features +} + // serverFeatures builds the server-level feature ID list: API surfaces that // are always present plus host-platform conditionals. func serverFeatures(goos string, emulation bool) []string { diff --git a/cmd/api/api/capabilities_machine_test.go b/cmd/api/api/capabilities_machine_test.go new file mode 100644 index 000000000..705eabf86 --- /dev/null +++ b/cmd/api/api/capabilities_machine_test.go @@ -0,0 +1,34 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHostMachineImageCapabilities(t *testing.T) { + for _, tc := range []struct { + name, os, arch, defaultPlatform string + emulation, macOSOnly bool + platforms []string + }{ + {"Linux unchanged", "linux", "amd64", "linux/amd64", false, false, []string{"linux/amd64"}}, + {"Linux arm unchanged", "linux", "arm64", "linux/arm64", false, false, []string{"linux/arm64"}}, + {"Mac mixed", "darwin", "arm64", "linux/arm64", false, false, []string{"linux/arm64", "darwin/arm64"}}, + {"Mac mixed Rosetta", "darwin", "arm64", "linux/arm64", true, false, []string{"linux/arm64", "linux/amd64", "darwin/arm64"}}, + {"Mac only", "darwin", "arm64", "darwin/arm64", true, true, []string{"darwin/arm64"}}, + } { + t.Run(tc.name, func(t *testing.T) { + caps := hostImageCapabilities(tc.os, tc.arch, tc.emulation, tc.macOSOnly) + require.Equal(t, tc.defaultPlatform, caps.DefaultPlatform) + require.Equal(t, tc.platforms, caps.Platforms) + }) + } +} + +func TestMachineImageFeatureActivation(t *testing.T) { + require.Empty(t, machineImageFeatures("linux", "arm64", true)) + require.Empty(t, machineImageFeatures("darwin", "amd64", true)) + require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64", false)) + require.Equal(t, []string{"macos-machine-images", "macos-machine-builds-v1"}, machineImageFeatures("darwin", "arm64", true)) +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d5aa47fcf..ced753723 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -63,7 +63,11 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image) + platform := derefString(request.Body.Platform) + if s.Config != nil && s.Config.MacOSOnly && platform == "" { + platform = "darwin/arm64" + } + macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform) if err != nil { log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{Code: "internal_error", Message: "failed to look up image"}, nil @@ -337,7 +341,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Name: request.Body.Name, Image: request.Body.Image, Size: size, - Platform: derefString(request.Body.Platform), + Platform: platform, HotplugSize: hotplugSize, OverlaySize: overlaySize, Vcpus: vcpus, diff --git a/cmd/api/api/macos.go b/cmd/api/api/macos.go index 5dc42cbdb..62e6652ce 100644 --- a/cmd/api/api/macos.go +++ b/cmd/api/api/macos.go @@ -16,11 +16,19 @@ import ( // macOSHost is true where macOS guests can exist. Tests override it. var macOSHost = runtime.GOOS == "darwin" -// isMacOSImage reports whether a create names an imported macOS image. Other -// hosts skip the lookup, so Linux creates are unchanged. An image that is not -// cached yet is not a macOS image; any other lookup error is returned. -func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error) { - if !macOSHost || s.ImageManager == nil { +// isMacOSImage identifies macOS creates before Linux defaults are applied. An +// explicit Darwin platform and macOS-only servers can resolve uncached images. +func (s *ApiService) isMacOSImage(ctx context.Context, name, platform string) (bool, error) { + if !macOSHost { + return false, nil + } + if s.Config != nil && s.Config.MacOSOnly { + return true, nil + } + if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { + return true, nil + } + if s.ImageManager == nil { return false, nil } img, err := s.ImageManager.GetImage(ctx, name) diff --git a/cmd/api/api/macos_build_contract_test.go b/cmd/api/api/macos_build_contract_test.go new file mode 100644 index 000000000..8f5dee8b9 --- /dev/null +++ b/cmd/api/api/macos_build_contract_test.go @@ -0,0 +1,26 @@ +package api + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/builds" + "github.com/stretchr/testify/require" +) + +func TestMachineRecipeHashInBuildResponse(t *testing.T) { + for _, hash := range []string{"", strings.Repeat("a", 64)} { + response := buildToOAPI(&builds.Build{Provenance: &builds.BuildProvenance{RecipeHash: hash}}) + require.NotNil(t, response.Provenance) + data, err := json.Marshal(response) + require.NoError(t, err) + if hash == "" { + require.Nil(t, response.Provenance.RecipeHash) + require.NotContains(t, string(data), "recipe_hash") + } else { + require.Equal(t, hash, *response.Provenance.RecipeHash) + require.Contains(t, string(data), `"recipe_hash":"`+hash+`"`) + } + } +} diff --git a/cmd/api/api/macos_builds.go b/cmd/api/api/macos_builds.go new file mode 100644 index 000000000..22e9ff46a --- /dev/null +++ b/cmd/api/api/macos_builds.go @@ -0,0 +1,130 @@ +package api + +import ( + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/scopes" +) + +// maxConcurrentMachineBuildUploads bounds how many machine build bodies, each up +// to 65 MiB, are read into memory at once. +const maxConcurrentMachineBuildUploads = 4 + +// LimitMachineBuildBody must wrap the validator, which also consumes request bodies. +// A build upload is authenticated before it takes an upload slot, so unauthenticated +// requests cannot hold the slots. +func LimitMachineBuildBody(jwtSecret string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + uploads := make(chan struct{}, maxConcurrentMachineBuildUploads) + upload := mw.JwtAuth(jwtSecret)(scopes.RequireScope(scopes.BuildWrite)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case uploads <- struct{}{}: + defer func() { <-uploads }() + default: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"code":"too_many_requests","message":"too many machine build uploads in progress"}`)) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 65<<20) + next.ServeHTTP(w, r) + }))) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/builds" { + upload.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// Machine-only HTTP admission is bounded regardless of multipart field order. +// Linux retains its existing parser and source contract. +func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildRequest, []byte, error) { + req := builds.CreateBuildRequest{BuildPolicy: &builds.BuildPolicy{}} + var source []byte + seen := map[string]bool{} + if body == nil { + return req, nil, fmt.Errorf("multipart source is required") + } + for { + part, err := body.NextPart() + if err == io.EOF { + break + } + if err != nil { + return req, nil, fmt.Errorf("invalid multipart form") + } + name := part.FormName() + if seen[name] { + return req, nil, fmt.Errorf("duplicate multipart field") + } + seen[name] = true + limit := int64(64 << 10) + switch name { + case "source": + limit = 64 << 20 + case "machine_base_image", "machine_network_mode", "timeout_seconds", "memory_mb", "cpus", "tags": + default: + return req, nil, fmt.Errorf("unsupported machine build field: %s", name) + } + data, err := io.ReadAll(io.LimitReader(part, limit+1)) + if err != nil || int64(len(data)) > limit { + return req, nil, fmt.Errorf("invalid or oversized %s field", name) + } + if err = part.Close(); err != nil { + return req, nil, fmt.Errorf("invalid multipart field") + } + switch name { + case "source": + source = data + case "machine_base_image": + req.MachineBaseImage = string(data) + case "machine_network_mode": + if string(data) != "isolated" && string(data) != "egress" { + return req, nil, fmt.Errorf("machine_network_mode must be isolated or egress") + } + req.BuildPolicy.NetworkMode = string(data) + case "tags": + req.Tags, err = parseTagsJSON(string(data)) + if err != nil { + return req, nil, fmt.Errorf("tags must be a JSON object of strings") + } + default: + value, err := strconv.Atoi(string(data)) + if err != nil || value < 0 { + return req, nil, fmt.Errorf("%s must be a nonnegative integer", name) + } + switch name { + case "timeout_seconds": + req.BuildPolicy.TimeoutSeconds = value + case "memory_mb": + req.BuildPolicy.MemoryMB = value + case "cpus": + req.BuildPolicy.CPUs = value + } + } + } + if len(source) == 0 { + return req, nil, fmt.Errorf("source is required") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !builds.ValidMachineDigest(ref.Digest()) { + return req, nil, fmt.Errorf("machine_base_image must be pinned by sha256 digest") + } + if req.BuildPolicy.TimeoutSeconds > 86400 { + return req, nil, fmt.Errorf("timeout_seconds exceeds 86400") + } + if err = req.BuildPolicy.Validate(); err != nil { + return req, nil, err + } + return req, source, nil +} diff --git a/cmd/api/api/macos_builds_test.go b/cmd/api/api/macos_builds_test.go new file mode 100644 index 000000000..149342101 --- /dev/null +++ b/cmd/api/api/macos_builds_test.go @@ -0,0 +1,218 @@ +package api + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +func machineMultipart(t *testing.T, fields ...[2]string) *multipart.Reader { + t.Helper() + var data bytes.Buffer + writer := multipart.NewWriter(&data) + for _, field := range fields { + require.NoError(t, writer.WriteField(field[0], field[1])) + } + require.NoError(t, writer.Close()) + return multipart.NewReader(&data, writer.Boundary()) +} + +type machineZeroReader struct{} + +func (machineZeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil } + +func TestMachineBuildBodyBoundBeforeValidation(t *testing.T) { + for _, path := range []string{"/builds", "/images"} { + r := authorizedMachineBuildRequest(t, path, io.LimitReader(machineZeroReader{}, (65<<20)+1)) + LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n, err := io.Copy(io.Discard, r.Body) + if path == "/builds" { + require.Error(t, err) + require.EqualValues(t, 65<<20, n) + } else { + require.NoError(t, err) + require.EqualValues(t, (65<<20)+1, n) + } + })).ServeHTTP(httptest.NewRecorder(), r) + } +} + +func TestMachineBuildUploadsAreBounded(t *testing.T) { + release := make(chan struct{}) + entered := make(chan struct{}, maxConcurrentMachineBuildUploads) + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entered <- struct{}{} + <-release + })) + done := make(chan struct{}) + for range maxConcurrentMachineBuildUploads { + go func() { + handler.ServeHTTP(httptest.NewRecorder(), authorizedMachineBuildRequest(t, "/builds", nil)) + done <- struct{}{} + }() + <-entered + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusTooManyRequests, w.Code) + + close(release) + for range maxConcurrentMachineBuildUploads { + <-done + } + w = httptest.NewRecorder() + go func() { <-entered }() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "a slot frees once an upload finishes") +} + +func TestMachineBuildMultipartAdmission(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + valid := [][2]string{{"source", "synthetic"}, {"machine_base_image", base}} + for _, tc := range []struct { + name string + extra [][2]string + ok bool + }{ + {name: "inherit", ok: true}, + {name: "policy", extra: [][2]string{{"cpus", "4"}, {"memory_mb", "8192"}, {"timeout_seconds", "1200"}}, ok: true}, + {name: "isolated network", extra: [][2]string{{"machine_network_mode", "isolated"}}, ok: true}, + {name: "egress network", extra: [][2]string{{"machine_network_mode", "egress"}}, ok: true}, + {name: "invalid network", extra: [][2]string{{"machine_network_mode", "bridge"}}}, + {name: "empty network", extra: [][2]string{{"machine_network_mode", ""}}}, + {name: "duplicate", extra: [][2]string{{"source", "second"}}}, + {name: "linux", extra: [][2]string{{"dockerfile", "FROM scratch"}}}, + {name: "secret", extra: [][2]string{{"secrets", "[]"}}}, + {name: "invalid number", extra: [][2]string{{"cpus", "nan"}}}, + {name: "negative", extra: [][2]string{{"memory_mb", "-1"}}}, + {name: "timeout", extra: [][2]string{{"timeout_seconds", "86401"}}}, + {name: "metadata bound", extra: [][2]string{{"tags", strings.Repeat("x", (64<<10)+1)}}}, + } { + t.Run(tc.name, func(t *testing.T) { + req, source, err := parseMachineBuildMultipart(machineMultipart(t, append(append([][2]string{}, valid...), tc.extra...)...)) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, base, req.MachineBaseImage) + require.Equal(t, []byte("synthetic"), source) + if tc.name == "egress network" { + require.Equal(t, "egress", req.BuildPolicy.NetworkMode) + } + }) + } + _, _, err := parseMachineBuildMultipart(machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", "example.test/base:latest"})) + require.Error(t, err) +} + +type machineHTTPManager struct { + builds.Manager + req builds.CreateBuildRequest + calls int +} + +func (m *machineHTTPManager) CreateBuild(_ context.Context, req builds.CreateBuildRequest, source []byte) (*builds.Build, error) { + m.calls++ + m.req = req + return &builds.Build{ID: "synthetic-build", Status: builds.StatusQueued}, nil +} + +func TestMachineBuildHTTPGateAndDispatch(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + for _, tc := range []struct { + name string + cfg *config.Config + accepted bool + }{ + {"disabled", &config.Config{MacOSOnly: true}, false}, + {"linux", &config.Config{}, false}, + {"enabled", &config.Config{MacOSOnly: true, MacOSBuilds: true}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := &machineHTTPManager{} + service := &ApiService{Config: tc.cfg, BuildManager: manager} + response, err := service.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{Body: machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", base})}) + require.NoError(t, err) + if tc.accepted { + require.IsType(t, oapi.CreateBuild202JSONResponse{}, response) + require.Equal(t, 1, manager.calls) + require.Equal(t, base, manager.req.MachineBaseImage) + } else { + // Disabled machine builds are unsupported (501) on macOS-only servers, and + // a Linux server rejects the machine selector as a bad request (400). + if tc.cfg.MacOSOnly { + require.IsType(t, oapi.CreateBuild501JSONResponse{}, response) + } else { + require.IsType(t, oapi.CreateBuild400JSONResponse{}, response) + } + require.Zero(t, manager.calls) + } + }) + } +} + +const machineBuildTestSecret = "machine-build-test-secret" + +func authorizedMachineBuildRequest(t *testing.T, path string, body io.Reader) *http.Request { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + }).SignedString([]byte(machineBuildTestSecret)) + require.NoError(t, err) + r := httptest.NewRequest(http.MethodPost, path, body) + r.Header.Set("Authorization", "Bearer "+token) + return r +} + +func TestMachineBuildUploadsRequireWriteScopeBeforeTakingASlot(t *testing.T) { + claims := jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + "permissions": []string{"build:read"}, + } + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(machineBuildTestSecret)) + require.NoError(t, err) + called := false + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })) + r := httptest.NewRequest(http.MethodPost, "/builds", nil) + r.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + require.Equal(t, http.StatusForbidden, w.Code) + require.False(t, called, "read-only token must be rejected before upload processing") +} + +func TestMachineBuildUploadsRequireAuthBeforeTakingASlot(t *testing.T) { + reached := false + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + reached = true + })) + for range maxConcurrentMachineBuildUploads + 1 { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/builds", nil)) + require.Equal(t, http.StatusUnauthorized, w.Code) + } + require.False(t, reached) + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "unauthenticated requests hold no upload slot") + require.True(t, reached) +} diff --git a/cmd/api/api/macos_test.go b/cmd/api/api/macos_test.go index ce19cce6b..9cdb9b795 100644 --- a/cmd/api/api/macos_test.go +++ b/cmd/api/api/macos_test.go @@ -84,6 +84,41 @@ func TestLinuxCreateDefaultsMatchSchema(t *testing.T) { require.EqualValues(t, linuxDefaultVcpus, props["vcpus"].Value.Default) } +func TestMacOSOnlyCreateDefaultsToDarwinPlatform(t *testing.T) { + useMacOSHost(t) + manager := newCaptureCreateManager(nil) + service := &ApiService{InstanceManager: manager, Config: &config.Config{MacOSOnly: true}} + ctx, status := recordCreateBody(t, `{"name":"macos","image":"registry.example/macos/app:latest"}`) + require.Equal(t, http.StatusOK, status) + size, overlay, vcpus := linuxDefaultSize, linuxDefaultOverlaySize, linuxDefaultVcpus + _, err := service.CreateInstance(ctx, oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{ + Name: "macos", Image: "registry.example/macos/app:latest", Size: &size, OverlaySize: &overlay, Vcpus: &vcpus, + }}) + require.NoError(t, err) + require.Equal(t, "darwin/arm64", manager.lastReq.Platform) + require.Zero(t, manager.lastReq.Size) + require.Zero(t, manager.lastReq.OverlaySize) + require.Zero(t, manager.lastReq.Vcpus) +} + +func TestMacOSCreateRecognizesUncachedDarwinImage(t *testing.T) { + useMacOSHost(t) + manager := newCaptureCreateManager(nil) + service := &ApiService{InstanceManager: manager, ImageManager: missingMacOSImageManager{}, Config: &config.Config{}} + ctx, status := recordCreateBody(t, `{"name":"macos","image":"registry.example/macos/app:latest","platform":"darwin/arm64"}`) + require.Equal(t, http.StatusOK, status) + platform, size, overlay, vcpus := "darwin/arm64", linuxDefaultSize, linuxDefaultOverlaySize, linuxDefaultVcpus + _, err := service.CreateInstance(ctx, oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{ + Name: "macos", Image: "registry.example/macos/app:latest", Platform: &platform, + Size: &size, OverlaySize: &overlay, Vcpus: &vcpus, + }}) + require.NoError(t, err) + require.Equal(t, "darwin/arm64", manager.lastReq.Platform) + require.Zero(t, manager.lastReq.Size) + require.Zero(t, manager.lastReq.OverlaySize) + require.Zero(t, manager.lastReq.Vcpus) +} + func TestMacOSCreateIgnoresInjectedLinuxDefaults(t *testing.T) { useMacOSHost(t) m := newCaptureCreateManager(nil) @@ -172,3 +207,9 @@ type macOSImageManager struct{ images.Manager } func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) { return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil } + +type missingMacOSImageManager struct{ images.Manager } + +func (missingMacOSImageManager) GetImage(context.Context, string) (*images.Image, error) { + return nil, images.ErrNotFound +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 0bffb655b..806704895 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -2,7 +2,9 @@ package config import ( "fmt" + "net" "net/netip" + "net/url" "os" "path/filepath" "runtime" @@ -285,7 +287,8 @@ type Config struct { JwtSecret string `koanf:"jwt_secret"` Env string `koanf:"env"` Version string `koanf:"version"` - MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. + MacOSBuilds bool `koanf:"macos_builds"` // Experimental: enable machine builds on the macOS-only server. + MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. Network NetworkConfig `koanf:"network"` Caddy CaddyConfig `koanf:"caddy"` @@ -606,6 +609,9 @@ func (c *Config) Validate() error { if err := c.validateAPIListenPorts(); err != nil { return err } + if err := c.validateMacOSBuilds(); err != nil { + return err + } if strings.TrimSpace(c.Metrics.ListenAddress) == "" { return fmt.Errorf("metrics.listen_address must not be empty") } @@ -852,3 +858,34 @@ func (c *Config) validateFirecrackerUFFDGraduation() error { func intPtr(v int) *int { return &v } + +// validateMacOSBuilds enforces the experimental machine-build registry boundary. +func (c *Config) validateMacOSBuilds() error { + if !c.MacOSBuilds { + return nil + } + if !c.MacOSOnly { + return fmt.Errorf("macos_builds requires macos_only") + } + if c.Registry.CACertFile != "" { + return fmt.Errorf("macos_builds requires system-trusted registry TLS or loopback HTTP") + } + + u, err := url.Parse(c.Registry.URL) + if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { + return fmt.Errorf("macos_builds requires an explicit registry URL without credentials, path, query, or fragment") + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("macos_builds registry URL must use http or https") + } + if u.Scheme == "https" && c.Registry.Insecure { + return fmt.Errorf("macos_builds requires system-trusted registry TLS or loopback HTTP") + } + if u.Scheme == "http" { + host := u.Hostname() + if host != "localhost" && !net.ParseIP(host).IsLoopback() { + return fmt.Errorf("macos_builds plaintext registry must be loopback") + } + } + return nil +} diff --git a/cmd/api/config/macos_builds_test.go b/cmd/api/config/macos_builds_test.go new file mode 100644 index 000000000..1c2fcd83d --- /dev/null +++ b/cmd/api/config/macos_builds_test.go @@ -0,0 +1,39 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildsFailClosedConfiguration(t *testing.T) { + require.False(t, (&Config{}).MacOSBuilds) + c := *defaultConfig() + c.MacOSBuilds = true + require.ErrorContains(t, c.validateMacOSBuilds(), "requires macos_only") + c.MacOSOnly = true + c.Hypervisor.Default = "vz" + c.Registry.CACertFile = "custom.pem" + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.CACertFile = "" + c.Registry.URL = "https://registry.example" + c.Registry.Insecure = true + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.Insecure = false + c.Registry.URL = "http://registry.example:4973" + require.ErrorContains(t, c.validateMacOSBuilds(), "must be loopback") + c.Registry.URL = "localhost:4973" + require.ErrorContains(t, c.validateMacOSBuilds(), "explicit registry URL") + c.Registry.URL = "https://registry.example/prefix" + require.ErrorContains(t, c.validateMacOSBuilds(), "without credentials") + c.Registry.URL = "https://user:pass@registry.example" + require.ErrorContains(t, c.validateMacOSBuilds(), "without credentials") + c.Registry.URL = "https://registry.example?debug=1" + require.ErrorContains(t, c.validateMacOSBuilds(), "without credentials") + c.Registry.URL = "http://127.0.0.1:4973" + require.NoError(t, c.validateMacOSBuilds()) + c.Registry.URL = "http://localhost:4973" + require.NoError(t, c.validateMacOSBuilds()) + c.Registry.URL = "http://[::1]" + require.NoError(t, c.validateMacOSBuilds()) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 347b395ff..1c8241323 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -535,6 +535,11 @@ func run() error { r.Use(api.RecordCreateFields) } + // Bound machine multipart input before OpenAPI validation reads the body. + if app.Config.MacOSBuilds { + r.Use(api.LimitMachineBuildBody(app.Config.JwtSecret)) + } + // OpenAPI request validation with authentication validatorOptions := &nethttpmiddleware.Options{ Options: openapi3filter.Options{ @@ -632,10 +637,10 @@ func run() error { logger.Error("failed to start builders manager", "error", err) return err } - if err := app.BuildManager.Start(gctx); err != nil { - logger.Error("failed to start build manager", "error", err) - return err - } + } + if err := app.BuildManager.Start(gctx); err != nil { + logger.Error("failed to start build manager", "error", err) + return err } grp.Go(func() error { diff --git a/cmd/publish-macos-base/main.go b/cmd/publish-macos-base/main.go new file mode 100644 index 000000000..4d9767cc6 --- /dev/null +++ b/cmd/publish-macos-base/main.go @@ -0,0 +1,219 @@ +// publish-macos-base is an offline administrator tool, not an HTTP filesystem API. +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "regexp" + "strings" + "syscall" + "time" + + "github.com/kernel/hypeman/lib/images" +) + +type audit struct { + Version int `json:"version"` + Status string `json:"status"` + OSVersion string `json:"os_version"` + OSBuild string `json:"os_build"` + Architecture string `json:"architecture"` + AgentSHA256 string `json:"agent_sha256"` + PlistSHA256 string `json:"plist_sha256"` + ToolchainSHA256 string `json:"toolchain_sha256"` + ToolchainVersion string `json:"toolchain_version"` + HomeBaselineSHA256 string `json:"home_baseline_sha256"` + AuditScriptSHA256 string `json:"audit_script_sha256"` +} + +type receipt struct { + Version int `json:"version"` + Reference string `json:"reference"` + Digest string `json:"digest"` + Audit audit `json:"audit"` + AuditSHA256 string `json:"audit_sha256"` + PayloadSHA256 map[string]string `json:"payload_sha256"` + PublishedAt time.Time `json:"published_at"` +} + +func main() { + source := flag.String("source", "", "exclusive stopped macvm bundle") + auditPath := flag.String("audit", "", "guest audit JSON from final GuestService exec") + registry := flag.String("registry", "", "configured internal registry URL") + key := flag.String("registry-secret-file", "", "private file containing internal registry signing secret") + id := flag.String("id", "", "unique lowercase base publication ID (base-...) ") + out := flag.String("output", "", "new receipt file (must not exist)") + stopped := flag.Bool("stopped", false, "attest graceful shutdown and VMM exit; no restart during export") + clean := flag.Bool("clean-install-reviewed", false, "attest fresh installation, no real credentials ever, final audit via GuestService") + timeout := flag.Duration("timeout", 2*time.Hour, "snapshot and publication deadline") + flag.Parse() + if *source == "" || *auditPath == "" || *registry == "" || *key == "" || *out == "" || !*stopped || !*clean || !strings.HasPrefix(*id, "base-") || len(*id) > 128 || strings.Trim(*id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || *timeout <= 0 || *timeout > 24*time.Hour { + fmt.Fprintln(os.Stderr, "required: --source --audit --registry --registry-secret-file --id base-... --output --stopped --clean-install-reviewed; timeout must be in (0,24h]") + os.Exit(2) + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + ctx, deadline := context.WithTimeout(ctx, *timeout) + defer deadline() + if err := run(ctx, *source, *auditPath, *registry, *key, *id, *out); err != nil { + // Registry/OS errors can contain credentials, paths or server response bodies. + fmt.Fprintln(os.Stderr, "base publication failed; no verified receipt (check private operator diagnostics)") + os.Exit(1) + } +} + +func readAudit(path string) (audit, string, error) { + var a audit + f, err := os.Open(path) + if err != nil { + return a, "", err + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, 65537)) + if err != nil || len(data) > 65536 { + return a, "", fmt.Errorf("invalid audit size") + } + dec := json.NewDecoder(strings.NewReader(string(data))) + dec.DisallowUnknownFields() + if err = dec.Decode(&a); err != nil { + return a, "", err + } + if dec.Decode(new(any)) != io.EOF || a.Version != 1 || a.Status != "pass" || a.Architecture != "arm64" { + return a, "", fmt.Errorf("audit rejected") + } + for _, hash := range []string{a.AgentSHA256, a.PlistSHA256, a.ToolchainSHA256, a.HomeBaselineSHA256, a.AuditScriptSHA256} { + if !hashValid(hash) { + return a, "", fmt.Errorf("audit hash rejected") + } + } + if !regexp.MustCompile(`^[0-9]+(\.[0-9]+)*$`).MatchString(a.OSVersion) || + !regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString(a.OSBuild) || + !regexp.MustCompile(`^Xcode [0-9.]+; Build version [a-zA-Z0-9]+$`).MatchString(a.ToolchainVersion) { + return a, "", fmt.Errorf("audit provenance rejected") + } + sum := sha256.Sum256(data) + return a, hex.EncodeToString(sum[:]), nil +} + +type deadlineReader struct { + ctx context.Context + reader io.Reader +} + +func (r deadlineReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +func hashValid(s string) bool { return len(s) == 64 && strings.Trim(s, "0123456789abcdef") == "" } + +func validateSource(root string) error { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode().Perm()&0077 != 0 { + return fmt.Errorf("private real bundle directory required") + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 3 { + return fmt.Errorf("only disk.img, aux.img and config.json allowed") + } + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, err := os.Lstat(filepath.Join(root, name)) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 { + return fmt.Errorf("private regular payload required") + } + } + platform, err := images.ValidateMacOSBundle(root) + if err != nil { + return err + } + if !platform.GuestAgent { + return fmt.Errorf("system guest agent required") + } + return nil +} + +func run(ctx context.Context, source, auditPath, registry, key, id, out string) error { + a, auditHash, err := readAudit(auditPath) + if err != nil { + return err + } + if err = validateSource(source); err != nil { + return err + } + // Reserve before publishing: never overwrite a prior release receipt. + output, err := os.OpenFile(out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + complete := false + defer func() { + output.Close() + if !complete { + os.Remove(out) + } + }() + info, err := os.Lstat(key) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() < 32 || info.Size() > 4096 { + return fmt.Errorf("private registry signing key required") + } + secret, err := os.ReadFile(key) + if err != nil { + return err + } + secretText := strings.TrimSpace(string(secret)) + if len(secretText) < 32 { + return fmt.Errorf("registry signing key is too short") + } + workspace, err := os.MkdirTemp("", "hypeman-base-") + if err != nil { + return err + } + defer os.RemoveAll(workspace) + bundle, err := snapshot(ctx, source, workspace) + if err != nil { + return err + } + if err = validateSource(bundle); err != nil { + return err + } + hashes := map[string]string{} + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + f, err := os.Open(filepath.Join(bundle, name)) + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(h, deadlineReader{ctx, f}) + f.Close() + if err != nil { + return err + } + hashes[name] = hex.EncodeToString(h.Sum(nil)) + } + reference, digest, err := publish(ctx, registry, secretText, id, bundle) + if err != nil { + return err + } + r := receipt{1, reference, digest, a, auditHash, hashes, time.Now().UTC()} + if err = json.NewEncoder(output).Encode(r); err != nil { + return err + } + if err = output.Sync(); err != nil { + return err + } + if err = output.Close(); err != nil { + return err + } + complete = true + return nil +} diff --git a/cmd/publish-macos-base/main_test.go b/cmd/publish-macos-base/main_test.go new file mode 100644 index 000000000..2af2c597e --- /dev/null +++ b/cmd/publish-macos-base/main_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +func goodAudit() audit { + h := strings.Repeat("a", 64) + return audit{Version: 1, Status: "pass", OSVersion: "15.5", OSBuild: "24F74", Architecture: "arm64", AgentSHA256: h, PlistSHA256: h, ToolchainSHA256: h, ToolchainVersion: "Xcode 16.4; Build version 16F6", HomeBaselineSHA256: h, AuditScriptSHA256: h} +} +func TestReadAudit(t *testing.T) { + a := goodAudit() + data, _ := json.Marshal(a) + tests := map[string][]byte{"valid": data, "trailing": append(append([]byte{}, data...), []byte(" {}")...), "unknown": []byte(strings.Replace(string(data), `"status":"pass"`, `"status":"pass","credential":"secret"`, 1)), "oversized": []byte(strings.Repeat(" ", 65537))} + bad := a + bad.Status = "fail" + tests["failed"], _ = json.Marshal(bad) + bad = a + bad.OSVersion = "secret-or-command-output" + tests["unsafe-provenance"], _ = json.Marshal(bad) + bad = a + bad.AgentSHA256 = "" + tests["missing-pin"], _ = json.Marshal(bad) + for name, data := range tests { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.json") + os.WriteFile(path, data, 0600) + _, sum, err := readAudit(path) + if name == "valid" { + if err != nil || !hashValid(sum) { + t.Fatalf("valid audit rejected: %v", err) + } + } else if err == nil { + t.Fatal("invalid audit accepted") + } + }) + } +} +func TestDeadlineReader(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := io.ReadAll(deadlineReader{ctx, strings.NewReader("data")}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation ignored: %v", err) + } +} + +func TestFailedPublicationEvidence(t *testing.T) { + root := bundleFixture(t) + dir := t.TempDir() + a, _ := json.Marshal(goodAudit()) + auditPath := filepath.Join(dir, "audit.json") + if err := os.WriteFile(auditPath, a, 0600); err != nil { + t.Fatal(err) + } + out := filepath.Join(dir, "receipt.json") + err := run(context.Background(), root, auditPath, "https://registry.example", filepath.Join(dir, "missing-key"), "base-test", out) + if err == nil { + t.Fatal("missing key accepted") + } + if _, err := os.Stat(out); !os.IsNotExist(err) { + t.Fatal("failed publication left receipt") + } + if err := os.WriteFile(out, []byte("existing"), 0600); err != nil { + t.Fatal(err) + } + if err := run(context.Background(), root, auditPath, "https://registry.example", "missing-key", "base-test", out); err == nil { + t.Fatal("existing receipt accepted") + } + data, _ := os.ReadFile(out) + if string(data) != "existing" { + t.Fatal("existing receipt changed") + } +} + +func bundleFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + os.Chmod(root, 0700) + platform := images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} + data, _ := json.Marshal(platform) + for name, data := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { + if err := os.WriteFile(filepath.Join(root, name), data, 0600); err != nil { + t.Fatal(err) + } + } + return root +} +func TestValidateSource(t *testing.T) { + for _, name := range []string{"valid", "extra", "symlink", "public", "unmanaged", "unknown-metadata", "hardlink"} { + t.Run(name, func(t *testing.T) { + root := bundleFixture(t) + switch name { + case "extra": + os.WriteFile(filepath.Join(root, "token"), []byte("secret"), 0600) + case "symlink": + os.Remove(filepath.Join(root, "aux.img")) + os.Symlink("disk.img", filepath.Join(root, "aux.img")) + case "hardlink": + os.Remove(filepath.Join(root, "aux.img")) + os.Link(filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img")) + case "public": + os.Chmod(filepath.Join(root, "disk.img"), 0644) + case "unmanaged", "unknown-metadata": + data, _ := os.ReadFile(filepath.Join(root, "config.json")) + if name == "unmanaged" { + data = []byte(strings.Replace(string(data), `"guest_agent":true`, `"guest_agent":false`, 1)) + } else { + data = []byte(strings.Replace(string(data), `"guest_agent":true`, `"guest_agent":true,"token":"secret"`, 1)) + } + os.WriteFile(filepath.Join(root, "config.json"), data, 0600) + } + err := validateSource(root) + if (err == nil) != (name == "valid") { + t.Fatalf("validation result: %v", err) + } + }) + } +} diff --git a/cmd/publish-macos-base/snapshot_darwin.go b/cmd/publish-macos-base/snapshot_darwin.go new file mode 100644 index 000000000..48e02eed5 --- /dev/null +++ b/cmd/publish-macos-base/snapshot_darwin.go @@ -0,0 +1,60 @@ +//go:build darwin && arm64 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" +) + +func publish(ctx context.Context, registry, secret, id, bundle string) (string, string, error) { + publisher := builds.StreamingMachinePublisher{RegistryURL: registry, Tokens: builds.NewRegistryTokenGenerator(secret)} + result, err := publisher.Publish(ctx, id, bundle) + if err != nil { + return "", "", err + } + return result.Reference, result.Digest, nil +} + +func snapshot(ctx context.Context, source, workspace string) (string, error) { + if os.Geteuid() != 0 { + return "", fmt.Errorf("root required to inspect all open storage") + } + p := paths.New(filepath.Join(workspace, "data")) + // Reuse the offline import's lsof fail-closed check and APFS clones. The + // operator must still exclude restart/writers for the whole snapshot window. + img, err := images.ImportMacOSImage(ctx, p, "localhost/macos-base:export", source) + if err != nil { + return "", err + } + ref, err := images.ParseNormalizedRef(img.Name) + if err != nil { + return "", err + } + imported := p.ImageDigestDir(ref.Repository(), img.Digest[7:]) + bundle := filepath.Join(workspace, "bundle") + if err = os.Mkdir(bundle, 0700); err != nil { + return "", err + } + for src, dst := range map[string]string{"rootfs.raw": "disk.img", "aux.img": "aux.img"} { + if err = os.Rename(filepath.Join(imported, src), filepath.Join(bundle, dst)); err != nil { + return "", err + } + } + // Canonical metadata from import; do not carry arbitrary host bundle files. + data, err := json.Marshal(img.MacOS) + if err != nil { + return "", err + } + if err = os.WriteFile(filepath.Join(bundle, "config.json"), data, 0600); err != nil { + return "", err + } + return bundle, nil +} diff --git a/cmd/publish-macos-base/snapshot_other.go b/cmd/publish-macos-base/snapshot_other.go new file mode 100644 index 000000000..8e30221fe --- /dev/null +++ b/cmd/publish-macos-base/snapshot_other.go @@ -0,0 +1,16 @@ +//go:build !darwin || !arm64 + +package main + +import ( + "context" + "fmt" +) + +func publish(context.Context, string, string, string, string) (string, string, error) { + return "", "", fmt.Errorf("base publication requires an Apple-silicon macOS operator host") +} + +func snapshot(context.Context, string, string) (string, error) { + return "", fmt.Errorf("base snapshot requires an Apple-silicon macOS operator host") +} diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index e7745fb1a..93203f95a 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -27,6 +27,14 @@ data_dir: ~/Library/Application Support/hypeman # Server configuration port: "4973" +# Native macOS guests/builds are opt-in; these defaults retain Linux guests on VZ. +macos_only: false +macos_builds: false +# For a dedicated native Mac builder, set both true after the release gates in +# docs/macos-builds.md pass, and replace registry.url below with an explicit +# system-trusted HTTPS address reachable by the build and destination hosts. +# Machine publication runs on the host, not at the Linux guest NAT gateway. + # ============================================================================= # Hypervisor Configuration (IMPORTANT FOR MACOS) # ============================================================================= @@ -129,8 +137,7 @@ registry: build: # builder_image: "" # empty (default) = built from Dockerfile on first run docker_socket: /var/run/docker.sock - max_concurrent_source_builds: 2 - timeout: 600 + max_concurrent_source_builds: 2 timeout: 600 # ============================================================================= # Resource Limits (same as Linux) diff --git a/docs/macos-base-images.md b/docs/macos-base-images.md new file mode 100644 index 000000000..9e3cce528 --- /dev/null +++ b/docs/macos-base-images.md @@ -0,0 +1,214 @@ +# macOS base image publication + +This is an **operator-controlled pipeline**, not an unattended Apple OS installer. +The first installation of a licensed Apple-silicon macOS guest, Setup Assistant, +local account policy, OS updates, Xcode/license acceptance, Python 3, and root +GuestService provisioning are prerequisites. No automated first-ever installation +is implemented here. See [Darwin GuestService](macos-guest-agent.md) and +[macOS builds](macos-builds.md) for the existing runtime/build contracts. + +Pristine macOS is not assumed to include Python 3 or full Xcode. Provision both +explicitly from pinned, reviewed sources before auditing; Command Line Tools +alone do not satisfy this audit's `xcodebuild -version` requirement. Missing tools +are a failed prerequisite, not a signal to install software during the final audit. + +A development guest is **not** a clean production base. No real base was installed, +sanitized, published, or boot-validated as part of adding this tooling. + +## Release boundary + +1. Provision a fresh, dedicated guest. Never sign into Apple ID, a browser, cloud + CLI, git credential helper, or a real developer/service account. Use no real + passwords/tokens in scripts, recipe arguments, environment, or shell history. +2. Run the reviewed audit over the root GuestService, collect its structured + evidence, gracefully shut down, and independently confirm VMM exit. Exclude + restart and all writers until snapshot completes. +3. Publish the offline bundle using the local administrator command. It creates + actual OCI machine content and verifies registry storage before writing an + immutable manifest-digest receipt. It does not expose arbitrary filesystem + reads over HTTP. +4. Submit a derivative through normal `/builds`; push it to a **candidate** registry with `POST /builds`. It is published to this server's registry, then pushed to the + external registry with `POST /pushes` and pulled by digest on each Mac with `POST /images`. +5. Cold-boot and validate the candidate sequentially. After approval, push the approved digest to the production tag with `/pushes`. Retain the previous digest + for rollback; production consumers should pin digests, not mutable tags. + +Identity is preserved, not rekeyed. Never boot the source guest, a derivative, or +another copy of the same identity concurrently, including on different hosts. +Do not assume distinct digests imply distinct VZ identity. CPU/memory must match +the installed base exactly; derivative builds inherit them by default. + +## Audit and limited sanitation (inside guest) + +Use Python 3 provisioned from a pinned, reviewed source. Run +`scripts/macos/audit-base.py --help` for arguments. The script requires root, +arm64, an exact list of non-system accounts, a pinned root-owned non-writable +agent binary and LaunchDaemon plist, the expected single-command root agent +job, and a running launchd service. The plist must not inject environment +variables. The normal host GuestService readiness gate and a successful final +GuestService exec of this script establish transport/exec functionality; the +plist flag or a readiness file alone does not. An audit run over SSH alone is +not sufficient evidence for publication. + +Stage the reviewed script and baseline outside all audited homes, for example +under root-owned `/opt/base-audit`, not `/var/root`, `/Users/Shared`, or an account's +home. Copy the JSON result directly to the operator through GuestService rather +than redirecting it into a guest home. Do not add the script, baseline, or report +to the home allowlist to make an audit pass. The example's hash placeholders must +be supplied as literal arguments to GuestService exec, not exported by opening an +interactive root shell. Close existing shells before the final audit: a shell can +rewrite history on exit after `--clean-transient` has removed it. The audit cannot +prove there are no future writers, so keep the guest quiescent through shutdown. + +Example final guest command, issued through the normal authenticated instance +exec interface (substitute reviewed paths/hashes; do not run on the Mac host): + +```sh +/usr/local/bin/python3 /opt/base-audit/audit-base.py \ + --agent /usr/local/sbin/hypeman-guest-agent \ + --agent-sha256 "$EXPECTED_AGENT_SHA256" \ + --plist /Library/LaunchDaemons/com.hypeman.guest-agent.plist \ + --plist-sha256 "$EXPECTED_PLIST_SHA256" \ + --users builder --home-baseline /opt/base-audit/home-baseline.json \ + --clean-transient +``` + +The account name/job path above are examples, not an invented existing installer. +The plist/binary must first be provisioned according to the actual agent contract. +Copy only the successful JSON stdout to a private operator `audit.json`; reject +nonzero exec status, truncated output, or any extra output. Keep the exec job ID, +agent source commit, native cgo build command and artifact SHA256 in the private +release record. Build the Darwin agent with cgo on macOS as documented; a +cgo-disabled executable is not a functional native vsock agent. + +`--clean-transient` only removes known shell-history files. It does not remove +credentials to turn a used guest into a clean one. The audit rejects credential, +browser, keychain, account, cloud and runtime-state locations, unknown accounts, +unknown home files, links/special files, SSH host keys, managed-device/autologin +configuration and Time Machine local snapshots. Missing tools, failed probes, +unrecognized Xcode output or incomplete checks fail closed. Diagnostics contain +fixed failure codes only, never file contents or command output. + +Home directories (including root and Shared) must contain no files by default. +A private reviewed `home-baseline.json` can allow **specific nonsecret OS preference +files by exact absolute path and SHA256**, not whole directories or globs. `{}` +allows no files. Credential/browser locations remain forbidden even when present +in the baseline. Do not generate an allow-all manifest from a development guest: +review each entry for secret/state content first. Fresh macOS defaults may require +baseline review or deliberate removal, and unsupported configurations deliberately +fail instead of silently passing. Additional system-account homes or custom +application state need policy extension and review before release. + +This is a policy audit, not a forensic erasure/secret detector or attestation of +all disk bytes. It cannot prove absence of arbitrary secrets in system files, +APFS deleted blocks, other APFS snapshots, OS keychains, provisioning artifacts, +or arbitrary custom paths. The operator must review those surfaces and enforce +the fresh/no-real-credentials prerequisite. If the guest has ever contained real +credentials, discard it and reinstall; deletion is not sufficient. Keep the guest +quiescent between final audit and graceful shutdown; do not reopen a browser or +run more provisioning commands. Review known OS-generated default keychains and +machine identity separately; never waive user credentials in the home baseline. + +Audit evidence contains OS product/build, architecture, agent/plist hashes, +Xcode version/build, a toolchain probe hash (Xcode + clang), the reviewed home +baseline hash, and the audit script hash. It contains no user/account lists, +home paths, process output, tokens, or logs. Keep script, baseline and toolchain +inputs under version control in the private release source for reproducibility. +The audit is trusted operator input, not a cryptographically signed guest proof; +protect its custody and associate it with the exact guest/stop receipt. + +## Offline base → verified OCI (Mac host) + +Build `./cmd/publish-macos-base` on the Apple-silicon operator host from the reviewed +commit, with the repository's real embedded binaries built. On the native Mac, +run `make build-embedded build-vz-shim` first (requires the macOS SDK/cgo). Do not +use dummy embedded binaries. No HTTP import endpoint is added. +The source must be an exclusively owned, stopped, clean macvm bundle with exactly +these nonempty, private regular files under a real 0700 directory: + +- `disk.img`: raw installed OS disk; +- `aux.img`: VZ auxiliary storage; +- `config.json`: strict `images.MacOSImage` metadata with `guest_agent: true`. + +Do not add audit reports, logs, credentials, provenance files or sockets to that +bundle. Symlinks, aliased payload inodes, unknown metadata and public permissions +are rejected. Do not put evidence inside an imported disk. Stage any necessary +layout conversion under operator control before the final audit/stop. + +```sh +umask 077 +go build -o ./bin/publish-macos-base ./cmd/publish-macos-base +sudo ./bin/publish-macos-base \ + --source /private/base-release/stopped-bundle \ + --audit /private/base-release/audit.json \ + --registry https://internal-registry.example \ + --registry-secret-file /private/operator/internal-registry-signing-secret \ + --id base-macos-toolchain-001 \ + --output /private/base-release/base-receipt.json \ + --timeout 2h --stopped --clean-install-reviewed +``` + +Run as root so the closed-storage probe can inspect other users' open files. +The two attestations are explicit operator assertions, not automated proof of +sanitation or graceful stop. Use the normal strict stop/export path for managed +instances when available; that path confirms shutdown acknowledgement and owned +VMM exit under the instance lock. Offline `lsof` detects open storage but cannot +prove graceful shutdown or exclude a future restart. Keep the source under +exclusive operator control; never publish a bundle still managed by another +running process/API that could restart it. There is no claim of a lock shared +with arbitrary external VM launchers. + +The command reuses `ImportMacOSImage`'s fail-closed closed-storage check and APFS +clones into an exclusive temporary data directory, canonicalizes imported machine +metadata, validates the strict snapshot, hashes its files, then reuses +`StreamingMachinePublisher`. This avoids inventing another OCI format/upload +implementation. Unlike import alone (which creates local raw-image metadata, +**not an OCI cache**), the publisher creates the OCI config/layer/manifest in +`builds/base-...:latest`. It hashes and verifies both stored blobs, commits the +manifest last and reads it back by digest. The receipt uses the immutable +`registry/builds/base-...@sha256:...` reference, never `latest`. + +The signing-secret file is local administrator access to the configured internal +registry, never guest/recipe input. It must be a private regular file of 32–4096 +bytes (surrounding whitespace is trimmed). The tool mints a deadline-bounded +repository-scoped token internally; never print it or pass the secret in argv. +Registry addresses follow the existing publisher policy: HTTPS/system trust or +loopback HTTP only, no embedded credentials, proxies or redirects. External +registry credentials belong to normal image push APIs, not this command. + +The successful new 0600 receipt contains the immutable OCI reference/digest, +allowlisted audit fields, audit JSON hash, exact snapshot file hashes and UTC +publication timestamp. It is written only after verification. Existing output +files are never overwritten. An interrupted run may leave an unreferenced blob, +a committed candidate or a temporary directory; no receipt means unverified, not +proof that nothing was uploaded. Do not retry under a production alias. Operator +cleanup must first exclude active publication processes. Error text is deliberately +masked; inspect private authenticated registry/host diagnostics, not public CI logs. + +## Derivative build, external push and pull + +Build the derivative with `POST /builds`, passing the pinned base digest, the source archive +and `machine_network_mode=egress` only when the recipe downloads pinned artifacts. The example +recipe in `scripts/macos/example/` needs no network. Verify the build's `provenance` before use. + +```sh +curl -H "Authorization: Bearer $HYPEMAN_TOKEN" \ + -F "machine_base_image=registry.example/base@sha256:$BASE_DIGEST" \ + -F "source=@source.tar.gz;type=application/gzip" \ + "$HYPEMAN_URL/builds" +``` + +Push the resulting digest with `POST /pushes`, giving the external repository and registry +credentials. Each Mac host then runs `POST /images` with the digest and `platform: darwin/arm64`. +Pre-pull proves readiness and content admission, not bootability. Cold-boot the digest on one +authorized Mac and run the marker check and UID0 exec before any production use. Keep the +previous validated digest for rollback. Pin digests, never tags. + +## Remaining production gates + +- An approved, fresh base installation with its final audit and confirmed graceful stop. +- Choice of external registry and a test push of a 26–28 GB machine layer to it. +- Cold-boot validation of the promoted digest on an authorized Mac. +- Disk capacity on the build host. The build path has no admission control in this change; + run machine builds serially on a dedicated host with enough free space for the base copy, + the export and the published layer. +- Apple licence review for distributing macOS-derived images. diff --git a/docs/macos-builds.md b/docs/macos-builds.md new file mode 100644 index 000000000..91ac98f13 --- /dev/null +++ b/docs/macos-builds.md @@ -0,0 +1,224 @@ +# macOS image builds (experimental, opt-in HTTP) + +Ordinary build jobs can provision a digest-pinned installed macOS base in an +isolated VZ VM and publish a complete cold-boot OCI machine image. This is not +macOS installation/bootstrap, Linux rootfs conversion, memory forking or identity +rekeying. This branch depends on PR3's shared system GuestService. + +`Config.MachineBuild` opts into `CreateBuildRequest.MachineBaseImage` at the Go +boundary. Queue, persisted request, deadline, source staging/hash, status, log +completion, provenance and image-readiness gate remain owned by the normal build +manager. The concrete implementations are `GuestMachineBuildDriver` and +`StreamingMachinePublisher`. HTTP activation is disabled by default. Set both +`macos_only: true` and `macos_builds: true` on an Apple-silicon VZ server, with +an explicit `registry.url` reachable from the host (not a guest gateway), including +its scheme. For example, use `http://localhost:4973` for loopback HTTP or a +system-trusted `https://` endpoint. Credentials, URL paths, queries and fragments +are rejected; custom CA/insecure HTTPS machine configurations are rejected. +Linux servers reject the machine selector. + +Use the existing authenticated `POST /builds` multipart endpoint: + +```sh +curl -H "Authorization: Bearer $TOKEN" \ + -F "machine_base_image=registry.example/base@sha256:$BASE_DIGEST" \ + -F "source=@source.tar.gz;type=application/gzip" \ + -F "timeout_seconds=1200" "$API/builds" +``` + +The installed base must already be Ready. A 202 response queues an ordinary job; +GET /builds/{id}, cancellation and status/provenance use the shared APIs. OpenAPI +and generated server request types include machine_base_image; external language +SDK releases are separate, not claimed here. Only source, machine_base_image, +machine_network_mode, timeout_seconds, memory_mb, cpus and tags are accepted for machine HTTP builds. +Duplicates/unknown fields, malformed numbers and mutable bases return 400. Metadata +is limited to 64 KiB per field, source 64 MiB and the entire HTTP body 65 MiB before OpenAPI +validation. These limits do not change Linux source parsing. + +## Use the built image on another Mac host + +After a build is ready, `GET /builds/{id}` returns a digest-pinned `image_ref` +that can be passed directly to the normal instance-create API. For use across +hosts, configure `registry.url` to a registry address reachable from each Mac +host; loopback is only suitable for same-host builds. On a macOS-only server, an +omitted platform defaults to `darwin/arm64`; on a mixed host, request that +platform explicitly. If the image isn't cached locally, Hypeman starts a +platform-specific pull. Large machine images may take longer than the create +request's readiness wait, so retry after the image becomes ready or pull it +explicitly first. + +```json +{ + "name": "macos-app", + "image": "registry.example/builds/@sha256:", + "platform": "darwin/arm64" +} +``` + +Each instance is pinned to the resolved image digest. Use the build's returned +image reference. Local tags stay within one repository. To distribute a build, push its +exact digest with `POST /pushes` using registry credentials. Use a digest, not a tag, for +every consumer. + +## Base and policy + +The ready installed base must be pinned by canonical SHA256, declare a provisioned +system agent on port 2222 and be `darwin/arm64`. Driver provisioning additionally checks +actual guest UID0. The driver never installs an agent or falls back to host +SSH/sudo. CPU/memory match the base exactly; zero request values inherit them. +Build limits, exact MiB precision and bounded deadlines still apply. Timeout +normally defaults to 600 s, capped at 24 hours; full machine-image compression/pull can need a +longer explicitly selected deadline. + +Networking defaults isolated. Explicit `machine_network_mode=egress` enables +unrestricted VZ NAT for pinned, checksum-verified artifact downloads, not Linux +TAP/domain-policy parity. Omit the selector or set it to `isolated` for offline recipes. Domain allowlists, Linux Dockerfile/builder/cache/ +build-argument/secret/admin options are rejected, not ignored. Template identity +is preserved: jobs pinned to the same base digest are serialized, and concurrent +same-identity guests remain forbidden. + +## Experimental versioned provisioning recipe + +Source is the normal build-owned tar.gz, privately staged and SHA256 verified, +including recovery. Compressed machine input is capped at 64 MiB; Linux staging gets +no new size cap. Host extraction into an exclusive 0700 temporary directory limits +uncompressed data 128 MiB, entries 1024 and individual regular files 64 MiB. Only regular +files/directories are accepted; duplicate names, links/devices and path escapes +fail before transfer. Files are 0600 or owner-executable 0700. Guest tar is not used. + +The archive must contain `hypeman-macos-build.json`, limited to 64 KiB. The +The server rejects duplicate/case-variant keys and enforces byte limits: + +```json +{"version":1,"steps":[{"command":["/bin/sh","build.sh"],"env":{"MODE":"release"}}]} +``` + +At at most 32 steps execute as root in one exclusive `/var/root/hypeman-build-{id}` +directory. Command executable paths must be absolute; arguments/environment are +bounded and reject NUL. Commands run only in the guest, never a host shell. +Unknown JSON fields, duplicate keys, null strings, case-variant field names and +extra JSON objects fail. The exact validated recipe bytes are SHA256-hashed in +`provenance.recipe_hash`, alongside the source archive hash and pinned base digest. +Version 1 remains experimental pending native production validation; no OS/compiler +version is inferred from the host or a base image name. + +Sanitation removes and verifies absence of the build-owned guest source directory +and removes extracted host source before stopping. There is no injected registry +token, host credential or secret in the guest. Guest stdout/stderr are discarded. +Guest stdout and stderr are discarded. Build logs contain no guest output; failures report +the phase and, for recipe steps, the step number and exit code. + +## Stop, export and cleanup safety + +The publication path is two instance operations, not one copy-inside-stop: +`StopMacOSBuildInstance` performs only the strict graceful stop (no storage copy) +and rejects forced fallback, a pre-existing Stopped state, unmanaged/non-macOS +guests and any stop without shutdown acknowledgement plus owned VMM exit. +`ExportStoppedMacOSBuildInstance` later copies the closed storage of a +still-Stopped instance under the instance lock, so a concurrent Start makes export +fail rather than race it. Each storage file is copied once, into a private +exclusive 0700 output that holds only 0600 disk.img, aux.img and bounded config.json. + +Failure cleanup uses `Abort`, not `Stop`. Nothing from a failed build is exported, +so cleanup first attempts the strict stop, then uses the ordinary forced +hypervisor stop and deletes storage only after confirmed VMM exit. A VM that still +has no confirmed exit remains quarantined for operator recovery. A re-enqueued +attempt of the same job first reaps the builder recorded by the interrupted attempt +(stop if running, then delete) and fails rather than start a second builder if that +delete fails. Builders orphaned outside this job path still have no general reaper. Cleanup has a 2-minute budget (strict stop window plus forced +fallback and deletion), separate from the build deadline. + +On macOS-only server startup, the build manager recovers queued, building and +pushing machine jobs without preparing a Linux builder image or starting Linux +builder services. Persisted Linux jobs, and machine jobs whose backend is disabled, +remain unchanged on disk for recovery when their supported mode is enabled. + +Darwin shutdown accepts a detached, short-delayed power-off command so gRPC can +send its acceptance reply before vsock disappears. RPC cancellation after +acceptance does not undo shutdown. The host still independently awaits VMM exit; +an acknowledgement alone is insufficient. + +The backend validates complete output and preserved identity/resources and destroys +the stopped builder **before** publication. Staged source is retained until a terminal +status is persisted, so a job interrupted while pushing or converting is rebuilt on +recovery; startup also reaps cancelled builders and removes terminal-job source. Each +attempt first removes export and guest workspaces left by an interrupted attempt of +the same job; every export holds a full disk image. Cancellation +interrupts the active normal machine job, not forced instance deletion. It records +the owned builder before provisioning and keeps Cancelled terminal. Potentially secret-bearing guest errors remain masked in +ordinary error/log strings while errors.Is/As work internally. + +## Streaming publication + +The publisher uses a server-configured registry and a token scoped only to +`builds/{id}`. Recipes cannot select destinations or credentials. HTTPS uses system +trust; plaintext is permitted only on loopback. Redirects are never followed and +environment HTTP proxies are disabled for these credential-bearing requests. + +The layer is never written to disk. A first pass generates the gzipped tar from +the exported bundle only to compute its compressed digest/size and uncompressed +DiffID; the upload pass generates the same bytes again (tar headers carry no +timestamps, so the output is deterministic) and the registry rejects any digest +mismatch. Configuration and layer blobs stream through monolithic POST uploads, +not go-containerregistry's large-buffer PATCH path. Each persisted +blob is independently GET/hash/size verified. The `latest` manifest is committed +last (the normal manager's image-readiness convention), then immutable digest +content is verified. Ambiguous errors do not return successful receipts. Disk +a compressed-size bound is not a storage quota. + +## Validation and remaining gates + +Repeated race tests cover source/recipe admission, shared queue/status/provenance, +cancellation without publication, phase failures/quarantine, bundle identity/modes, +closed-storage/hardlink rejection and authenticated production-registry publication +with manifest-last ordering and redirect isolation. Tiny test disks are synthetic, +not bootability evidence. + +Native Apple-silicon QA has run real builds end to end through the HTTP API: +provisioning from a pinned installed base, sanitation, strict stop, export, +publication, the readiness gate and a cold boot of the result with the build +marker present. It also covered recipe-step failures, cancellation, API crashes +during provisioning, publication and conversion (each rebuilt on recovery), and +host admission errors. These runs used a development base, not a sanitized +production template. + +Remaining gates are native execution of the build path, +a reviewed fresh base with approved artifact pins, cross-host publication/pull and +sequential cold-boot validation, operational monitoring and SDK/CLI release. The +broader Linux runtime suite must also pass in an environment with its required +hypervisor/network permissions. Keep experimental HTTP activation disabled by default. + + +## Rollout gates and rollback + +1. **Contract gate:** pass build, image, registry/push, config and API contract + tests with real embedded binaries; run the audit tests. The Linux suite + must pass on a host with its normal hypervisor/network prerequisites. Fixtures + prove admission/transport, not a bootable macOS installation. +2. **Base/recipe gate:** review a fresh installation and final GuestService audit, + retain the strict stop evidence, publish the base receipt, and approve all + downstream application artifact hashes. Never promote a development guest + or the placeholder example pins. The initial Apple install remains an explicit + operator prerequisite. +3. **Native failure gate:** on an isolated authorized builder, exercise success, + recipe failure, cancellation and process interruption during provisioning, + export, publication and import. Restart and verify shared job recovery; verify + failed or cancelled jobs never report Ready, and the builder is destroyed only after confirmed + VMM exit. Check that build logs contain no guest output or test secrets. +4. **Distribution/canary gate:** publish only a candidate tag, pre-pull the exact + digest on a second approved Mac, then cold-boot sequentially with no other copy + of the identity running. Require agent readiness, root exec, expected OS/tool + versions and workload health. Preserve the previous validated image digest and + successful private evidence before changing any production selection. +5. **Operations gate:** enable one explicit builder and canary first. Record build success/failure counts and free disk space with operator tooling. This change + exposes no machine-build storage metrics. Treat missing disk telemetry as a failed gate. Configure operator-owned alerts and stop + expansion on any unresolved quarantine, low free disk space, missed + deadline, wrong digest/platform, or unexpected workload failure. No production + alert routing or inventory is fabricated by this change. + +For rollback, first revoke new build submissions at the authenticated admission +boundary and drain/cancel existing jobs through normal APIs while keeping the +backend available for cleanup/recovery. Repoint workload selection to the prior +validated immutable digest and pre-pull it through the same image API. Only after active jobs finish or are cancelled, set `macos_builds: false` on +the dedicated builder. Disabling the backend does not delete persisted jobs or +images. Do not delete the old digest or stop an unrelated instance to free space. diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go new file mode 100644 index 000000000..294f8b0c5 --- /dev/null +++ b/lib/builds/macos_build.go @@ -0,0 +1,305 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// MachineBuildRequest is an internal backend contract, not a new recipe syntax +// or an HTTP API. The normal build manager owns ID, source staging and publish +// authorization. A concrete driver must require a provisioned system agent. +type MachineBuildRequest struct { + ID string + BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. + SourceHash string // Optional expected lowercase SHA256 of the source archive. + Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. + onStarted func(string) error // Normal manager records the owned builder before provisioning. + onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. +} + +type MachineStopReceipt = instances.MacOSBuildExportReceipt +type MachinePublication struct{ Reference, Digest string } +type MachineBuildResult struct { + Publication MachinePublication + Provenance BuildProvenance +} + +type MachineBuildDriver interface { + // Start owns an isolated instance and identity lease, named name. A partially + // created session must be returned even with an error so cleanup can stop it. + Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) +} + +type MachineBuildSession interface { + ID() string + Provision(context.Context, string) error // Build-owned staged source, never a client host path. + Sanitize(context.Context) error // Remove build credentials, source and secret injection artifacts. + // Stop must await actual VMM exit. Forced-stop fallback must not claim graceful. + Stop(context.Context) (MachineStopReceipt, error) + // Abort is failure-only cleanup. It may force-stop because nothing is exported. + Abort(context.Context) (MachineStopReceipt, error) + Export(context.Context, string) error // Stopped disk.img, aux.img, config.json only. + // Destroy removes instance storage only after proven VMM exit, not the export. + Destroy(context.Context) error +} + +type MachineBuildPublisher interface { + // Publisher owns registry address, scoped token and builds/{id} destination. + // It must stream blobs and commit/verify the manifest last. An ambiguous + // network error is not a verified successful publication. + Publish(context.Context, string, string) (MachinePublication, error) +} + +type MachineBuildBackend struct { + Driver MachineBuildDriver + Publisher MachineBuildPublisher +} + +// machinePhaseError keeps potentially secret-bearing guest/command errors out of +// ordinary log/error text, while retaining errors.Is/As for internal handling. +type machinePhaseError struct { + phase string + cause error +} + +func (e *machinePhaseError) Error() string { + if step := (*machineStepError)(nil); errors.As(e.cause, &step) { + return "machine build " + e.phase + " failed: " + step.Error() + } + // Host admission errors, such as a template identity already in use, carry + // no guest output. + if errors.Is(e.cause, instances.ErrInsufficientResources) || errors.Is(e.cause, instances.ErrInvalidState) { + return "machine build " + e.phase + " failed: " + e.cause.Error() + } + return "machine build " + e.phase + " failed" +} +func (e *machinePhaseError) Unwrap() error { return e.cause } +func machineFailure(phase string, err error) error { return &machinePhaseError{phase, err} } + +// machineStepError names the recipe step that exited nonzero. It holds no guest +// output, so it is safe to show. +type machineStepError struct{ step, code int } + +func (e *machineStepError) Error() string { + return fmt.Sprintf("recipe step %d exited with code %d", e.step, e.code) +} + +// normalizeMachinePolicy applies machine build policy defaults and rejects what +// machine builds do not support. Admission, the runner and the guest driver share it. +func normalizeMachinePolicy(policy *BuildPolicy) error { + if policy.TimeoutSeconds == 0 { + policy.TimeoutSeconds = 600 + } + if policy.TimeoutSeconds < 1 || policy.TimeoutSeconds > 86400 { + return fmt.Errorf("invalid machine build timeout") + } + if policy.NetworkMode == "" { + policy.NetworkMode = "isolated" + } + if len(policy.AllowedDomains) != 0 || (policy.NetworkMode != "isolated" && policy.NetworkMode != "egress") { + return fmt.Errorf("unsupported machine network policy") + } + return nil +} + +// runPrepared executes only machine-specific phases. The shared build manager +// owns resolution, source staging/hash verification, deadline, queue and status. +func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath, workDir string) (result *MachineBuildResult, err error) { + if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(workDir) || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { + return nil, fmt.Errorf("machine driver, publisher, private workspace and verified source required") + } + if _, bounded := ctx.Deadline(); !bounded { + return nil, fmt.Errorf("prepared machine job requires a deadline") + } + if len(req.ID) == 0 || len(req.ID) > 128 || strings.Trim(req.ID, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" { + return nil, fmt.Errorf("invalid machine build ID") + } + ref, e := images.ParseNormalizedRef(req.BaseImage) + if e != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := req.Policy + if e = normalizeMachinePolicy(&policy); e != nil { + return nil, e + } + if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || base.Digest != ref.Digest() || base.MacOS.CPUs < 2 || base.MacOS.CPUs > MaxBuildCPUs || base.MacOS.Memory < 4<<30 || base.MacOS.Memory > uint64(MaxBuildMemoryMB)<<20 || base.MacOS.Memory%(1<<20) != 0 { + return nil, fmt.Errorf("base is not the pinned ready macOS machine") + } + if base.MacOS.Validate() != nil { + return nil, fmt.Errorf("base has invalid machine identity") + } + memory := int(base.MacOS.Memory >> 20) + cpus := int(base.MacOS.CPUs) + if (policy.MemoryMB != 0 && policy.MemoryMB != memory) || (policy.CPUs != 0 && policy.CPUs != cpus) { + return nil, fmt.Errorf("machine resources must match installed base") + } + policy.MemoryMB = memory + policy.CPUs = cpus + if e = policy.Validate(); e != nil { + return nil, e + } + workspace, e := os.MkdirTemp(workDir, "machine-build-") + if e != nil { + return nil, machineFailure("workspace", e) + } + defer func() { + if cleanupErr := os.RemoveAll(workspace); cleanupErr != nil { + err = errors.Join(err, machineFailure("cleanup_workspace", cleanupErr)) + result = nil + } + }() + if e = verifyBuildSource(ctx, sourcePath, req.SourceHash); e != nil { + return nil, machineFailure("verify_source", e) + } + if e = ctx.Err(); e != nil { + return nil, e + } + session, e := r.Driver.Start(ctx, machineBuilderName(req.ID), base, policy) + if session == nil { + if e == nil { + e = fmt.Errorf("driver returned no session") + } + return nil, machineFailure("start", e) + } + stopped, destroyed := false, false + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), machineCleanupTimeout) + defer cancel() + if !stopped { + receipt, stopErr := session.Abort(cleanup) + stopped = receipt.VMMExited + if !stopped { + err = errors.Join(err, machineFailure("quarantine_unconfirmed_vmm_exit", stopErr)) + result = nil + return + } + } + if !destroyed { + if destroyErr := session.Destroy(cleanup); destroyErr != nil { + err = errors.Join(err, machineFailure("cleanup", destroyErr)) + result = nil + } + } + }() + instanceID := session.ID() + if req.onStarted != nil { + if recordErr := req.onStarted(instanceID); recordErr != nil { + return nil, machineFailure("record_builder", recordErr) + } + } + if e != nil { + return nil, machineFailure("start", e) + } + if e = session.Provision(ctx, sourcePath); e != nil { + return nil, machineFailure("provision", e) + } + if e = session.Sanitize(ctx); e != nil { + return nil, machineFailure("sanitize", e) + } + receipt, e := session.Stop(ctx) + stopped = receipt.VMMExited + if e != nil || !receipt.Graceful || !stopped { + return nil, machineFailure("graceful_stop", e) + } + bundle := filepath.Join(workspace, "bundle") + if e = os.Mkdir(bundle, 0700); e != nil { + return nil, machineFailure("export", e) + } + if e = session.Export(ctx, bundle); e != nil { + return nil, machineFailure("export", e) + } + if e = validateMachineExport(bundle, base.MacOS); e != nil { + return nil, machineFailure("validate_export", e) + } + if e = session.Destroy(ctx); e != nil { + return nil, machineFailure("cleanup", e) + } + destroyed = true + if e = ctx.Err(); e != nil { + return nil, e + } + publication, e := r.Publisher.Publish(ctx, req.ID, bundle) + if e != nil { + return nil, machineFailure("publish", e) + } + output, e := images.ParseNormalizedRef(publication.Reference) + if e != nil || !output.IsDigest() || !ValidMachineDigest(publication.Digest) || output.Digest() != publication.Digest { + return nil, fmt.Errorf("machine publication receipt is unverified") + } + if req.onPublished != nil { + if e = req.onPublished(ctx, publication); e != nil { + return nil, machineFailure("image_ready", e) + } + } + provenance := BuildProvenance{BaseImageDigest: base.Digest, SourceHash: req.SourceHash, Timestamp: time.Now().UTC()} + // Only the driver that actually parsed the executed recipe can attest it. + // No guest OS/compiler versions are inferred from the host or base name. + if evidence, ok := session.(interface{ machineRecipeHash() string }); ok { + provenance.RecipeHash = evidence.machineRecipeHash() + } + return &MachineBuildResult{Publication: publication, Provenance: provenance}, nil +} + +// machineCleanupTimeout covers a strict graceful stop attempt (30s power-off +// budget) plus forced fallback and storage deletion. Export is outside cleanup. +const machineCleanupTimeout = 2 * time.Minute + +// ValidMachineDigest reports whether s is a canonical lowercase sha256 digest. +// It is the single validation rule for machine base and output digests. +func ValidMachineDigest(s string) bool { + if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { + return false + } + return strings.Trim(s[7:], "0123456789abcdef") == "" +} + +func validateMachineExport(root string, base *images.MacOSImage) error { + directory, err := os.Lstat(root) + if err != nil || !directory.IsDir() { + return fmt.Errorf("export root must be a real directory") + } + entries, err := os.ReadDir(root) + if err != nil { + return err + } + if len(entries) != 3 { + return fmt.Errorf("export must contain only machine payload files") + } + for _, entry := range entries { + if entry.Name() != "disk.img" && entry.Name() != "aux.img" && entry.Name() != "config.json" { + return fmt.Errorf("unexpected export file") + } + info, err := os.Lstat(filepath.Join(root, entry.Name())) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("export files must be regular") + } + } + platform, err := images.ValidateMacOSBundle(root) + if err != nil { + return err + } + if base != nil && (!bytes.Equal(platform.HardwareModel, base.HardwareModel) || !bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || !strings.EqualFold(platform.MAC, base.MAC) || platform.CPUs != base.CPUs || platform.Memory != base.Memory) { + return fmt.Errorf("machine export changed preserved identity/resources") + } + for _, entry := range entries { + if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { + return err + } + } + return nil +} + +// machineBuilderName names a build's builder instance after the build, so a retry +// can find and reap a builder that an interrupted attempt created but never recorded. +func machineBuilderName(buildID string) string { + return "macos-build-" + strings.ToLower(buildID) +} diff --git a/lib/builds/macos_build_cancel.go b/lib/builds/macos_build_cancel.go new file mode 100644 index 000000000..09b984000 --- /dev/null +++ b/lib/builds/macos_build_cancel.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "fmt" + "sync" + "time" +) + +type machineBuildControl struct { + mu sync.Mutex + cancel context.CancelFunc +} + +func (m *manager) recordMachineBuilder(id, instanceID string) error { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status == StatusCancelled { + return context.Canceled + } + meta.BuilderInstance = &instanceID + return writeMetadata(m.paths, meta) +} + +func (m *manager) cancelQueuedMachineBuild(meta *buildMetadata) error { + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + m.removeMachineSource(meta.ID) + m.notifyStatusChange(meta.ID, StatusCancelled) + return nil +} + +func (m *manager) cancelMachineBuild(id string) error { + active, ok := m.machineBuilds.Load(id) + if !ok { + return ErrBuildInProgress + } + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status != StatusBuilding && meta.Status != StatusPushing { + return fmt.Errorf("machine build no longer running") + } + // Persist cancellation before interrupting phases. The backend performs bounded + // graceful cleanup or quarantine; Cancel must not force-delete mutable storage. + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + control.cancel() + m.notifyStatusChange(id, StatusCancelled) + return nil +} diff --git a/lib/builds/macos_build_cancel_test.go b/lib/builds/macos_build_cancel_test.go new file mode 100644 index 000000000..292983fbe --- /dev/null +++ b/lib/builds/macos_build_cancel_test.go @@ -0,0 +1,57 @@ +package builds + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestQueuedMachineCancellationRemovesSource(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) + require.NoError(t, err) + t.Cleanup(func() { + _ = m.CancelBuild(context.Background(), first.ID) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) + }) + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, first.ID) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) + require.NoError(t, err) + source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") + require.FileExists(t, source) + require.NoError(t, m.CancelBuild(context.Background(), queued.ID)) + done, err := m.GetBuild(context.Background(), queued.ID) + require.NoError(t, err) + require.Equal(t, StatusCancelled, done.Status) + require.NotNil(t, done.CompletedAt) + require.NoFileExists(t, source) +} + +func TestMachineCancellationInterruptsPhasesWithoutPublishing(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + build, e := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("synthetic source")) + require.NoError(t, e) + require.Eventually(t, func() bool { + meta, e := readMetadata(m.paths, build.ID) + return e == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), build.ID)) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(build.ID); return !active }, 5*time.Second, 10*time.Millisecond) + done, e := m.GetBuild(context.Background(), build.ID) + require.NoError(t, e) + require.Equal(t, StatusCancelled, done.Status) + require.False(t, fixture.publisherCalled) + require.Contains(t, fixture.calls, "abort") + require.Contains(t, fixture.calls, "destroy") +} diff --git a/lib/builds/macos_build_contract_test.go b/lib/builds/macos_build_contract_test.go new file mode 100644 index 000000000..6186f633b --- /dev/null +++ b/lib/builds/macos_build_contract_test.go @@ -0,0 +1,102 @@ +package builds + +import ( + "archive/tar" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/stretchr/testify/require" +) + +func TestMachineRecipeContract(t *testing.T) { + valid := `{"version":1,"steps":[{"command":["/bin/sh","build.sh"],"env":{"MODE":"qa"}}]}` + cases := map[string]string{ + "valid": valid, + "case variant version": `{"Version":1,"steps":[{"command":["/bin/true"]}]}`, + "case variant command": `{"version":1,"steps":[{"Command":["/bin/true"]}]}`, + "duplicate version": `{"version":2,"version":1,"steps":[{"command":["/bin/true"]}]}`, + "duplicate command": `{"version":1,"steps":[{"command":["/bin/false"],"command":["/bin/true"]}]}`, + "duplicate env": `{"version":1,"steps":[{"command":["/bin/true"],"env":{"KEY":"one","KEY":"two"}}]}`, + "null argument": `{"version":1,"steps":[{"command":["/bin/echo",null]}]}`, + "null env": `{"version":1,"steps":[{"command":["/bin/true"],"env":{"KEY":null}}]}`, + "empty steps": `{"version":1,"steps":[]}`, + "relative executable": `{"version":1,"steps":[{"command":["sh"]}]}`, + "unknown step field": `{"version":1,"steps":[{"command":["/bin/true"],"cwd":"/tmp"}]}`, + "unknown top field": `{"version":1,"steps":[{"command":["/bin/true"]}],"secret":"no"}`, + "invalid env name": `{"version":1,"steps":[{"command":["/bin/true"],"env":{"A=B":"x"}}]}`, + "nul argument": `{"version":1,"steps":[{"command":["/bin/echo","\u0000"]}]}`, + "invalid UTF-8": strings.Replace(valid, "qa", "\xff", 1), + "trailing object": valid + `{}`, + "oversize": strings.Replace(valid, "qa", strings.Repeat("x", 64<<10), 1), + "oversize argument": strings.Replace(valid, "build.sh", strings.Repeat("x", 8193), 1), + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + source := writeMachineSource(t, []*tar.Header{{Name: machineRecipeName, Typeflag: tar.TypeReg, Size: int64(len(body))}}, []string{body}) + root := filepath.Join(t.TempDir(), "source") + recipe, err := extractMachineRecipe(context.Background(), source, root) + if name != "valid" { + require.Error(t, err) + require.NoDirExists(t, root) + return + } + require.NoError(t, err) + digest := sha256.Sum256([]byte(body)) + require.Equal(t, hex.EncodeToString(digest[:]), recipe.hash) + }) + } + for _, count := range []int{32, 33} { + body := `{"version":1,"steps":[` + strings.TrimSuffix(strings.Repeat(`{"command":["/bin/true"]},`, count), ",") + `]}` + source := writeMachineSource(t, []*tar.Header{{Name: machineRecipeName, Typeflag: tar.TypeReg, Size: int64(len(body))}}, []string{body}) + _, err := extractMachineRecipe(context.Background(), source, filepath.Join(t.TempDir(), "source")) + require.Equal(t, count > 32, err != nil) + } +} + +type machineEvidenceFixture struct { + *machineFixture +} + +func (f *machineEvidenceFixture) machineRecipeHash() string { return strings.Repeat("a", 64) } +func (f *machineEvidenceFixture) Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) { + _, err := f.machineFixture.Start(ctx, name, base, policy) + return f, err +} + +func TestMachineRecipeProvenanceRoundTrip(t *testing.T) { + f := &machineEvidenceFixture{machineFixture: newMachineFixture()} + backend := MachineBuildBackend{Driver: f, Publisher: f} + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + root := t.TempDir() + source := filepath.Join(root, "source.tar.gz") + hash, err := stageBuildSource(ctx, strings.NewReader("source"), source, maxBuildSourceBytes) + require.NoError(t, err) + req := machineTestRequest() + req.SourceHash = hash + result, err := backend.runPrepared(ctx, req, f.image, source, root) + require.NoError(t, err) + require.Equal(t, f.machineRecipeHash(), result.Provenance.RecipeHash) + require.Equal(t, machineTestDigest, result.Provenance.BaseImageDigest) + require.Empty(t, result.Provenance.BuildkitVersion) + require.Empty(t, result.Provenance.LockfileHashes) + + // Recipe bytes are hashed before execution; no command output or inferred + // toolchain version belongs in provenance. + session := &guestMachineSession{recipeHash: strings.Repeat("a", 64)} + provenance := BuildProvenance{RecipeHash: session.machineRecipeHash()} + encoded, err := json.Marshal(provenance) + require.NoError(t, err) + require.Contains(t, string(encoded), `"recipe_hash":"`+session.recipeHash+`"`) + require.NotContains(t, string(encoded), "buildkit_version") + var decoded BuildProvenance + require.NoError(t, json.Unmarshal(encoded, &decoded)) + require.Equal(t, provenance, decoded) +} diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go new file mode 100644 index 000000000..10bef47c2 --- /dev/null +++ b/lib/builds/macos_build_failure_test.go @@ -0,0 +1,164 @@ +package builds + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +func runMachineBuildFailure(t *testing.T, f *machineFixture, ctx context.Context, req MachineBuildRequest, source io.Reader) (*machineRunnerFixture, error) { + t.Helper() + runner := machineTestRunner(t, f) + result, err := runner.Run(ctx, req, source) + if err == nil || result != nil { + t.Fatal("failed build returned a result") + } + return runner, err +} + +type failingMachineSource struct{} + +func (failingMachineSource) Read(p []byte) (int, error) { + return copy(p, "partial"), errors.New("secret-source-error") +} + +func TestMachineBuildSourceReadFailure(t *testing.T) { + f := newMachineFixture() + r, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), failingMachineSource{}) + if strings.Contains(err.Error(), "secret-source-error") { + t.Fatal("source failure was not masked") + } + if len(f.calls) != 0 { + t.Fatalf("source failure started guest: %v", f.calls) + } + entries, err := os.ReadDir(r.WorkDir) + if err != nil || len(entries) != 0 { + t.Fatal("partial source retained") + } +} + +func TestMachineBuildTimeoutCleanup(t *testing.T) { + f := newMachineFixture() + f.waitForCancel = true + req := machineTestRequest() + req.Policy.TimeoutSeconds = 1 + parent, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := runMachineBuildFailure(t, f, parent, req, strings.NewReader("source")) + if !errors.Is(err, context.DeadlineExceeded) || f.publisherCalled || !f.cleanupContextLive { + t.Fatal("timeout cleanup failed") + } + if !reflect.DeepEqual(f.calls, []string{"start", "abort", "destroy"}) { + t.Fatalf("timeout cleanup order: %v", f.calls) + } +} + +func TestMachineBuildPublicationReceipt(t *testing.T) { + for _, kind := range []string{"ambiguous error", "tag receipt", "digest mismatch", "invalid digest"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + switch kind { + case "ambiguous error": + f.fail = "publish" + case "tag receipt": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test:latest", Digest: machineTestDigest} + case "digest mismatch": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "sha256:" + strings.Repeat("b", 64)} + case "invalid digest": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "not-a-digest"} + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if !f.publisherCalled { + t.Fatal("unverified receipt became ready") + } + if strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("publisher error leaked") + } + count := 0 + for _, phase := range f.calls { + if phase == "publish" { + count++ + } + } + if count != 1 { + t.Fatal("ambiguous publication was retried") + } + }) + } +} + +func TestMachineBuildMalformedExport(t *testing.T) { + for _, kind := range []string{"empty disk", "empty aux", "symlink disk", "hardlink payloads", "truncated config", "oversized config", "resource change", "bad MAC", "identity", "extra file", "unknown metadata"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + f.mutateExport = func(root string) error { + disk, aux, config := filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"), filepath.Join(root, "config.json") + switch kind { + case "empty disk": + return os.Truncate(disk, 0) + case "empty aux": + return os.Truncate(aux, 0) + case "symlink disk": + if err := os.Remove(disk); err != nil { + return err + } + return os.Symlink(aux, disk) + case "hardlink payloads": + if err := os.Remove(aux); err != nil { + return err + } + return os.Link(disk, aux) + case "truncated config": + return os.WriteFile(config, []byte(`{"hardware_model":`), 0600) + case "oversized config": + return os.WriteFile(config, []byte(strings.Repeat(" ", 65<<10)), 0600) + case "resource change": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), `"cpus":4`, `"cpus":2`, 1)), 0600) + case "bad MAC": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:01", "02:00:00:00:00:01:02:03", 1)), 0600) + case "identity", "unknown metadata": + var raw map[string]any + data, err := os.ReadFile(config) + if err == nil { + err = json.Unmarshal(data, &raw) + } + if err != nil { + return err + } + if kind == "identity" { + raw["machine_identifier"] = []byte{9} + } else { + raw["password"] = "must-not-publish" + } + data, _ = json.Marshal(raw) + return os.WriteFile(config, data, 0600) + case "extra file": + return os.WriteFile(filepath.Join(root, "secret.txt"), []byte("must-not-publish"), 0600) + } + return nil + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("malformed export published") + } + if strings.Contains(err.Error(), "must-not-publish") { + t.Fatal("export secret leaked in failure") + } + }) + } +} diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go new file mode 100644 index 000000000..d4e63a6f4 --- /dev/null +++ b/lib/builds/macos_build_guest.go @@ -0,0 +1,217 @@ +package builds + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +type machineExportManager interface { + StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) + ExportStoppedMacOSBuildInstance(context.Context, string, string) error +} + +// GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system +// GuestService. It never installs an agent or silently upgrades an unmanaged base. +type GuestMachineBuildDriver struct { + Instances instances.Manager + WorkDir string +} + +func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) { + exporter, ok := d.Instances.(machineExportManager) + if !ok || !filepath.IsAbs(d.WorkDir) { + return nil, fmt.Errorf("strict machine export and private workspace required") + } + if _, ok = ctx.Deadline(); !ok { + return nil, fmt.Errorf("machine start requires a deadline") + } + if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || !base.MacOS.GuestAgent || base.MacOS.Validate() != nil || !ValidMachineDigest(base.Digest) { + return nil, fmt.Errorf("provisioned pinned macOS base required") + } + if err := policy.Validate(); err != nil { + return nil, err + } + if err := normalizeMachinePolicy(&policy); err != nil { + return nil, err + } + if policy.CPUs != int(base.MacOS.CPUs) || policy.MemoryMB != int(base.MacOS.Memory>>20) { + return nil, fmt.Errorf("machine policy must preserve resources and supported networking") + } + ref, err := images.ParseNormalizedRef(base.Name) + if err != nil { + return nil, err + } + // Workspaces are named per builder, so one left by an interrupted attempt + // of this job is removed here. + prefix := "guest-machine-" + name + "-" + stale, _ := filepath.Glob(filepath.Join(d.WorkDir, prefix+"*")) + for _, dir := range stale { + _ = os.RemoveAll(dir) + } + dir, err := os.MkdirTemp(d.WorkDir, prefix) + if err != nil { + return nil, err + } + inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) + if inst == nil { + _ = os.RemoveAll(dir) + if err == nil { + err = fmt.Errorf("instance creation returned no instance") + } + return nil, err + } + session := &guestMachineSession{manager: d.Instances, exporter: exporter, instance: inst, workspace: dir, guestDir: "/var/root/hypeman-build-" + inst.Id} + return session, err +} + +type guestMachineSession struct { + manager instances.Manager + exporter machineExportManager + instance *instances.Instance + workspace, guestDir string + receipt MachineStopReceipt + sanitized bool + recipeHash string +} + +func (s *guestMachineSession) ID() string { return s.instance.Id } +func (s *guestMachineSession) machineRecipeHash() string { return s.recipeHash } +func (s *guestMachineSession) dialer() (hypervisor.VsockDialer, error) { + return hypervisor.NewVsockDialer(s.instance.HypervisorType, s.instance.VsockSocket, s.instance.VsockCID) +} +func (s *guestMachineSession) exec(ctx context.Context, command []string, cwd string, env map[string]string, wait time.Duration) error { + dialer, err := s.dialer() + if err != nil { + return err + } + // Drain output without retaining or publishing it. Redaction cannot reliably + // make arbitrary guest output safe: even successful commands may print secrets. + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{Command: command, Cwd: cwd, Env: env, WaitForAgent: wait, Stdout: io.Discard, Stderr: io.Discard}) + if err != nil { + return err + } + if exit == nil { + return fmt.Errorf("machine guest command did not succeed") + } + if exit.Code != 0 { + return &machineExitError{code: exit.Code} + } + return nil +} + +// machineExitError is a guest command's nonzero exit code. +type machineExitError struct{ code int } + +func (e *machineExitError) Error() string { return "machine guest command did not succeed" } + +func (s *guestMachineSession) Provision(ctx context.Context, source string) error { + root := filepath.Join(s.workspace, "source") + recipe, err := extractMachineRecipe(ctx, source, root) + if err != nil { + return err + } + s.recipeHash = recipe.hash + if err = s.exec(ctx, []string{"/bin/sh", "-c", `test "$(/usr/bin/id -u)" -eq 0`}, "", nil, 30*time.Second); err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 30*time.Second); err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/mkdir", "-m", "700", s.guestDir}, "", nil, 0); err != nil { + return err + } + dialer, err := s.dialer() + if err != nil { + return err + } + if err = guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{SrcPath: root, DstPath: s.guestDir}); err != nil { + return err + } + for i, step := range recipe.Steps { + if err = s.exec(ctx, step.Command, s.guestDir, step.Env, 0); err != nil { + if exitErr := (*machineExitError)(nil); errors.As(err, &exitErr) { + return &machineStepError{step: i + 1, code: exitErr.code} + } + return err + } + } + return nil +} + +func (s *guestMachineSession) Sanitize(ctx context.Context) error { + if err := s.exec(ctx, []string{"/bin/rm", "-rf", "--", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(s.workspace, "source")); err != nil { + return err + } + s.sanitized = true + return nil +} + +// Stop is the publication path: only an acknowledged graceful power-off with +// confirmed VMM exit may produce a receipt that permits export. +func (s *guestMachineSession) Stop(ctx context.Context) (MachineStopReceipt, error) { + if s.receipt.VMMExited { + return s.receipt, nil + } + receipt, err := s.exporter.StopMacOSBuildInstance(ctx, s.ID()) + s.receipt = receipt + return s.receipt, err +} + +// Abort is failure cleanup only. Nothing from a failed build is exported, so after +// a strict attempt it may use the ordinary forced hypervisor stop. Destroy still +// requires confirmed VMM exit; a non-exited instance remains quarantined. +func (s *guestMachineSession) Abort(ctx context.Context) (MachineStopReceipt, error) { + if s.receipt.VMMExited { + return s.receipt, nil + } + if receipt, err := s.exporter.StopMacOSBuildInstance(ctx, s.ID()); err == nil && receipt.VMMExited { + s.receipt = receipt + return s.receipt, nil + } + if inst, err := s.manager.GetInstance(ctx, s.ID()); err != nil || inst.State != instances.StateStopped { + inst, err = s.manager.StopInstance(ctx, s.ID()) + if err != nil { + return s.receipt, err + } + if inst == nil || inst.State != instances.StateStopped { + return s.receipt, fmt.Errorf("forced stop did not confirm VMM exit") + } + } + s.receipt = MachineStopReceipt{VMMExited: true} + return s.receipt, nil +} + +// Export copies only from closed storage of a Stopped instance, under the +// instance lock, after a strict graceful receipt and sanitation. +func (s *guestMachineSession) Export(ctx context.Context, destination string) error { + if !s.receipt.Graceful || !s.receipt.VMMExited || !s.sanitized { + return fmt.Errorf("sanitized strict stop receipt required") + } + return s.exporter.ExportStoppedMacOSBuildInstance(ctx, s.ID(), destination) +} + +func (s *guestMachineSession) Destroy(ctx context.Context) error { + if !s.receipt.VMMExited { + return fmt.Errorf("unconfirmed VMM exit: instance quarantined") + } + if err := s.manager.DeleteInstance(ctx, s.ID()); err != nil { + return err + } + return os.RemoveAll(s.workspace) +} diff --git a/lib/builds/macos_build_job_test.go b/lib/builds/macos_build_job_test.go new file mode 100644 index 000000000..12394b213 --- /dev/null +++ b/lib/builds/macos_build_job_test.go @@ -0,0 +1,37 @@ +package builds + +import ( + "context" + "io" + "os" + "path/filepath" + "time" +) + +type machineRunnerFixture struct { + MachineBuildBackend + WorkDir string +} + +func (r *machineRunnerFixture) Run(ctx context.Context, req MachineBuildRequest, source io.Reader) (*MachineBuildResult, error) { + timeout := req.Policy.TimeoutSeconds + if timeout == 0 { + timeout = 600 + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + job, err := os.MkdirTemp(r.WorkDir, "prepared-job-") + if err != nil { + return nil, err + } + defer os.RemoveAll(job) + sourcePath := filepath.Join(job, "source.tar.gz") + hash, err := stageBuildSource(ctx, source, sourcePath, maxBuildSourceBytes) + if err != nil { + return nil, machineFailure("stage_source", err) + } + if req.SourceHash == "" { + req.SourceHash = hash + } + return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath, r.WorkDir) +} diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_build_layer_test.go new file mode 100644 index 000000000..6798c5582 --- /dev/null +++ b/lib/builds/macos_build_layer_test.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { + parent := t.TempDir() + bundle := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + disk := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk + copy(disk[1<<20:], "data") + for name, data := range map[string][]byte{"disk.img": disk, "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.NotEmpty(t, diffID.Hex) + digest, err := layer.Digest() + require.NoError(t, err) + size, err := layer.Size() + require.NoError(t, err) + for range 2 { + body, err := layer.Compressed() + require.NoError(t, err) + sum := sha256.New() + n, err := io.Copy(sum, body) + require.NoError(t, err) + require.NoError(t, body.Close()) + require.Equal(t, size, n) + require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) + } + entries, err := os.ReadDir(parent) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing is spooled beside the bundle") +} + +func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { + bundle := t.TempDir() + for name, data := range map[string][]byte{"disk.img": make([]byte, 8<<20), "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + before := runtime.NumGoroutine() + body, err := layer.Compressed() + require.NoError(t, err) + _, err = io.CopyN(io.Discard, body, 16) + require.NoError(t, err) + require.NoError(t, body.Close()) + // Polled directly: require.Eventually runs its condition in another goroutine. + deadline := time.Now().Add(5 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), before, "the layer generator exits when the upload is abandoned") +} diff --git a/lib/builds/macos_build_lifecycle_test.go b/lib/builds/macos_build_lifecycle_test.go new file mode 100644 index 000000000..d987685f5 --- /dev/null +++ b/lib/builds/macos_build_lifecycle_test.go @@ -0,0 +1,135 @@ +package builds + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineLifecycleInstances struct { + instances.Manager + lookedUp string + state, stoppedState instances.State + missing, stopped bool + deleted bool + stopErr, deleteErr error +} + +func (m *machineLifecycleInstances) GetInstance(_ context.Context, id string) (*instances.Instance, error) { + m.lookedUp = id + if m.missing { + return nil, instances.ErrNotFound + } + return &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "prior"}, State: m.state}, nil +} +func (m *machineLifecycleInstances) StopInstance(context.Context, string) (*instances.Instance, error) { + m.stopped = true + if m.stopErr != nil { + return nil, m.stopErr + } + if m.stoppedState != "" { + m.state = m.stoppedState + } + return &instances.Instance{State: m.state}, nil +} +func (m *machineLifecycleInstances) DeleteInstance(context.Context, string) error { + m.deleted = true + return m.deleteErr +} + +type abortExporter struct { + receipt instances.MacOSBuildExportReceipt + err error +} + +func (e abortExporter) StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) { + return e.receipt, e.err +} +func (abortExporter) ExportStoppedMacOSBuildInstance(context.Context, string, string) error { + return errors.New("abort must not export") +} + +func TestMachineAbortCleanup(t *testing.T) { + for _, tc := range []struct { + name string + state, stoppedState instances.State + graceful, forceError, wantExit bool + }{ + {"graceful exit", instances.StateRunning, "", true, false, true}, + {"already stopped", instances.StateStopped, "", false, false, true}, + {"force running VM", instances.StateRunning, instances.StateStopped, false, false, true}, + {"unconfirmed stop", instances.StateRunning, "", false, false, false}, + {"failed force stop", instances.StateRunning, "", false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + m := &machineLifecycleInstances{state: tc.state, stoppedState: tc.stoppedState} + stop := MachineStopReceipt{Graceful: tc.graceful, VMMExited: tc.graceful} + var stopErr error + if !tc.graceful { + stopErr = errors.New("agent never became ready") + } + if tc.forceError { + m.stopErr = errors.New("hypervisor still alive") + } + s := &guestMachineSession{manager: m, exporter: abortExporter{receipt: stop, err: stopErr}, instance: &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "builder"}}} + receipt, err := s.Abort(context.Background()) + if tc.wantExit { + require.NoError(t, err) + require.Equal(t, MachineStopReceipt{Graceful: tc.graceful, VMMExited: true}, receipt) + } else { + wantErr := "forced stop did not confirm VMM exit" + if tc.forceError { + wantErr = "hypervisor still alive" + } + require.EqualError(t, err, wantErr) + require.False(t, receipt.VMMExited) + require.Error(t, s.Destroy(context.Background()), "unconfirmed exit must remain quarantined") + } + require.Equal(t, !tc.graceful && tc.state != instances.StateStopped, m.stopped) + }) + } +} + +func TestReapPriorMachineBuilder(t *testing.T) { + for _, tc := range []struct { + name string + state instances.State + missing bool + unrecorded bool + deleteErr error + wantStop bool + wantDelete bool + wantError bool + }{ + {"stop and delete", instances.StateRunning, false, false, nil, true, true, false}, + {"delete failure blocks retry", instances.StateStopped, false, false, errors.New("storage still open"), false, true, true}, + {"missing builder", "", true, false, nil, false, false, false}, + {"find unrecorded builder", instances.StateRunning, false, true, nil, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := paths.New(t.TempDir()) + meta := &buildMetadata{ID: "job", Status: StatusQueued} + if !tc.unrecorded { + prior := "prior" + meta.BuilderInstance = &prior + } + require.NoError(t, writeMetadata(p, meta)) + fake := &machineLifecycleInstances{state: tc.state, missing: tc.missing, deleteErr: tc.deleteErr} + m := &manager{paths: p, instanceManager: fake, logger: slog.Default()} + err := m.reapPriorMachineBuilder(context.Background(), "job") + if tc.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.unrecorded, fake.lookedUp == machineBuilderName("job")) + require.Equal(t, tc.wantStop, fake.stopped) + require.Equal(t, tc.wantDelete, fake.deleted) + }) + } +} diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go new file mode 100644 index 000000000..08045c8df --- /dev/null +++ b/lib/builds/macos_build_manager.go @@ -0,0 +1,212 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +func (m *manager) skipMachineBuildRecovery(meta *buildMetadata) bool { + if !m.config.MachineBuildOnly { + return false + } + if meta.Request != nil && meta.Request.MachineBaseImage != "" && m.config.MachineBuild != nil { + return false + } + m.logger.Warn("leaving unsupported build pending in machine-build-only mode", "id", meta.ID) + return true +} + +// executeMachineBuild is a backend of the same queued build job. runBuild owns +// deadline/status/log completion, and CreateBuild already staged verified source. +func (m *manager) executeMachineBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + backend := m.config.MachineBuild + if backend == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if err := m.reapPriorMachineBuilder(ctx, id); err != nil { + return nil, machineFailure("reap_prior_builder", err) + } + base, err := m.imageManager.GetImage(ctx, req.MachineBaseImage) + if err != nil { + return nil, machineFailure("resolve_base", err) + } + sourcePath, err := filepath.Abs(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")) + if err != nil { + return nil, err + } + // An interrupted attempt skips its own cleanup; drop its workspace first. + removeMachineWorkspaces(m.paths.BuildDir(id)) + defer removeMachineWorkspaces(m.paths.BuildDir(id)) + job := MachineBuildRequest{ + ID: id, BaseImage: req.MachineBaseImage, SourceHash: req.SourceHash, Policy: *policy, + onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, + onPublished: func(ctx context.Context, publication MachinePublication) error { + return m.waitMachinePublication(ctx, id, publication.Digest) + }, + } + result, err := backend.runPrepared(ctx, job, base, sourcePath, filepath.Dir(filepath.Dir(sourcePath))) + if err != nil { + return nil, err + } + ref, err := images.ParseNormalizedRef(result.Publication.Reference) + if err != nil || ref.Repository() != strings.TrimSuffix(stripRegistryScheme(m.config.RegistryURL), "/")+"/builds/"+id { + return nil, fmt.Errorf("machine publication is outside the authorized job repository") + } + return &BuildResult{Success: true, ImageDigest: result.Publication.Digest, ImageRef: result.Publication.Reference, Provenance: result.Provenance}, nil +} + +func (m *manager) waitMachinePublication(ctx context.Context, id, digest string) error { + if !ValidMachineDigest(digest) { + return fmt.Errorf("canonical machine publication digest required") + } + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + ref := "builds/" + id + "@" + digest + for { + image, err := m.imageManager.GetImage(ctx, ref) + if err != nil && !errors.Is(err, images.ErrNotFound) { + return err + } + if err == nil && image.Digest == digest { + if image.Status == images.StatusReady { + return nil + } + if image.Status == images.StatusFailed { + return fmt.Errorf("machine image import failed") + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// reapPriorMachineBuilder removes the builder recorded by an interrupted attempt of +// this same job before a re-enqueued attempt starts. Nothing from that builder is +// published, so a forced stop is acceptable; a failed delete fails this attempt +// rather than leaving two builders with storage in play. +func (m *manager) reapPriorMachineBuilder(ctx context.Context, id string) error { + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + // An attempt interrupted before it recorded its builder still named it after the build. + prior := machineBuilderName(id) + if meta.BuilderInstance != nil && *meta.BuilderInstance != "" { + prior = *meta.BuilderInstance + } + inst, err := m.instanceManager.GetInstance(ctx, prior) + if errors.Is(err, instances.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if inst.State != instances.StateStopped { + if _, err := m.instanceManager.StopInstance(ctx, inst.Id); err != nil { + return fmt.Errorf("stop prior builder: %w", err) + } + } + if err := m.instanceManager.DeleteInstance(ctx, inst.Id); err != nil && !errors.Is(err, instances.ErrNotFound) { + return fmt.Errorf("delete prior builder: %w", err) + } + m.logger.Info("reaped prior machine builder", "build_id", id, "instance_id", inst.Id) + return nil +} + +// storeMachineSource stages machine source privately and records its SHA256 +// provenance. A caller-supplied source hash must match. +func (m *manager) storeMachineSource(ctx context.Context, meta *buildMetadata, data []byte) error { + sourceDir := m.paths.BuildSourceDir(meta.ID) + if err := os.MkdirAll(sourceDir, 0700); err != nil { + return fmt.Errorf("store source: %w", err) + } + hash, err := stageBuildSource(ctx, bytes.NewReader(data), filepath.Join(sourceDir, "source.tar.gz"), int64(len(data))) + if err != nil { + return fmt.Errorf("store source: %w", err) + } + if meta.Request.SourceHash != "" && meta.Request.SourceHash != hash { + return ErrSourceHashMismatch + } + meta.Request.SourceHash = hash + if err := writeMetadata(m.paths, meta); err != nil { + return fmt.Errorf("write source provenance: %w", err) + } + return nil +} + +// admitMachineBuild validates a machine build request and returns its policy. +// Machine resources inherit the installed base, not Linux builder defaults. +func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) (*BuildPolicy, error) { + if m.config.MachineBuild == nil || m.config.MachineBuild.Driver == nil || m.config.MachineBuild.Publisher == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if req.Dockerfile != "" || req.BuilderID != "" || len(req.BuildArgs) != 0 || len(req.Secrets) != 0 || req.CacheScope != "" || req.GlobalCacheKey != "" || req.IsAdminBuild { + return nil, fmt.Errorf("machine builds do not support Linux builder/cache/secret options") + } + if len(sourceData) > maxBuildSourceBytes { + return nil, fmt.Errorf("machine build source too large") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := &BuildPolicy{} + if req.BuildPolicy != nil { + requested := *req.BuildPolicy + policy = &requested + } + if err := normalizeMachinePolicy(policy); err != nil { + return nil, err + } + req.BuildPolicy = policy + return policy, nil +} + +// enqueueMachineBuild queues a machine build whose source is already staged. +func machineBuildSerialKey(req CreateBuildRequest) string { + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() { + return "machine:" + req.MachineBaseImage + } + return "machine:" + ref.Digest() +} + +func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { + queuePos := m.queue.EnqueueSerial(meta.ID, req, machineBuildSerialKey(req), func() { + m.runBuild(context.Background(), meta.ID, req, policy) + }) + build := meta.toBuild() + if queuePos > 0 { + build.QueuePosition = &queuePos + } + m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) + return build +} + +// removeMachineWorkspaces deletes export workspaces under a build directory. Each +// holds a full exported disk. +func removeMachineWorkspaces(buildDir string) { + matches, _ := filepath.Glob(filepath.Join(buildDir, "machine-build-*")) + for _, dir := range matches { + _ = os.RemoveAll(dir) + } +} + +// removeMachineSource deletes a machine build's staged source after the job is terminal. +func (m *manager) removeMachineSource(id string) { + if err := os.Remove(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")); err != nil && !errors.Is(err, os.ErrNotExist) { + m.logger.Warn("failed to remove machine build source", "id", id, "error", err) + } +} diff --git a/lib/builds/macos_build_manager_failure_test.go b/lib/builds/macos_build_manager_failure_test.go new file mode 100644 index 000000000..f8d01567c --- /dev/null +++ b/lib/builds/macos_build_manager_failure_test.go @@ -0,0 +1,56 @@ +package builds + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/kernel/hypeman/lib/instances" + "github.com/stretchr/testify/require" +) + +func TestMachineFailureUsesSharedFailedStatusWithoutPublication(t *testing.T) { + for _, tc := range []struct { + name string + phase string + resolveErr error + wantError string + }{ + {name: "base resolution", resolveErr: errors.New("sensitive-resolution-error"), wantError: "machine build resolve_base failed"}, + {name: "provision", phase: "provision", wantError: "machine build provision failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + m := newMachineTestManager(t, f, f, machineJobImages{fixture: f, resolveErr: tc.resolveErr}, false) + build, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && b.Status == StatusFailed + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusFailed, done.Status) + require.Nil(t, done.ImageDigest) + require.NotContains(t, f.calls, "publish") + require.NotContains(t, *done.Error, "sensitive-resolution-error") + require.Equal(t, tc.wantError, *done.Error) + if tc.phase == "provision" { + require.Contains(t, f.calls, "destroy") + } else { + require.Empty(t, f.calls) + } + }) + } +} + +func TestMachineFailureShowsOnlyRecipeStepAndExitCode(t *testing.T) { + err := machineFailure("provision", fmt.Errorf("guest said secret-token: %w", &machineStepError{step: 2, code: 7})) + require.Equal(t, "machine build provision failed: recipe step 2 exited with code 7", err.Error()) + require.Equal(t, "machine build provision failed", machineFailure("provision", fmt.Errorf("secret-token")).Error()) + admission := fmt.Errorf("%w: macOS template identity is already in use", instances.ErrInsufficientResources) + require.Contains(t, machineFailure("start", admission).Error(), "machine build start failed: insufficient resources") +} diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go new file mode 100644 index 000000000..042916b5a --- /dev/null +++ b/lib/builds/macos_build_manager_test.go @@ -0,0 +1,141 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineJobImages struct { + images.Manager + fixture *machineFixture + resolveErr error +} + +func (m machineJobImages) GetImage(ctx context.Context, ref string) (*images.Image, error) { + if m.resolveErr != nil { + return nil, m.resolveErr + } + if ref == "registry/base@"+machineTestDigest { + return m.fixture.ResolveBase(ctx, ref) + } + return &images.Image{Status: images.StatusReady, Digest: machineTestDigest}, nil +} + +func (m machineJobImages) WaitForReady(context.Context, string) error { return nil } + +func newMachineTestManager(t *testing.T, f *machineFixture, publisher MachineBuildPublisher, imageManager images.Manager, machineOnly bool) *manager { + t.Helper() + config := DefaultConfig() + config.RegistrySecret = "synthetic-test-secret" + config.MachineBuildOnly = machineOnly + if f != nil && publisher != nil { + config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: publisher} + } + if imageManager == nil && f != nil { + imageManager = machineJobImages{fixture: f} + } + api, err := NewManager(paths.New(t.TempDir()), config, noPriorBuilder{}, nil, nil, imageManager, nil, nil, nil) + require.NoError(t, err) + return api.(*manager) +} + +type machineJobPublisher struct { + fixture *machineFixture + publishedID string + // sourcePath, when set, must still exist at publish time. + sourcePath func(id string) string +} + +func (p *machineJobPublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + p.publishedID = id + if p.sourcePath != nil { + if _, err := os.Stat(p.sourcePath(id)); err != nil { + return MachinePublication{}, errors.New("source removed before publish") + } + } + // Reuse the fixture's export assertions with its fixed ID. + receipt, err := p.fixture.Publish(ctx, "build-test", bundle) + if err == nil { + receipt.Reference = "localhost:4973/builds/" + id + "@" + receipt.Digest + } + return receipt, err +} +func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { + f := newMachineFixture() + publisher := &machineJobPublisher{fixture: f} + m := newMachineTestManager(t, f, publisher, nil, false) + publisher.sourcePath = func(id string) string { return filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz") } + source := []byte("synthetic compressed-source bytes") + sum := sha256.Sum256(source) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: hex.EncodeToString(sum[:])} + build, err := m.CreateBuild(context.Background(), req, source) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && (b.Status == StatusReady || b.Status == StatusFailed) + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, done.Status, "failure: %v", done.Error) + require.NotNil(t, done.ImageRef) + require.Equal(t, "localhost:4973/builds/"+build.ID+"@"+machineTestDigest, *done.ImageRef) + require.Equal(t, build.ID, publisher.publishedID) + require.Equal(t, req.SourceHash, done.Provenance.SourceHash) + require.Equal(t, machineTestDigest, done.Provenance.BaseImageDigest) + require.Equal(t, f.ID(), *done.BuilderInstanceID) + meta, err := readMetadata(m.paths, build.ID) + require.NoError(t, err) + require.Equal(t, req.MachineBaseImage, meta.Request.MachineBaseImage) + require.Equal(t, "isolated", meta.Request.BuildPolicy.NetworkMode) + require.Equal(t, 0, meta.Request.BuildPolicy.CPUs, "machine defaults must not use Linux CPU defaults") + _, err = readBuildConfig(m.paths, build.ID) + require.Error(t, err, "machine builds must not persist a builder config or registry token") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) +} +func TestMachineBuildSerializesByBaseDigest(t *testing.T) { + first := CreateBuildRequest{MachineBaseImage: "registry-a/base@" + machineTestDigest} + second := CreateBuildRequest{MachineBaseImage: "registry-b/another-name@" + machineTestDigest} + require.Equal(t, machineBuildSerialKey(first), machineBuildSerialKey(second)) +} + +func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, false) + _, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.ErrorContains(t, err, "not configured") + f := newMachineFixture() + m.config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: f} + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base:mutable"}, []byte("source")) + require.ErrorContains(t, err, "pinned") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, Secrets: []SecretRef{{ID: "unsupported"}}}, []byte("source")) + require.ErrorContains(t, err, "secret options") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: "wrong"}, []byte("source")) + require.True(t, errors.Is(err, ErrSourceHashMismatch)) + _, err = m.admitMachineBuild(&CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, make([]byte, maxBuildSourceBytes+1)) + require.ErrorContains(t, err, "source too large") + for _, policy := range []BuildPolicy{{TimeoutSeconds: -1}, {TimeoutSeconds: 86401}, {NetworkMode: "host"}, {AllowedDomains: []string{"example.test"}}} { + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: &policy} + _, err := m.admitMachineBuild(&req, []byte("source")) + require.Error(t, err) + } + list, err := m.ListBuilds(context.Background()) + require.NoError(t, err) + require.Empty(t, list) +} + +// noPriorBuilder reports that no earlier attempt left a builder instance behind. +type noPriorBuilder struct{ instances.Manager } + +func (noPriorBuilder) GetInstance(context.Context, string) (*instances.Instance, error) { + return nil, instances.ErrNotFound +} diff --git a/lib/builds/macos_build_policy_test.go b/lib/builds/macos_build_policy_test.go new file mode 100644 index 000000000..32c1592f6 --- /dev/null +++ b/lib/builds/macos_build_policy_test.go @@ -0,0 +1,44 @@ +package builds + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachinePolicyNormalization(t *testing.T) { + for _, tc := range []struct { + name string + policy BuildPolicy + err string + }{ + {name: "defaults"}, + {name: "minimum timeout", policy: BuildPolicy{TimeoutSeconds: 1}}, + {name: "maximum timeout", policy: BuildPolicy{TimeoutSeconds: 86400}}, + {name: "egress", policy: BuildPolicy{NetworkMode: "egress"}}, + {name: "negative timeout", policy: BuildPolicy{TimeoutSeconds: -1}, err: "invalid machine build timeout"}, + {name: "excessive timeout", policy: BuildPolicy{TimeoutSeconds: 86401}, err: "invalid machine build timeout"}, + {name: "unsupported mode", policy: BuildPolicy{NetworkMode: "host"}, err: "unsupported machine network policy"}, + {name: "domains", policy: BuildPolicy{NetworkMode: "egress", AllowedDomains: []string{"example.test"}}, err: "unsupported machine network policy"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := tc.policy + err := normalizeMachinePolicy(&policy) + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + require.NoError(t, err) + want := tc.policy + if want.TimeoutSeconds == 0 { + want.TimeoutSeconds = 600 + } + if want.NetworkMode == "" { + want.NetworkMode = "isolated" + } + require.Equal(t, want, policy) + require.NoError(t, normalizeMachinePolicy(&policy)) + require.Equal(t, want, policy) + }) + } +} diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go new file mode 100644 index 000000000..ce49a097d --- /dev/null +++ b/lib/builds/macos_build_publisher.go @@ -0,0 +1,214 @@ +package builds + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// StreamingMachinePublisher targets only the configured internal registry and a +// build-scoped repository. go-containerregistry hashes and compresses the layer +// from a reopenable tar stream, so it is never written to disk. Blobs use the +// registry's disk-streaming monolithic upload and are independently verified before +// the manifest is committed last. +// No credentials, redirect destinations or arbitrary output repositories come from recipes. +type StreamingMachinePublisher struct { + RegistryURL string + Tokens *RegistryTokenGenerator +} + +type machineBlob struct { + open func() (io.ReadCloser, error) + descriptor v1.Descriptor +} + +func (p *StreamingMachinePublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return MachinePublication{}, fmt.Errorf("bounded publication required") + } + if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil { + return MachinePublication{}, fmt.Errorf("scoped machine publisher required") + } + address := p.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { + return MachinePublication{}, fmt.Errorf("invalid internal registry address") + } + if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { + return MachinePublication{}, fmt.Errorf("plaintext registry must be loopback") + } + if err := validateMachineExport(bundle, nil); err != nil { + return MachinePublication{}, err + } + layer, err := machineTarballLayer(ctx, bundle) + if err != nil { + return MachinePublication{}, err + } + diffID, err := layer.DiffID() + if err != nil { + return MachinePublication{}, err + } + layerDigest, err := layer.Digest() + if err != nil { + return MachinePublication{}, err + } + layerSize, err := layer.Size() + if err != nil { + return MachinePublication{}, err + } + layerBlob := machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: layerDigest, Size: layerSize}} + config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: []v1.Hash{diffID}}, Config: v1.Config{Labels: map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + }}} + configBytes, err := json.Marshal(config) + if err != nil { + return MachinePublication{}, err + } + configBlob := machineBlob{open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(configBytes)), nil }, descriptor: v1.Descriptor{MediaType: types.OCIConfigJSON, Digest: machineHash(configBytes), Size: int64(len(configBytes))}} + manifest := v1.Manifest{SchemaVersion: 2, MediaType: types.OCIManifestSchema1, Config: configBlob.descriptor, Layers: []v1.Descriptor{layerBlob.descriptor}} + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return MachinePublication{}, err + } + digest := machineHash(manifestBytes).String() + repo := "builds/" + id + token, err := p.Tokens.GeneratePushToken(id, []string{repo}, time.Until(deadline)) + if err != nil { + return MachinePublication{}, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + base := strings.TrimRight(address, "/") + "/v2/" + repo + for _, blob := range []machineBlob{configBlob, layerBlob} { + body, e := blob.open() + if e != nil { + return MachinePublication{}, e + } + response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, base+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) + closeErr := body.Close() + if e != nil { + return MachinePublication{}, e + } + if closeErr != nil { + response.Body.Close() + return MachinePublication{}, closeErr + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") + } + response, e = machineRegistryRequest(ctx, client, token, http.MethodGet, base+"/blobs/"+blob.descriptor.Digest.String(), "", nil, 0) + if e != nil { + return MachinePublication{}, e + } + sum := sha256.New() + n, e := io.Copy(sum, io.LimitReader(response.Body, blob.descriptor.Size+1)) + response.Body.Close() + if e != nil || response.StatusCode != http.StatusOK || n != blob.descriptor.Size || hex.EncodeToString(sum.Sum(nil)) != blob.descriptor.Digest.Hex { + return MachinePublication{}, fmt.Errorf("machine blob verification failed") + } + } + target := base + "/manifests/latest" + response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target, string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) + if err != nil { + return MachinePublication{}, err + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") + } + response, err = machineRegistryRequest(ctx, client, token, http.MethodGet, base+"/manifests/"+digest, "", nil, 0) + if err != nil { + return MachinePublication{}, err + } + published, err := io.ReadAll(io.LimitReader(response.Body, int64(len(manifestBytes))+1)) + response.Body.Close() + if err != nil || response.StatusCode != http.StatusOK || !bytes.Equal(published, manifestBytes) { + return MachinePublication{}, fmt.Errorf("machine manifest verification failed") + } + return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil +} + +func machineRegistryRequest(ctx context.Context, client *http.Client, token, method, target, media string, body io.Reader, size int64) (*http.Response, error) { + request, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return nil, err + } + request.ContentLength = size + request.Header.Set("Authorization", "Bearer "+token) + if media != "" { + request.Header.Set("Content-Type", media) + } + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("machine registry request failed") + } + return response, nil +} + +func machineHash(data []byte) v1.Hash { + sum := sha256.Sum256(data) + return v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum[:])} +} + +func machineTarballLayer(ctx context.Context, bundle string) (v1.Layer, error) { + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(writeMachineTar(ctx, bundle, writer)) }() + return reader, nil + }, tarball.WithMediaType(types.OCILayer)) +} + +// writeMachineTar writes a deterministic tar of the exported bundle. +func writeMachineTar(ctx context.Context, bundle string, w io.Writer) error { + archive := tar.NewWriter(w) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + source, e := os.Open(filepath.Join(bundle, name)) + if e != nil { + return e + } + info, e := source.Stat() + if e != nil { + source.Close() + return e + } + e = archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) + if e == nil { + _, e = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) + } + closeErr := source.Close() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + } + return archive.Close() +} diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go new file mode 100644 index 000000000..c710a49e7 --- /dev/null +++ b/lib/builds/macos_build_publisher_test.go @@ -0,0 +1,96 @@ +package builds_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +type machineImportStub struct { + images.Manager + imported chan string +} + +func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, digest string) (*images.Image, error) { + m.imported <- repo + return &images.Image{Status: images.StatusReady, Digest: digest}, nil +} + +func machinePublisherBundle(t *testing.T) string { + t.Helper() + root := t.TempDir() + platform := images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} + data, e := json.Marshal(platform) + require.NoError(t, e) + for name, data := range map[string][]byte{"config.json": data, "disk.img": []byte("synthetic disk"), "aux.img": []byte("synthetic aux")} { + require.NoError(t, os.WriteFile(filepath.Join(root, name), data, 0600)) + } + return root +} + +func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { + // Tiny storage is intentional: this proves auth/streaming/commit ordering, not bootability. + imports := &machineImportStub{imported: make(chan string, 1)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads, verified atomic.Int32 + handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NotEqual(t, http.MethodPatch, r.Method, "uploads must use the disk-streaming monolithic endpoint") + if r.Method == http.MethodPost { + require.NotEmpty(t, r.URL.Query().Get("digest")) + require.Greater(t, r.ContentLength, int64(0)) + uploads.Add(1) + } + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { + verified.Add(1) + } + if r.Method == http.MethodPut { + require.Equal(t, int32(2), uploads.Load(), "manifest must be last") + } + reg.Handler().ServeHTTP(w, r) + })) + server := httptest.NewServer(handler) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + result, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.NoError(t, e) + require.Contains(t, result.Reference, "/builds/qa-build@sha256:") + select { + case repo := <-imports.imported: + require.Equal(t, "builds/qa-build", repo) + case <-ctx.Done(): + t.Fatal("manifest did not reach production conversion boundary") + } +} + +func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing.T) { + var visited atomic.Bool + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { visited.Store(true) })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic secret")} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.Error(t, e) + require.False(t, visited.Load()) +} diff --git a/lib/builds/macos_build_recipe.go b/lib/builds/macos_build_recipe.go new file mode 100644 index 000000000..a752534f0 --- /dev/null +++ b/lib/builds/macos_build_recipe.go @@ -0,0 +1,188 @@ +package builds + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "unicode/utf8" +) + +// MachineRecipeVersion is the supported hypeman-macos-build.json schema version. +// It describes recipe syntax, not a measured guest agent or compiler version. +const MachineRecipeVersion = 1 + +const machineRecipeName = "hypeman-macos-build.json" +const maxMachineExtractBytes = 128 << 20 + +// machineRecipe v1 runs 1–32 ordered commands as root in the private source +// directory. Commands use argv (no implicit shell); env augments the guest +// environment. There are no interpolation, secret, cwd or output-capture fields. +// Use an explicit /bin/sh invocation when shell syntax is needed. +type machineRecipe struct { + hash string + Version int `json:"version"` + Steps []machineRecipeStep `json:"steps"` +} +type machineRecipeStep struct { + Command []string `json:"command"` + Env map[string]string `json:"env,omitempty"` +} + +var machineEnvName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// Extract on the host into an exclusive build-owned directory, never with guest +// tar or a caller-supplied destination. Links/devices, duplicates and bombs fail +// before any build source is copied to the VM. +func extractMachineRecipe(ctx context.Context, source, destination string) (*machineRecipe, error) { + file, err := os.Open(source) + if err != nil { + return nil, err + } + defer file.Close() + compressed, err := gzip.NewReader(file) + if err != nil { + return nil, err + } + defer compressed.Close() + compressed.Multistream(false) + if err = os.Mkdir(destination, 0700); err != nil { + return nil, err + } + complete := false + defer func() { + if !complete { + _ = os.RemoveAll(destination) + } + }() + limited := &io.LimitedReader{R: compressed, N: maxMachineExtractBytes + 1} + reader := tar.NewReader(limited) + seen := map[string]bool{} + var total int64 + for count := 0; ; count++ { + if err = ctx.Err(); err != nil { + return nil, err + } + header, e := reader.Next() + if e == io.EOF { + break + } + if e != nil { + return nil, e + } + if count >= 1024 { + return nil, fmt.Errorf("too many machine source entries") + } + name := path.Clean(header.Name) + if name == "." && header.Typeflag == tar.TypeDir { + continue + } + if !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || !filepath.IsLocal(name) || seen[name] { + return nil, fmt.Errorf("invalid or duplicate machine source path") + } + seen[name] = true + if header.Size < 0 || header.Size > 64<<20 || total+header.Size > maxMachineExtractBytes { + return nil, fmt.Errorf("machine source expands beyond limit") + } + total += header.Size + target := filepath.Join(destination, name) + switch header.Typeflag { + case tar.TypeDir: + if header.Size != 0 { + return nil, fmt.Errorf("directory has data") + } + if err = os.MkdirAll(target, 0700); err != nil { + return nil, err + } + case tar.TypeReg, tar.TypeRegA: + if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return nil, err + } + output, e := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600|os.FileMode(header.Mode)&0100) + if e != nil { + return nil, e + } + _, copyErr := io.Copy(output, reader) + closeErr := output.Close() + if copyErr != nil { + return nil, copyErr + } + if closeErr != nil { + return nil, closeErr + } + default: + return nil, fmt.Errorf("machine source must contain only regular files and directories") + } + } + // Consume bounded trailing tar padding to verify gzip integrity without allowing + // an unbounded post-tar decompression stream. + _, err = io.Copy(io.Discard, limited) + if err != nil { + return nil, err + } + if limited.N <= 0 { + return nil, fmt.Errorf("machine source expands beyond limit") + } + metadata, err := os.Open(filepath.Join(destination, machineRecipeName)) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(io.LimitReader(metadata, 64<<10+1)) + closeErr := metadata.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + if len(data) > 64<<10 { + return nil, fmt.Errorf("machine recipe too large") + } + if !utf8.Valid(data) { + return nil, fmt.Errorf("machine recipe must be UTF-8") + } + if err = validateMachineRecipeJSON(data); err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var recipe machineRecipe + if err = decoder.Decode(&recipe); err != nil { + return nil, err + } + if err = decoder.Decode(new(any)); err != io.EOF { + return nil, fmt.Errorf("recipe must contain one JSON object") + } + if recipe.Version != MachineRecipeVersion || len(recipe.Steps) == 0 || len(recipe.Steps) > 32 { + return nil, fmt.Errorf("invalid machine recipe version or step count") + } + for _, step := range recipe.Steps { + if len(step.Command) == 0 || len(step.Command) > 128 || !filepath.IsAbs(step.Command[0]) || len(step.Env) > 128 { + return nil, fmt.Errorf("invalid machine command") + } + for _, arg := range step.Command { + if len(arg) > 8192 || !utf8.ValidString(arg) || strings.ContainsRune(arg, 0) { + return nil, fmt.Errorf("invalid machine argument") + } + } + for key, value := range step.Env { + if !machineEnvName.MatchString(key) || len(value) > 8192 || !utf8.ValidString(value) || strings.ContainsRune(value, 0) { + return nil, fmt.Errorf("invalid machine environment") + } + } + } + digest := sha256.Sum256(data) + recipe.hash = hex.EncodeToString(digest[:]) + complete = true + return &recipe, nil +} diff --git a/lib/builds/macos_build_recipe_json.go b/lib/builds/macos_build_recipe_json.go new file mode 100644 index 000000000..c226386ba --- /dev/null +++ b/lib/builds/macos_build_recipe_json.go @@ -0,0 +1,68 @@ +package builds + +import ( + "bytes" + "encoding/json" + "fmt" + "io" +) + +// encoding/json otherwise silently accepts duplicate keys and null strings. +// Reject both so a recipe has one unambiguous interpretation. +func validateMachineRecipeJSON(data []byte) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + var value func(int) error + value = func(depth int) error { + if depth > 8 { + return fmt.Errorf("machine recipe nesting exceeds limit") + } + token, err := decoder.Token() + if err != nil { + return fmt.Errorf("invalid machine recipe JSON") + } + if token == nil { + return fmt.Errorf("machine recipe does not support null") + } + switch token { + case json.Delim('{'): + seen := make(map[string]bool) + for decoder.More() { + key, err := decoder.Token() + if err != nil { + return fmt.Errorf("invalid machine recipe object") + } + name, ok := key.(string) + if !ok || seen[name] { + return fmt.Errorf("duplicate or invalid machine recipe key") + } + // Struct decoding matches keys case-insensitively; the recipe + // contract accepts only the exact documented field names. + if (depth == 0 && name != "version" && name != "steps") || (depth == 2 && name != "command" && name != "env") { + return fmt.Errorf("unknown machine recipe field") + } + seen[name] = true + if err := value(depth + 1); err != nil { + return err + } + } + _, err = decoder.Token() + return err + case json.Delim('['): + for decoder.More() { + if err := value(depth + 1); err != nil { + return err + } + } + _, err = decoder.Token() + return err + } + return nil + } + if err := value(0); err != nil { + return err + } + if _, err := decoder.Token(); err != io.EOF { + return fmt.Errorf("recipe must contain one JSON object") + } + return nil +} diff --git a/lib/builds/macos_build_recipe_test.go b/lib/builds/macos_build_recipe_test.go new file mode 100644 index 000000000..cce154654 --- /dev/null +++ b/lib/builds/macos_build_recipe_test.go @@ -0,0 +1,70 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeMachineSource(t *testing.T, headers []*tar.Header, bodies []string) string { + t.Helper() + name := filepath.Join(t.TempDir(), "source.tar.gz") + f, e := os.Create(name) + require.NoError(t, e) + gz := gzip.NewWriter(f) + writer := tar.NewWriter(gz) + for i, h := range headers { + require.NoError(t, writer.WriteHeader(h)) + if h.Typeflag == tar.TypeReg { + _, e = writer.Write([]byte(bodies[i])) + require.NoError(t, e) + } + } + require.NoError(t, writer.Close()) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) + return name +} + +func TestMachineRecipeExtraction(t *testing.T) { + recipe := `{"version":1,"steps":[{"command":["/bin/sh","build.sh"],"env":{"MODE":"qa"}}]}` + source := writeMachineSource(t, []*tar.Header{{Name: machineRecipeName, Size: int64(len(recipe)), Typeflag: tar.TypeReg}, {Name: "build.sh", Size: 4, Typeflag: tar.TypeReg, Mode: 0755}}, []string{recipe, "true"}) + root := filepath.Join(t.TempDir(), "source") + result, e := extractMachineRecipe(context.Background(), source, root) + require.NoError(t, e) + require.Len(t, result.Steps, 1) + info, e := os.Stat(filepath.Join(root, "build.sh")) + require.NoError(t, e) + require.Equal(t, os.FileMode(0700), info.Mode().Perm()) +} + +func TestMachineRecipeRejectsUnsafeArchivesAndCommands(t *testing.T) { + for _, tc := range []struct { + name string + headers []*tar.Header + bodies []string + }{ + {"traversal", []*tar.Header{{Name: "../outside", Typeflag: tar.TypeReg, Size: 1}}, []string{"x"}}, + {"symlink", []*tar.Header{{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc"}}, []string{""}}, + {"hardlink", []*tar.Header{{Name: "link", Typeflag: tar.TypeLink, Linkname: "other"}}, []string{""}}, + {"duplicate", []*tar.Header{{Name: "a", Typeflag: tar.TypeReg, Size: 1}, {Name: "a", Typeflag: tar.TypeReg, Size: 1}}, []string{"x", "y"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := filepath.Join(t.TempDir(), "source") + _, e := extractMachineRecipe(context.Background(), writeMachineSource(t, tc.headers, tc.bodies), root) + require.Error(t, e) + _, e = os.Stat(root) + require.True(t, os.IsNotExist(e)) + }) + } + for _, body := range []string{`{"version":2,"steps":[]}`, `{"version":1,"steps":[{"command":["sh"]}]}`, `{"version":1,"steps":[{"command":["/bin/true"],"secret":"no"}]}`, `{"version":1,"steps":[{"command":["/bin/true"]}]} {}`} { + source := writeMachineSource(t, []*tar.Header{{Name: machineRecipeName, Typeflag: tar.TypeReg, Size: int64(len(body))}}, []string{body}) + _, e := extractMachineRecipe(context.Background(), source, filepath.Join(t.TempDir(), "source")) + require.Error(t, e) + } +} diff --git a/lib/builds/macos_build_recovery_test.go b/lib/builds/macos_build_recovery_test.go new file mode 100644 index 000000000..411c12102 --- /dev/null +++ b/lib/builds/macos_build_recovery_test.go @@ -0,0 +1,88 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { + for _, status := range []string{StatusQueued, StatusBuilding, StatusPushing} { + t.Run(status, func(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, true) + m.config.BuilderImage = "linux-builder-must-not-start" + policy := &BuildPolicy{} + require.NoError(t, normalizeMachinePolicy(policy)) + meta := &buildMetadata{ID: "build-test", Status: status, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: policy}} + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("synthetic compressed-source bytes"))) + // A workspace left by the interrupted attempt. + stale := filepath.Join(m.paths.BuildDir(meta.ID), "machine-build-interrupted") + require.NoError(t, os.MkdirAll(filepath.Join(stale, "bundle"), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(stale, "bundle", "disk.img"), []byte("disk"), 0600)) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "linux-job", Status: StatusQueued, Request: &CreateBuildRequest{BuilderID: "linux-builder"}})) + t.Cleanup(func() { + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) + }) + + // Hold the recovered build active so the second recovery sees it running. + f.startGate = make(chan struct{}) + require.NoError(t, m.Start(context.Background())) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 1 }, 5*time.Second, 10*time.Millisecond) + m.RecoverPendingBuilds() + close(f.startGate) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 && m.queue.PendingCount() == 0 }, 5*time.Second, 10*time.Millisecond) + build, err := m.GetBuild(context.Background(), meta.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, build.Status, "failure: %v", build.Error) + require.NoDirExists(t, stale, "the interrupted attempt's workspace is removed") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz"), "source is removed once the image is ready") + require.False(t, m.ReadyForBuilds(), "Linux builder image must not be prepared") + var starts int + for _, call := range f.calls { + if call == "start" { + starts++ + } + } + require.Equal(t, 1, starts, "recovery must not duplicate an active build") + linux, err := readMetadata(m.paths, "linux-job") + require.NoError(t, err) + require.Equal(t, StatusQueued, linux.Status, "Linux jobs are preserved for Linux-mode recovery") + require.True(t, m.BuilderHasBuilds("linux-builder"), "unrecovered disk jobs still protect their builders") + }) + } +} + +func TestRecoveryCleansCancelledMachineBuildSource(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, false) + meta := &buildMetadata{ + ID: "cancelled-build", + Status: StatusCancelled, + Request: &CreateBuildRequest{ + MachineBaseImage: "registry/base@" + machineTestDigest, + }, + } + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("source"))) + meta.Status = StatusCancelled + require.NoError(t, writeMetadata(m.paths, meta)) + + m.RecoverPendingBuilds() + + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz")) +} + +func TestMachineOnlyStartPreservesBuildsWhenBackendDisabled(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, true) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "disabled", Status: StatusBuilding, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}})) + require.NoError(t, m.Start(context.Background())) + require.Equal(t, 0, m.queue.ActiveCount()) + meta, err := readMetadata(m.paths, "disabled") + require.NoError(t, err) + require.Equal(t, StatusBuilding, meta.Status) + require.False(t, m.ReadyForBuilds()) +} diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go new file mode 100644 index 000000000..b8d9ce5dc --- /dev/null +++ b/lib/builds/macos_build_test.go @@ -0,0 +1,259 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +type machineFixture struct { + image *images.Image + calls []string + fail string + stop MachineStopReceipt + cancel context.CancelFunc + waitForCancel bool + mutateExport func(string) error + publicationOverride *MachinePublication + publisherCalled bool + cleanupContextLive bool + startGate chan struct{} +} + +func newMachineFixture() *machineFixture { + return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} +} +func (f *machineFixture) record(phase string) error { + f.calls = append(f.calls, phase) + if f.fail == phase { + return errors.New("sensitive-command-argument") + } + return nil +} +func (f *machineFixture) ResolveBase(context.Context, string) (*images.Image, error) { + if err := f.record("resolve"); err != nil { + return nil, err + } + return f.image, nil +} +func (f *machineFixture) Start(_ context.Context, _ string, _ *images.Image, p BuildPolicy) (MachineBuildSession, error) { + if p.CPUs != 4 || p.MemoryMB != 8192 { + return nil, errors.New("wrong resource defaults") + } + if f.startGate != nil { + <-f.startGate + } + return f, f.record("start") +} +func (f *machineFixture) ID() string { return "isolated-test-instance" } +func (f *machineFixture) Provision(ctx context.Context, source string) error { + if _, err := os.ReadFile(source); err != nil { + return err + } + if f.cancel != nil { + f.cancel() + return ctx.Err() + } + if f.waitForCancel { + <-ctx.Done() + return ctx.Err() + } + return f.record("provision") +} +func (f *machineFixture) Sanitize(context.Context) error { return f.record("sanitize") } +func (f *machineFixture) Stop(ctx context.Context) (MachineStopReceipt, error) { + f.cleanupContextLive = ctx.Err() == nil + return f.stop, f.record("stop") +} +func (f *machineFixture) Abort(ctx context.Context) (MachineStopReceipt, error) { + f.cleanupContextLive = ctx.Err() == nil + return f.stop, f.record("abort") +} +func (f *machineFixture) Export(_ context.Context, root string) error { + if err := f.record("export"); err != nil { + return err + } + data, _ := json.Marshal(*f.image.MacOS) + for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { + if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { + return err + } + } + if f.mutateExport != nil { + return f.mutateExport(root) + } + return nil +} +func (f *machineFixture) Destroy(ctx context.Context) error { + f.cleanupContextLive = ctx.Err() == nil + return f.record("destroy") +} +func (f *machineFixture) Publish(ctx context.Context, id, root string) (MachinePublication, error) { + f.publisherCalled = true + if err := ctx.Err(); err != nil { + return MachinePublication{}, err + } + if id != "build-test" { + return MachinePublication{}, errors.New("wrong job namespace") + } + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, err := os.Stat(filepath.Join(root, name)) + if err != nil || info.Mode().Perm() != 0600 { + return MachinePublication{}, errors.New("unsafe export mode") + } + } + publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} + if f.publicationOverride != nil { + publication = *f.publicationOverride + } + return publication, f.record("publish") +} +func machineTestRunner(t *testing.T, f *machineFixture) *machineRunnerFixture { + t.Helper() + return &machineRunnerFixture{MachineBuildBackend: MachineBuildBackend{Driver: f, Publisher: f}, WorkDir: t.TempDir()} +} +func machineTestRequest() MachineBuildRequest { + return MachineBuildRequest{ID: "build-test", BaseImage: "localhost/macos@" + machineTestDigest} +} + +func TestMachineBuildPublicationOrder(t *testing.T) { + f := newMachineFixture() + r := machineTestRunner(t, f) + result, err := r.Run(context.Background(), machineTestRequest(), strings.NewReader("source")) + if err != nil { + t.Fatal(err) + } + expected := []string{"start", "provision", "sanitize", "stop", "export", "destroy", "publish"} + if !reflect.DeepEqual(f.calls, expected) { + t.Fatalf("wrong phase order: %v", f.calls) + } + hash := sha256.Sum256([]byte("source")) + if result.Provenance.SourceHash != hex.EncodeToString(hash[:]) || result.Provenance.BaseImageDigest != machineTestDigest { + t.Fatal("unverified provenance") + } + entries, err := os.ReadDir(r.WorkDir) + if err != nil || len(entries) != 0 { + t.Fatal("workspace not cleaned") + } +} + +func TestMachineBuildFailuresDoNotPublish(t *testing.T) { + for _, tc := range []struct { + name string + phase string + receipt *MachineStopReceipt + }{ + {name: "start", phase: "start"}, + {name: "provision", phase: "provision"}, + {name: "sanitize", phase: "sanitize"}, + {name: "stop", phase: "stop"}, + {name: "export", phase: "export"}, + {name: "destroy", phase: "destroy"}, + {name: "unconfirmed stop", receipt: &MachineStopReceipt{}}, + {name: "forced stop", receipt: &MachineStopReceipt{VMMExited: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + if tc.receipt != nil { + f.stop = *tc.receipt + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("failed build reached publication") + } + if tc.phase != "" && strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("raw guest error leaked") + } + for _, phase := range f.calls { + if phase == "export" && tc.receipt != nil { + t.Fatal("unconfirmed stop exported") + } + if phase == "destroy" && tc.receipt != nil && !tc.receipt.VMMExited { + t.Fatal("destroyed unconfirmed live storage") + } + } + }) + } +} + +func TestMachineBuildCancellationUsesIndependentCleanup(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + f.cancel = cancel + _, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), strings.NewReader("source")) + if !errors.Is(err, context.Canceled) || !f.cleanupContextLive || f.publisherCalled { + t.Fatal("cancellation did not retain safe cleanup") + } + if !reflect.DeepEqual(f.calls, []string{"start", "abort", "destroy"}) { + t.Fatalf("cancel cleanup: %v", f.calls) + } +} + +func TestMachineBuildInputAdmission(t *testing.T) { + for _, kind := range []string{"floating base", "wrong digest", "linux", "pending", "shaping", "domains", "source hash", "CPU overflow", "memory precision"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + req := machineTestRequest() + switch kind { + case "floating base": + req.BaseImage = "localhost/macos:latest" + case "wrong digest": + f.image.Digest = "sha256:" + strings.Repeat("b", 64) + case "linux": + f.image.Platform = "linux/arm64" + case "pending": + f.image.Status = images.StatusPending + case "shaping": + req.Policy.MemoryMB = 4096 + case "domains": + req.Policy.AllowedDomains = []string{"example.com"} + case "source hash": + req.SourceHash = strings.Repeat("0", 64) + case "CPU overflow": + f.image.MacOS.CPUs = ^uint(0) + case "memory precision": + f.image.MacOS.Memory++ + } + runMachineBuildFailure(t, f, context.Background(), req, strings.NewReader("source")) + for _, phase := range f.calls { + if phase == "start" { + t.Fatal("bad input started a VM") + } + } + }) + } +} + +type cancelMachineSource struct{ cancel context.CancelFunc } + +func (s cancelMachineSource) Read(p []byte) (int, error) { + s.cancel() + return copy(p, "source"), io.EOF +} +func TestMachineBuildCanceledSourceDoesNotStart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + _, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), cancelMachineSource{cancel}) + if !errors.Is(err, context.Canceled) || len(f.calls) != 0 { + t.Fatalf("canceled source started builder: %v", f.calls) + } +} + +var _ MachineBuildDriver = (*machineFixture)(nil) +var _ MachineBuildSession = (*machineFixture)(nil) +var _ MachineBuildPublisher = (*machineFixture)(nil) diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 806ae5622..df6350b9e 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -87,6 +87,12 @@ type Manager interface { // Config holds configuration for the build manager type Config struct { + // MachineBuild is an opt-in internal executor. No HTTP recipe mode is exposed. + MachineBuild *MachineBuildBackend + + // MachineBuildOnly disables Linux builder preparation and recovery. + MachineBuildOnly bool + // MaxConcurrentBuilds is the maximum number of concurrent builds MaxConcurrentBuilds int @@ -149,6 +155,7 @@ type manager struct { logger *slog.Logger metrics *Metrics createMu sync.Mutex + machineBuilds sync.Map // build ID -> *machineBuildControl builderReady atomic.Bool // pendingRecovered is set once RecoverPendingBuilds has re-enqueued // persisted pending builds; until then BuilderHasBuilds also scans @@ -204,6 +211,10 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { + if m.config.MachineBuildOnly { + m.RecoverPendingBuilds() + return nil + } go func() { for { m.ensureBuilderImage(ctx) @@ -444,7 +455,12 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc // Apply defaults to build policy policy := req.BuildPolicy - if policy == nil { + if req.MachineBaseImage != "" { + var err error + if policy, err = m.admitMachineBuild(&req, sourceData); err != nil { + return nil, err + } + } else if policy == nil { defaultPolicy := DefaultBuildPolicy() policy = &defaultPolicy } else { @@ -481,6 +497,15 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc CreatedAt: time.Now(), } + if req.MachineBaseImage != "" { + // Stage verified input before making the job visible to recovery. + if err := m.storeMachineSource(ctx, meta, sourceData); err != nil { + deleteBuild(m.paths, id) + return nil, err + } + return m.enqueueMachineBuild(meta, req, policy), nil + } + // Write initial metadata if err := writeMetadata(m.paths, meta); err != nil { return nil, fmt.Errorf("write metadata: %w", err) @@ -678,18 +703,29 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques start := time.Now() m.logger.Info("starting build", "id", id) - // Update status to building - m.updateStatus(id, StatusBuilding, nil) - - // Create timeout context + // Register machine cancellation before exposing the running status. Ordinary + // Linux builder cancellation retains its existing instance-delete behavior. buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + if req.MachineBaseImage != "" { + m.machineBuilds.Store(id, &machineBuildControl{cancel: cancel}) + defer m.machineBuilds.Delete(id) + defer func() { + meta, err := readMetadata(m.paths, id) + if err == nil && (meta.Status == StatusReady || meta.Status == StatusFailed || meta.Status == StatusCancelled) { + m.removeMachineSource(id) + } + }() + } + m.updateStatus(id, StatusBuilding, nil) // Mirror base images to the local registry before launching the VM. // BuildKit is configured with our registry as a mirror for docker.io, // so pre-cached images will be served locally without pulling from Docker Hub. - if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { - m.logger.Warn("failed to mirror base images", "id", id, "error", err) + if req.MachineBaseImage == "" { + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } } // Run the build in a builder VM @@ -754,6 +790,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // If image_name is set, re-tag the image so it's accessible by that name. // The builder pushed to builds/{id} but the user wants it as image_name. imageRef := buildRepo + if req.MachineBaseImage != "" && result.ImageRef != "" { + imageRef = result.ImageRef + } if req.ImageName != "" { ref, err := images.ParseNormalizedRef(req.ImageName) if err != nil { @@ -798,6 +837,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // executeBuild runs the build in a builder VM func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + if req.MachineBaseImage != "" { + return m.executeMachineBuild(ctx, id, req, policy) + } if !m.builderReady.Load() { return nil, fmt.Errorf("builder image is being prepared, please retry shortly") } @@ -1106,6 +1148,11 @@ func (m *manager) updateStatus(id string, status string, err error) { // updateBuildComplete updates the build with final results func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } meta, readErr := readMetadata(m.paths, id) if readErr != nil { m.logger.Error("read metadata for completion", "id", id, "error", readErr) @@ -1246,12 +1293,18 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { case StatusQueued: // Remove from queue if m.queue.Cancel(id) { + if meta.Request != nil && meta.Request.MachineBaseImage != "" { + return m.cancelQueuedMachineBuild(meta) + } m.updateStatus(id, StatusCancelled, nil) return nil } return ErrBuildInProgress // Was already picked up case StatusBuilding, StatusPushing: + if meta.Request != nil && meta.Request.MachineBaseImage != "" { + return m.cancelMachineBuild(id) + } // Can't cancel a running build easily // Would need to terminate the builder instance if meta.BuilderInstance != nil { @@ -1449,15 +1502,36 @@ func (m *manager) StreamBuildEvents(ctx context.Context, id string, follow bool) // RecoverPendingBuilds recovers builds that were interrupted on restart func (m *manager) RecoverPendingBuilds() { - pending, err := listPendingBuilds(m.paths) + all, err := listAllBuilds(m.paths) if err != nil { - m.logger.Error("list pending builds for recovery", "error", err) + m.logger.Error("list builds for machine cleanup", "error", err) return } + for _, meta := range all { + if meta.Request == nil || meta.Request.MachineBaseImage == "" { + continue + } + switch meta.Status { + case StatusReady, StatusFailed: + m.removeMachineSource(meta.ID) + case StatusCancelled: + if m.config.MachineBuild != nil { + if err := m.reapPriorMachineBuilder(context.Background(), meta.ID); err != nil { + m.logger.Error("failed to reap cancelled machine build", "id", meta.ID, "error", err) + continue + } + } + m.removeMachineSource(meta.ID) + } + } + pending := pendingBuilds(all) for _, meta := range pending { meta := meta // Shadow loop variable for closure capture m.logger.Info("recovering build", "id", meta.ID, "status", meta.Status) + if m.skipMachineBuildRecovery(meta) { + continue + } // Re-enqueue the build if meta.Request != nil { @@ -1472,7 +1546,11 @@ func (m *manager) RecoverPendingBuilds() { continue } - m.queue.EnqueueSerial(meta.ID, *meta.Request, meta.Request.BuilderID, func() { + serialKey := meta.Request.BuilderID + if meta.Request.MachineBaseImage != "" { + serialKey = machineBuildSerialKey(*meta.Request) + } + m.queue.EnqueueSerial(meta.ID, *meta.Request, serialKey, func() { policy := DefaultBuildPolicy() if meta.Request.BuildPolicy != nil { policy = *meta.Request.BuildPolicy @@ -1485,11 +1563,16 @@ func (m *manager) RecoverPendingBuilds() { if len(pending) > 0 { m.logger.Info("recovered pending builds", "count", len(pending)) } - m.pendingRecovered.Store(true) + if !m.config.MachineBuildOnly { + m.pendingRecovered.Store(true) + } } // refreshBuildToken regenerates the registry token for a build and updates the config file func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) error { + if req.MachineBaseImage != "" { + return nil // Machine builds have no builder config or registry token. + } // Read existing build config config, err := readBuildConfig(m.paths, buildID) if err != nil { diff --git a/lib/builds/source.go b/lib/builds/source.go new file mode 100644 index 000000000..b2075607f --- /dev/null +++ b/lib/builds/source.go @@ -0,0 +1,75 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" +) + +const maxBuildSourceBytes = 64 << 20 + +type buildContextReader struct { + ctx context.Context + r io.Reader +} + +func (r buildContextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +// stageBuildSource streams compressed source once into a private, exclusive job +// file. Both backends consume this file and use its hash for input verification. +func stageBuildSource(ctx context.Context, source io.Reader, path string, limit int64) (hash string, err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return "", err + } + defer func() { + if closeErr := file.Close(); err == nil { + err = closeErr + } + }() + h := sha256.New() + n, err := io.Copy(io.MultiWriter(file, h), io.LimitReader(buildContextReader{ctx, source}, limit+1)) + if err != nil { + return "", err + } + if n > limit { + return "", fmt.Errorf("build source too large") + } + if err = file.Sync(); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// verifyBuildSource rejects missing/replaced source on recovery before a machine +// is launched. It never writes a second source copy. +func verifyBuildSource(ctx context.Context, path, expected string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxBuildSourceBytes { + return fmt.Errorf("invalid staged source") + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + h := sha256.New() + if _, err = io.Copy(h, io.LimitReader(buildContextReader{ctx, file}, maxBuildSourceBytes+1)); err != nil { + return err + } + if hex.EncodeToString(h.Sum(nil)) != expected { + return ErrSourceHashMismatch + } + return nil +} diff --git a/lib/builds/source_test.go b/lib/builds/source_test.go new file mode 100644 index 000000000..1bb7e8a84 --- /dev/null +++ b/lib/builds/source_test.go @@ -0,0 +1,36 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSharedBuildSourceStagingAndRecoveryVerification(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.tar.gz") + hash, err := stageBuildSource(ctx, strings.NewReader("source"), path, maxBuildSourceBytes) + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.NoError(t, verifyBuildSource(ctx, path, hash)) + _, err = stageBuildSource(ctx, strings.NewReader("replacement"), path, maxBuildSourceBytes) + require.Error(t, err, "exclusive staging must preserve existing source") + require.NoError(t, verifyBuildSource(ctx, path, hash)) + canceled, cancel := context.WithCancel(ctx) + cancel() + require.ErrorIs(t, verifyBuildSource(canceled, path, hash), context.Canceled) + require.NoError(t, os.WriteFile(path, []byte("changed"), 0600)) + require.ErrorIs(t, verifyBuildSource(ctx, path, hash), ErrSourceHashMismatch) + require.NoError(t, os.Remove(path)) + require.Error(t, verifyBuildSource(ctx, path, hash)) + target := filepath.Join(filepath.Dir(path), "target") + require.NoError(t, os.WriteFile(target, []byte("source"), 0600)) + require.NoError(t, os.Symlink(target, path)) + require.ErrorContains(t, verifyBuildSource(ctx, path, hash), "invalid staged source") +} diff --git a/lib/builds/storage.go b/lib/builds/storage.go index ce555aac9..1add7a3c1 100644 --- a/lib/builds/storage.go +++ b/lib/builds/storage.go @@ -144,7 +144,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { if err != nil { return nil, err } + return pendingBuilds(all), nil +} +func pendingBuilds(all []*buildMetadata) []*buildMetadata { var pending []*buildMetadata for _, meta := range all { switch meta.Status { @@ -152,13 +155,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { pending = append(pending, meta) } } - - // Sort by created_at ascending (oldest first for FIFO recovery) sort.Slice(pending, func(i, j int) bool { return pending[i].CreatedAt.Before(pending[j].CreatedAt) }) - - return pending, nil + return pending } // deleteBuild removes a build's data from disk diff --git a/lib/builds/types.go b/lib/builds/types.go index 4ad6008e9..c6172449c 100644 --- a/lib/builds/types.go +++ b/lib/builds/types.go @@ -43,6 +43,10 @@ type CreateBuildRequest struct { // The Dockerfile specifies the runtime (e.g., FROM node:20-alpine). Dockerfile string `json:"dockerfile,omitempty"` + // MachineBaseImage selects the installed-machine backend. The HTTP multipart + // field machine_base_image maps here; it is enabled only with macos_builds. + MachineBaseImage string `json:"machine_base_image,omitempty"` + // BaseImageDigest optionally pins the base image by digest for reproducibility BaseImageDigest string `json:"base_image_digest,omitempty"` @@ -120,6 +124,11 @@ type BuildProvenance struct { // SourceHash is the SHA256 of the source tarball SourceHash string `json:"source_hash"` + // RecipeHash is the SHA256 of the exact validated machine recipe bytes. + // Omitted when the executor provides no recipe evidence. Machine toolchain + // versions are not inferred; BuildkitVersion remains empty for machine builds. + RecipeHash string `json:"recipe_hash,omitempty"` + // LockfileHashes maps lockfile names to their SHA256 hashes LockfileHashes map[string]string `json:"lockfile_hashes,omitempty"` @@ -219,6 +228,9 @@ type BuildResult struct { // ImageDigest is the digest of the pushed image (only on success) ImageDigest string `json:"image_digest,omitempty"` + // ImageRef is the pullable output reference for machine builds. + ImageRef string `json:"image_ref,omitempty"` + // Error is the error message (only on failure) Error string `json:"error,omitempty"` diff --git a/lib/instances/create.go b/lib/instances/create.go index 9477b2202..e7ecdde66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,6 +99,9 @@ func (m *manager) createInstance( if err := validateExpiresAt(req.ExpiresAt, m.nowUTC()); err != nil { return nil, err } + if m.macOSOnly && strings.TrimSpace(req.Platform) == "" { + req.Platform = "darwin/arm64" + } if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index cd605597f..825196d8b 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -20,8 +20,22 @@ type createImageResolver interface { func resolveImageForCreate(ctx context.Context, imageManager createImageResolver, imageName, platform string, log *slog.Logger) (*images.Image, error) { if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { img, err := imageManager.GetImage(ctx, imageName) + if err != nil && !errors.Is(err, images.ErrNotFound) { + return nil, fmt.Errorf("get macOS image: %w", err) + } + if err != nil || img.MacOS == nil { + img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}) + if err != nil { + return nil, fmt.Errorf("pull macOS image: %w", err) + } + } + pinned, err := pinnedImageName(imageName, img.Digest) if err != nil { - return nil, fmt.Errorf("get imported macOS image: %w", err) + return nil, err + } + img, err = awaitReady(ctx, imageManager, img, pinned, log) + if err != nil { + return nil, err } if img.MacOS == nil { return nil, fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) diff --git a/lib/instances/create_image_test.go b/lib/instances/create_image_test.go index d964bafe2..fea162437 100644 --- a/lib/instances/create_image_test.go +++ b/lib/instances/create_image_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "strings" "testing" "github.com/kernel/hypeman/lib/images" @@ -30,6 +31,55 @@ func (f createImageResolverFake) WaitForReady(ctx context.Context, name string) return f.waitReady(ctx, name) } +func TestResolveImageForCreatePullsUncachedMacOSImage(t *testing.T) { + t.Parallel() + + const imageName = "registry.example/macos/app:latest" + digest := "sha256:" + strings.Repeat("a", 64) + pinnedName := "registry.example/macos/app@" + digest + createPlatform := "" + waitName := "" + resolver := createImageResolverFake{ + getImage: func(_ context.Context, name string) (*images.Image, error) { + if name == imageName { + return nil, images.ErrNotFound + } + if name != pinnedName { + t.Fatalf("unexpected image lookup %q", name) + } + return &images.Image{ + Name: imageName, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusReady, MacOS: &images.MacOSImage{}, + }, nil + }, + createImage: func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createPlatform = req.Platform + return &images.Image{ + Name: req.Name, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusPending, MacOS: &images.MacOSImage{}, + }, nil + }, + waitReady: func(_ context.Context, name string) error { + waitName = name + return nil + }, + } + + img, err := resolveImageForCreate(context.Background(), resolver, imageName, "darwin/arm64", slog.Default()) + if err != nil { + t.Fatalf("resolve image: %v", err) + } + if createPlatform != "darwin/arm64" { + t.Fatalf("expected Darwin pull, got %q", createPlatform) + } + if waitName != pinnedName { + t.Fatalf("expected readiness wait on %q, got %q", pinnedName, waitName) + } + if img.MacOS == nil || img.Platform != "darwin/arm64" { + t.Fatalf("expected ready macOS image, got %+v", img) + } +} + func TestResolveImageForCreateWithPlatformForcesPlatformResolve(t *testing.T) { t.Parallel() diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 9e8e4d901..909972aef 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -24,20 +24,17 @@ import ( // macOSAuxStorageFile is the instance's copy of the image's auxiliary storage. const macOSAuxStorageFile = "mac-aux.img" -// requireMacOSImageName refuses a create on a macOS-only server unless the image -// is an already imported macOS image, so a Linux image is never pulled there. +// requireMacOSImageName checks lookup errors before resolving the reference as +// darwin/arm64. The resolved image is validated by checkMacOSCreate. func (m *manager) requireMacOSImageName(ctx context.Context, name string) error { if !m.macOSOnly { return nil } - cached, err := m.imageManager.GetImage(ctx, name) - if err != nil && !errors.Is(err, images.ErrNotFound) { - return err - } - if err != nil || cached.MacOS == nil { - return fmt.Errorf("%w: macOS-only server requires an already imported macOS image", ErrInvalidRequest) + _, err := m.imageManager.GetImage(ctx, name) + if errors.Is(err, images.ErrNotFound) { + return nil } - return nil + return err } // checkMacOSCreate applies macOS machine defaults to a create from a macOS image. diff --git a/lib/instances/macos_build_export.go b/lib/instances/macos_build_export.go new file mode 100644 index 000000000..13539cd15 --- /dev/null +++ b/lib/instances/macos_build_export.go @@ -0,0 +1,5 @@ +package instances + +// MacOSBuildExportReceipt describes one stop/export operation, not persisted state. +// A failed export may still confirm that instance cleanup is safe. +type MacOSBuildExportReceipt struct{ Graceful, VMMExited bool } diff --git a/lib/instances/macos_build_export_darwin.go b/lib/instances/macos_build_export_darwin.go new file mode 100644 index 000000000..9419ae9ab --- /dev/null +++ b/lib/instances/macos_build_export_darwin.go @@ -0,0 +1,136 @@ +//go:build darwin + +package instances + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/images" +) + +// StopMacOSBuildInstance performs only the strict graceful stop used by build +// export. It copies no storage and never falls back to a forced hypervisor stop. +// Success requires the shutdown acknowledgement and confirmed VMM exit. +func (m *manager) StopMacOSBuildInstance(ctx context.Context, id string) (receipt MacOSBuildExportReceipt, err error) { + if err = ctx.Err(); err != nil { + return + } + if _, bounded := ctx.Deadline(); !bounded { + return receipt, fmt.Errorf("machine stop requires a deadline") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return receipt, err + } + if current.MacOS == nil || !current.GuestAgentEnabled() || (current.State != StateRunning && current.State != StateInitializing) { + return receipt, fmt.Errorf("strict stop requires an active macOS guest with a system agent") + } + if err = m.markRestartManualStopLocked(ctx, id); err != nil { + return + } + stopped, err := m.stopInstanceWithFallback(ctx, id, true) + if err != nil { + return receipt, err + } + receipt = MacOSBuildExportReceipt{Graceful: true, VMMExited: true} + m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) + return receipt, nil +} + +// ExportStoppedMacOSBuildInstance copies closed storage for an instance that is +// still Stopped when the copy starts. The instance lock excludes Start during the +// check and copy; a concurrently restarted instance fails export instead. +func (m *manager) ExportStoppedMacOSBuildInstance(ctx context.Context, id, destination string) error { + if err := ctx.Err(); err != nil { + return err + } + if _, bounded := ctx.Deadline(); !bounded { + return fmt.Errorf("machine export requires a deadline") + } + if !filepath.IsAbs(destination) { + return fmt.Errorf("export destination must be absolute") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return err + } + if current.State != StateStopped || current.MacOS == nil { + return fmt.Errorf("export requires a stopped macOS instance") + } + if rel, e := filepath.Rel(current.DataDir, destination); e != nil || filepath.IsLocal(rel) { + return fmt.Errorf("export must be outside instance storage") + } + return writeStoppedMacOSBuildBundle(ctx, current, m.paths.InstanceMacOSBootDisk(id), destination) +} + +func writeStoppedMacOSBuildBundle(ctx context.Context, inst *Instance, disk, destination string) error { + if inst.State != StateStopped || inst.MacOS == nil { + return fmt.Errorf("machine must be stopped") + } + aux := filepath.Join(inst.DataDir, "mac-aux.img") + var diskInfo os.FileInfo + for _, source := range []string{disk, aux} { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + if diskInfo != nil && os.SameFile(diskInfo, info) { + return fmt.Errorf("machine storage files must be distinct") + } + diskInfo = info + err = exec.CommandContext(ctx, "/usr/sbin/lsof", "-t", source).Run() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + return fmt.Errorf("cannot confirm closed machine storage") + } + } + for _, file := range []struct{ source, name string }{{disk, "disk.img"}, {aux, "aux.img"}} { + if err := ctx.Err(); err != nil { + return err + } + path := filepath.Join(destination, file.name) + if err := forkvm.CopyRegularFile(file.source, path); err != nil { + return err + } + if err := os.Chmod(path, 0600); err != nil { + return err + } + } + data, err := json.Marshal(inst.MacOS) + if err != nil { + return err + } + path := filepath.Join(destination, "config.json") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + if err = ctx.Err(); err != nil { + return err + } + _, err = images.ValidateMacOSBundle(destination) + return err +} diff --git a/lib/instances/macos_build_export_darwin_test.go b/lib/instances/macos_build_export_darwin_test.go new file mode 100644 index 000000000..0d357bed1 --- /dev/null +++ b/lib/instances/macos_build_export_darwin_test.go @@ -0,0 +1,53 @@ +//go:build darwin + +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildBundleRequiresClosedDistinctStorage(t *testing.T) { + root := t.TempDir() + disk := filepath.Join(root, paths.MacOSBootDiskFile) + aux := filepath.Join(root, "mac-aux.img") + require.NoError(t, os.WriteFile(disk, []byte("synthetic disk"), 0600)) + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + inst := &Instance{State: StateStopped, StoredMetadata: StoredMetadata{DataDir: root, MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}} + file, e := os.Open(disk) + require.NoError(t, e) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + require.NoError(t, file.Close()) + out := t.TempDir() + require.NoError(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, out)) + _, e = images.ValidateMacOSBundle(out) + require.NoError(t, e) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, e := os.Stat(filepath.Join(out, name)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + } + require.NoError(t, os.Remove(aux)) + require.NoError(t, os.Link(disk, aux)) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + inst.State = StateRunning + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) +} + +func TestMacOSBuildStopAndExportRequireBoundedInputs(t *testing.T) { + m := &manager{} + _, e := m.StopMacOSBuildInstance(context.Background(), "unused") + require.Error(t, e) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, e = m.StopMacOSBuildInstance(ctx, "unused") + require.ErrorIs(t, e, context.Canceled) + require.Error(t, m.ExportStoppedMacOSBuildInstance(context.Background(), "unused", t.TempDir())) + require.Error(t, m.ExportStoppedMacOSBuildInstance(ctx, "unused", t.TempDir())) +} diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 2d8ee7a39..96d983a7a 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -42,6 +42,10 @@ func hasVFIODevices(stored *StoredMetadata) bool { // tryGracefulGuestShutdown asks guest init to shut down and waits for the // hypervisor process to exit. Returns true if the process exited in time. func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int) bool { + return m.tryGuestShutdown(ctx, inst, stopTimeout, false) +} + +func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int, strict bool) bool { log := logger.FromContext(ctx) if !inst.GuestAgentEnabled() { @@ -67,7 +71,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, if pid == 0 { // Nothing was signalled. Linux treats an already-gone VMM as stopped, but // macOS only counts a stop as graceful after a shutdown attempt. - return inst.MacOS == nil + return !strict && inst.MacOS == nil } inst.HypervisorProcessIdentity.Set(pid) if inst.MacOS != nil { @@ -104,6 +108,14 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // If we couldn't signal the guest, don't burn the full graceful timeout. waitTimeout = shutdownFailureFallbackWait } + if strict { + if ctx.Err() != nil { + return false + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < waitTimeout { + waitTimeout = max(0, time.Until(deadline)) + } + } if WaitForProcessExit(pid, waitTimeout) { // The shim exits after a forced stop too, so macOS also needs its receipt that @@ -113,7 +125,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, return false } log.DebugContext(ctx, "VM shut down gracefully", "instance_id", inst.Id) - return true + return shutdownSent || !strict } log.WarnContext(ctx, "graceful shutdown timed out, falling back to hypervisor shutdown", "instance_id", inst.Id) @@ -127,7 +139,11 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, func (m *manager) stopInstance( ctx context.Context, id string, -) (_ *Instance, retErr error) { +) (*Instance, error) { + return m.stopInstanceWithFallback(ctx, id, false) +} + +func (m *manager) stopInstanceWithFallback(ctx context.Context, id string, strict bool) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) log.InfoContext(ctx, "stopping instance", "instance_id", id) @@ -180,7 +196,7 @@ func (m *manager) stopInstance( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown := m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown := m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, strict) if gracefulShutdown { gracefulSpanEnd(nil) } else { @@ -189,6 +205,10 @@ func (m *manager) stopInstance( // 5. Fallback hypervisor shutdown if guest graceful shutdown didn't work if !gracefulShutdown { + // Export callers must never turn a forced stop into a graceful receipt. + if strict { + return nil, errGracefulShutdownFailed + } log.DebugContext(ctx, "shutting down hypervisor (fallback)", "instance_id", id) shutdownCtx, shutdownSpanEnd := m.startLifecycleStep(ctx, "shutdown_hypervisor", attribute.String("instance_id", id), diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 4c604c755..43d633a93 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -274,6 +274,12 @@ const ( SnapshotTargetStateStopped SnapshotTargetState = "Stopped" ) +// Defines values for CreateBuildMultipartBodyMachineNetworkMode. +const ( + Egress CreateBuildMultipartBodyMachineNetworkMode = "egress" + Isolated CreateBuildMultipartBodyMachineNetworkMode = "isolated" +) + // Defines values for GetInstanceLogsParamsSource. const ( App GetInstanceLogsParamsSource = "app" @@ -451,6 +457,9 @@ type BuildProvenance struct { // LockfileHashes Map of lockfile names to SHA256 hashes LockfileHashes *map[string]string `json:"lockfile_hashes,omitempty"` + // RecipeHash SHA256 of the exact validated machine recipe bytes, omitted without executor evidence. Machine builds do not infer toolchain versions. + RecipeHash *string `json:"recipe_hash,omitempty"` + // SourceHash SHA256 hash of source tarball SourceHash *string `json:"source_hash,omitempty"` @@ -2002,6 +2011,20 @@ type CreateBuildMultipartBody struct { // Admin builds can populate the shared global cache that all tenant builds read from. IsAdminBuild *string `json:"is_admin_build,omitempty"` + // MachineBaseImage Experimental installed darwin/arm64 base pinned by sha256 digest. + // Requires server macos_only and macos_builds. Source must contain + // hypeman-macos-build.json (version 1). Machine source is limited to + // 64 MiB compressed; only machine_network_mode, timeout_seconds, memory_mb, cpus and tags + // are accepted alongside source and this selector. CPU/memory zero + // inherit the base. Linux/cache/secrets/admin options are rejected. + // Output retains the base identity and cannot run concurrently with it. + MachineBaseImage *string `json:"machine_base_image,omitempty"` + + // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables + // unrestricted VZ NAT for checksum-verified artifact downloads; it does not + // enforce domain allowlists. Linux builds do not use this selector. + MachineNetworkMode *CreateBuildMultipartBodyMachineNetworkMode `json:"machine_network_mode,omitempty"` + // MemoryMb Memory limit for builder VM in MB (default 2048) MemoryMb *int `json:"memory_mb,omitempty"` @@ -2021,6 +2044,9 @@ type CreateBuildMultipartBody struct { TimeoutSeconds *int `json:"timeout_seconds,omitempty"` } +// CreateBuildMultipartBodyMachineNetworkMode defines parameters for CreateBuild. +type CreateBuildMultipartBodyMachineNetworkMode string + // GetBuildEventsParams defines parameters for GetBuildEvents. type GetBuildEventsParams struct { // Follow Continue streaming new events after initial output @@ -19326,397 +19352,404 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", - "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", - "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", - "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", - "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", - "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", - "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", - "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", - "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", - "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", - "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", - "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", - "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", - "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", - "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", - "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", - "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", - "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", - "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", - "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", - "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", - "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", - "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", - "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", - "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", - "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", - "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", - "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", - "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", - "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", - "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", - "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", - "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", - "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", - "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", - "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", - "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", - "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", - "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", - "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", - "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", - "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", - "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", - "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", - "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", - "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", - "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", - "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", - "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", - "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", - "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", - "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", - "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", - "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", - "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", - "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", - "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", - "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", - "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", - "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", - "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", - "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", - "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", - "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", - "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", - "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", - "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", - "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", - "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", - "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", - "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", - "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", - "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", - "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", - "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", - "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", - "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", - "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", - "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", - "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", - "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", - "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", - "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", - "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", - "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", - "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", - "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", - "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", - "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", - "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", - "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", - "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", - "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", - "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", - "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", - "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", - "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", - "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", - "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", - "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", - "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", - "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", - "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", - "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", - "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", - "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", - "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", - "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", - "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", - "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", - "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", - "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", - "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", - "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", - "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", - "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", - "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", - "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", - "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", - "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", - "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", - "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", - "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", - "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", - "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", - "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", - "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", - "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", - "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", - "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", - "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", - "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", - "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", - "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", - "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", - "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", - "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", - "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", - "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", - "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", - "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", - "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", - "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", - "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", - "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", - "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", - "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", - "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", - "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", - "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", - "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", - "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", - "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", - "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", - "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", - "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", - "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", - "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", - "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", - "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", - "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", - "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", - "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", - "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", - "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", - "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", - "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", - "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", - "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", - "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", - "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", - "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", - "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", - "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", - "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", - "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", - "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", - "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", - "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", - "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", - "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", - "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", - "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", - "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", - "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", - "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", - "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", - "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", - "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", - "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", - "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", - "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", - "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", - "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", - "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", - "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", - "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", - "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", - "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", - "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", - "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", - "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", - "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", - "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", - "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", - "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", - "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", - "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", - "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", - "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", - "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", - "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", - "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", - "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", - "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", - "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", - "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", - "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", - "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", - "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", - "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", - "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", - "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", - "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", - "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", - "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", - "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", - "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", - "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", - "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", - "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", - "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", - "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", - "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", - "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", - "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", - "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", - "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", - "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", - "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", - "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", - "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", - "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", - "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", - "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", - "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", - "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", - "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", - "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", - "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", - "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", - "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", - "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", - "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", - "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", - "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", - "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", - "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", - "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", - "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", - "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", - "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", - "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", - "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", - "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", - "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", - "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", - "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", - "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", - "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", - "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", - "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", - "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", - "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", - "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", - "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", - "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", - "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", - "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", - "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", - "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", - "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", - "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", - "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", - "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", - "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", - "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", - "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", - "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", - "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", - "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", - "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", - "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", - "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", - "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", - "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", - "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", - "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", - "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", - "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", - "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", - "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", - "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", - "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", - "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", - "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", - "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", - "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", - "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", - "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", - "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", - "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", - "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", - "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", - "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", - "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", - "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", - "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", - "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", - "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", - "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", - "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", - "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", - "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", - "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", - "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", - "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", - "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", - "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", - "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", - "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", - "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", - "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", - "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", - "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", - "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", - "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", - "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", - "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", - "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", - "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", - "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", - "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", - "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", - "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", - "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", - "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", - "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", - "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", - "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", - "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", - "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", - "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", - "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", - "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", - "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", - "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", - "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", - "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", - "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", - "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", - "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", - "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", - "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", - "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", - "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", - "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", - "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", - "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", - "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", - "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", - "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", - "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", - "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", - "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", - "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", - "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", - "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", - "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", - "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", - "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", - "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", - "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", - "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", - "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", - "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", - "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", - "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", - "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", - "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", - "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV0s20qlvuPYjqMdK9a2bM+3J8yhwG6QxKgb6ABoyUzK", + "VeeveYCp84TzJKfWAtA3osmWbEn2xDVVE5mNOxYWFtblt/7oRTLNpGDC6N7RHz0dLVhK8c8nxtBo8U4m", + "ecpes99ypg38nCmZMWU4w0KpzIWZZNQs4F8x05HimeFS9I56p9QsyNWCKUYusRWiFzJPYjJlBOuxuNfv", + "sfc0zRLWO+rtpMLsxNTQXr9nlhn8pI3iYt770O8pRmMpkqXtZkbzxPSOZjTRrN/o9gSaJlQTqDLAOkV7", + "UykTRkXvA7b4W84Vi3tHv1Sn8WtRWE7/ziIDnT/JjTwzVMTT5alMeLRcnexLLvL32BuhuZEpNTwi2tYh", + "GVYiU6pZTKQgNDL8khEupjIXMXnz9JREUggWQWN6LORUM3XJYjJTMiVmwchCaoNljKLRBTF0mrDhWPT6", + "jf1gAr7Em1fprwtmFkwFBss1ca2QmVTELLgmXMDXiA2rG2ZUzlZXtt/jccImhqdM5mZ1oX6UVySRYo7T", + "8u2SNNeGLOglI78zJclvOU34bMnFvH2RpmwmFSM/LjOWUkGyhEZME24IF0b62dg1KmnsQRoiLj4XUrFJ", + "zLThgkL7k0wqo1dH/wr/oAmplMWhYXliFtR4KhfSkAvGsvpE6RW9qC/jL3t7/cej0ejXfo8bltpjRd/z", + "NE97R4cPHuw/6PdSLuy/d4vRc2HYnKneh+IXqhRdVqajZa4iNol4rNbNJEo4E4Y8PX72+oYT6O2Ohvi/", + "nUe9fm/38d5w9/AR/nv3sFed1srC10f+Yf3ROzPU5HqVB9nTNHGEMqkQyeqsf87TKVNEzkiUK8WESZYE", + "jxSLOxBdbdqj0FZEUsz4PFcsXu3bH7naci6oJlRYpjFo8IuysU7nLgImFssrMVEspVzAGq8M4rX/ROCE", + "EneIYEiRFEbJJAGmYAxLM6P9KeoDGxeEZlnCI2Q9tUN1kI50r98TeZLQ6coIy91mCZ9zLNBpabiubJKv", + "S4wkTBimihPeZWlqbLGt43K5g7tR8sVOXS5kEk9yYXiy2utb+NmtabUnrsmCJTGRs1mf8BmhBFqBny2N", + "15d9b7R3OBgdDEaHb3YfHo0eH40e/K3X782kSqnpHfViatgAdrnL3iDT1lxE4d0RTRJJ4UJSLLK7U1xY", + "tQ2cskimjEDTa0d+8PEjb71unpabCgWJK1jerIGtHjaujG7UnVBtCiaE+8XNckIDY3rDU6YNTTPgQzCG", + "ymK2cSHfYHMf/MqvXeDdj1pgwd6biVuh4HxC9MHeZyyCG1F6blIIGNCeI++CZd0JjStGtRRWTILL9Jde", + "LnSewdXN4kmWUAPt9vo9SwaTlGsNVYsfYq4tH+n3PJFPhDQTlQthCwpmrqS6qJZ0rUx41uv3FlRPLudZ", + "3uuvu7bqRI1dsIRmGttzO64mTCmpelY0Xk5mUvlN6v1aXcI1Ta2skC6u2MAK9fq92gIU7NzPxY+72NXg", + "4Po999dE2VcFNmcnszrwalurwy2Gtp6x21vECtF+m4mrrOscIOZ0LqQ2PNKd2DwKD7C9qYwDrPNZ0Rzh", + "MROGzzhTTq5mRMH9kDLiGyHQCOGC5LpxDgrRf8IumTB6cnkwMVG2uiiNh0118yqySXkjVm7lYvuLk7KB", + "SOtzDz6cLinHM/mMXfKIrcpubmsmseKXTAXYdyEAWFZoy5EtOOvAQoQUbLu2UuKSx5x2YQcxjmnCA9Rz", + "+vSY2M/k+BnZWrD39U72Hk4f9dqbFDQN0MKPeUrFAA4EDMu3j2Wrbb88CLXMZZrmk7mSebba8vGrk5O3", + "BD8SgRJutcVHeyFJNYv4hMaxYlqH5+8/Vsc2Go1GR3TvaDQajkKjvGQilqp1Se3n8JLujmK2pslOS+ra", + "X1nSn98dPzt+Qp5KlUmFb7aNB6e6PNV5Vcmmvish+v8+50m8SvVT+Jm1rBNTmmvDhCHf22JEMfuUI1cL", + "qRmJaLRgZGofLfg8x/a6kHzRsb+9QiN4xnUmkc+TdyelNIdPQ/aeRbmp9fstibk2XEROqKpMrsOIQP2U", + "MLh/Q5IFrgBxZbgUxHjR6cZCQKQY3dAdlOjU2er5zy15TVLd1rovQrggKU8SrlkkRayrfXBhDg/aJ1M5", + "xfbaXOnqOfxMUqY1nTOyhWopfG5YDk+4JjPKExZvd5Ow2ybzdzmt3Gu1M4dkMKDTaHdvP8jQUjpnk5jP", + "nV6xSYPwO8jH0I4hPG2dCEoY3eaBXSo2W+3vB7xPsBPFZkwxEX10d5mSl0xQ96T6D+y393/tlArXHadt", + "3cHFPC2Lf+j3fstZziaZ1NyOcIVNuC9ARrjUBGuEx4yf4u1OFKUNVevPB5b4BCexFDY3ro1T/YC8Recb", + "q7yBMk2GjuyoEHAqXKCVbz8HSSsgskhhmAiszks5JwkX9hnEhHF7gcLeMmPfJXK+3ftk61As/+rhh3Hf", + "gHnZH1pag2/94lWQyHl1NReMKjNltcVsuVddQ+XoWpf/tHZ8Ghco1WyynoOcciFYjDp3d7BtSZCtg28f", + "PEUX3Ewu4QYOnTkc1k/cEFeitalERhcznrDJguqFHTGNY24Vrqe1mQREyJoxg6KSwDeIog0+qs9+fLL3", + "4JC4DgJrqFjEMzuC1Zm42k79wN7TyJBLmnCgj5ikNFoAHdsmyHRpmO4TmXIDX6+4WcjciwJSEXYJV0DE", + "huTEVcSl1CSWqEPmYsYUMVIm0YJy4VdPD8thVw6DVVqvHTZ8hLHbssRQNaVJEiTp9lNyfdFilbDDhFvq", + "qduuzOLg+PNkGXTPEaHTOeR6Yf/CKwdGhVd2r9+L4FQk8PfKeev33g+g1cElVUgs0HxlWP/te6r89n3Z", + "aeXX06L/yo+v3VAqP/3gR1X57Wk5QL8qTK0eY/e6xEmHhWHJhRlwgVsA7wZHsVO7ecXj0ClgiBQVwZSp", + "LrfQOnnw6YokSLZe//B0f3//8XZ3mZDri4nmv7PJfLpW3HeDdvI9VCNQjXBB5nxO8RgOyVMq4FBNGYkW", + "VMxZTOjMQC031pre4MFok+2oVbZjqk2yM78f0MeP3r+n5vEhv9KPf0+nav73fdprU4rmumV9UXFI3W5C", + "UaKoCGzizeSLlL4HhYWlkpDN9sRa2UhRyHjO1RjCkJR6iBl/D2tuyG5tpYOLG360FpY3IcUgF/y3HHc7", + "S+hy9elqGE0HSBGh5bV8wx4gvfn4aDg/to6f6RXlBk5OYep1U+4TmcRwXc640qbX3Yh3LXmGqU8l2dVO", + "WVjQW6WI5gK2yiLFOFvPSsJnLFpGCVtl7p59Zyp3WuKYwZ2DfxZ6z65snKkAJ/Y/nhZd1H5+VvZX+/25", + "7fxDv/eUZnTKE+45c51PO1eCidNXbtqlamPPbNXXruaHfm/GqMkVC6zlGbo1EKveHiTskiXElQbiPSJP", + "To+JztUMLf1IrbYsWBukZnosFJtTFSdMI6lfLXi0KLSsoDHRhIrSm4E8Sa7oUpNMMc2EORqLcaHY1+Ne", + "n4ztm9H9banE/m09WnwhMVdMu3+AXDTu9aGtRM71uDckP0ptBvDKdyLgERk7NZIe98iW/ZNkVGuzUDKf", + "L8YipYLOWcqE6RP0LUHtNXCmZLlNqIjJuKekZsbQAUvzBHk/NPYkyxI2Fmc84ZEUIMu9OnN1QXIjr22l", + "yuWJM4bLuk8ypzHXNGVjQa32FnZySTIlp4wkNBfRgmnCxEyqiG0PySlTA7/Efm8JVQw2w+qeSS7giDBa", + "bob+5VfChFHOQaM7c4GZXIf+YOmLt7++Ts1jW+NDYdS5TuWfXRVgVW7GwaeVWpZGgAo92xuxsFD466hm", + "wOiPBa4o7iqHUleC1LZsltA5kkrlEFli1GMRyTTF96nrhWh2yRRNih2CO06iGgTuBncjjUWUyDweLJYZ", + "U5dcS9WHG4JF6EOh+uQ3luZ97BT+GqQ8UvIyJVKMRQJkvEPT+PBge0iekIRrIA4/f26J25EYcgJUY3AY", + "7bhHvSlh3LMT45rAxT8kz4VR3NIc0ZbgpvYmbRBX182rMKuVuw335zqNndkaK4YZ+7Mj6P4Ki63QTUmA", + "BRlXuGjozlrDgFdlcL+u621obnzFbkVUuJ2qUecR4YbQLGNUacLFWJTUJGKkUlcpUwyXQ3Pjti5lhmyl", + "1CB7AWXzWCCH+EaT2u5vDwnKjdatreRIUIUoliztgNhY+DFf8SRBfSfJ0fECOZw9aKjQqzh+4GGiouB9", + "SWmkczY7Z86tTaB2RGzf//385C3RS21YSqZcULXcJlyXvNaS5qo9MSwyut1DhQOKavVZu4c21CVS1AXI", + "5nndaPlwImhJF5so7EcZ8gylKlqEvP+0IU9P3xL4zA2LgIhr46UqPQxawKRuaQ56pSjE2vWuNYdMZ+Oc", + "JRwpHPGmyR4X90hYRPLcOWCkg5oV8zNLWIT6FPsIsoIqcTZx1Lbolc10LLRtiXzjelPvmmxJvQMT3q7c", + "LHCmVS6G5JUgKEaQmhQxFliUiyjJY6ZJhZ+TUuvs5Yua91ZB9mPxr//3/7OyREwuOS2kDfKOKwOuL7/b", + "V+xM0ZQB06tfadGCRRdj0RRDCLSqJaH+UimWmevqhaL4fGGIkFfDsTi2YyoGvKXlzFxRxfIMnplkMHCj", + "Hjg5a9v6nmFD3DQ8S39p7E1lca7pAlmzQhYb2l+lsE2k+nMpttRpdU4Nu6KBB/ELIL3BJdfoaYc04cqS", + "49Mhwc9oAYkW9itwUZBiyZalCi7mY4FfnEi8TZxEa8nMWVSH5JVXJgLJCFlcL+6mGwvwi5wyJtD4mVxa", + "v+mSVJesuQHghLoLdunhbuhozGHsEyMn8zyoLvZ3XcwViwx5dzIwcvDuhBhFZzMeAR1lTLlR2wumOegg", + "PwfXiKRlpX1F7/mRDOGRoXg8ByEHlsuJ/vY3ezllTMG43jw5hQsOV/9bMu4JamydktMPwADFYxaTn5+8", + "gas1enW27VbNPU1twz24dkK6+35P51PBTNv47Vd0HQ5vBXoAb2K+dolWtmgTeX+cRAMXv23gmzUyScnB", + "QDoBIbYgwSF5WZNfbYXC3wZE0FWJvmjBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKVVUu", + "LJey9+Ggwn1RLLHyCEpkRWzBJfz/4FLL6MJ5pthXZsFpBU3xdFeXbkWy4qaTkLNRBVB5sASW1PdeEzzh", + "jaAFzfRCGnxV+3/sKKaNVGwbHufOzQu+ZzTXjPyFpCyVajkWvnzfnrOEGut6UdSdAWPqka0okYJV9QjW", + "74GOhTYyy1jsDBPfkpk73LTwfZSq0FXbQmPhutA7VgLQpJgFbkBpOEqW5SZXZrKNqgacTTEtmHOe2mEv", + "pMmSfD5w8+yRrYRfMjdtmCD/ndk2wIoipRg4N04oWupO8BMxVM2Zcapnje0jzYx7Y7GF55bgv8GwkuaC", + "R3ibY7l5lg+qKo4e2Xpx+nandLUai8pnrAJKvQGXg4SnHNjbFvxAjndeEQU3NP7MxdyOHgvb6RSTxPJu", + "ikPyjia5O6dI7bC+Y+GlfxzpwKjcLGZ5ckQuf3dCWH0/ip0spKLd/b7TNCWoYWqVZN6dEE0vmSdIrAVP", + "am3gGNH4kinDtdPGXpAtR04DZ+dCutNEMBYTIcei2hYcZCRTGGJT5CdWEaNJY5HGohTbGg7w/mrzls5C", + "FeEFn+4am/UvmhYbw8e/Wq7xRD4rHvWNiyTj7cZg0EXiQUHbqVumyvuVkVcZE1AqllGeMoGGUOmL1p3f", + "RsO9Noe6lu7PqtdK0b1eSGXInBuiGIhxUjhuNu4NYq4MHH+dz2b8Pb4ic9QCWZkGWwIzryLjXi4uhLwS", + "4159mA8P6d4snu5v3A0/7H5tDYP7gEzPaaVbwwzX29CerjWYgahJjIS3c/Xl7yj8E5jLykgquPTUAM5K", + "wllcoe2+dTKeM8EUmtjRddUQL6G1mHwGQVH21q1K17LEtOyodfht3dANc5gncooXnp9I6Ss7JMf1tesT", + "ih9QYK6vchHkUfFnJVtsOB/C7ZJFfAAOrQO6NxiNBqMmtScHA+upn1FjmIIB/j+/0MHvTwZ/Gw0e/1r+", + "ORkOfv3LfwQf5B2dbL1d281zy5+lPvGDrXreNge6ySv3xpa16vDbDy+qFlp3OlIMjwFNNo4BPA6eVoq3", + "0/rT41VnPbtSsQQ99JDLnYRPFVXLHTHn4v0RiHPa1Ndtfdl1CpZATI8VjHwBQjVxGpZfdi6p4lSYX+0Y", + "ybiqHAD5jeCtAff+MxwRGQx8O459+YCBmgFgSH6WhlBnkYYn3REWWu13rmiaUtR2ll4+zsymecysclbg", + "N3JppSTFTK4EORiN/JdJZVAgN0jtpzLuOcWpihbEKYQUQVWIDeyJWAZdGhnTRnRVZSlWTxocs7/sFH9t", + "+T+Hv/5l+3//xycldSS0NTRu1RnX5GcN93QoRLYSecVURDUjCYPZ6j6J+ZwbbYW3GD26UKH2LYmss4f1", + "uJSKMOFV1FiuTs7pckAzPnCaF2vufsnE3Cx6R4f7wcXdcn8Mfv1P/1PLwqo8Cb3XXsscNa/4uRov7sfQ", + "yfjiVzdP0OiScnFsq+2uipMft8N+Iut22j7qWrca7pgimGnDQFaBA8DjKA0JEZdMKTiKlq89PXkGT5gL", + "5hgavBrJOB+N9iMsgH8y9wsIcfCUwd+26xKPbS4k7PzSY9FCovdmksjrKCn7Tabe5ty41kYWWu2S+6+6", + "QqIR3ZrF0UffFSRTJS+YQpMMLDFnmlywpVUXzGsKTSYuySVV3jT4BqMpsIj/hNYghs9jeMQipgCE4DLl", + "2KJ7s9noLUaTklumFMxd+K7UkcwYMFRXDKdGzpm4PCcpzeyzXjGrNE+pYYrDc7FULKYs5sCjx4LhwSAZ", + "hTMfRVKBihX2FnWw5Sp8o8m5fSaeY/PnXABZn9uD2VCV/tF79fbN96/e/vxs8ur0+c9Pjic/Pf8f+NlW", + "6h398kfPYoAUjlffM6qYIv/xB873g/X5RZ8+iNBfSOVeu95Ej/4qNONDmTFB+TCSaa/f+8/qP3/98Kt3", + "97QBh5e9o+DAPgTFTKf7DIQO+LgtTaQqHWYpwpkgi/Jqh4q6oaHKt5LftY4EPlG4nEwzHQypaeotSjl0", + "dzQ6+X7HeY888P/YHhJnusXhAw+SyonHegHkU8CJoD0tSWTkAktmbcgBvqsQg0dbaya5MBuZU1l0lUcN", + "BuXXa7CinSkXOxq2YRBdb92Rbm7sX/1cXHIlBT7PUVyaJkzXz8rPr549nzz/+V3vCC6COMfwx16/d/rq", + "9ZveUW9/NBr1QgTK3mdcMR10fHwy1TLJDbheZ1yVPqZDcpJrdO50zGOWox4U5TNN0hwUS8mSsPdRkmvg", + "UygNGJOEwqYfDfZ23+w+OhqNjkajv3V2WAXS38C8X5y+tYH1eN4ZTcxigqa5TRV/xLJPseiHvtdN4tM+", + "sErFnnptJZxeV4dsLerSlZ0cQYIc9/ZffG8P1O6Lca9CiC6CuWjFNtxYvRffh9aloo1alfSKbzYwT8Se", + "6lcxdKy+PtdOTVS1hA8ERWQgKBlTFZOppCr+tu5LgzVBkf+NJqitoImzE1nRG+tYWVIyjW9l7+NTKlt1", + "v7EEqPlJrSqRCsLQUnrJgeUM0pRLX6/OlpwypaZDHVYMTAE9XsVdCBwwWZq7//gJ9vq9y9/roeCb1YHO", + "NabTW3HDI5AmGRdszSvwMxH2QaecSBoPdj+xrF/xd2ug59gP9e0uHqeewFdAqaZUxFc8NosJBPLDkAM3", + "pPtCisLFNfkeZkKTf/3jn+9OSoXI7otp5u7M3b0HH3lnNm5JaDoYIlRMJM/C03ibhSfx7uRf//inn8n9", + "TsKKlDcSz93+P7ctNHF1HNyY9cZvsbsWclgZVixbbOkboBdCV668ZCqhy8p14sbU2x0hT2+MSnFrbnT1", + "Kirk9ZcLtObFter1Amx+xllifbSs6y2+L7S3EFnbIZ4W4EnfaB+BxMhUSmNHAGvD6AqkS/hW+jiV1NDd", + "HJEUhnLBlBtvcWPYK8W+WNLDgyF5/j5jioPARBMSU3XFhXV68TXdm5tQAThGCRdswFNnwbVLYieJPBka", + "BkObdzjS1uFoVWuPyuqYuPvRVpaqoRC7Tc2SYsiYJ1kBALju+Ly2pctHf4AkAxT5PdWFZbYLHRZkuLt3", + "4v7cswRZJb7icNmzxlVBfLarb0EMhaezwU61ReObMgI9MaoNOSAv+PeEC0AgSBg54d8T6m4nLsi7v9l9", + "QF6nO8lS+oJn1s9jQudFNO06wMKzC5659zzW8F4G9g6Kc2gZz9BwLCwgEbANF9/IIm+7NRBEoK3jAty2", + "eJ+tymigHShvNFvcmmlVLvpkmht48kvDiFO+uHBIGAsWnjKSl06kjQe4m+CqYwQuywVTgiUT+8DWHVfG", + "ViKuUuvi4FRnVDsANWXyrL5ez346OSNbz5aCpjwiP9lWT2Scgzeg5Qnb9dXrj0WmEJUGOoGzwF2/ckZk", + "bgZyNjCKMT/EFBsr7DAOm+Pyxelbh+6iwUnpNYOFZSJ2Dq9e2Ck8XsU3xprCa81W+28sehuCjzfvdzzZ", + "Z654ebS7ayT7PWOSECZfYtkaBBHZOAowXBXU50P7+nACX8gSQsJygyF5q4EbjMDtRSr0WiBTaRbV1yVe", + "S7r6AiqRP8ti8P4MPzHL12zjfB8sgnbrKMvrpLvXb4VivLQOEyDO6K6c6xIKVxgX4kQFONdeR0YVxMhx", + "QT6rO2ZRcat6JTvAcs+oqeOIdFWF25YRw3ajd6jTaOOSdNdoV3SsK7ptrwD8o2PQxPr2j4UfyFqNfqkD", + "/Ii+zmwjzSVybff9zG6wSsfFmjTsAJ9meZ7ois60E9yslXmsPKPJ1jmoWR0dg2L1vE/O/7P2A/BTL0/B", + "c+GK2NVAHi3gp2r7esWzeYMe98bezVTffD+e6FawMHK5S4yiQls/+wXN2JD8iBcjMSwFaRFuKl51LRTy", + "6lsi7RvFVx0LGJqu+1p6bb7mcwGuZ+g0Dwwnjq3K32nsvP+NDqE0ez3cisRuR+ewlpCXOW97cu5V7+d1", + "/ruqmF/VWzlN/YrCAn+3igpU5ZidNDfQvX29OJu0zI3FXnNTbwh4dfX/JscgN5ZCFXmD/T8r2EUT+voy", + "oLGAyclZaa+pGI7a7DNO+Avbji7YErfc24noiqWoaiIKG3KcS7sTZSpVUXdorxKuK/YlaylyxiF0Pq4f", + "0aDZZNNWwHp1Xv76yz8YhzPwky0pxj3mPayq50IwOdsfKh01w8VHTcKRDes671vVB0P9IhFALIn3zW82", + "j27J6MR47n4ZutbOiyDBTwGVXkZ/VLeWVHbWi9LYjBfFDIv7ddkAQLX15klZ99RFi9lTsSvFPSPzuJwd", + "RV4XNZO6h9fH6YGeVxpb4xDWtYnVKFm7vpUxO3qyeNQ+iMTuB9q/akDNzXwFceMZbpFj6l2e0yQ5J1uu", + "0DZR7O8IZuv2Cjzsiu0HcF5HAoW72buT/lhYLnC+MCabwP/pCZzi82Zjrq4/4SWW/KMRiq4HB/v1mA07", + "4EazdXV5EA2nfWvA7+qW/LdalPIetB8/FzcQIN6QLUPncHQcWhPcE9sfq7DnQrMoV2zzWX+C4pGD3sEx", + "JZSLwY9v3pwSxeZcY6hxMKjBusa3QMs5ZUHpruana2Rda7u3f/Dg8OGjx8P4Qg1ZpIa5HsBDZrA7pCn9", + "XQp6pUHs2kmXNMuOLnc38nm7BcX42lm+f9m2k4JMgSV7h+Qur+SnZZXSYHjBRdy1gZ+g7GanV/+Iv22z", + "S6bYIM/misbsExtdbuzXhKvZvqsbErisd2jOtQFFRIlcvLXej7m+WJcyGcTU0Ou4Mdvhrnosp0vblH2S", + "h9prdQ8/a/qE13B7gmkcPtbJzI8ltC1tUMhWkcDiiZGB8F0vOxw/gzX3ZTthviJys5GTyxmXm4L8YWj1", + "0F2s7tUb0MQgi7jT1HmrBi+0/lZZ9+6k6lszHIsBSmFH5FnRQdFs0STFJwaNrZV/S6pa/DAaAqYAfULe", + "nQzJm2K032hvFLBjIkXkaI4GRRSKBlYkqw4gx0AbbprVnVrSwkhvowuRdN+GRbIZVIAWCjTUl015Yz42", + "rhs3yt0lVeiZjgrhdehkr+19tA6hrOYR8mAw2h3sPnizO7quR8gtIGWH2npS5y3O97/KfZ6+PX6259Se", + "2zcGJfvkWNphVvasDFogW7lmlYBcqYKhCpWIgJZQhBtHGNxSwECJGrqurF2JN1DyNkDCQ3hgWKR/Axjv", + "JsPciBVbmdyqQmeZofqhPCUVQd7uaBbxYOQ1uAx+rxi9wOQHq/c24lBM7H0W9jdEsJDpEsGxUMmlpDQz", + "TQoMmVL5vXvw8ODR/uHBo9EoAEO9SvAy4hO0yHYaALjgJHRZQBtuoRtDTKaJnNYJ/cH+4aOHo8e7e13H", + "YQ333daheH/7WmTLrchffII6/6U2qL29h4f7+/ujw8O9g06jso11G5QrWxdJHu4/PNh9tHfQaRVC77rn", + "Hha8KcCHclQ8sWmd4F8DnbGIz3hEEJeOQAVEDliAGd9boetnckpjn8kjfHcYyhO91rvPduZKWn1rmieG", + "Zwmz33BDOpkkcObPsKVgajQhiswp12vJgalv9PvycymKoDgTs2k+nzvIv2LpTlw0fyk8cZbER6R4sq3l", + "c7ib5cB+baMDN4eO1PBSXhXgexUisFeXRapQjBR0YjetNisfLsRFlgdJonUpf8iVBeHARgmdytxYC75L", + "dVN2YvFsBFq0cxF3U3f8INXFxvgOuImLjD4dFQY+0H+GQACNEP+q0FdY2q0/gvuuyWtbwyoKy5+zvJ7O", + "sI89OYWi8OgEcaE3ds10lS7DcgvqzL1Tn+2v5J135NE4mFknsE/7wlZzhpmszEaJBSjF+k+dYfHO4WJQ", + "caMipcO6C3Z1F4uOAY8DINsBqFBuZ8XXOU4XuoayEJEu3ADwOWHmcQVeqnHSzjzihlvy4Vi4eKviJ+cU", + "ABXtOlhHAqn4nNc7rutZ79B1+TqU6YnrxtRZrdjmM6gLM2CIB1jA6Zn1BLbei1W8W7cnvX7vrEj55RhT", + "fWn8j4EVKQMMVqGHTt9e1wE5U3LGQ3hA6HXkvrqHmnfNfXkwOhvs/rf17gTyQ4mNC+upBHaEYSNtE5bv", + "dhG9OH172jamImcWqY5uZU5rUI5Kz5oyKqZYHBdrzrVvuZJfmApi6AUDDROeuFoxC9/F4iF5an1tUK7h", + "M9RoG1/MwqsAA7J9X7KjGlQO+IQJVjRqTalxDlRVxXmDYiAKaG/1RIy4F6dv63gNj0PiNwKuTHMY2CQN", + "6AN/gO/EFigAI06+r4vgewehpsOM+7RGQPjSn9EIvAM6U0hAh9iYxiZExhenb1+7PFFtWNVATkUuKQtT", + "PSQ/F5nUYHl1iUc1DCgXGyRoXc+hwEQnMuTA8oNijOC3MoMr0A0avS26ceFoQGNEt5LC55EE33+HNfhb", + "ThUVhgsWk3c/6G/JyFnZ3/1AbPyR41lcV/0u64/Lh6EtbY0jPF0sNej27LLYXC5cVBWbyAU6v0xOy4pO", + "BRx4n4SzCHqOQ7YgdyQu4NnrwfGrdztpzC77tTHBR+ul++L07XaFLV96MIuibJ0bX7ZpmPxWFUktJ2Ui", + "zeZBQBxtp/psUgfszYhMWUS9U3dz86Ii7/QU3TVQUQZmbsMEUdTpqamoNYKsCxzv8yQZOp7ph0xQMz1D", + "1l3BfrW3m2NqimUUH1+F5PXeYKeM6DyKGIsbHi/NUVfo7YjkIqVKL2jSnNxRAbAQLaiikWGKfPP+m144", + "cRSy0q53Q4U6i8upM1lWrqIAPVbOXdsRf/dD/Xha6cCGVTudOhfcIEwcYoNvFYO0aOaQN2bpVC8ZXrOO", + "Xsodo7OZBUst7hHFEkY1JnzX5N0Pm305jTQ0aZvDG/jo+NQWTAhOEQyzT7IaG4Dfqyhl1W4Pg1dGOZ+A", + "ZQo7bJo5huRYhNmedRhCnEELGaY1nwt4DmrnMwT3qV/Jlb2r351BTVITl7FXX7jadPqBCyBEMaHbysaB", + "rr6MVhPQyos6m5IXm5Oe2kba+33qQ1UbpjD3/PTnFiNavfMSsVUdx7AKVc0yqqhhydK+RwrqrGeDYKt+", + "KBAtcI1Y2edQ/IPNAJQrNjELxTT4jNUUFPv91eTXGqMSLpnL92fnVDHXGUlSqi7wlPnnN8mFXYF6XOz+", + "JnCshTHZNSYFThVWJ2eYuqRJM2JLr/jlPGMQpTVl5oox4adCNaFVr/hmxLtuybkGsTxMcVlfw95+oN8z", + "GylB5opGjNhaPnV+wdbg6HVdyt5+y7DgwtG6ZX931+2vqzrLk257HBrW7qYdNtF1NvjN01OfhKvIze6X", + "eW91lSGRhD1yrsaGrd3T67PY+a6ERdFekdmnDNPauVizahS790rEtH0CI/YqkeMV5qCrsc6uHzwFdqn6", + "9pz/2ulN2DzuIfcbiDQK5bjHDy5Kb45ggxiboHJ8efHYSkr2Tq7czy6SCiQtLgCXue73HeUq6fV7g5mb", + "1dHODvD7BPFWD/Z3H+2sd/5e6/XvnBwnMV+nFXKlnLOchwCwYJU46TpJ7NAs66A3x5qb7gdkTyubYPPa", + "w91WeWT5EBew1zScFzG1ni1rFek1RwlaPbaIil4XZKDB1Gapgtfu4+r5HAU9V8ANtU7+Oyu0DzOznshA", + "I1ZJ2VhHS+S/B6VTqULpFqUyLlx5yvyjvbgPvSOwx/+umdRHj6qzPHzwYP/BJj6EzKZxzN25C0zVlq4D", + "MrDKcrvz6xqA/+i6zOG+rCcrXJcNNPXmaYCk1i+ozJi41no+ONjfu956dp3IsXfmbPClEJ4VAFihTFRG", + "FqfM0JgaWmcyqIEGLoOJsShLMXBw9u161tLi9VQFqLppVr9P5TPTkpf0tcezT6ngM3wk2ZLVnvWC7j04", + "PLLZk2M2O3hwOBwOrwvb87z41m0rdqxrbQXBZ6gXH7cPt4DO02Uuf/ROn7z5ERhZrpW9tHb0lIujyr+L", + "f5Yf8A/7zykXYVSfLgm3+Wwl0XZte7Pcwaaz+AhmIlwwlpeUOnj1tZiQMKYAocSCGJk13+rbA8MsaJzX", + "c55cAxRzTSg/iCugkW9dmxvnzhaFLtRUcmZX9Qod8meD6+c6zw+vccYyrk+r6ShSi6/6w9woObxem4h2", + "JU9hxkSRejZJ7F+RFIhKHspDW7si/bcOaQzxGnH5Cosuqz8WvVd+fFodSOV3nwax8pNLSPvrNR3Z1gqk", + "f12VQzdzIS+ObjjMYfV/cSt0zVnuQDSDERz3fBfexNmz3vur+X/99n/06cO/7/728t27/7l88V/Pfub/", + "8y45ffVRAE3r8VPvFQT1k+GeoodjDfy0KymdgGVmlZ7g+deywu6L1TiYaIHpismUHYEL9ktumLLJLxsh", + "y5CvwSYhx1pECsxS6NAstqHyqfVDgMp/eDH4Q7ON2MFWKLchBVCSzqexTCkX22MxFq4t4iei8V0Af8Uk", + "opnNYMkFKGbBsVthulFnGC4775M/aJZ92IaUKNQQ9t6mAsgoZmycFbYJ4d063Kis87orzgq8ZRtHPBbF", + "bV0AiFrvgGGZbYSzpBnp17Io699v7uX0aBSCWsUoNNhIzKiFT5CCsoGMivA48mi0vfqe2/DGKGhoDfk5", + "9xkbpgxhxgFSbAltfsZiHiFf8dG9tmQlsNpSmlPiZUq+X+LevLYhpzGhuVkwYZxjZJ9EUl5w1sct7RMw", + "zqK/Fta0XjgLmQ2my8FCZgUMS2ELt34s9Uf2/xm4iQ7eMcVnrqcgmgaQSEDoxCPjZoZlSu3CysTe2GRE", + "QqNR3hW3Wa20TWRkAzBMroTPtcMw4y6CDRXUx0Iy+bckSjhqnfQCbXgLRAs10EwI8LM3KgLd6DSK2az5", + "77oj0t6DQ3zB+n/v73WOM7dLt47K8iTwpk096+vAsS2bxAFY8WDiFeEbXAfhBrScmKCewkj87xnxDZUn", + "rmAk1tJnI1+1S92S6ErM63YwONEdAwcqNKG56Tq71VNYi3Xs0IL1KcJqSQc4o+c4D/Lm5RkxTKUe9WIr", + "gt3BU2KxawZc69xlEXzy9OT59rAXRImrmbRwq9bGQtYHHUBIseeg1XWq1NHQlPXJ8TOMcXfXSvkWw6Ck", + "H6Qiib0Vy8voCOF86uoeavOPHj9zAmiyLB2VrNgy7m37FrPm9XZEXvtuCS2GUkQrl7TlmywvE2zWua3a", + "iKmV1hvB7Wgfc88/dx9jfBQmZ7W8GJFuW++v7jpHt+Lwsakhu/aFVKnYrv+q7P2nxoT/9DL6/vVkdGeF", + "nmQLqkPUvahaNbHQyr5XDdl1dq/aPf1q/U40D56tv/oUVZVryDC452rVQyDIDwa7u292D66vvrsunHcd", + "qa+C8VkgeneH4r4NSOsAQDU3k9aQEAKfXQCI14u8O4HoSkBeg48N7cju/sMuSgnstWswReUjiNQ4pIJL", + "eeS9IgjAAiBe8CSxAozmc0ET8phsnR2/+On45cttMiCvXp00t2JdjeD+3ADZG28BXEfrHBUAX6sBeZAi", + "4vfNm5fwJUoYBk1ZOfzi5njfG1WLHfC//eBenL6FGguqJ97duj3CmJZR+uw910av4i52ilr4GLxxW7VU", + "jHWZpG3DOVVuBC3/sYaqHUTx3L4FtHEfcrKynPcAxH2fob2fHwj4Wtjuj8XednqGW4Lebr3SQrDVDRyT", + "B223281BtG9lODUYqBDbqko4HnfhxqjV/R4PxJw/cX585Pi0TFNXGiN88405Pd4b7h4+wkzLu6MujD2l", + "0Zq+T5487d75aM/eMkd0ehTFR/hgv6nNyhG2fYLQ5Aoic8f+kTju2Quz8rqtHFtbplvMxSo4+M2wwJti", + "XAssN4qz3nFJT9K1iZI6BBU3oQ7T3IHGpjxJuGaRFLGuy8gLqonOLFayTRhUSPBjgQPskyJvO0ophEaR", + "ykvVo5OunbyfZ47uMWlxJoVmLmvJT2ypScrRCFp0j66PmhSxa/FYbCkf51gENGK+4hh+wKihPnFPoT56", + "DWNiJKgwFnqRG2Bi2xCAIiC9sXJaWTLlaDHaJjq3T1ocL67GEhim5jFTYwHFAmjMfxTPk6NDQEbo94qX", + "HCR3GY1C1HSrxs+hg13HFjAEhgqXTg+93VUuSC5ipshrqZkxlDBLDk0XuWsaTj8S49xX7yZeueqlXBU+", + "mJsg0TsJUx+N8YxDbXmfo3foDR7nD24uoneKFPTyq4sRdLUm1/FgcCEZ8OKbwm1nFXIsdmpIzVysRxF7", + "89YmDq5P3bkeGwlGdbUk705Oam4PLkKj28SRS7Tsg8yutQ17G3QkG0dzHfNyBRF7Mwr2xyNTNyWVioT4", + "yXGoqxZGD11gKbSm2Ko9nNfp1+BVGYyB4sLuExDNmgk2ckdboJfSMbLaql7qmUNldaK0j6lwiPQFaEtp", + "1NdLvTPN9U4W8R37Xe8gos4jRNQ5CEIexOxykuehpxF8cv2Qt28B9gZ/QURoGyBT657Sw91Ho0ePB4+m", + "u4eDg3i0O6C7+4eDvQd0NNuPHu7v7u2vCSrqEAN787DW4Is54MdceK1PvPd8yKm5LXahIZs4f+wrLmJ5", + "Vbv+gg6y1d6d8+2m7ldd6zsPIRgSlFBtrPqihZOd4CXPImgby/k46yJPXFjRefhmtPux2h8cXMsd8Ubl", + "wppVsUBpQkgrA65uVn2cN2P5OCAf+LJptaqdd1+00dGDx0cPPnbRfPDGpjE2yekON7fNI8wjpzeiQ3yQ", + "cEWP5BWUPScTWa2+Lb7s9Xu5qP6NwkDDl7r43CmIq+3A9sNsZN210gJ5cFx7r6CnikXOjI9AUvHvEcz5", + "UgBrgAj0FHRhpKKLs0CCaIc7rrxdoBk0izkVnQ1ctsEYGBnJtYvn4wIYMdofoRGHj3BEXmBZ/ERT+6xz", + "g7CJu6umNxovrb8MnC/ftX1krR/ymXtfYR14bBH4F04blsGpbNc3YaWzI/KzxDrFa0/Ipu7XFsdn1mrx", + "Rlmy5UD2Pd4NduZEzSPyQyFeFgKqE0i3NHN/ThzDKmGmtmtgH27HwZGwsnMV4Ip+z65or9/zCwV/BqAu", + "3pZUv3L+qqQYciRjNMGzXIbp54YnDlwfZ8K14ZF2wSOwuW1ij4vNZPHEPp7afFJt6KstQ4pKXqp6d0K2", + "ED/1L8QptuFf24X/au2u23t88Pjw4d7jw04oaeUAN4vGTxGZYHVwG+XkKMsnTjfSNnVIoodrHlmtQuH7", + "AtaukswzJYH1wMx9g9XOHw8fV8HhYpkDHy6G5JAkLRY1blgQ/7DgRS1+kL/x5JLPZuK336OLvb8rnu6+", + "P9R7090W0GvbUVjtdlx1LljRUbPpwKb2DON3IUEp3Qpx95ppnAE5Y4Yg/QwIjfB5U8RTO5LLdW3Fg4R1", + "sL+//+jhg71OdOVGVzk4E1TCBS5lN4KyJMGSZOv12RnZqRCcbdMjwWQBmIPGOSMuQfuoLpAOd0f7ISpp", + "ubhLqnFtX6atS/7OPR/dpNyiY/B28bRcOeXB1d7fHz08ePDoQbdj7NTDE/V+PYdx5Wx/Pn1Gdee3UJp8", + "8+SUQOtqRiPIZf/E6oNHrRkVc83Iz0/eWGgnIUnG1KC8EX1L9TBX52t2rfmZa80Pk8jY5A8bpljzVHl4", + "CNFbu91QH0PuKw7PtHb061wwcHwD5BXY18BSrDLxftu9ExLMLKm+ZlFCefok8sEajXvMJnmYKFus3IQu", + "V4x7y69cgR3qdlKTFcovG+qDdYhUJBdFaqHhZuPuJ7HRtvN/e9Fsvh9CAT4CVs/Bk9l0jTdYykyxSy5z", + "/QkaksZG384SKdW16rY9fV4zDRY8D4r17uQbTRytEW1YVn+NOWpcA+J2w8ld6zzXSCRM5G2L1Wk3umz9", + "ugn3W05tfx00R40btEInxsC5crHZjfQpTaIck2nRYj9hVnhRILhAliVLGyWQJFIKEi2oQHOLcqn7xJxQ", + "AigMw6BPLXyZzIL+GfKKJNKCvkOyH5dnyg4CqoEwBJqlrWo2R0tKjfTPD1LLZFwmoTo1PkjDmWrD6ExF", + "UD2sJzWygkduq9R0qomca3xeGox/GDbTYGRU2bAGKmzetMvUvkoDrtuBITaYeehGxY9wMO1T2QkvGLJu", + "V5JGSuoKsNi7Ez3sGj1XxENvdg2tD7YD6VrTaOAqwytOd06vGLofA5FBH3NDIg1j9OEap0uv5kypyDHz", + "VIWQnUp/2NmxEvTakwIw65qD1WaC6WRAFVjYwIr4/UKz5MsE70XP2m6yXM6D+Ua1V6gq3FTbANt5anBF", + "w6vVL2gwRMarCHVrQfFKqMAmLNZ1wCrL/CNcY6u8gkFItjB6psKWKliS213cbcKPX+hn5d3rMhGDxaMr", + "RuN6SMZTahbHYiZXOcF1bLBep+38XjOmPExizARnsX+VFsZYpzTj2uWhi3PmVg67rQHpobMExZeT07ZB", + "sFCd1zc77KJotmNYn20G+3UFO+wk1+HQ3Dcqx7VCAgLiKYN0O7mTcj0JG8ZWG1ZsnidUIXRgxyHrZZpw", + "cdGldb1MpzLhEYEKTQv7TEII2QQ+6e9wLtudZgcVJm2pyc7s4FyEod2QRr/lFL6DWW434ptRp7Nj6+9A", + "/U5+aEGf8x94whzK5VvB31cIvZ6O4WBv1BZ339JoLeJ+Feb1upzbkWzwxOc6EKW4VsrxSdVY7HJkYGlM", + "DodHrcOt5EGYvTHxZrahegjKx4GcPLX8ugFxQqYMI4j81Faa68IWu0wlmFoGsu39XU7rqtWuDsSBhIVb", + "ogTbUGwWjBTAHV2r2rYlVtaksrvXQbNAtgoTxUrXBInYlNqx9NRq4yevV7IcLphbMu7naDMedsjg4z1J", + "CiAE12t3RIRmPsqA5zPi3WizxMzOmE5rWUmyCmnKlEIEepBwpPCzQQAXqexaewQtiOteEsVSysVYcFGo", + "WxGWjRHBLpmqxNtKBY+sOQR7/LXyxIPfWZqZpUsGgWr4bzQBY5wf41hUBwmNgxp9SJ4Iq1lUeWZq6WJZ", + "7F59QCg+Sb5iRmH+VG4WZKaYXlTnHsqZCzLelVRxazKyJfFFYBoUrbXEyAsmqqysaCb4NLQNTWytVX9A", + "m/Aav7r3J6nloCbNHNPr+8s1U2EhsZhSUaSTE0zlqFQeJ3ggYQAut6j7y7L4Ak+lA3pK2fx/+ybLn06L", + "xuu/NYpVEFI8ZvgTq7YNqmCjLF9dnYZNbqPTCwbUrcPFWbVKkC3vjO0TNOntlkRJne7JbjF9zbADP5od", + "zaJ67wePHjw87Jip6qPMfhYH7FMb+S7TNca9lp066WJBevTg0ePH+wcPHu9dy8LiI1Ra9qctSqW6PwAj", + "YuCwJv/6xz/fndSpZe8BenOPrjWoPGsf0tusw4DenfzrH//0o7rxgEKMZhXuv8UDoNXfJ6nupHc5qBsD", + "uxnJ1rzvn9SUBLRgM2SLIag3uO/YdRuUg2kAinSTgmlGI26WAUZOr3yCC1ekgdrdofXGYEMir23b4asC", + "59L5tCgBPiyuwH9aK3ODFh51Tnin82mbRftVs1csR0qrRdVbooOzgqWIsIK7mM8V1TXX8JhZu0O/iFVb", + "jduxJdbj9zaDKmAoCMlbaysKZS1oyJOuUnX7G9tZsVvW1DrNFf91zTlsP4LX0voGbuSA0jfK8q4NlXI7", + "XIA3qzWZVlNRrs31WctbWd661++3QxzyatKT4ga7fn+V0MvrVGzQmqVHNwa35GXb/RpJtFBTJaomoECT", + "CRtU0iBgaaJzaxGEM+8A6wOxoNGFnM3qqLkP2lHWEUAIw8Z8L9QYeJn0QevvdBZNiG6LFjTuPdDjHpEK", + "QjLTca9htgoGYqb0/cR1UEeJGa2DPS8D6RuD1H4G00RGFzbLIibvH5IRSRkVmuQCD3/DqrY7Wm8d6vey", + "yt74dcTH3cpSus0kU7aglxwzszibyrzm0snec6PR9RTbOSKxtLhRtRTTboZQzIZJHpWTxkuHiqVrGBqE", + "clJ439iyLCr4ZpjYWvzOlOw72APg2K9enfStAwM6MdqB1Twl/UTtCHr9XtlFI1FD+XvYExmMFTjuJvB/", + "urqO1eh2tKwqppnRDgm8JIcGEUD4kjDNjABpt4dcPUBt9UrKBboNOvcPWFDfuyUQErMIT6RePYt1Qr8B", + "cTciENxKh0IQ9kMkjIdiTSaf184g3ByAVTZUssPbdqoe4tZKONFGZr36qZ6w9xFjcRM6NFykq9e9qxn0", + "un9JHdpQkbjdlUbP6dXZDW8vVAzH2rba1egAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgQ7jF+z9", + "+rX+mb03iLQe5wmLV9e2OJKWVbnLaNOK3zhGsu1AS8U2pue8hbSV1nP9RokrbVV2H7krXalbyVe5sjtn", + "zPiyZ46MWneonjKmZtJyTRBfpO5jY0mpT9wFT3bT7QYJHizCWhEH79sx2hP0LWB8n/H3a4jHFrAP4zpA", + "SnmQPDUQi1S6ldL35OAhJhHTjbELPl+YZFl3wDkIoDJ9VFJXxQwTXlHYZefL3fQVV73d3HZWWw8Jx2cV", + "kKGV5ChOJJ2sQ+B+WhTz2vmMLlGL02okfLh/MBrt741uBMHth3WN5XpaVnEpUOvttAXnVeoRVrWYWQ5U", + "baEI117Nq20T35WrSbRRjKZHGMKT0YiRhM0Qbq8Ijd1sWWx2vX7wTqBy8bgF/fuNcvvmbfD15DtFV2Je", + "+ZdVfeBc62hG1e8bDKItbCZaAecLRO/tD0aHb3b3jx4cHu3u3gZsdrFIbXEjD3/fvXqY7NHZQfJo+fC3", + "3cXD+V66H3yHXXARd6XVn6Bsq5dNeUnWUZFqLI1suTlkTDUTpte1RoDmnHDBBrqItdoc8LiGF1j7+8bz", + "fz09v53BWtnhrD7JqghBDdFByrofJC83mbW2i+Zsjp+tn8WNYpmaAwnTW3MoUKrjYDDXxW7vIzEectHx", + "GnpbKdj5IlobX7fpKgpZ2PGkB3e5ZcVD5N2AeKjMet0FvnrJBXSnc6m4WaTrb4uiWAFIjn7Tv2sT15Gj", + "huR4LqRi9Z8LN7nqIwoq9/q95PeD+plxv3fHEHNYxgUBuq2uSgUd3MgSdsmS9auARcqHh7Ke7FQxXIjv", + "dge7j9EPIfn94LvR4HHd46BvV6u6fLu+dO3XUZc1rCYT9Emodh9fy+Par+c6CvqJh1LhlfeyQzl2NF7m", + "pvdXhw/drW1w+XlljxuYQK0C6MdKeu5ym1SFphg0siGU+4qiVjdej1UiI1M250J30dvujwrF7QNQ3JIn", + "DmrcJoSXRT/V5oFWqnTC05TFnBoclTJrIhj2Ourimm+J62U58bUC0towLK493gy2sCngatM1OfyIyN6P", + "ev12e/GuwwFBvZp/oiIaGRbsEz4DHXS9fZ/22j46MDAS/dWOPORbSbKOB+hSDvR6kj6ZS0PKYPyO+rZc", + "tOsFi/FDjs58LfqGJYi9TwKtUkCR8XXs6/gZyZSM86iMH01w0CV2iMrFsKuQv3Fot6rfwBBvUHJs1m+0", + "KTS66Sfb9ruhmwSCbd/q3dHmrb4VpUi/l2fxZh5mC3XjYNfCgN8QghhQ0dSXvSEJVibzaweO/rq6gqtv", + "XqtLjkAkyjNvYAGaWqWkgLkFTQwhv95nLGGGhRohMonLKAmuSy66maXuHj5atJk40SK1OpCfGMvgrYJI", + "SthfCta80MB8AtPiLtkaedxwbQ1eA5v4yK1WfXAPN0pirVtVVeG2JWOwXL6h87bIKxcV9befTU00W8VS", + "8Qy/JqS9bk8m4Gp6t7dW/fFtqOXuU0h75UwPDQBYD09c4Kz7/ktfYBDr6sR7EDLPh8jiDZ1j1r0NdpRW", + "UgllqEQjByWGujRZmKoT/kkwH6n20C9eQMDqPqLXpS0k3BA6p80cFG0JLq0lerMPUXvKmDd0vhbdtxkO", + "W1W/Pxn8zarbyWR4tPPdX/7vwa//GVS7NxQImqlBzGbocXXBlgObzwmUFcM6ti0mk+hpQ+fuzDCaojIN", + "ceMdV6qO98Go4J7Ln2m6MgV0VaskY9rdOKG//Ee7o1dlGd/ihbHx7H50rpXbyElrpL+Xt1Km5t6p3kfU", + "bQ/HAoP0LthSk0qKt+mydmK/0UWViis+ObfkPmTi8pxMOebM1GMBz3saRQzs2UOXNYjjWJRENqwYTart", + "uFRz/rw4y6x1rIDw6RVg5Fdv33z/6u3PzyavTp///OR48tPz/+kd9QS7Gtge4gHQ3sGDQ5fuvbqSu8GU", + "I9fPnDEkJy5ewfk8zHJ82SP0GdhIIQQtWRL2PkpyDW4TuPTGJDfPkbEatXzznBMfCapsTBJyz3Dg3wDo", + "DWO296rzLuLaEyPXmCffaXm4IKuiiyWccQ+vFCO9JBHMUAJbEV7tcmPri3646HaXWLjZwGHH0F84bAEt", + "6kuuEbjDO8FUCpMtDKHxyXx9BPH29eBvnxQNBl0wP3HOptHjT5FX9e3aRKqXMhnAO68l+URQrW7XIhhC", + "gE3Z0Ixem/VlPg08ZpyOe87nNGBwCRlWPkn+Uz+gjaFjK/vfmgguHNDxrJmZwx5Lu1SNTBINbYk2g/Z4", + "jxTE+xYsYwRrtkG8vOJkWGt+JxVmx+UpDgGHxBLx4deFbZenzANODrDS5mjktQ/MyswqI2nfmxOZh7Zl", + "3QJBPD7wSMUqG4EVWHzDJXMBSh3gaJDe6uhkrjKK52h312SrUIH5JSjCrlftAuszWpzQ90UPUAJeE01D", + "LM6jzKi1++J7zD7w2u0SsE/XBA6j8coNY+3XqWjdmniqWt2MKlWtztuWDx48x6vWcL+2s9UgzrKPGmmG", + "6PGvlJsfpMJ3cTv4y61D9uPlHzOFsHpNQP5OaPY8ZfFE5mb9+YdiID3aK7/INFtmKvY6AIpEHNXimtt4", + "gYcnKcewutKwHCzKFTdLUBq5l8kU40F9emBcSOwIfy47xpSsHz6g9nwWiJx5wQRTPMKEtxZqUOCLCWD7", + "yryHNgXmCgQuikCvnh47vYtHUcZ3NDdIet4B9cnpMRjjmbK6h95ouD8c4WHOmKAZ7x319oe7w1EPX1UL", + "nOLOFBL2u0By96IuXvDHsZOEvveFoKaiKTNY45cAJAI6YLri+ASh88ojMqNcuVdklsi4IBgOtTGDg79Q", + "j+yt3LfL3ll5jPHWGAbEslduc39FQRnPDk5zbzRyWPHGXb8YxGQjJ3b+7txoy347SXVuiQIJDVaeeV62", + "LJb+Q793MNq91pjWDQXPbqjjt4K6aGaGr/MHo9Htd3osbHyiizZ3fmHVE4eEVD1rv/wKe6bzNKVq6Res", + "ulqZ1G2CMQO9litr33FGkwhYBaZdAkAbZr+DSpRaF26VC8xW7SsChdZPgW3bb3KB1vS9jJefbAlrfXgd", + "xYc6O4Pj8mGFnj8d7RRkvLqR7pMHLbdUewcE9D0tUq3f20k5GD2+/U5B95vwCKJVPQE7x2yu0fcpQQh2", + "D8IkFfktl4aSIq7hCzrSTmadFuTWL6+inT94/MEe74SF7AGnTKVU2CgRW2bDoV85ztY2Ux7ntbeaJ/zj", + "Zz13U3k0IntRoSBXP6LVa6spDK5eRwe9o7Y+7fTieyT8gzs44W6yRbbd+zxymOMU7Mpf0nFytsZpKYQE", + "ZbkXzHwuND+6yyvL5WX4E56iL4WAX7BCwit3a+VS2MlULuwDOCgBvi4jN129b+rC35vyS8VdCO0a0DRm", + "CDHWwEzj5ZD4NbWPflD6agvRry6tu1n9fJ3C8D6XE7Z3FycMZ1xYir5eU1+vqXWn3FKLnwIezMop76CD", + "uJYG4s+nf7i29uGr7qG77qGT5kGwK1sa8CCHxLnmRjJmRC8g3ykYmy3wk3fCMVQN578TqqIFv2SI7od5", + "7/LE8IwqdLFJCVgJrA29VTGxVi1RNLcDzQ28Q2a5wE1AD80m6AIzaQPiLF0xuRCgrKXae82UuIorGdrt", + "2Q8q2IsGy6uRXC2kZgWwoTCV2xzjvLV9HWOzw7F44xBvYQHRq9zzGs0SxO1do/+RgtCxcBW+9SzEe8Rp", + "mpaciyoET+QWotNuy8pUcaQTHckQ6NAbJqgwA52xiM945KZ1wZbOsTXYYKdUVjBgP07IC+Mdtfa2w8B1", + "6LoURil+VnwjjpLq9hshjUvyXRq5PJYSVZD7IJihZJ7IKU0mdn0uWMAm+AJLuEUpDS6lNUnImNms/NnS", + "LKSwf+fTXJjc/j1V8kozBTanscCIFLfW4K1QCIjkCnPjpZlUxvo5YJ87dog7f1yw5YfhWDyJUy5sDW2r", + "0ERLMFNAPSM9eIjlXi30YE9T2A7+NNdGplUIWE93dpgyN1luXGiNZqYfgj8dCyPJHx7k8sPOH2WPH9BY", + "zGgMdFIpYqeEsnXbqPWEwuwnWDRgbme4AOMeXKTWzWOuqDA4phKlk8yrW7pVpInAJLTNFY6oIJnMbIoN", + "JKoFBZKrtYGgFTRJiMGj5OuC4I472TKflEYLLtikZHAB49f7jCmeMgFBnGhSTcBNNqbqiosdmywduZ3n", + "fEsY4d6DQ8f4hmPxugg3sDdPSiOpJ8iSQD9l/2mHXNwSCMkABw1BXhfWqjPAogPL4OBaJFvOwkN2t4fk", + "xE7HHzugQos7RIwci8MDcsK/r4Rcf2u5ol8Ej0SYypj1vfXNp3Pqu5CySTrtE+A9OHJDm+5hhAJqjuZx", + "MQosByei5LpPT9/u2OYI4ALBLbdgihtL41SzIXmJCeXt+dMsUszoHSQ9IjP7SoNeFfs74ruB1xseCqIY", + "LJguWiofazCOiApgUyoXsLQlugxqVrnZQCXVBQqwXuso5Qp7CkS311qiFa4lkHI8JM/fg4zDDWGYidNF", + "uemxyIVi0D1Mjbz7G6YdAxZgkwIDTitTyLwIVYbPaGRTaKIp8FvC4V9MA0MeCyZmEnYhlinlNoncVcK1", + "0W6J/TgdDlSuWWOzanAffuzwE445mPiioJRWWE8ky+blxAU5+b5yRY0OHoVvKUcPq63/19mrnwnKusDZ", + "bLHSCdKuvgCKAd80zMjjJIXnAD5mzb+Y9XTc4/G4VzhJxNs4VlgcpJTBAC3138HQvrPd9Hn83XAITVkn", + "gCPyyx+2lSO4obLUwgyPex/6pPJhzs0inxbffg0TYBs64lnteiVbVnjc9mwDZliRo63gCcdAOtkK+A8p", + "7/WqO9iUC6qW6+KUA0vvVlDOrH9wZTH+GKM/8Lh3NPYeweNef9xj4hJ/c27D496H8Ao0uFGLirxwGSiI", + "6HA02t4MtO/WN+AJ0MHc9ok1K626hiI/MOygQ3n+c1nd/q21Ot8XehC8osq7yd6s3qR1369l6P0ONv9n", + "aUAQT1jK0E9rUM10CDcFZugcWPkFruOb2BsrD/Sq4ilgcWy8sqmIWOJf2Zv1undvKwQqihjIqXd7cu7r", + "3BTW8CJDyRdl/sbNKs/3WmvdPVPc6K5uu5qV7n7o94szlwWMZc5Uxi59ZEM4PxNibznNGbGFQR95hmMa", + "nDFhyHP8dej+61U9iCV7nsj5+ZHV1EGqEZJw4cJ/KnEJ+ACza4mVLPxWUc/+s0iOuWVFnH/94584KC7m", + "//rHP11Oi3/945943HcsrCRm6T9fMKrMlFFzfkQgyHVAE3iPuclgHm12ydSS7I+ciQ8/wZAEi4oYCpkD", + "Di282U2uRCVMyuap1K5BZxmE+XCRM+3gy6Agn7kkWtaVOaCm9WfZLuWdnuh+AAYWZ1CZANyKngYQQ5ML", + "jmFdVvHUC1tI7JxrNpKmV/aKb+5m/mLYe2Opd2AHeE0Gg0scOnf4wU2abJ2dPd8eEnwGWqrARGn4qCmb", + "ce+b4VeetJknWY5SZyi4ypY3RTSjU55wb2FoyfJkj6BTqZThBEWOBd/EkR8p8BjwrHZasT4WHYtXZzto", + "UTEsMrlifccJlENWLtNAShfWhj04rRA6gw5c2bGYMQptkONnlglUkg8UcdBFwwIBjNClnZtaxsn+WFgE", + "badVgoMHkmyClbD/OTXsii77pCL52qxQCTXwUtd9KDwWNrLYrcEAIZpIZZhD5Gd2SAPvuO9CNBWbgaIH", + "J2KT7Nm+t0Dh4QIa+mUYue/OBpfbYRHpxHGY3xyfqNKq/7GlV2d+N7b7REsSJRypIaJiLObo9+dBy6Wo", + "7WoRP7qgKh5EEi6BKogdBBwkLJ638dinlbZ6tyjJ1PoJHKcfm+T6pQkXi9UJwCG2wJzrDfXPXJlulnrX", + "4p/JVG8X6Dq2emuwYZbf2NX9arfvYLcPr5u34YcM6c888uztOfjbLu7Jv9/T3uqa2y+VJbsPVSPZ8pBe", + "aASVipw+PSY0jhXTevvfWxEJM7VUWsp/cD8CK74PTzM3FqnsXnh9S51AvhR28NqNmlA/r2Ze8er9tlNL", + "OtZ60xX5x8or7/Zvj0an17lGSqG3pLWvN8lG33quI4hUrlLLAEWjhBXiS3FOq1S0SatsvfaLK2etuGRL", + "gUuUO5B3p192XeeieTfcAVN81mCI98gI68gKhIoCgOFLoua3xS66ea1TP39epDm6OynorlXRITL/kp6L", + "cWPZgAtaXKPWC/QFMxbN6Dbf6a6HwMRBz+1OtR3o0s66mJatah1w7IRQE7P+7Xtsi3R7+tr2/kwvX1ye", + "60gsbsm/iigdHrvlWq174B671Pe3976tATfesT+NI7DAIsMHp5tGFDTIJkv1UkTbX11qPjlF42JXHrHK", + "z5vFhSYbv5fvrLuS654Ir1HPwfNbF+9aLsgsAVBoawSI+QydCE0F5NAyor07GOVr56FNFDXM+U5+iWH+", + "p3lirRuRFJdMVYBZq1fqzh/oo775qeSZ19rb9e3rlwMmIhkXxpN2mdR9+cQPJkv/tdD9uz91X2D4Ovfi", + "QZvA+BH7b2NHSIHU+7/2fnBYvf9r7weaZFyw/7X/JKGGabN9a8Qyuqub7q4fMF8w8cH7hdcXbYU17Rg6", + "r4ad1ynTo1VfhzJrKNNe2V3EzX0y6vv0Ql4TmruThHdndA8vqbsGKDr2WUjslkrlweoaCOT/3kLfWY2g", + "71xfV+ueYyiMBXH4AlGRpPIgy9Sh1HNP3I43CQzF2aSMKEp11Ef48n8qlYSd9LWUEsW6ftVLdNFLVJdr", + "rWrCFrxd5YTt456s7wWxhVYbP31F17sDg46jyAq6Xs3CXeLrLaQ2+OnzgFr5d4rE4cVRqN5nHU2mJadY", + "e6/5M3X8rI873Ic9PX5WQizckQHVj+PONQKu37uXxp6kUz7PZa6r6A0pNdGCaYdskrD6zfCl6SpKuaFV", + "W/EZU+noLu+0O1dGfKX7W1KTNDfUMm/rBbHpMeJLdX2MuPKIroGo7w79xeVo6/v8ndst0Ske970rGdfg", + "8T/0u4wr9EiCnDealO8Ygk+bo7H43/Aw+sUwmv76nQ+Iz0ejvUP8nYnLX7/zMfHixJMKE0Zxl275yc/P", + "0PVkjtEFmJG5BLVpjsOiiSDp+dRL/3Yvt9L7pvvTzVPh16dbp6dbZbnWP91swVt+u9XTt935483TW2jB", + "7bc/6fPtT25Trj0tdT6b8YgzgUnwUCupV5yo7RPzqzn5ZvpRKpwTR8UDsyaJdH5GFlxrg4ReZFb/pC6O", + "/dZcOJJQY1iaGQBTixjk5Lahg3qRGwB+8qlxcII+yyIv5xO63n1TE99IOHK3lsJnXeBu8KVbpIe966eu", + "6/gLDZ2VNlg+9o/LUrRpf13eL/He7puyw1V796/KL5nE7PNtdekyeCEEUj1a+1Oa2zjjomaJEjskb968", + "9DHFFpnPJQ410mcL9Rnlx6KaLRRh8HwaVlugju3nMAigvs+/GTMaJ1wwDMJgOhT+W8/xe6/H4tNLwOEE", + "xndte+9yLO2G3qMEfG+s4E5kzcLZwDhLcXE0K7mNi9Pi5U08NV8Uv3IMKMB4QrLeDs2NHDiUgp2FtEi1", + "YbDu04RGiNUNxSyMrAOGsRii1aYQ7UXJJGHKwgNnufHiFmCYusFxYWSRAN9JZufQ/CQXhifnfQ9FGrFE", + "EyqWDs1vLGqdOZkPwRsQmARHqFhmR9zI5g2D5qBqhFKIs1DtktDkii71WDi4B1s9lRgVHVkk7SQZkh8l", + "Iu3YrPYVxmtTSn+jx+KcxwmbOKCcc8I10QupDBMsJqm8ZLreL6Mq4UzhJJ5SWDlNUroscUbd+siMWbjK", + "GhyPhH9TEXNMTgw9F1M+GgtK9kYjkjIqtAPX0HSGF45rg+AgagP6llByMHrsajX2DZMo+OXfgtOkFLuU", + "EZ1CHmqgYpv0fxs3MHXJwhXTRirYvhlX2u5Xod90WWBrG8u1z30d90kuSvgQ1PVXrGuwXSZXAufpzJOM", + "q+IadIhJUxZRWE8h6/0gNLWMolyFLkjY6krW+n9HwbEyvTNcqjA4R4IqgwhiACQiPy7wTEs8StvftlBV", + "SVR/josmeEikIpRU6LrUaLAoR9a4haCz52UKZjFx5c+3v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ADdfTfYA", + "rwuKWyXhP+s5feqYXo3FxZzOhdSGR54ZeqT86qP564Ow04Nw/coGqXkm1UW7J/QPUl10fYF5j80v6yFW", + "neFnaIiA4aGTzv3bI1Abbh8rQDR3/khr0ldxSlHo4kYXvsoEEhgwVdXK37nZoPqq27JIm3CZKmvsLnDR", + "4BEycT/a9P0Vf3M0MUSu1T+tsxoSG2x26bM2XdqnR4kkdz1eCaeqCvhg34La+h/0vTiEdOU3bAul99Xt", + "CjJVABrdiNRadO5hSQNQrWPxVtuUKefW9HROCh4MAq3N8UCuFjxaQDv4G7ZvUV1plp0XUPrbR+QFHuTK", + "ttnOt2zyI6A1LRNm0Vgv0/T8aDWB/buTE6yEZVxSk/Mj4pPWF/eHhlJVGFaYRUK1IT87cNmt4jGOO3oO", + "OszK/LYdQGuZ6mAsQmCtgHVqG+Qzcl7BbT1vARX0/PalnOvPxlRUpn2yczGSuKcj0iYTca/NyYMnYcPP", + "7mgUypvQET7WDuOW0WNXBvNSzouUUzVSplnWlXzdMJGKL9N0DQ2TrZKDEG1imZu/aBMzpbCyo+424iZb", + "NLL/MPQCCNUhj/qDvT0WLUtlZxheKmCqlXwt9l+Xadrr99x4oN6VydJA5paPh+NtNvihH9qhCubun1g8", + "v7cr0aUnK46KTxZkJR73ggIoVzwvuKU3gf6t31AV7N/Gded0AO3PiNe2wJ/exOkVjXcmvrsXTUN8CtLH", + "vZ1iFLMauvDPSZKvjLeuNxOyQFXGkT2+i5Eh9DKJEqlZzU725QA3On1hQ/Ru17f5NR7A8OLcJ7bs4gh0", + "5qqe+ZqfgQpjk8uNHzPx071z35vVEXzJIBR6ZTbA/hpof5uccj57Qvp0W7Iy1S4U8pU2r6+s7USY8Nxa", + "ZRGuQmwTHdPcyJQaHmE6wGghpa6QfQHNb9PhOh18QZmoobLKAheIcQ6keu60+edOxDxymkdCq59cH0Os", + "7sI3wjX8p7LGDxXlSsHx+/4FhZlpNKRMU5zNSEZzzUDOzEEjtowS5vM/MrCrRzQzuWKYMJqRlAue5mk1", + "5wLs2CVFfKjz3fS8T6a5IQlo2VTx0fssRTJNmYitnDQWC0YvOVOwKAk1TETLgWZCc3SNgrQPmLMTNDVZ", + "TNFghoGUigEFYgKLlBkaU0NR0DmHEz+xsWDnJFMMichqRwR7X1ID2I5z8a3NpgPNnvuBnhOG6SK4XhQ5", + "yiMaMxEF0yicfd5s7NOr9M+YaU70nhysbsRL79Pjqqq69sP5PJyxvljIjw5sfo3Qq9sf1fUgGk9G/55H", + "2s7Vz/Ge7HTFEq87xZ+Hga4gus/GSHf/VjipSJzb7iqnEsn8z6pHLBhKDRECAnTtNt7UvlZkZy2W+Vo8", + "b+cP/+fxDbSLnwkn7Lc+7NvyAJaT/hxYrlvVG/Hce1KrOl1SVSd3fyzYDer+xCepKlzuS1G21pD3Cr5d", + "5U5GUXx9SfGVbTfZtvMbuSnb9rrZFc+ICiPnYoCutmEO7tS4razaqQ7+TYN6GrOrsMx7Z5Gl5eLOYTQ9", + "a8zoElQlfwZf6zX2o0gqZVFEqGFfFvR3RWtYjbJA3VyZYLTvg17fnZxst3EJZdbyCGW+YA5RiWyCamm8", + "qsB9dcmU4rGHN3168qy0yapcDMmrlBtiJLlgLCsDgzA4E1A7CzyVxrCbwCn9HhNGLTPJhdk4irLo7Qym", + "/EE6WJnPUJR0uSS+msM7m8NRs//lsTPkMoQWE1j/MjXUbMxxzcVMqtTKZXQqc2gdeBAsE+yndbaY8YTp", + "pTYstc6dAAUBxw3zErnc866e3eU+ujbDybFRhxlTKdeaS6HHwoXcZExB31Ad2q/4qQUNAoYW/PXUMsnP", + "wwcSBmPd/qhpWzVEvsKc1r2j3g7Nsp2YGtriZ+eG9xFD+gGdGoleplMIoASvyAtNthJ+YZ8n5FKTBP7Y", + "XusVOcF6nzqz/s1PFqz0sZjJYN5SS7MFMf+pguMcW/OGyc+Crd3TO/OjvQhfsOrB9bxwJltZrN7IYxWj", + "yQDz8nssJpIbnvDfLduFRrg2PLJRZLTYx3cnBYMfjsUJM4pH1t4bgf85glTgS3cnUzLaARy//SjjCOi3", + "z3BwyHzbP6fY49PTt1guZalUy/5YUOEafvPk1FqaZzRizYEKZsCSTI53Xm3wWT/DZfo39p+0E1wLSBHc", + "8K/myevDxrSeId1yRGW27tkmsz+9g6+TJr/qOL5MHQfQcDmbrQKrzQOshfUZlzLJU/iH/eN4E1SdodHi", + "HRb9bCRvO5yN3fgJfhGH0s0pZjbH870YYOyCfan+s7BwfgooxNQ8E4O3wBPzZ6TuT29KqK7jZ2h6dSvq", + "86d/Nmfrrm8+NwYPmlJdjy/lmFtK8zMxcr0m7Irydk3Y94mMLrRD16mqMIliFCHz4ccS4tyZK1FMwGBf", + "4lCpLPYZ0/2xaChDLYiTJpQYplIuaLKDc7aNIFi716jRS8kx5j7CmJmBBlV3JpMEEdkR0RBmAyVmvoGK", + "dVm7DJPVMlXDqJEA2yRT5gHst0NPt79Sbn6Qqo5G/7nwxTeV9UeUR8pR978BgL+9x48C5D+h79FtO86d", + "cduPaOuFLH+0aqk+wb0Z9/ZHetzrk3FvLx33YAeeUlTnUkMekJSL3ACy5TOra8Oo6sMR0SySItYeR99r", + "E/dHumXmjixbAnYPsd5dij2OqnApX7tOQuwByhGojwFEZKt64NyZjPt46GIic2NND+5cuVIxM6ge2b5z", + "u3HljHx923fh5H91x7fGo3CXgV1Wtt5y9izXC9aucntpk2blZgrrUCTZ1gvydznVfYg6sJp5QMxb4XtQ", + "+9R2cBe5I6Cr6+SNcHP/mjSiQ9KIcq3C+JvW2ROuZE8dFoSTvc+kMvC7wy1wNIQvCQQDwVSYr54ej0UE", + "rMiiRSqWSuRODuLe3sJP/npGnj993SfPMOEz+TGfbg/JK4i0wVa1sxeNhZXELPOKKACr2xnEoevZjh2p", + "5zYd16GDa0nxe5/O0oMnI2Dl8XvlHdb7vQWjMUokf/ReSttZAEj69Us4QLB/rmax7b21wgfo8NVy8GRm", + "mFpt9sTFbIkCBsVd0h5V0Alu2DF2qD2YXtmnlQ0s2sn+Xi8AfvLhax6P208UfjcWO+zXpXaEaL4vNUcw", + "HsSCOYZYYPW6LjJhtEUsO1629oGBXR4/+ySPilu1Uq/jXbV0Af+upwtn+sUamrLaPgERFxl0Nlp6faDy", + "wiJc22okohmNuFn2CU0Sd0e5m6DwjhkU4u9UMXoBevshoJC5nn1wMdhq+95QS2KuL2wLzhY7JOAep/Np", + "MTiCB81ajXHNIUTXSBLRJMoTahhhsxmLMC4YU/LoFltuMZTeLZ6dspMgLEzFwz7/4tIWhmkCd68kiybF", + "7dit3lEsSihP2/HknaCGzo/oajCFRqUA34XEUhyNlNSauKYGLOFzDmk10FMCMmmAmoumbCyyhArBFMm1", + "9dCHoQ8yxbTOFfMNwN3lKKpPSqzITEnjXBMSKZUeC0/h706INixbQ2avbcsnOOdbkm1t466ne1JSN8bQ", + "rgpxRQhsiKUUu+BAR6Aauge3eDug+5YSv5SD/0bxOfi+ZkpSy2StX5I91n457aGvRU+3pjA9K0p1S2Fa", + "tFqJkKxED67F+puU8Olx73oeiIHOL3grHKT7dL2I5p+gUse+65Gz4UG4Tx85y1A21n/HxKdnlYDFrgqs", + "ksK/NHVSZeS1o1oL+t0M8dU5yvc2o247Y3ndG4TXl4zcRWuhvG0P3s+PEEZ3izhx13nzvmzaqiFv1d6m", + "LfADmxMkfBYUeDuZEe4ZceUGmRE+KwwAoJ17xGIJHtT7iumv2Z6l+prc4FZD+W2GA4SGawvlt1zPOa+u", + "fSi9c2W6PZNci38mCd4u0HXkd7/sX1/9HZ4MlcXaZIIGgmdpZpaukrdVlk5nmv/Ohi2G4MJv9fZMwTdw", + "6fx05OHptNWh8x7Ry+7RkHMvPqMuGyTX4OZXZt3zgd1fGN5h9czVLpYduHUGVEULfsnale71E+yWKFNs", + "kMkMjSuxXTC3Hv4uM1QN578T17zDf3X/wnSimMiAxSTmikUGvEWEkcgRbB/faKKkNO67VMt2LxF7RH5Q", + "Mn3iZrPhPnRnyinDSj/DdDmIqaGDS89t1qjQPsK70/tTAsMjXJAX35Mt9t4om7SEzODlA9HmfknZ+4ix", + "WCNNblcHvDsKDxEansynXUa5Jv3MK/yDJiTKtZGp3/vjZ2QL09nNmYC9AFF/hpJspuQlj1lcGyM479tV", + "3W1Z0OvqXUGoKHIR+seFHdy9yDBdLqT57zyrs4XCJWbKBcXBbUzwUj9TFlAA+qNceAcct0d+FF+vMPfy", + "2/KPHaBETK3qFtFIaeGmt79ec1/yNVcNhvJ3Wu228+4565XX3eKjOoYt3UYSiiJ27m7V1u8+n5Aerr/I", + "aB6nOr8sHqRtavPPiwRHd3c/3LW6/N0XHAIKyvLLxrLZBtRlmGBeok93zC5ZIrMUU9xj2V6/l6ukd9Rb", + "GJMd7eyg7/dCanN08Pjhfu/Drx/+/wEASc4tyF8CAgA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 8de7c33d3..fdbd7fed2 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/netip" "os" + "path/filepath" "strconv" "strings" "time" @@ -425,15 +426,20 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } - registryURL, err := builderRegistryURL(cfg.Registry.URL, networkManager) - if err != nil { - return nil, err + registryURL := cfg.Registry.URL + if !cfg.MacOSBuilds { + var err error + registryURL, err = builderRegistryURL(registryURL, networkManager) + if err != nil { + return nil, err + } } if registryURL != cfg.Registry.URL { log.Info("resolved registry URL for builder VMs", "original", cfg.Registry.URL, "resolved", registryURL) } buildConfig := builds.Config{ + MachineBuildOnly: cfg.MacOSOnly, MaxConcurrentBuilds: cfg.Build.MaxConcurrentSourceBuilds, BuilderImage: cfg.Build.BuilderImage, RegistryURL: registryURL, @@ -444,6 +450,20 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins DockerSocket: cfg.Build.DockerSocket, } + if cfg.MacOSBuilds { + if !cfg.MacOSOnly || registryURL == "" { + return nil, fmt.Errorf("machine builds require macOS-only mode and an explicit host registry URL") + } + workDir := filepath.Join(p.BuildsDir(), "machine-work") + if err := os.MkdirAll(workDir, 0700); err != nil { + return nil, err + } + buildConfig.MachineBuild = &builds.MachineBuildBackend{ + Driver: &builds.GuestMachineBuildDriver{Instances: instanceManager, WorkDir: workDir}, + Publisher: &builds.StreamingMachinePublisher{RegistryURL: registryURL, Tokens: builds.NewRegistryTokenGenerator(cfg.JwtSecret)}, + } + } + // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics) var secretProvider builds.SecretProvider if cfg.Build.SecretsDir != "" { diff --git a/lib/registrypush/macos_test.go b/lib/registrypush/macos_test.go new file mode 100644 index 000000000..a9acd4275 --- /dev/null +++ b/lib/registrypush/macos_test.go @@ -0,0 +1,48 @@ +package registrypush + +import ( + "context" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/ocicache/testutil" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestPushMacOSImagePreservesManifestAndPlatform(t *testing.T) { + host, _ := newTestRegistry(t) + img, err := random.Image(256, 1) + require.NoError(t, err) + config, err := img.ConfigFile() + require.NoError(t, err) + config.OS, config.Architecture = "darwin", "arm64" + config.Config.Labels = map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", + images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", + images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + } + img, err = mutate.ConfigFile(img, config) + require.NoError(t, err) + img = mutate.MediaType(img, types.OCIManifestSchema1) + img = mutate.ConfigMediaType(img, types.OCIConfigJSON) + p := paths.New(t.TempDir()) + digest, err := testutil.WriteImage(p, img) + require.NoError(t, err) + target := host + "/releases/macos:canary" + result, err := PushFromCache(context.Background(), p, digest, target, nil, Options{Insecure: true}) + require.NoError(t, err) + require.Equal(t, digest, result.Digest) + ref, err := name.ParseReference(host+"/releases/macos@"+digest, name.Insecure) + require.NoError(t, err) + published, err := remote.Image(ref) + require.NoError(t, err) + actual, err := published.ConfigFile() + require.NoError(t, err) + require.Equal(t, config, actual) +} diff --git a/openapi.yaml b/openapi.yaml index 73aa90cde..4e4d1d461 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1983,6 +1983,9 @@ components: source_hash: type: string description: SHA256 hash of source tarball + recipe_hash: + type: string + description: SHA256 of the exact validated machine recipe bytes, omitted without executor evidence. Machine builds do not infer toolchain versions. lockfile_hashes: type: object additionalProperties: @@ -4683,6 +4686,23 @@ paths: dockerfile: type: string description: Dockerfile content. Required if not included in the source tarball. + machine_base_image: + type: string + description: | + Experimental installed darwin/arm64 base pinned by sha256 digest. + Requires server macos_only and macos_builds. Source must contain + hypeman-macos-build.json (version 1). Machine source is limited to + 64 MiB compressed; only machine_network_mode, timeout_seconds, memory_mb, cpus and tags + are accepted alongside source and this selector. CPU/memory zero + inherit the base. Linux/cache/secrets/admin options are rejected. + Output retains the base identity and cannot run concurrently with it. + machine_network_mode: + type: string + enum: [isolated, egress] + description: | + Native machine builds only. Defaults to isolated. Explicit egress enables + unrestricted VZ NAT for checksum-verified artifact downloads; it does not + enforce domain allowlists. Linux builds do not use this selector. base_image_digest: type: string description: Optional pinned base image digest diff --git a/scripts/macos/.gitignore b/scripts/macos/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/scripts/macos/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/scripts/macos/audit-base.py b/scripts/macos/audit-base.py new file mode 100755 index 000000000..0dbf8c9a2 --- /dev/null +++ b/scripts/macos/audit-base.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Run INSIDE a fresh provisioning guest as root, preferably via GuestService. +Never emits command output, paths, file contents, account names or environment. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import plistlib +import re +import stat +import subprocess +import sys + + +class Rejected(Exception): + pass + + +def require(ok, code): + if not ok: + raise Rejected(code) + + +def command(*args): + result = subprocess.run(args, capture_output=True, timeout=30, check=False) + require(result.returncode == 0, "probe-failed") + return result.stdout.decode("utf-8", errors="strict").strip() + + +def digest(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def trusted(path): + path = Path(path) + for parent in (path, *path.parents): + s = parent.lstat() + require(not stat.S_ISLNK(s.st_mode) and s.st_uid == 0 and not s.st_mode & 0o022, + "untrusted-agent-path") + require(path.is_file(), "agent-not-regular") + + +def walk_error(_error): + raise Rejected("home-scan-failed") + + +FORBIDDEN = {".ssh", ".aws", ".azure", ".config", ".docker", ".kube", ".gnupg", + ".npmrc", ".netrc", ".git-credentials", "Keychains", "Accounts", "Cookies", + "Safari", "Google", "Chromium", "Firefox", "CloudStorage", "Mobile Documents", + ".bash_history", ".zsh_history", ".python_history", ".lesshst", ".zsh_sessions"} + + +def audit_inputs_outside_homes(homes, paths): + roots = [home.resolve(strict=True) for home in homes] + for path in paths: + resolved = Path(path).resolve(strict=True) + require(not any(resolved == home or home in resolved.parents for home in roots), + "audit-input-in-home") + + +def audit_homes(homes, baseline): + seen = set() + for home in homes: + require(home.is_dir() and not home.is_symlink(), "invalid-home") + for root, dirs, files in os.walk(home, followlinks=False, onerror=walk_error): + for name in dirs + files: + require(name not in FORBIDDEN, "credential-or-browser-state") + path = Path(root) / name + require(not path.is_symlink(), "home-link") + if path.is_dir(): + continue + require(path.is_file(), "home-special-file") + key = str(path) + require(key in baseline and digest(path) == baseline[key], "unreviewed-home-state") + seen.add(key) + require(seen == set(baseline), "baseline-mismatch") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--agent", required=True) + p.add_argument("--agent-sha256", required=True) + p.add_argument("--plist", required=True) + p.add_argument("--plist-sha256", required=True) + p.add_argument("--users", required=True, help="exact comma-separated UID>=500 accounts, excluding nobody") + p.add_argument("--home-baseline", required=True, help="reviewed JSON: absolute home file -> SHA256; {} allows no files") + p.add_argument("--clean-transient", action="store_true", help="remove shell histories only, never credential files") + p.add_argument("--runtime-state-prefix", action="append", default=[], help="additional /private/tmp, /private/var/tmp or /private/var/run entry prefix that must not exist; repeatable") + args = p.parse_args() + require(sys.platform == "darwin" and os.geteuid() == 0, "guest-root-required") + require(command("/usr/bin/uname", "-m") == "arm64", "arm64-required") + for value in (args.agent_sha256, args.plist_sha256): + require(re.fullmatch(r"[0-9a-f]{64}", value), "invalid-pin") + trusted(args.agent) + trusted(args.plist) + require(digest(args.agent) == args.agent_sha256 and digest(args.plist) == args.plist_sha256, + "agent-pin-mismatch") + with open(args.plist, "rb") as f: + job = plistlib.load(f) + require(job.get("UserName", "root") == "root" and job.get("ProgramArguments") == [args.agent] + and not job.get("EnvironmentVariables") and not job.get("Program"), "agent-job-policy") + label = job.get("Label", "") + require(re.fullmatch(r"[a-zA-Z0-9.-]+", label), "invalid-agent-label") + job_state = command("/bin/launchctl", "print", "system/" + label) + require("state = running" in job_state and ("program = " + args.agent) in job_state, + "agent-not-running") + pid = re.search(r"\bpid = ([0-9]+)\b", job_state) + require(pid is not None, "agent-pid-missing") + process = command("/bin/ps", "-p", pid.group(1), "-o", "uid=", "-o", "comm=").split(None, 1) + require(process == ["0", args.agent], "agent-process-mismatch") + # A readiness file alone is not RPC proof. The operator must execute this + # script over GuestService and require normal instance readiness before stop. + users = command("/usr/bin/dscl", ".", "-list", "/Users", "UniqueID") + actual = {line.split()[0] for line in users.splitlines() + if int(line.split()[-1]) >= 500 and line.split()[0] != "nobody"} + require(actual == set(args.users.split(",")), "unexpected-accounts") + homes = [Path("/var/root"), Path("/Users/Shared")] + # /var is a standard macOS symlink; homes themselves must be real directories. + for user in sorted(actual): + entry = command("/usr/bin/dscl", ".", "-read", "/Users/" + user, "NFSHomeDirectory") + home = Path(entry.split(": ", 1)[1]) + require(home == Path("/Users") / user, "unexpected-home") + homes.append(home) + require({x.name for x in Path("/Users").iterdir()} <= actual | {"Shared", ".localized"}, + "unknown-home") + for home in homes: + require(home.is_dir() and not home.is_symlink(), "invalid-home") + audit_inputs_outside_homes(homes, [__file__, args.home_baseline]) + if args.clean_transient: + for home in homes: + for name in (".bash_history", ".zsh_history", ".python_history", ".lesshst"): + path = home / name + require(not path.is_symlink(), "history-link") + if path.exists(): + require(path.is_file(), "history-not-file") + path.unlink() + with open(args.home_baseline) as f: + baseline = json.load(f) + require(isinstance(baseline, dict) and all(isinstance(k, str) and isinstance(v, str) + and re.fullmatch(r"[0-9a-f]{64}", v) for k, v in baseline.items()), "invalid-baseline") + audit_homes(homes, baseline) + for root in ("/private/tmp", "/private/var/tmp", "/private/var/run"): + for item in Path(root).iterdir(): + require(not item.name.startswith(("hypeman-build-", "profile", *args.runtime_state_prefix)), "runtime-state") + require(not list(Path("/private/etc/ssh").glob("ssh_host_*")), "host-ssh-keys") + require(not Path("/Library/Preferences/com.apple.loginwindow.plist").exists(), "review-autologin-state") + require(not Path("/Library/Managed Preferences").exists(), "managed-device-state") + snapshots = command("/usr/bin/tmutil", "listlocalsnapshots", "/") + require(not any("com.apple.TimeMachine." in line for line in snapshots.splitlines()), "local-snapshots") + toolchain = command("/usr/bin/xcodebuild", "-version") + require(re.fullmatch(r"Xcode [0-9.]+\nBuild version [a-zA-Z0-9]+", toolchain), "unknown-toolchain") + tools = {"xcode": toolchain, "clang": command("/usr/bin/clang", "--version")} + report = {"version": 1, "status": "pass", "architecture": "arm64", + "os_version": command("/usr/bin/sw_vers", "-productVersion"), + "os_build": command("/usr/bin/sw_vers", "-buildVersion"), + "agent_sha256": args.agent_sha256, "plist_sha256": args.plist_sha256, + "toolchain_sha256": hashlib.sha256(json.dumps(tools, sort_keys=True).encode()).hexdigest(), + "toolchain_version": toolchain.replace("\n", "; "), + "home_baseline_sha256": digest(args.home_baseline), + "audit_script_sha256": digest(__file__)} + print(json.dumps(report, sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except Rejected as exc: + print("base audit rejected: " + str(exc), file=sys.stderr) + sys.exit(1) + except Exception: + print("base audit rejected: probe-error", file=sys.stderr) + sys.exit(1) diff --git a/scripts/macos/example/hypeman-macos-build.json b/scripts/macos/example/hypeman-macos-build.json new file mode 100644 index 000000000..7e4accf48 --- /dev/null +++ b/scripts/macos/example/hypeman-macos-build.json @@ -0,0 +1 @@ +{"version":1,"steps":[{"command":["/bin/sh","provision.sh"]}]} diff --git a/scripts/macos/example/provision.sh b/scripts/macos/example/provision.sh new file mode 100644 index 000000000..19d8d1a22 --- /dev/null +++ b/scripts/macos/example/provision.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# This example performs no downloads and inherits the pinned base toolchain. +[ "$(/usr/bin/uname -m)" = arm64 ] +/usr/bin/xcodebuild -version >/dev/null +/usr/bin/install -d -m 0755 /usr/local/share/hypeman +printf '%s\n' 'macos-base-pipeline-v1' > /usr/local/share/hypeman/base-pipeline-marker +/bin/chmod 0644 /usr/local/share/hypeman/base-pipeline-marker diff --git a/scripts/macos/test_audit.py b/scripts/macos/test_audit.py new file mode 100644 index 000000000..ac6e29341 --- /dev/null +++ b/scripts/macos/test_audit.py @@ -0,0 +1,68 @@ +import importlib.util +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(filename)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +audit = load("audit", "audit-base.py") + + +class AuditTests(unittest.TestCase): + def test_homes_fail_closed(self): + with tempfile.TemporaryDirectory() as root: + home = Path(root) + audit.audit_homes([home], {}) + private = home / "preferences" + private.write_text("never-print-this") + with self.assertRaises(audit.Rejected) as ctx: + audit.audit_homes([home], {}) + self.assertNotIn("never-print-this", str(ctx.exception)) + baseline = {str(private): audit.digest(private)} + audit.audit_homes([home], baseline) + private.write_text("changed") + with self.assertRaises(audit.Rejected): + audit.audit_homes([home], baseline) + private.unlink() + private.symlink_to("/etc/passwd") + with self.assertRaises(audit.Rejected): + audit.audit_homes([home], baseline) + + def test_credentials_cannot_be_baselined(self): + with tempfile.TemporaryDirectory() as root: + home = Path(root) + credentials = home / ".netrc" + credentials.write_text("never-print-this") + with self.assertRaises(audit.Rejected): + audit.audit_homes([home], {str(credentials): audit.digest(credentials)}) + with patch.object(audit.os, "walk", side_effect=lambda *args, **kwargs: kwargs["onerror"](OSError())): + with tempfile.TemporaryDirectory() as root: + with self.assertRaises(audit.Rejected): + audit.audit_homes([Path(root)], {}) + + def test_audit_inputs_outside_homes(self): + with tempfile.TemporaryDirectory() as root: + home = Path(root) / "home" + home.mkdir() + outside = Path(root) / "audit.py" + outside.write_text("reviewed") + audit.audit_inputs_outside_homes([home], [outside]) + inside = home / "baseline.json" + inside.write_text("{}") + with self.assertRaisesRegex(audit.Rejected, "audit-input-in-home"): + audit.audit_inputs_outside_homes([home], [inside]) + alias = Path(root) / "alias.json" + alias.symlink_to(inside) + with self.assertRaisesRegex(audit.Rejected, "audit-input-in-home"): + audit.audit_inputs_outside_homes([home], [alias]) + + +if __name__ == "__main__": + unittest.main()