From afc6904000d8e2ba86e13e5d344ea0749a13c42b Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Tue, 30 Jun 2026 23:20:41 +0000 Subject: [PATCH 1/2] feat: generate sources from custom origin --- internal/app/azldev/cmds/advanced/mock.go | 3 +- .../azldev/cmds/component/preparesources.go | 45 ++-- .../app/azldev/core/sources/mockprocessor.go | 2 +- internal/buildenv/mockroot.go | 2 +- internal/projectconfig/component.go | 31 ++- internal/projectconfig/configfile.go | 55 ++++ internal/projectconfig/configfile_test.go | 122 +++++++++ .../sourceproviders/customsourceprovider.go | 252 ++++++++++++++++++ .../customsourceprovider_internal_test.go | 123 +++++++++ .../sourceproviders/sourcemanager.go | 111 +++++++- internal/rpm/mock/mock.go | 19 +- internal/rpm/mock/mock_test.go | 14 +- ...ainer_config_generate-schema_stdout_1.snap | 16 +- ...shots_config_generate-schema_stdout_1.snap | 16 +- schemas/azldev.schema.json | 16 +- 15 files changed, 789 insertions(+), 38 deletions(-) create mode 100644 internal/providers/sourceproviders/customsourceprovider.go create mode 100644 internal/providers/sourceproviders/customsourceprovider_internal_test.go diff --git a/internal/app/azldev/cmds/advanced/mock.go b/internal/app/azldev/cmds/advanced/mock.go index 18e4f806..6ea70345 100644 --- a/internal/app/azldev/cmds/advanced/mock.go +++ b/internal/app/azldev/cmds/advanced/mock.go @@ -183,7 +183,8 @@ func RunShell(env *azldev.Env, options *ShellOptions, extraArgs []string) error } } - cmd, err := runner.CmdInChroot(env, extraArgs, true /*interactive*/) + // Caller manually wires stdin/stdout/stderr via cmd.SetStdin/SetStdout/SetStderr. + cmd, err := runner.CmdInChroot(env, extraArgs, true /*interactive*/, false /*pipeOutput*/) if err != nil { return fmt.Errorf("failed to create shell command: %w", err) } diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 2f0ffcaa..aac15169 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -115,6 +115,8 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er } // Create source manager to handle all source fetching, both local and upstream. + // Custom source generation (origin type 'custom') is automatically enabled when + // the distro has a 'mock-config' configured — no per-command wiring required. sourceManager, err = sourceproviders.NewSourceManager(env, distro) if err != nil { return fmt.Errorf("failed to create source manager:\n%w", err) @@ -130,21 +132,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er "synthetic history requires overlays to be applied") } - var preparerOpts []sources.PreparerOption - if !options.WithoutGitRepo && !options.SkipOverlays { - preparerOpts = append(preparerOpts, - sources.WithGitRepo(env, env.LockReader(), distro.Version.ReleaseVer), - sources.WithDirtyDetection(), - ) - } - - if options.AllowNoHashes { - preparerOpts = append(preparerOpts, sources.WithAllowNoHashes()) - } - - if options.SkipSources { - preparerOpts = append(preparerOpts, sources.WithSkipLookaside()) - } + preparerOpts := buildPreparerOptions(env, distro, options) preparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env, preparerOpts...) if err != nil { @@ -159,6 +147,33 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er return nil } +// buildPreparerOptions assembles [sources.PreparerOption] values based on the resolved +// distro and command-line options. +func buildPreparerOptions( + env *azldev.Env, + distro sourceproviders.ResolvedDistro, + options *PrepareSourcesOptions, +) []sources.PreparerOption { + var opts []sources.PreparerOption + + if !options.WithoutGitRepo && !options.SkipOverlays { + opts = append(opts, + sources.WithGitRepo(env, env.LockReader(), distro.Version.ReleaseVer), + sources.WithDirtyDetection(), + ) + } + + if options.AllowNoHashes { + opts = append(opts, sources.WithAllowNoHashes()) + } + + if options.SkipSources { + opts = append(opts, sources.WithSkipLookaside()) + } + + return opts +} + // CheckOutputDir verifies the output directory state before source preparation. // If the directory exists and is non-empty, it either removes it (when Force is set) // or returns an actionable error suggesting --force. diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index 0951779e..8e62d20b 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -184,7 +184,7 @@ func (p *MockProcessor) BatchProcess( workers := strconv.Itoa(max(1, maxWorkers)) // 1x CPU; mock work is CPU-bound args := []string{"python3", chrootScript, chrootStagingPath, workers} - cmd, err := runner.CmdInChroot(ctx, args, false) + cmd, err := runner.CmdInChroot(ctx, args, false, false /*pipeOutput: uses SetRealTimeStdoutListener*/) if err != nil { return nil, fmt.Errorf("failed to create batch command in mock:\n%w", err) } diff --git a/internal/buildenv/mockroot.go b/internal/buildenv/mockroot.go index 8711f7bc..bd4d0b9f 100644 --- a/internal/buildenv/mockroot.go +++ b/internal/buildenv/mockroot.go @@ -82,7 +82,7 @@ func (r *MockRoot) CreateCmd(ctx context.Context, args []string, options RunOpti mockRunner.AddBindMount(bindMount.PathInHost, bindMount.PathInBuildEnv) } - cmd, err = mockRunner.CmdInChroot(ctx, args, options.Interactive) + cmd, err = mockRunner.CmdInChroot(ctx, args, options.Interactive, false /*pipeOutput: returned to caller*/) if err != nil { return nil, fmt.Errorf("failed to create command to run in mock root:\n%w", err) } diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index b481b35b..d9f395b4 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -43,13 +43,18 @@ type OriginType string const ( // OriginTypeURI indicates that the source file is fetched from a URI. OriginTypeURI OriginType = "download" + + // OriginTypeCustom indicates that the source file is generated by running a local + // shell script inside a mock chroot. The script is expected to populate a specific + // output directory; azldev then packages that directory into a deterministic archive. + OriginTypeCustom OriginType = "custom" ) // Origin describes where a source file comes from and how to retrieve it. // When omitted from a source file reference, the file will be resolved via the lookaside cache. type Origin struct { // Type indicates how the source file should be acquired. - Type OriginType `toml:"type" json:"type" jsonschema:"required,enum=download,title=Origin type,description=Type of origin for this source file"` + Type OriginType `toml:"type" json:"type" jsonschema:"required,enum=download,enum=custom,title=Origin type,description=Type of origin for this source file"` // Uri to download the source file from if origin type is 'download'. Ignored for other origin types. Uri string `toml:"uri,omitempty" json:"uri,omitempty" jsonschema:"title=URI,description=URI to download the source file from if origin type is 'download',example=https://example.com/source.tar.gz"` } @@ -82,6 +87,30 @@ type SourceFileReference struct { // being replaced. Required when [SourceFileReference.ReplaceUpstream] is true; must be // empty otherwise. Excluded from the fingerprint because it is documentation only. ReplaceReason string `toml:"replace-reason,omitempty" json:"replaceReason,omitempty" jsonschema:"title=Replace reason,description=Required when 'replace-upstream' is true. Human-readable explanation for the replacement." fingerprint:"-"` + + // Script is the filename of a shell script, relative to the component's spec directory, + // that is run inside a mock chroot to generate this source file. + // Required when [SourceFileReference.Origin.Type] is 'custom'; must be empty otherwise. + Script string `toml:"script,omitempty" json:"script,omitempty" jsonschema:"title=Script,description=Shell script filename (relative to the component spec directory) to run in mock to generate this source file. Required when origin type is 'custom'."` + + // MockPackages is a list of RPM package names to install in the mock chroot before + // running [SourceFileReference.Script]. Only valid when [SourceFileReference.Origin.Type] + // is 'custom'. + MockPackages []string `toml:"mock-packages,omitempty" json:"mockPackages,omitempty" jsonschema:"title=Mock packages,description=RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'."` +} + +// HashInclude implements the hashstructure [Includable] interface so that +// [SourceFileReference.Script] and [SourceFileReference.MockPackages] are omitted +// from the component fingerprint when they hold their zero values. +func (r SourceFileReference) HashInclude(field string, _ any) (bool, error) { + switch field { + case "Script": + return r.Script != "", nil + case "MockPackages": + return len(r.MockPackages) > 0, nil + } + + return true, nil } // ComponentPublishConfig holds publish channel settings for a component's packages. diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index e16b79b7..f313361c 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -189,6 +189,10 @@ func validateSourceFiles(sourceFiles []SourceFileReference, componentName string return err } + if err := validateCustomSourceRef(ref, componentName); err != nil { + return err + } + if err := validateOrigin(ref.Origin, ref.Filename, componentName); err != nil { return err } @@ -223,6 +227,46 @@ func validateReplaceUpstream(ref SourceFileReference, componentName string) erro return nil } +// validateCustomSourceRef enforces the pairing rules for the 'script' and 'mock-packages' +// fields on a [SourceFileReference]: +// - 'script' is required when 'origin.type' is 'custom'. +// - 'script' must be empty when 'origin.type' is not 'custom'. +// - 'mock-packages' must be empty when 'origin.type' is not 'custom'. +func validateCustomSourceRef(ref SourceFileReference, componentName string) error { + if ref.Origin.Type == OriginTypeCustom { + if ref.Script == "" { + return fmt.Errorf( + "source file %#q in component %#q has 'custom' origin but no 'script'; "+ + "a non-empty 'script' filename is required for 'custom' origin", + ref.Filename, componentName) + } + + if err := fileutils.ValidateFilename(ref.Script); err != nil { + return fmt.Errorf( + "invalid 'script' value %#q for source file %#q in component %#q:\n%w", + ref.Script, ref.Filename, componentName, err) + } + + return nil + } + + if ref.Script != "" { + return fmt.Errorf( + "source file %#q in component %#q has 'script' set but origin type is %#q; "+ + "'script' is only valid when origin type is 'custom'", + ref.Filename, componentName, string(ref.Origin.Type)) + } + + if len(ref.MockPackages) > 0 { + return fmt.Errorf( + "source file %#q in component %#q has 'mock-packages' set but origin type is %#q; "+ + "'mock-packages' is only valid when origin type is 'custom'", + ref.Filename, componentName, string(ref.Origin.Type)) + } + + return nil +} + // validateOrigin checks that a source file [Origin] is present and valid for its type. // For [OriginTypeURI] ('download'), the [Origin.Uri] field must be a valid URI with a scheme. func validateOrigin(origin Origin, filename string, componentName string) error { @@ -255,6 +299,17 @@ func validateOrigin(origin Origin, filename string, componentName string) error "URI %#q is missing a scheme (e.g. 'https://')", filename, componentName, origin.Uri) } + + case OriginTypeCustom: + // Script validation is handled by validateCustomSourceRef on SourceFileReference. + // Reject 'uri' since it is meaningless for custom-generated sources. + if origin.Uri != "" { + return fmt.Errorf( + "source file %#q in component %#q has 'uri' set but origin type is 'custom'; "+ + "'uri' is only valid when origin type is 'download'", + filename, componentName) + } + default: return fmt.Errorf( "unsupported 'origin' type %#q for source file %#q, component %#q", diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index d61f2866..5bf15ec3 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -452,3 +452,125 @@ func TestProjectConfigFileValidation_PerComponentSnapshotDisallowed(t *testing.T assert.Contains(t, err.Error(), "snapshot") assert.Contains(t, err.Error(), "test-component") } + +// --- Custom origin source file validation --- + +func TestValidateCustomSourceRef_ValidCustomOrigin(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + Script: "gen.sh", + }, + }, + }, + }, + } + assert.NoError(t, file.Validate()) +} + +func TestValidateCustomSourceRef_MissingScript(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + // Script intentionally absent + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "script") + assert.Contains(t, err.Error(), "gen.tar.gz") + assert.Contains(t, err.Error(), "comp") +} + +func TestValidateCustomSourceRef_ScriptOnDownloadOrigin(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "src.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/src.tar.gz"}, + Script: "gen.sh", + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'script'") + assert.Contains(t, err.Error(), "custom") +} + +func TestValidateCustomSourceRef_MockPackagesOnDownloadOrigin(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "src.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/src.tar.gz"}, + MockPackages: []string{"curl"}, + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'mock-packages'") + assert.Contains(t, err.Error(), "custom") +} + +func TestValidateCustomSourceRef_UriOnCustomOrigin(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Uri: "https://example.com/should-not-be-here", + }, + Script: "gen.sh", + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "'uri'") + assert.Contains(t, err.Error(), "custom") +} + +func TestValidateCustomSourceRef_InvalidScriptFilename(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "comp": { + SourceFiles: []projectconfig.SourceFileReference{ + { + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + Script: "../../escape.sh", // path traversal attempt + }, + }, + }, + }, + } + err := file.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "script") +} diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go new file mode 100644 index 00000000..1af602c6 --- /dev/null +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sourceproviders + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/archive" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// customGenScriptDir is the path inside the mock chroot where the generation script +// is bind-mounted (read-only). +const customGenScriptDir = "/azldev-gen/script" + +// customGenOutputDir is the path inside the mock chroot where the generation script +// must write its output (read-write). +const customGenOutputDir = "/azldev-gen/output" + +// customFileSourceProvider implements [FileSourceProvider] for source files with +// [projectconfig.OriginTypeCustom]. It executes a user-supplied script inside a +// fresh mock chroot and packages the output directory as a deterministic archive. +type customFileSourceProvider struct { + fs opctx.FS + runner *mock.Runner +} + +var _ FileSourceProvider = (*customFileSourceProvider)(nil) + +// GetFile implements [FileSourceProvider]. It returns [ErrNotFound] for any file +// reference whose origin type is not [projectconfig.OriginTypeCustom], allowing +// the source manager to fall through to lookaside and other configured origins. +func (p *customFileSourceProvider) GetFile( + ctx context.Context, + component components.Component, + ref projectconfig.SourceFileReference, + destDirPath string, +) error { + if ref.Origin.Type != projectconfig.OriginTypeCustom { + return ErrNotFound + } + + destPath := filepath.Join(destDirPath, ref.Filename) + + return generateCustomSourceFile(ctx, p.fs, p.runner, component, &ref, destPath) +} + +// generateCustomSourceFile runs a single [projectconfig.SourceFileReference] through a +// fresh mock chroot and places the resulting deterministic archive at destPath. +// +// The runner must be configured with the distro's mock config path. A clone of the runner +// is created for each call so bind mounts and package installs do not bleed across invocations. +func generateCustomSourceFile( + ctx context.Context, + fs opctx.FS, + baseRunner *mock.Runner, + component components.Component, + ref *projectconfig.SourceFileReference, + destPath string, +) error { + slog.Info("Generating custom source file", + "filename", ref.Filename, + "script", ref.Script, + "component", component.GetName()) + + specDir, err := resolveComponentSpecDir(component) + if err != nil { + return fmt.Errorf("failed to resolve spec directory for component %#q:\n%w", + component.GetName(), err) + } + + // Verify the generation script is present before spinning up a mock chroot. + scriptHostPath := filepath.Join(specDir, ref.Script) + + if _, statErr := fs.Stat(scriptHostPath); statErr != nil { + return fmt.Errorf("generation script %#q not found at %#q:\n%w", + ref.Script, scriptHostPath, statErr) + } + + scriptTmpDir, genOutputTmpDir, cleanup, err := prepareStagingDirs(fs, scriptHostPath, ref.Script) + if err != nil { + return err + } + + defer cleanup() + + // Clone the base runner so bind mounts added here don't persist to other calls. + // Network access is always enabled — custom source scripts commonly need to + // download upstream tarballs or toolchain artifacts. + runner := baseRunner.Clone() + runner.EnableNetwork() + runner.AddBindMount(scriptTmpDir, customGenScriptDir) + runner.AddBindMount(genOutputTmpDir, customGenOutputDir) + + if err := execScriptInChroot(ctx, runner, ref); err != nil { + return err + } + + // Package the output directory as a deterministic archive whose format is + // inferred from the filename extension (e.g., .tar.gz, .tar.xz). + comp, compErr := archive.DetectCompression(ref.Filename) + if compErr != nil { + return fmt.Errorf("cannot determine archive format for custom source %#q:\n%w", + ref.Filename, compErr) + } + + // Ensure the destination directory exists. FetchFiles runs before FetchComponent, + // so the output directory may not have been created yet when this is called. + if mkdirErr := fileutils.MkdirAll(fs, filepath.Dir(destPath)); mkdirErr != nil { + return fmt.Errorf("failed to create destination directory for %#q:\n%w", + ref.Filename, mkdirErr) + } + + if archiveErr := archive.CreateDeterministicArchive(destPath, genOutputTmpDir, comp); archiveErr != nil { + return fmt.Errorf("failed to create archive %#q:\n%w", ref.Filename, archiveErr) + } + + slog.Info("Custom source file generated successfully", + "filename", ref.Filename, + "path", destPath) + + return nil +} + +// resolveComponentSpecDir returns the directory on the host filesystem that contains the +// component's spec file and its sidecar files (patches, generation scripts, etc.). +// +// For local components the spec path is explicit; its parent directory is returned directly. +// For upstream components the directory is inferred from the config file that defines the +// component, because the script is a project-local file rather than something fetched from +// the upstream dist-git. +func resolveComponentSpecDir(component components.Component) (string, error) { + config := component.GetConfig() + + // Local components: the spec path is an absolute path on disk. + if config.Spec.SourceType == projectconfig.SpecSourceTypeLocal && config.Spec.Path != "" { + return filepath.Dir(config.Spec.Path), nil + } + + // Upstream components: derive from the config file that defines the component. + if config.SourceConfigFile != nil { + return config.SourceConfigFile.Dir(), nil + } + + return "", fmt.Errorf( + "cannot determine spec directory for component %#q: "+ + "neither a local spec path nor a config file directory is available", + config.Name) +} + +// prepareStagingDirs creates two temporary host directories: +// - a read-only script directory containing a copy of the generation script +// - a read-write output directory to be bind-mounted inside the chroot +// +// The returned cleanup function removes both directories; callers must defer it. +func prepareStagingDirs( + fs opctx.FS, + scriptHostPath string, + scriptName string, +) (scriptDir string, outputDir string, cleanup func(), err error) { + scriptDir, err = os.MkdirTemp("", "azldev-script-*") + if err != nil { + return "", "", nil, fmt.Errorf("failed to create temporary script directory:\n%w", err) + } + + outputDir, err = os.MkdirTemp("", "azldev-output-*") + if err != nil { + _ = os.RemoveAll(scriptDir) + + return "", "", nil, fmt.Errorf("failed to create temporary output directory:\n%w", err) + } + + cleanup = func() { + _ = os.RemoveAll(scriptDir) + _ = os.RemoveAll(outputDir) + } + + // Copy the script into the staging directory so we bind-mount only that file, + // without leaking other sidecar files from the spec directory. + scriptData, readErr := fileutils.ReadFile(fs, scriptHostPath) + if readErr != nil { + cleanup() + + return "", "", nil, fmt.Errorf("failed to read generation script %#q:\n%w", + scriptName, readErr) + } + + hostScriptCopy := filepath.Join(scriptDir, scriptName) + + if writeErr := os.WriteFile(hostScriptCopy, scriptData, fileperms.PublicExecutable); writeErr != nil { + cleanup() + + return "", "", nil, fmt.Errorf("failed to stage generation script to %#q:\n%w", + hostScriptCopy, writeErr) + } + + return scriptDir, outputDir, cleanup, nil +} + +// execScriptInChroot initialises a fresh mock root, optionally installs extra packages, +// runs the generation script, and scrubs the root on return. +func execScriptInChroot( + ctx context.Context, + runner *mock.Runner, + ref *projectconfig.SourceFileReference, +) error { + if initErr := runner.InitRoot(ctx); initErr != nil { + return fmt.Errorf("failed to initialize mock root for generating %#q:\n%w", + ref.Filename, initErr) + } + + defer func() { + // Best-effort cleanup. Log on failure rather than overwriting the primary error. + if scrubErr := runner.ScrubRoot(ctx); scrubErr != nil { + slog.Warn("Failed to scrub mock root after custom source generation", + "filename", ref.Filename, + "error", scrubErr) + } + }() + + if len(ref.MockPackages) > 0 { + if installErr := runner.InstallPackages(ctx, ref.MockPackages); installErr != nil { + return fmt.Errorf("failed to install mock packages for generating %#q:\n%w", + ref.Filename, installErr) + } + } + + scriptChrootPath := filepath.Join(customGenScriptDir, ref.Script) + + cmd, cmdErr := runner.CmdInChroot(ctx, []string{scriptChrootPath}, false /* interactive */, true /* pipeOutput */) + if cmdErr != nil { + return fmt.Errorf("failed to create chroot command for generating %#q:\n%w", + ref.Filename, cmdErr) + } + + if runErr := cmd.Run(ctx); runErr != nil { + return fmt.Errorf("generation script %#q failed for source %#q:\n%w", + ref.Script, ref.Filename, runErr) + } + + return nil +} diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go new file mode 100644 index 00000000..055be123 --- /dev/null +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sourceproviders + +import ( + "context" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestCustomFileSourceProvider_GetFile_NonCustomOriginReturnsNotFound(t *testing.T) { + provider := &customFileSourceProvider{ + fs: afero.NewMemMapFs(), + runner: nil, // never reached + } + + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + + ref := projectconfig.SourceFileReference{ + Filename: "src.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI}, + } + + err := provider.GetFile(context.Background(), comp, ref, "/output") + assert.ErrorIs(t, err, ErrNotFound) +} + +func TestCustomFileSourceProvider_GetFile_MissingScriptReturnsError(t *testing.T) { + provider := &customFileSourceProvider{ + fs: afero.NewMemMapFs(), + runner: nil, // never reached — script stat check fails first + } + + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().Return("yara").AnyTimes() + comp.EXPECT().GetConfig().Return(&projectconfig.ComponentConfig{ + Name: "yara", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/specs/yara/yara.spec", + }, + }).AnyTimes() + + ref := projectconfig.SourceFileReference{ + Filename: "gen.tar.gz", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeCustom}, + Script: "gen.sh", + } + + err := provider.GetFile(context.Background(), comp, ref, "/output") + require.Error(t, err) + assert.Contains(t, err.Error(), "gen.sh") + assert.NotErrorIs(t, err, ErrNotFound) +} + +func TestResolveComponentSpecDir_LocalComponent(t *testing.T) { + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetConfig().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/specs/yara/yara.spec", + }, + }).AnyTimes() + + dir, err := resolveComponentSpecDir(comp) + require.NoError(t, err) + assert.Equal(t, filepath.FromSlash("/specs/yara"), dir) +} + +func TestResolveComponentSpecDir_NoSpecInfoReturnsError(t *testing.T) { + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetConfig().Return(&projectconfig.ComponentConfig{ + Name: "yara", + // No Spec.Path, no SourceConfigFile + }).AnyTimes() + + _, err := resolveComponentSpecDir(comp) + require.Error(t, err) + assert.Contains(t, err.Error(), "yara") +} + +func TestPrepareStagingDirs_ScriptIsStagedAndExecutable(t *testing.T) { + memFS := afero.NewMemMapFs() + + const scriptContent = "#!/bin/bash\necho hello" + + require.NoError(t, afero.WriteFile(memFS, "/scripts/gen.sh", []byte(scriptContent), fileperms.PublicExecutable)) + + scriptDir, outputDir, cleanup, err := prepareStagingDirs(memFS, "/scripts/gen.sh", "gen.sh") + require.NoError(t, err) + require.NotEmpty(t, scriptDir) + require.NotEmpty(t, outputDir) + require.NotNil(t, cleanup) + + defer cleanup() + + // Script should have been copied into the staging dir. + data, readErr := afero.ReadFile(afero.NewOsFs(), filepath.Join(scriptDir, "gen.sh")) + require.NoError(t, readErr) + assert.Equal(t, scriptContent, string(data)) +} + +func TestPrepareStagingDirs_MissingScriptReturnsError(t *testing.T) { + emptyFS := afero.NewMemMapFs() // empty — no script present + + _, _, cleanup, err := prepareStagingDirs(emptyFS, "/scripts/missing.sh", "missing.sh") + require.Error(t, err) + assert.Contains(t, err.Error(), "missing.sh") + assert.Nil(t, cleanup) +} diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index b5431d84..a50386c7 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -15,6 +15,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/utils/downloader" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" @@ -26,14 +27,26 @@ import ( // Provider is an abstract interface implemented by a source provider. type Provider interface{} +// ErrNotFound is returned by a [FileSourceProvider] when it does not handle the +// given file reference. The source manager tries the next registered provider on +// this error, eventually falling back to lookaside cache and configured origins. +var ErrNotFound = errors.New("file not handled by this provider") + // FileSourceProvider is an abstract interface implemented by a source provider that can retrieve individual // source files. type FileSourceProvider interface { Provider - // GetFiles retrieves the specified source files and places them in the provided directory. If a file - // is not known to (or handled by) the providers, the error will be (or will wrap) ErrNotFound. - GetFiles(ctx context.Context, fileRefs []projectconfig.SourceFileReference, destDirPath string) error + // GetFile retrieves a single source file and places it in destDirPath. + // Implementations must return [ErrNotFound] (or an error wrapping it) when the + // provider does not handle the given file reference, so the manager can try the + // next registered provider before falling back to lookaside and configured origins. + GetFile( + ctx context.Context, + component components.Component, + fileRef projectconfig.SourceFileReference, + destDirPath string, + ) error } // SourceIdentityProvider resolves a reproducible identity string for a component's source. @@ -232,6 +245,13 @@ func NewSourceManager(env *azldev.Env, distro ResolvedDistro) (SourceManager, er // Create component providers manager.createComponentProviders(distro) + // Automatically register the custom file source provider when the distro has + // a mock config path. This makes 'custom' origin source files available to all + // commands (build, prep-sources, diff-sources, etc.) without any per-command + // wiring. The provider only spins up a mock chroot when GetFile is actually + // called for a custom-origin file, so registering it upfront is cheap. + manager.createFileProviders(env) + // Ensure at least one provider was created successfully if len(manager.upstreamComponentProviders) == 0 && len(manager.fileProviders) == 0 { @@ -260,6 +280,48 @@ func (m *sourceManager) createComponentProviders(distro ResolvedDistro) { slog.Debug("Registered Fedora component provider") } +// createFileProviders registers [FileSourceProvider] implementations based on the +// project's default distro configuration. This follows the same pattern as the +// render and build commands, which both use [azldev.Env.Distro] (the project-level +// default) rather than the per-component resolved distro for mock operations. +// Currently this registers a [customFileSourceProvider] when the project distro +// has a 'mock-config' path configured, enabling 'custom' origin source files for +// all commands without per-command wiring. +// +// Failures are logged and the provider is skipped, matching the tolerant +// registration pattern used by [createComponentProviders]. +func (m *sourceManager) createFileProviders(env *azldev.Env) { + _, distroVerDef, err := env.Distro() + if err != nil { + slog.Debug("Cannot resolve project distro; 'custom' origin source generation will be unavailable", + "error", err) + + return + } + + mockConfigPath := distroVerDef.MockConfigPath + if mockConfigPath == "" { + slog.Debug("No 'mock-config' set on the project distro version; 'custom' origin source generation is unavailable") + + return + } + + if _, statErr := env.FS().Stat(mockConfigPath); statErr != nil { + slog.Warn("Mock config not accessible; 'custom' origin source generation will be unavailable", + "path", mockConfigPath, + "error", statErr) + + return + } + + m.fileProviders = append(m.fileProviders, &customFileSourceProvider{ + fs: m.fs, + runner: mock.NewRunner(env, mockConfigPath), + }) + + slog.Debug("Registered custom file source provider", "mockConfig", mockConfigPath) +} + func (m *sourceManager) FetchFiles( ctx context.Context, component components.Component, @@ -280,6 +342,18 @@ func (m *sourceManager) FetchFiles( for i := range sourceFiles { fileRef := &sourceFiles[i] + // Fail fast when a 'custom' origin source file has no registered provider. + // This means the distro has no 'mock-config' set (or the file was inaccessible), + // so no generation can happen. Surfacing the error here — before any network + // or disk work — gives a clearer diagnosis than the message produced deep in + // the fetch fallback path. + if fileRef.Origin.Type == projectconfig.OriginTypeCustom && len(m.fileProviders) == 0 { + return fmt.Errorf( + "source file %#q has 'custom' origin but no file provider is available; "+ + "set 'mock-config' on the distro %#q version definition to enable custom source generation", + fileRef.Filename, component.GetConfig().Spec.UpstreamDistro.Name) + } + err := m.fetchSourceFile(ctx, httpDownloader, component, fileRef, destDirPath) if err != nil { return fmt.Errorf("failed to fetch source file %#q:\n%w", fileRef.Filename, err) @@ -318,6 +392,19 @@ func (m *sourceManager) fetchSourceFile( return nil } + // Try each registered file provider. Providers return [ErrNotFound] to signal + // they don't handle this reference; any other error is fatal. + for _, provider := range m.fileProviders { + err := provider.GetFile(ctx, component, *fileRef, destDirPath) + if err == nil { + return nil + } + + if !errors.Is(err, ErrNotFound) { + return fmt.Errorf("file provider failed for %#q:\n%w", fileRef.Filename, err) + } + } + // Phase 1: Try lookaside cache if hash info is available if fileRef.Hash != "" && fileRef.HashType != "" { lookasideErr := m.tryLookasideDownload(ctx, httpDownloader, component, fileRef, destPath) @@ -341,7 +428,7 @@ func (m *sourceManager) fetchSourceFile( fileRef.Filename) } - return m.downloadFromOrigin(ctx, httpDownloader, fileRef, destPath) + return m.fetchFromOrigin(ctx, httpDownloader, fileRef, destPath) } // tryLookasideDownload attempts to download a source file from the lookaside cache. @@ -377,8 +464,12 @@ func (m *sourceManager) tryLookasideDownload( return nil } -// downloadFromOrigin downloads a source file using its configured origin. -func (m *sourceManager) downloadFromOrigin( +// fetchFromOrigin acquires a source file using its configured origin. +// For [projectconfig.OriginTypeCustom], callers should have already dispatched +// to a registered [FileSourceProvider] — reaching this function for a custom +// origin means no provider was configured. +// For [projectconfig.OriginTypeURI], the file is downloaded from the configured URI. +func (m *sourceManager) fetchFromOrigin( ctx context.Context, httpDownloader downloader.Downloader, fileRef *projectconfig.SourceFileReference, @@ -403,6 +494,14 @@ func (m *sourceManager) downloadFromOrigin( return nil + case projectconfig.OriginTypeCustom: + // The file provider dispatch in fetchSourceFile should have handled this. + // Reaching here means no [FileSourceProvider] was registered for 'custom' origin. + return fmt.Errorf( + "source file %#q has 'custom' origin but no provider handled it; "+ + "ensure the distro has a 'mock-config' configured", + fileRef.Filename) + default: return fmt.Errorf("unsupported origin type %#q for source file %#q", fileRef.Origin.Type, fileRef.Filename) diff --git a/internal/rpm/mock/mock.go b/internal/rpm/mock/mock.go index 2ed9a0a8..0791b817 100644 --- a/internal/rpm/mock/mock.go +++ b/internal/rpm/mock/mock.go @@ -545,8 +545,14 @@ func (r *Runner) ensureMockPresentAndConfigured() error { return nil } -// Builds a wrapper command that will run the specified inside a mock chroot. -func (r *Runner) CmdInChroot(ctx context.Context, args []string, interactive bool) (cmd opctx.Cmd, err error) { +// CmdInChroot builds a wrapper command that will run the specified args inside a mock chroot. +// When pipeOutput is true, the command's stdout and stderr are connected to the process +// stdout and stderr so output is visible to the user in real time. Leave pipeOutput false +// when the caller intends to capture output itself (e.g. via [opctx.Cmd.RunAndGetOutput] or +// [opctx.Cmd.SetRealTimeStdoutListener]). +func (r *Runner) CmdInChroot( + ctx context.Context, args []string, interactive bool, pipeOutput bool, +) (cmd opctx.Cmd, err error) { // We're going to need to run mock, so make sure we can. err = r.ensureMockPresentAndConfigured() if err != nil { @@ -569,7 +575,14 @@ func (r *Runner) CmdInChroot(ctx context.Context, args []string, interactive boo mockArgs = append(mockArgs, shellquote.Join(args...)) } - cmd, err = r.cmdFactory.Command(exec.CommandContext(ctx, MockBinary, mockArgs...)) + rawCmd := exec.CommandContext(ctx, MockBinary, mockArgs...) + + if pipeOutput { + rawCmd.Stdout = os.Stdout + rawCmd.Stderr = os.Stderr + } + + cmd, err = r.cmdFactory.Command(rawCmd) if err != nil { return nil, fmt.Errorf("failed to create command to run in mock root:\n%w", err) } diff --git a/internal/rpm/mock/mock_test.go b/internal/rpm/mock/mock_test.go index 96d5f148..55639a32 100644 --- a/internal/rpm/mock/mock_test.go +++ b/internal/rpm/mock/mock_test.go @@ -221,7 +221,7 @@ func TestCmdInChroot_MockNotPresent(t *testing.T) { runner := mock.NewRunner(ctx, testMockConfigPath) - _, err := runner.CmdInChroot(ctx, []string{"arg1", "arg2"}, true /*interactive*/) + _, err := runner.CmdInChroot(ctx, []string{"arg1", "arg2"}, true /*interactive*/, false /*pipeOutput*/) require.Error(t, err) } @@ -230,7 +230,7 @@ func TestCmdInChroot_Success(t *testing.T) { runner := mock.NewRunner(ctx, testMockConfigPath) - cmd, err := runner.CmdInChroot(ctx, []string{"arg1", "arg2 with spaces"}, true /*interactive*/) + cmd, err := runner.CmdInChroot(ctx, []string{"arg1", "arg2 with spaces"}, true /*interactive*/, false /*pipeOutput*/) require.NoError(t, err) require.NotNil(t, cmd) @@ -249,7 +249,7 @@ func TestCmdInChroot_BindMount(t *testing.T) { runner.AddBindMount("/host-path", "/mock-path") assert.Equal(t, map[string]string{"/host-path": "/mock-path"}, runner.BindMounts()) - cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/) + cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/, false /*pipeOutput*/) require.NoError(t, err) require.NotNil(t, cmd) @@ -264,7 +264,7 @@ func TestCmdInChroot_NoPreClean(t *testing.T) { runner.WithNoPreClean() assert.True(t, runner.HasNoPreClean()) - cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/) + cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/, false /*pipeOutput*/) require.NoError(t, err) require.NotNil(t, cmd) @@ -279,7 +279,7 @@ func TestCmdInChroot_EnableNetworking(t *testing.T) { runner.EnableNetwork() assert.True(t, runner.HasNetworkEnabled()) - cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/) + cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/, false /*pipeOutput*/) require.NoError(t, err) require.NotNil(t, cmd) @@ -294,7 +294,7 @@ func TestCmdInChroot_ConfigOpts(t *testing.T) { runner.WithConfigOpts(map[string]string{"cleanup_on_success": "True", "cleanup_on_failure": "False"}) assert.Equal(t, map[string]string{"cleanup_on_success": "True", "cleanup_on_failure": "False"}, runner.ConfigOpts()) - cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/) + cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/, false /*pipeOutput*/) require.NoError(t, err) require.NotNil(t, cmd) @@ -311,7 +311,7 @@ func TestCmdInChroot_Unprivileged(t *testing.T) { runner := mock.NewRunner(ctx, testMockConfigPath) runner.WithUnprivileged() - cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/) + cmd, err := runner.CmdInChroot(ctx, []string{"arg"}, false /*interactive*/, false /*pipeOutput*/) require.NoError(t, err) require.NotNil(t, cmd) diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 7b824f4e..49f074dc 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -720,7 +720,8 @@ "type": { "type": "string", "enum": [ - "download" + "download", + "custom" ], "title": "Origin type", "description": "Type of origin for this source file" @@ -1199,6 +1200,19 @@ "type": "string", "title": "Replace reason", "description": "Required when 'replace-upstream' is true. Human-readable explanation for the replacement." + }, + "script": { + "type": "string", + "title": "Script", + "description": "Shell script filename (relative to the component spec directory) to run in mock to generate this source file. Required when origin type is 'custom'." + }, + "mock-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Mock packages", + "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 7b824f4e..49f074dc 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -720,7 +720,8 @@ "type": { "type": "string", "enum": [ - "download" + "download", + "custom" ], "title": "Origin type", "description": "Type of origin for this source file" @@ -1199,6 +1200,19 @@ "type": "string", "title": "Replace reason", "description": "Required when 'replace-upstream' is true. Human-readable explanation for the replacement." + }, + "script": { + "type": "string", + "title": "Script", + "description": "Shell script filename (relative to the component spec directory) to run in mock to generate this source file. Required when origin type is 'custom'." + }, + "mock-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Mock packages", + "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." } }, "additionalProperties": false, diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7b824f4e..49f074dc 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -720,7 +720,8 @@ "type": { "type": "string", "enum": [ - "download" + "download", + "custom" ], "title": "Origin type", "description": "Type of origin for this source file" @@ -1199,6 +1200,19 @@ "type": "string", "title": "Replace reason", "description": "Required when 'replace-upstream' is true. Human-readable explanation for the replacement." + }, + "script": { + "type": "string", + "title": "Script", + "description": "Shell script filename (relative to the component spec directory) to run in mock to generate this source file. Required when origin type is 'custom'." + }, + "mock-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Mock packages", + "description": "RPM packages to install in the mock chroot before running the generation script. Only valid when origin type is 'custom'." } }, "additionalProperties": false, From 24c1de2e694c82749741cff305ceffa01e38e0ad Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 1 Jul 2026 17:53:14 +0000 Subject: [PATCH 2/2] Improved logging and instructions --- docs/user/reference/config/components.md | 94 ++++++++++--------- .../sourceproviders/customsourceprovider.go | 15 ++- .../customsourceprovider_internal_test.go | 4 +- .../sourceproviders/sourcemanager.go | 4 +- 4 files changed, 63 insertions(+), 54 deletions(-) diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 35344a8d..60105f80 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -302,71 +302,81 @@ rpm-channel = "none" ## Source File References -The `[[components..source-files]]` array defines additional source files that azldev should download before building. These are files not available in the dist-git repository or lookaside cache — typically binaries, pre-built artifacts, or files from custom hosting. +The `[[components..source-files]]` array defines additional source files to fetch or generate before building — binaries, pre-built artifacts, or archives generated on-the-fly by a script. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| -| Filename | `filename` | string | **Yes** | Name of the file as it will appear in the sources directory | -| Hash | `hash` | string | Conditional | Expected hash of the downloaded file for integrity verification. Required for the `prep-sources` command unless `--allow-no-hashes` is used, in which case the hash is computed automatically from the downloaded file. | -| Hash type | `hash-type` | string | Conditional | Hash algorithm used (examples: `"SHA256"`, `"SHA512"`). Required when `hash` is specified. When omitted alongside `hash` for the `prep-sources` command and `--allow-no-hashes` is used, defaults to `"SHA512"`. | -| Origin | `origin` | [Origin](#origin) | **Yes** | Where to download the file from | -| Replace upstream | `replace-upstream` | bool | No | When `true`, intentionally replaces an entry with the same `filename` in the upstream dist-git `sources` file. The matching upstream entry **must exist**, otherwise source preparation fails (this guards against stale configs and filename typos). When `false` (default) a filename collision with an upstream entry is also an error. Requires `replace-reason`. | -| Replace reason | `replace-reason` | string | Conditional | Human-readable explanation for the replacement. **Required** when `replace-upstream = true`; **must be empty** otherwise. Captured in the `prep-sources` warning log so the override is auditable. | +| Filename | `filename` | string | **Yes** | Name of the file in the sources directory | +| Hash | `hash` | string | Conditional | Expected hash. Required unless `--allow-no-hashes` is passed to `prep-sources` (which computes and prints the hash). | +| Hash type | `hash-type` | string | Conditional | Hash algorithm (`"SHA256"`, `"SHA512"`). Required with `hash`; defaults to `"SHA512"` when auto-computed. | +| Origin | `origin` | [Origin](#origin) | **Yes** | How to obtain the file | +| Script | `script` | string | Conditional | Script filename (relative to the component's spec dir) to run in mock. **Required** for `origin.type = "custom"`; must be empty otherwise. | +| Mock packages | `mock-packages` | array of string | No | Extra RPM packages to install in the mock chroot before the script runs. Only valid for `"custom"` origin. | +| Replace upstream | `replace-upstream` | bool | No | Replace the same-named entry in the upstream `sources` file. The upstream entry must exist. Requires `replace-reason`. | +| Replace reason | `replace-reason` | string | Conditional | Required when `replace-upstream = true`. Logged by `prep-sources` for auditability. | ### Origin -The `origin` field specifies how to obtain the source file. +Two origin types are supported. -| Field | TOML Key | Type | Required | Description | -|-------|----------|------|----------|-------------| -| Type | `type` | string | **Yes** | Origin type. Currently only `"download"` is supported. | -| URI | `uri` | string | No | URI to download the file from (required when type is `"download"`) | +#### `"download"` — fetch from a URI -### Example +`origin = { type = "download", uri = "https://..." }`. The `uri` field is required. ```toml [[components.shim.source-files]] -filename = "shimx64.efi" -hash = "7741013d9a24ce554bf6a9df6b776a57b114055e..." +filename = "shimx64.efi" +hash = "7741013d9a24ce554bf6a9df6b776a57b114055e..." hash-type = "SHA512" -origin = { type = "download", uri = "https://example.com/repo/pkgs/shim/shimx64.efi/sha512/.../shimx64.efi" } +origin = { type = "download", uri = "https://example.com/repo/.../shimx64.efi" } +``` -[[components.shim.source-files]] -filename = "shimaa64.efi" -hash = "57aa116d1c91a9ec36ab8b46c9164ae19af192b..." -hash-type = "SHA512" -origin = { type = "download", uri = "https://example.com/repo/pkgs/shim/shimaa64.efi/sha512/.../shimaa64.efi" } +#### `"custom"` — generate via a mock script + +Use `origin.type = "custom"` when a source archive must be assembled or modified (e.g. stripping sensitive test fixtures from an upstream tarball). azldev runs a script inside a fresh mock chroot and packages the output as a deterministic archive. + +The script must write its output to `/azldev-gen/output/`. azldev bind-mounts the script read-only at `/azldev-gen/script/` and the output directory read-write at `/azldev-gen/output/`, then packages the result. Network access is always enabled so scripts can download upstream tarballs. The mock config comes from the project's default distro — no extra config is needed beyond the existing `mock-config` setting. + +On first use, omit `hash` and run `prep-sources --allow-no-hashes` to generate the archive and print its hash, then copy it into the TOML. + +```toml +[[components.yara.source-files]] +filename = "yara-4.5.4-azl-stripped.tar.gz" +origin = { type = "custom" } +script = "gen-yara-stripped.sh" # relative to the component's spec directory +mock-packages = ["cmake"] # omit if not needed +hash-type = "SHA512" +hash = "abc123..." # from: prep-sources --allow-no-hashes ``` -### Replacing an upstream `sources` entry +To replace an existing upstream `sources` entry with the generated file, add `replace-upstream = true`: + +```toml +[[components.yara.source-files]] +filename = "yara-4.5.4.tar.gz" # matches the upstream entry +origin = { type = "custom" } +script = "gen-yara-stripped.sh" +replace-upstream = true +replace-reason = "Strip malware samples from test corpus (CVE hygiene)" +hash-type = "SHA512" +hash = "abc123..." +``` -By default, declaring a `source-files` entry whose `filename` matches one already -listed in the upstream dist-git `sources` file is an **error** — this protects -against accidental shadowing of the upstream tarball. +### Replacing an upstream `sources` entry -To intentionally substitute a different artifact for an upstream entry (e.g. to -ship a locally-patched tarball), set `replace-upstream = true` together with a -non-empty `replace-reason`: +A `source-files` entry whose `filename` collides with an upstream `sources` entry is an error by default. Set `replace-upstream = true` (with a non-empty `replace-reason`) to intentionally substitute it: ```toml [[components.example.source-files]] -filename = "example-1.0.tar.gz" # same filename as the upstream 'sources' entry -hash = "deadbeef..." -hash-type = "SHA512" -origin = { type = "download", uri = "https://internal.example.com/example-1.0-patched.tar.gz" } +filename = "example-1.0.tar.gz" +hash = "deadbeef..." +hash-type = "SHA512" +origin = { type = "download", uri = "https://internal.example.com/example-1.0-patched.tar.gz" } replace-upstream = true -replace-reason = "patched to fix CVE-2026-0001 before upstream" +replace-reason = "patched to fix CVE-2026-0001 before upstream" ``` -When `prep-sources` runs: - -- The matching upstream entry is removed from the generated `sources` file and - the user-defined entry takes its place. -- A `WARN`-level event is logged citing the `replace-reason`, both upstream and - new hashes, so the override is auditable. -- If **no upstream entry** exists with that `filename`, `prep-sources` fails — - this is almost always a stale config or filename typo. Drop - `replace-upstream` (and `replace-reason`) if you intended a brand-new +`prep-sources` removes the matching upstream entry, inserts the new one in its place, and logs a `WARN` with both hashes and the reason. If no upstream entry with that filename exists, `prep-sources` fails — this is almost always a stale config or filename typo. Drop `replace-upstream` if you intended a brand-new artifact instead. `replace-upstream` and `replace-reason` are per-entry switches, not a diff --git a/internal/providers/sourceproviders/customsourceprovider.go b/internal/providers/sourceproviders/customsourceprovider.go index 1af602c6..edc7529b 100644 --- a/internal/providers/sourceproviders/customsourceprovider.go +++ b/internal/providers/sourceproviders/customsourceprovider.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "log/slog" - "os" "path/filepath" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" @@ -24,7 +23,7 @@ import ( const customGenScriptDir = "/azldev-gen/script" // customGenOutputDir is the path inside the mock chroot where the generation script -// must write its output (read-write). +// must write its output. const customGenOutputDir = "/azldev-gen/output" // customFileSourceProvider implements [FileSourceProvider] for source files with @@ -168,21 +167,21 @@ func prepareStagingDirs( scriptHostPath string, scriptName string, ) (scriptDir string, outputDir string, cleanup func(), err error) { - scriptDir, err = os.MkdirTemp("", "azldev-script-*") + scriptDir, err = fileutils.MkdirTempInTempDir(fs, "azldev-script-*") if err != nil { return "", "", nil, fmt.Errorf("failed to create temporary script directory:\n%w", err) } - outputDir, err = os.MkdirTemp("", "azldev-output-*") + outputDir, err = fileutils.MkdirTempInTempDir(fs, "azldev-output-*") if err != nil { - _ = os.RemoveAll(scriptDir) + _ = fs.RemoveAll(scriptDir) return "", "", nil, fmt.Errorf("failed to create temporary output directory:\n%w", err) } cleanup = func() { - _ = os.RemoveAll(scriptDir) - _ = os.RemoveAll(outputDir) + _ = fs.RemoveAll(scriptDir) + _ = fs.RemoveAll(outputDir) } // Copy the script into the staging directory so we bind-mount only that file, @@ -197,7 +196,7 @@ func prepareStagingDirs( hostScriptCopy := filepath.Join(scriptDir, scriptName) - if writeErr := os.WriteFile(hostScriptCopy, scriptData, fileperms.PublicExecutable); writeErr != nil { + if writeErr := fileutils.WriteFile(fs, hostScriptCopy, scriptData, fileperms.PublicExecutable); writeErr != nil { cleanup() return "", "", nil, fmt.Errorf("failed to stage generation script to %#q:\n%w", diff --git a/internal/providers/sourceproviders/customsourceprovider_internal_test.go b/internal/providers/sourceproviders/customsourceprovider_internal_test.go index 055be123..c7b9a796 100644 --- a/internal/providers/sourceproviders/customsourceprovider_internal_test.go +++ b/internal/providers/sourceproviders/customsourceprovider_internal_test.go @@ -107,8 +107,8 @@ func TestPrepareStagingDirs_ScriptIsStagedAndExecutable(t *testing.T) { defer cleanup() - // Script should have been copied into the staging dir. - data, readErr := afero.ReadFile(afero.NewOsFs(), filepath.Join(scriptDir, "gen.sh")) + // Script should have been copied into the staging dir on the same FS. + data, readErr := afero.ReadFile(memFS, filepath.Join(scriptDir, "gen.sh")) require.NoError(t, readErr) assert.Equal(t, scriptContent, string(data)) } diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index a50386c7..89ce4b20 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -350,8 +350,8 @@ func (m *sourceManager) FetchFiles( if fileRef.Origin.Type == projectconfig.OriginTypeCustom && len(m.fileProviders) == 0 { return fmt.Errorf( "source file %#q has 'custom' origin but no file provider is available; "+ - "set 'mock-config' on the distro %#q version definition to enable custom source generation", - fileRef.Filename, component.GetConfig().Spec.UpstreamDistro.Name) + "set 'mock-config' on the project distro version definition to enable custom source generation", + fileRef.Filename) } err := m.fetchSourceFile(ctx, httpDownloader, component, fileRef, destDirPath)