diff --git a/README.md b/README.md index c56f2f1..ccaecb1 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,90 @@ The handler identity is retained in the local terminal-status cache for reportin after a service restart. Legacy cache entries without a handler identity are ignored with a warning rather than reported under an assumed handler name. +## Error clarification + +RunCommand V2 reports classified failures using the Azure extension platform's +`ErrorWithClarification`. Positive clarification codes indicate customer input, +permissions, connectivity, policy, or VM problems. Negative codes indicate handler +or Azure infrastructure failures. Clarification codes are separate from script +exit codes and handler process exit codes. + +Output-blob creation or replacement failures return exit code `-108` +(`ExitCode_OutputBlobCreateOrReplaceFailed`); error-blob creation or replacement +failures return `-109` (`ExitCode_ErrorBlobCreateOrReplaceFailed`). Once blobs +are created, periodic and final appends are best effort: failures do not change +the command result or exit code. Only the first append failure per command is +logged and written as an extension error event, across both output and error +blobs. Subsequent appends are still attempted. + +Managed and immediate status reports include a `substatus` entry with +`name: "ErrorClarification"`, `status: "error"`, and the numeric `code`. The +existing instance-view message, output, script exit code, and +`treatFailureAsDeploymentFailure` behavior are preserved. A failed script can +therefore have a successful top-level deployment status and still carry its +error clarification. + +Immediate commands retain the script's exit code and full terminal instance view. +An execution result already reported to the observer is not replaced by a fallback +status. If terminal reporting fails, the fallback retries the original instance +view and top-level status. Script failures carry the positive +`CommandExecution_failureExitCode` clarification, not a system-error code. + +Codes are declared centrally as named `ErrorClarificationCode` constants in +[errorclarificationcodes.go](pkg/errorutil/errorclarificationcodes.go), grouped +into system and user errors like CustomScript. Shared classifications use +CustomScript's constant names and numeric values; additional RunCommand failure +sites have distinct constants to preserve source-location uniqueness. + +Each code is allocated to exactly one production classification site and must +not be reused or renumbered. Higher layers add context without replacing a code +from a lower layer. The shared [error helper](pkg/errorutil/errors.go) preserves +both pointer and value platform errors, wrapped causes, and joined errors. Tests +check constant-value uniqueness, named usage at every classification site, test +references or unreachable-path explanations, and CustomScript compatibility. + +The allocation ranges are: + +| Absolute code range | Area | +| --- | --- | +| CustomScript values | Shared execution, input, download, identity, policy, and OS classifications | +| 1000-1999 | Download, blob/URL parsing, and preprocessing | +| 2000-2999 | Handler settings and extension policy | +| 3000-4999 | Lifecycle support, services, machine operations, and communication | +| 5000-5099 | Execution and RunAs | +| 5100-5199 | Instance-view serialization | +| 5200-5299 | Local and immediate status | +| 5300-5399 | Command processing | +| 5400-5499 | Command orchestration, upgrades, and output blobs | + +Error-path tests inject filesystem, process, identity, and HTTP failures instead +of depending on the host configuration or Azure endpoints. Defensive paths that +cannot fail with the current concrete types are documented by code in the tests. +Tests expecting a production error must assert its exact named clarification code, +not only that an error occurred or that its message matches. Propagation tests +must verify the original lower-level code. Control-flow sentinels are explicitly +tested as unclassified rather than treated as failures. +Tests expecting a failed execution in status must also check the serialized +`ErrorClarification` substatus and its exact code, including when +`treatFailureAsDeploymentFailure` leaves the top-level status successful. + +### Running tests on Windows + +The handler runtime and full integration suite require Linux. Mocked clarification +tests can also run natively on Windows, including from VS Code: + +```powershell +go test .\cmd\main -run TestCommandLineClarifications +``` + +Cleanup uses shared Go filesystem helpers on Linux and Windows; its error-path +tests inject filesystem failures. Only OS-specific primitives such as Linux +process-group signaling have separate implementations. The Windows signaling +default explicitly returns an unsupported-operation error; tests inject mocks. +Immediate-status tests that exercise persistence use a separate `t.TempDir()` +and `t.Setenv()` extension path per test, so existing local status files cannot +affect their results. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/cmd/main/clarification_test.go b/cmd/main/clarification_test.go new file mode 100644 index 0000000..1222eb0 --- /dev/null +++ b/cmd/main/clarification_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func TestCommandLineClarifications(t *testing.T) { + for _, tc := range []struct { + args []string + code int + }{ + {nil, errorutil.CustomerInput_invalidCommandLineArguments}, + {[]string{"handler"}, errorutil.CustomerInput_invalidCommandLineArguments}, + {[]string{"handler", "enable", "extra"}, errorutil.CustomerInput_invalidCommandLineArguments}, + {[]string{"handler", "invalid"}, errorutil.CustomerInput_unknownCommandLineOperation}, + } { + _, err := parseCommand(tc.args) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) + } + command, err := parseCommand([]string{"handler", "enable"}) + require.NoError(t, err) + require.NotEmpty(t, command.Name) +} diff --git a/cmd/main/main.go b/cmd/main/main.go index 3df2e0f..792ac11 100644 --- a/cmd/main/main.go +++ b/cmd/main/main.go @@ -7,6 +7,7 @@ import ( commands "github.com/Azure/run-command-handler-linux/internal/cmds" "github.com/Azure/run-command-handler-linux/internal/commandProcessor" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/versionutil" ) @@ -33,25 +34,31 @@ func main() { } } -// parseCmd looks at os.Args and parses the subcommand. If it is invalid, -// it prints the usage string and an error message and exits with code 0. +// parseCmd parses the subcommand. If it is invalid, it prints the usage string +// and a clarified error message and exits with code 2. func parseCmd(args []string) types.Cmd { - if len(os.Args) != 2 { + cmd, err := parseCommand(args) + if err != nil { printUsage(args) - fmt.Println("Incorrect usage.") + fmt.Println(err) os.Exit(2) } - op := os.Args[1] + return cmd +} + +func parseCommand(args []string) (types.Cmd, error) { + if len(args) != 2 { + return types.Cmd{}, errorutil.EnsureClarification(errorutil.CustomerInput_invalidCommandLineArguments, fmt.Errorf("Incorrect usage.")) + } + op := args[1] cmd, ok := commands.Cmds[op] // If no command was found, then exit if !ok { - printUsage(args) - fmt.Printf("Incorrect command: %q\n", op) - os.Exit(2) + return types.Cmd{}, errorutil.EnsureClarification(errorutil.CustomerInput_unknownCommandLineOperation, fmt.Errorf("Incorrect command: %q", op)) } - return cmd + return cmd, nil } // printUsage prints the help string and version of the program to stdout with a diff --git a/internal/cleanup/clarification_test.go b/internal/cleanup/clarification_test.go new file mode 100644 index 0000000..6460561 --- /dev/null +++ b/internal/cleanup/clarification_test.go @@ -0,0 +1,87 @@ +package cleanup + +import ( + "errors" + "io" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type cleanupErrorLogger struct{ err error } + +type cleanupDirectory struct{} + +func (cleanupDirectory) Close() error { return nil } + +func (l *cleanupErrorLogger) Log(values ...interface{}) error { + for _, value := range values { + if err, ok := value.(error); ok { + l.err = err + } + } + return nil +} + +func TestCleanupClarifications(t *testing.T) { + oldClear, oldDelete, oldOpen := clearExceptMostRecent, deleteDirectoriesExcept, openDownloadDirectory + t.Cleanup(func() { + clearExceptMostRecent, deleteDirectoriesExcept, openDownloadDirectory = oldClear, oldDelete, oldOpen + }) + cause := errors.New("machine failure") + for _, want := range []int{errorutil.Os_FailedToCleanOldScriptsAndSettings, errorutil.Os_FailedToOpenDownloadDirectory, errorutil.Os_FailedToCleanOldRunAsDirectories} { + clearExceptMostRecent = func(*log.Context, string, string, string, uint64, string, string) error { + if want == errorutil.Os_FailedToCleanOldScriptsAndSettings { + return cause + } + return nil + } + deleteDirectoriesExcept = func(*log.Context, string, string) error { return cause } + openDownloadDirectory = func(string) (io.Closer, error) { + if want == errorutil.Os_FailedToOpenDownloadDirectory { + return nil, cause + } + return cleanupDirectory{}, nil + } + logger := &cleanupErrorLogger{} + user := "" + if want == errorutil.Os_FailedToCleanOldRunAsDirectories { + user = "test" + } + RunCommandCleanup(log.NewContext(logger), types.RCMetadata{DownloadPath: "."}, types.HandlerEnvironment{}, user) + code, ok := errorutil.FindClarification(logger.err) + require.True(t, ok) + require.Equal(t, want, code) + require.ErrorIs(t, logger.err, cause) + } +} + +func TestCleanupPreservesLowerClarification(t *testing.T) { + oldClear, oldDelete, oldOpen := clearExceptMostRecent, deleteDirectoriesExcept, openDownloadDirectory + t.Cleanup(func() { + clearExceptMostRecent, deleteDirectoriesExcept, openDownloadDirectory = oldClear, oldDelete, oldOpen + }) + cause := errors.New("directory read failed") + classified := errorutil.EnsureClarification(errorutil.Os_FailedToReadCleanupParentDirectory, cause) + openDownloadDirectory = func(string) (io.Closer, error) { return cleanupDirectory{}, nil } + for _, user := range []string{"", "test"} { + t.Run("runAsUser="+user, func(t *testing.T) { + clearExceptMostRecent = func(*log.Context, string, string, string, uint64, string, string) error { + if user == "" { + return classified + } + return nil + } + deleteDirectoriesExcept = func(*log.Context, string, string) error { return classified } + logger := &cleanupErrorLogger{} + RunCommandCleanup(log.NewContext(logger), types.RCMetadata{DownloadPath: "."}, types.HandlerEnvironment{}, user) + code, ok := errorutil.FindClarification(logger.err) + require.True(t, ok) + require.Equal(t, errorutil.Os_FailedToReadCleanupParentDirectory, code) + require.ErrorIs(t, logger.err, cause) + }) + } +} diff --git a/internal/cleanup/cleanup.go b/internal/cleanup/cleanup.go index 32bcf6a..8af88b6 100644 --- a/internal/cleanup/cleanup.go +++ b/internal/cleanup/cleanup.go @@ -2,17 +2,24 @@ package cleanup import ( "fmt" + "io" "os" "path/filepath" "strconv" - "github.com/Azure/azure-extension-platform/pkg/utils" "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/linuxutils" "github.com/go-kit/kit/log" ) +var ( + openDownloadDirectory = func(name string) (io.Closer, error) { return os.Open(name) } + clearExceptMostRecent = linuxutils.TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent + deleteDirectoriesExcept = linuxutils.TryDeleteDirectoriesExcept +) + func ImmediateRunCommandCleanup(ctx *log.Context, metadata types.RCMetadata, h types.HandlerEnvironment, runAsUser string) { deleteAllScriptsAndSettings(ctx, metadata, h, runAsUser) } @@ -35,7 +42,7 @@ func deleteAllScriptsAndSettings(ctx *log.Context, metadata types.RCMetadata, h ctx.Log("message", "removing all files from the download 'runas' directory "+runAsDownloadParent) err = linuxutils.TryDeleteDirectories(ctx, runAsDownloadParent) if err != nil { - ctx.Log("event", "could not clear runas script") + ctx.Log("event", "could not clear runas script", "error", err) } } } @@ -45,13 +52,16 @@ func deleteScriptsAndSettingsExceptMostRecent(ctx *log.Context, metadata types.R runtimeSettingsLastSeqNumFormat := metadata.ExtName + ".%d.settings" // check if directory exists - _, err := os.Open(metadata.DownloadPath) + dir, err := openDownloadDirectory(metadata.DownloadPath) if err == nil { - err := utils.TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent(metadata.DownloadPath, h.HandlerEnvironment.ConfigFolder, "", + dir.Close() + err := clearExceptMostRecent(ctx, metadata.DownloadPath, h.HandlerEnvironment.ConfigFolder, "", uint64(metadata.SeqNum), runtimeSettingsRegexFormat, runtimeSettingsLastSeqNumFormat) if err != nil { - ctx.Log("event", "could not clear settings and script files", "error", err) + ctx.Log("event", "could not clear settings and script files", "error", errorutil.EnsureClarification(errorutil.Os_FailedToCleanOldScriptsAndSettings, err)) } + } else if !os.IsNotExist(err) { + ctx.Log("event", "could not open download directory", "error", errorutil.EnsureClarification(errorutil.Os_FailedToOpenDownloadDirectory, err)) } else { ctx.Log("message", "directory does not exist. Skipping cleanup") } @@ -60,9 +70,9 @@ func deleteScriptsAndSettingsExceptMostRecent(ctx *log.Context, metadata types.R runAsDownloadParent := filepath.Join(fmt.Sprintf(constants.RunAsDir, runAsUser), metadata.DownloadDir) seqNumString := strconv.Itoa(metadata.SeqNum) ctx.Log("message", "removing all files from the download 'runas' directory "+runAsDownloadParent) - err = utils.TryDeleteDirectoriesExcept(runAsDownloadParent, seqNumString) + err = deleteDirectoriesExcept(ctx, runAsDownloadParent, seqNumString) if err != nil { - ctx.Log("event", "could not clear runas script") + ctx.Log("event", "could not clear runas script", "error", errorutil.EnsureClarification(errorutil.Os_FailedToCleanOldRunAsDirectories, err)) } } } diff --git a/internal/cmds/clarification_test.go b/internal/cmds/clarification_test.go new file mode 100644 index 0000000..65a5b41 --- /dev/null +++ b/internal/cmds/clarification_test.go @@ -0,0 +1,497 @@ +package commands + +import ( + "bytes" + "compress/gzip" + "container/list" + "encoding/base64" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/Azure/azure-extension-platform/pkg/extensionevents" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob" + "github.com/Azure/azure-sdk-for-go/storage" + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/exec" + "github.com/Azure/run-command-handler-linux/internal/extensionpolicysettingsrc" + "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type mockCommandFile struct { + bytes.Buffer + entries []os.DirEntry + dirErr error +} + +func (f *mockCommandFile) Close() error { return nil } +func (f *mockCommandFile) ReadDir(int) ([]os.DirEntry, error) { return f.entries, f.dirErr } + +type mockCommandEntry string + +func (e mockCommandEntry) Name() string { return string(e) } +func (e mockCommandEntry) IsDir() bool { return false } +func (e mockCommandEntry) Type() os.FileMode { return 0 } +func (e mockCommandEntry) Info() (os.FileInfo, error) { return nil, nil } + +func mockCommandDependencies(t *testing.T) { + t.Helper() + mkdirAll, mkdir, remove, removeAll := commandMkdirAll, commandMkdir, commandRemove, commandRemoveAll + read, write, stat, copy, open, create := commandReadFile, commandWriteFile, commandStat, commandCopy, commandOpen, commandCreate + install, uninstall, enable, settings := immediateInstall, immediateUninstall, immediateEnable, readCommandSettings + readOutput, process, validate := readOutputFromPosition, processDownloadedScript, validateDownloadedScript + credential, client, createBlob, appendSAS, appendManaged := newManagedIdentityCredential, newAppendBlobClient, createManagedAppendBlob, appendSASBlock, appendManagedBlock + t.Cleanup(func() { + commandMkdirAll, commandMkdir, commandRemove, commandRemoveAll = mkdirAll, mkdir, remove, removeAll + commandReadFile, commandWriteFile, commandStat, commandCopy, commandOpen, commandCreate = read, write, stat, copy, open, create + immediateInstall, immediateUninstall, immediateEnable, readCommandSettings = install, uninstall, enable, settings + readOutputFromPosition, processDownloadedScript, validateDownloadedScript = readOutput, process, validate + newManagedIdentityCredential, newAppendBlobClient, createManagedAppendBlob, appendSASBlock, appendManagedBlock = credential, client, createBlob, appendSAS, appendManaged + }) + commandMkdirAll = func(string, os.FileMode) error { return nil } + commandMkdir = func(string, os.FileMode) error { return nil } + commandRemove = func(string) error { return nil } + commandRemoveAll = func(string) error { return nil } + commandReadFile = func(string) ([]byte, error) { return []byte("0"), nil } + commandWriteFile = func(string, []byte, os.FileMode) error { return nil } + commandStat = func(string) (os.FileInfo, error) { return nil, nil } + commandCopy = func(io.Writer, io.Reader) (int64, error) { return 0, nil } + commandOpen = func(string) (commandFile, error) { return &mockCommandFile{}, nil } + commandCreate = func(string) (commandFile, error) { return &mockCommandFile{}, nil } + immediateInstall = func() (int, error) { return 0, nil } + immediateUninstall = func(*log.Context, types.HandlerEnvironment, string, int, *extensionevents.ExtensionEventManager) (int, error) { + return 0, nil + } + immediateEnable = func(*log.Context, types.HandlerEnvironment, string, int, handlersettings.HandlerSettings, *extensionevents.ExtensionEventManager) (int, error) { + return 0, nil + } + readCommandSettings = func(string, string, int, *log.Context) (handlersettings.HandlerSettings, error) { + return handlersettings.HandlerSettings{}, nil + } + readOutputFromPosition = func(string, int64) ([]byte, error) { return []byte("output"), nil } + processDownloadedScript = func(*log.Context, string, string, *handlersettings.HandlerSettings) (string, error) { + return "/mock/script.sh", nil + } + validateDownloadedScript = func(string, *extensionpolicysettingsrc.RCv2ExtensionPolicySettings) error { return nil } + newManagedIdentityCredential = func(*azidentity.ManagedIdentityCredentialOptions) (*azidentity.ManagedIdentityCredential, error) { + return &azidentity.ManagedIdentityCredential{}, nil + } + newAppendBlobClient = func(string, azcore.TokenCredential, *appendblob.ClientOptions) (*appendblob.Client, error) { + return &appendblob.Client{}, nil + } + createManagedAppendBlob = func(*appendblob.Client) error { return nil } + appendSASBlock = func(*storage.Blob, []byte) error { return nil } + appendManagedBlock = func(*appendblob.Client, []byte) error { return nil } +} + +func requireCommandCode(t *testing.T, expected int, err error) { + t.Helper() + require.Error(t, err) + actual, ok := errorutil.FindClarification(err) + require.True(t, ok, "unclassified error: %v", err) + require.Equal(t, expected, actual) +} + +func requireReportedClarification(t *testing.T, expected int, err error, report types.StatusReport) { + t.Helper() + requireCommandCode(t, expected, err) + requireStatusClarification(t, expected, report) +} + +func requireStatusClarification(t *testing.T, expected int, report types.StatusReport) { + t.Helper() + require.Positive(t, expected, "policy violations are customer errors") + require.Len(t, report, 1) + require.Len(t, report[0].Status.Substatuses, 1) + require.Equal(t, "ErrorClarification", report[0].Status.Substatuses[0].Name) + require.Equal(t, string(types.StatusError), report[0].Status.Substatuses[0].Status) + require.Equal(t, expected, report[0].Status.Substatuses[0].Code) +} + +func TestLifecycleClarifications(t *testing.T) { + for _, code := range []int{errorutil.Os_FailedToCreateDataDir, errorutil.Os_FailedToDeleteDataDir, errorutil.Os_FailedToStatRuntimePolicy, errorutil.Storage_failedToAppendOutputWithSas, errorutil.Storage_failedToAppendOutputWithManagedIdentity, errorutil.Os_FailedToResetSequenceNumber, errorutil.FileDownload_unableToCreateDownloadDirectory, errorutil.Os_FailedToDeleteBlockedDownloadedScript, errorutil.ExtensionPolicySettings_downloadedScriptNotAllowed} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockCommandDependencies(t) + failure := errors.New("injected machine/network failure") + ctx := log.NewContext(log.NewNopLogger()) + var err error + switch code { + case errorutil.Os_FailedToCreateDataDir: + commandMkdirAll = func(string, os.FileMode) error { return failure } + _, _, err, _, _ = install(ctx, types.HandlerEnvironment{}, nil, types.RCMetadata{}, types.CmdInstallTemplate) + case errorutil.Os_FailedToDeleteDataDir: + commandRemoveAll = func(string) error { return failure } + _, _, err, _, _ = uninstall(ctx, types.HandlerEnvironment{}, nil, types.RCMetadata{}, types.CmdUninstallTemplate) + case errorutil.Os_FailedToStatRuntimePolicy: + commandStat = func(string) (os.FileInfo, error) { return nil, failure } + _, _, err, _, _ = enable(ctx, types.HandlerEnvironment{}, nil, types.RCMetadata{}, types.CmdEnableTemplate) + case errorutil.Storage_failedToAppendOutputWithSas: + appendSASBlock = func(*storage.Blob, []byte) error { return failure } + _, err = appendToBlob("mock", &storage.Blob{}, nil, 0) + case errorutil.Storage_failedToAppendOutputWithManagedIdentity: + appendManagedBlock = func(*appendblob.Client, []byte) error { return failure } + _, err = appendToBlob("mock", nil, &appendblob.Client{}, 0) + case errorutil.Os_FailedToResetSequenceNumber: + var output bytes.Buffer + ctx = log.NewContext(log.NewLogfmtLogger(&output)) + commandRemove = func(string) error { return failure } + resetSeqNum(ctx, "/mock/mrseq", createExtensionEventManager(ctx, types.HandlerEnvironment{})) + require.Contains(t, output.String(), "clarificationCode="+strconv.Itoa(errorutil.Os_FailedToResetSequenceNumber)) + return + case errorutil.FileDownload_unableToCreateDownloadDirectory: + commandMkdirAll = func(string, os.FileMode) error { return failure } + _, err = downloadScript(ctx, "/mock", &handlersettings.HandlerSettings{}, nil) + case errorutil.Os_FailedToDeleteBlockedDownloadedScript, errorutil.ExtensionPolicySettings_downloadedScriptNotAllowed: + cfg := handlersettings.HandlerSettings{} + cfg.PublicSettings.Source = &handlersettings.ScriptSource{ScriptURI: "https://mock/script.sh"} + validateDownloadedScript = func(string, *extensionpolicysettingsrc.RCv2ExtensionPolicySettings) error { return failure } + var output bytes.Buffer + ctx = log.NewContext(log.NewLogfmtLogger(&output)) + if code == errorutil.Os_FailedToDeleteBlockedDownloadedScript { + commandRemove = func(string) error { return failure } + } + _, err = downloadScript(ctx, "/mock", &cfg, &extensionpolicysettingsrc.RCv2ExtensionPolicySettings{}) + if code == errorutil.Os_FailedToDeleteBlockedDownloadedScript { + require.Contains(t, output.String(), "clarificationCode="+strconv.Itoa(errorutil.Os_FailedToDeleteBlockedDownloadedScript)) + requireCommandCode(t, errorutil.ExtensionPolicySettings_downloadedScriptNotAllowed, err) + return + } + } + requireCommandCode(t, code, err) + }) + } +} + +func TestRehydrationClarifications(t *testing.T) { + for _, code := range []int{errorutil.Os_FailedToOpenRehydrationStatusDirectory, errorutil.Os_FailedToReadRehydrationStatusDirectory, errorutil.Internal_invalidStatusFileSequenceNumber, errorutil.Os_FailedToRehydrateSequenceNumber, errorutil.Os_FailedToStatRehydrationSequenceFile, errorutil.Os_FailedToReadRehydrationSequenceFile, errorutil.Internal_invalidMrseqFileSequenceNumber, errorutil.Os_FailedToUpdateRehydratedSequenceFile} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockCommandDependencies(t) + failure := errors.New("injected failure") + directory := &mockCommandFile{entries: []os.DirEntry{mockCommandEntry("command.5.status")}} + commandOpen = func(string) (commandFile, error) { return directory, nil } + switch code { + case errorutil.Os_FailedToOpenRehydrationStatusDirectory: + commandOpen = func(string) (commandFile, error) { return nil, failure } + case errorutil.Os_FailedToReadRehydrationStatusDirectory: + directory.dirErr = failure + case errorutil.Internal_invalidStatusFileSequenceNumber: + directory.entries = []os.DirEntry{mockCommandEntry("command.invalid.status")} + case errorutil.Os_FailedToRehydrateSequenceNumber: + commandStat = func(string) (os.FileInfo, error) { return nil, os.ErrNotExist } + commandWriteFile = func(string, []byte, os.FileMode) error { return failure } + case errorutil.Os_FailedToStatRehydrationSequenceFile: + commandStat = func(string) (os.FileInfo, error) { return nil, failure } + case errorutil.Os_FailedToReadRehydrationSequenceFile: + commandReadFile = func(string) ([]byte, error) { return nil, failure } + case errorutil.Internal_invalidMrseqFileSequenceNumber: + commandReadFile = func(string) ([]byte, error) { return []byte("invalid"), nil } + case errorutil.Os_FailedToUpdateRehydratedSequenceFile: + commandWriteFile = func(string, []byte, os.FileMode) error { return failure } + } + ctx := log.NewContext(log.NewNopLogger()) + err := rehydrateMrSeqFilesForProblematicUpgrades(ctx, "/mock/old", "/mock/new", createExtensionEventManager(ctx, types.HandlerEnvironment{})) + requireCommandCode(t, code, err) + }) + } +} + +func TestStateCopyClarifications(t *testing.T) { + for _, code := range []int{errorutil.Os_FailedToCreateUpgradeStatusDirectory, errorutil.Internal_emptyUpgradeDirectories, errorutil.Os_FailedToOpenUpgradeSourceDirectory, errorutil.Os_FailedToReadUpgradeSourceDirectory, errorutil.Os_FailedToOpenUpgradeSourceFile, errorutil.Os_FailedToCreateUpgradeDestinationFile, errorutil.Os_FailedToCopyUpgradeStateFile} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockCommandDependencies(t) + failure := errors.New("injected failure") + subdir, source := "", "/mock/source" + directory := &mockCommandFile{entries: []os.DirEntry{mockCommandEntry("command.mrseq")}} + commandOpen = func(string) (commandFile, error) { return directory, nil } + switch code { + case errorutil.Os_FailedToCreateUpgradeStatusDirectory: + subdir = "status" + commandOpen = func(string) (commandFile, error) { return nil, failure } + commandMkdir = func(string, os.FileMode) error { return failure } + case errorutil.Internal_emptyUpgradeDirectories: + source = "" + case errorutil.Os_FailedToOpenUpgradeSourceDirectory: + commandOpen = func(string) (commandFile, error) { return nil, failure } + case errorutil.Os_FailedToReadUpgradeSourceDirectory: + directory.dirErr = failure + case errorutil.Os_FailedToOpenUpgradeSourceFile: + calls := 0 + commandOpen = func(string) (commandFile, error) { + calls++ + if calls == 2 { + return nil, failure + } + return directory, nil + } + case errorutil.Os_FailedToCreateUpgradeDestinationFile: + commandCreate = func(string) (commandFile, error) { return nil, failure } + case errorutil.Os_FailedToCopyUpgradeStateFile: + commandCopy = func(io.Writer, io.Reader) (int64, error) { return 0, failure } + } + ctx := log.NewContext(log.NewNopLogger()) + _, err := copyFiles(ctx, ".mrseq", subdir, source, "/mock/destination", createExtensionEventManager(ctx, types.HandlerEnvironment{})) + requireCommandCode(t, code, err) + }) + } +} + +func TestDummyStatusClarifications(t *testing.T) { + for _, code := range []int{errorutil.Os_FailedToReadDummyStatusSequenceFile, errorutil.Internal_invalidDummyStatusSequenceNumber, errorutil.Internal_emptyDummyStatusSequenceFile, errorutil.Internal_invalidDummyStatusSequenceFileName} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockCommandDependencies(t) + filename := "command.mrseq" + switch code { + case errorutil.Os_FailedToReadDummyStatusSequenceFile: + commandReadFile = func(string) ([]byte, error) { return nil, errors.New("read failed") } + case errorutil.Internal_invalidDummyStatusSequenceNumber: + commandReadFile = func(string) ([]byte, error) { return []byte("invalid"), nil } + case errorutil.Internal_emptyDummyStatusSequenceFile: + commandReadFile = func(string) ([]byte, error) { return nil, nil } + case errorutil.Internal_invalidDummyStatusSequenceFileName: + filename = "invalid" + } + names := list.New() + names.PushBack(filename) + ctx := log.NewContext(log.NewNopLogger()) + err := createDummyStatusFilesIfNeeded(ctx, names, createExtensionEventManager(ctx, types.HandlerEnvironment{})) + requireCommandCode(t, code, err) + }) + } +} + +func TestScriptDecodeClarifications(t *testing.T) { + _, _, err := decodeScript("not base64!") + requireCommandCode(t, errorutil.CustomerInput_invalidEmbeddedScriptBase64, err) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err = writer.Write([]byte("script")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + truncated := compressed.Bytes()[:compressed.Len()-8] + _, _, err = decodeScript(base64.StdEncoding.EncodeToString(truncated)) + requireCommandCode(t, errorutil.CustomerInput_invalidCompressedScript, err) +} + +func TestManagedBlobClarifications(t *testing.T) { + for _, code := range []int{errorutil.CustomerInput_outputIdentityObjectIdNotSupported, errorutil.Storage_failedToCreateManagedAppendBlobClient, errorutil.Storage_failedToCreateManagedAppendBlob, errorutil.Imds_failedToCreateManagedIdentityCredential} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockCommandDependencies(t) + failure := errors.New("injected Azure failure") + var identity *handlersettings.RunCommandManagedIdentity + switch code { + case errorutil.CustomerInput_outputIdentityObjectIdNotSupported: + identity = &handlersettings.RunCommandManagedIdentity{ObjectId: "unsupported"} + case errorutil.Storage_failedToCreateManagedAppendBlobClient: + newAppendBlobClient = func(string, azcore.TokenCredential, *appendblob.ClientOptions) (*appendblob.Client, error) { + return nil, failure + } + case errorutil.Storage_failedToCreateManagedAppendBlob: + createManagedAppendBlob = func(*appendblob.Client) error { return failure } + case errorutil.Imds_failedToCreateManagedIdentityCredential: + newManagedIdentityCredential = func(*azidentity.ManagedIdentityCredentialOptions) (*azidentity.ManagedIdentityCredential, error) { + return nil, failure + } + } + + _, err := createOrReplaceAppendBlobUsingManagedIdentity("https://mock/container/blob", identity) + requireCommandCode(t, code, err) + }) + } +} + +func TestEnableBlobCreateOrReplaceExitCodes(t *testing.T) { + for _, tc := range []struct { + name string + outputURI string + errorURI string + wantExitCode int + }{ + {"output", "https://mock/container/output", "", constants.ExitCode_OutputBlobCreateOrReplaceFailed}, + {"error", "", "https://mock/container/error", constants.ExitCode_ErrorBlobCreateOrReplaceFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + mockCommandDependencies(t) + commandStat = func(string) (os.FileInfo, error) { return nil, os.ErrNotExist } + readCommandSettings = func(string, string, int, *log.Context) (handlersettings.HandlerSettings, error) { + return handlersettings.HandlerSettings{ + PublicSettings: handlersettings.PublicSettings{ + OutputBlobURI: tc.outputURI, + ErrorBlobURI: tc.errorURI, + Source: &handlersettings.ScriptSource{}, + }, + }, nil + } + failure := errors.New("injected blob creation failure") + createManagedAppendBlob = func(*appendblob.Client) error { return failure } + originalRunCmd := RunCmd + t.Cleanup(func() { RunCmd = originalRunCmd }) + RunCmd = func(*log.Context, string, string, *handlersettings.HandlerSettings, types.RCMetadata) (error, int) { + t.Fatal("script must not run after blob creation fails") + return nil, constants.ExitCode_Okay + } + + stdout, stderr, err, exitCode, cleanup := enable( + log.NewContext(log.NewNopLogger()), types.HandlerEnvironment{}, + &types.RunCommandInstanceView{}, types.RCMetadata{DownloadPath: t.TempDir()}, Cmds["enable"]) + + require.ErrorIs(t, err, failure) + requireCommandCode(t, errorutil.Storage_failedToCreateManagedAppendBlob, err) + require.Equal(t, tc.wantExitCode, exitCode) + require.NotEqual(t, constants.ExitCode_BlobCreateOrReplaceFailed, exitCode) + require.Empty(t, stdout) + require.Empty(t, stderr) + require.Nil(t, cleanup) + }) + } + require.NotEqual(t, constants.ExitCode_OutputBlobCreateOrReplaceFailed, constants.ExitCode_ErrorBlobCreateOrReplaceFailed) +} + +func TestEnableBlobAppendsBestEffort(t *testing.T) { + scriptFailure := errors.New("script failed") + for _, tc := range []struct { + name string + failOutput bool + failError bool + failRead bool + waitForTick bool + scriptError error + scriptExitCode int + }{ + {name: "success"}, + {name: "output failure", failOutput: true}, + {name: "error failure", failError: true}, + {name: "both fail", failOutput: true, failError: true}, + {name: "read failure", failRead: true}, + {name: "script failure preserved", failOutput: true, failError: true, scriptError: scriptFailure, scriptExitCode: 42}, + {name: "periodic and final failures", failOutput: true, failError: true, waitForTick: true}, + } { + t.Run(tc.name, func(t *testing.T) { + mockCommandDependencies(t) + commandStat = func(string) (os.FileInfo, error) { return nil, os.ErrNotExist } + readCommandSettings = func(string, string, int, *log.Context) (handlersettings.HandlerSettings, error) { + return handlersettings.HandlerSettings{ + PublicSettings: handlersettings.PublicSettings{ + OutputBlobURI: "https://mock/container/output", + ErrorBlobURI: "https://mock/container/error", + Source: &handlersettings.ScriptSource{}, + }, + }, nil + } + + dataDir := t.TempDir() + eventsDir := t.TempDir() + dir := filepath.Join(dataDir, "0") + require.NoError(t, os.MkdirAll(dir, 0700)) + stdoutPath, stderrPath := exec.LogPaths(dir) + require.NoError(t, os.WriteFile(stdoutPath, []byte("stdout"), 0600)) + require.NoError(t, os.WriteFile(stderrPath, []byte("stderr"), 0600)) + blobFailure := errors.New("injected append failure") + var outputClient, errorClient *appendblob.Client + newAppendBlobClient = func(uri string, _ azcore.TokenCredential, _ *appendblob.ClientOptions) (*appendblob.Client, error) { + client := &appendblob.Client{} + if strings.HasSuffix(uri, "/output") { + outputClient = client + } else { + errorClient = client + } + return client, nil + } + readCount := 0 + readOutputFromPosition = func(_ string, position int64) ([]byte, error) { + readCount++ + if tc.failRead { + return nil, blobFailure + } + if position > 0 { + return nil, nil + } + return []byte("output"), nil + } + periodicAppended := make(chan struct{}) + outputAttempts, errorAttempts := 0, 0 + appendManagedBlock = func(client *appendblob.Client, _ []byte) error { + if client == outputClient { + outputAttempts++ + if tc.failOutput { + return blobFailure + } + } else if client == errorClient { + errorAttempts++ + if tc.waitForTick && errorAttempts == 1 { + close(periodicAppended) + } + if tc.failError { + return blobFailure + } + } + return nil + } + originalRunCmd := RunCmd + t.Cleanup(func() { RunCmd = originalRunCmd }) + RunCmd = func(*log.Context, string, string, *handlersettings.HandlerSettings, types.RCMetadata) (error, int) { + if tc.waitForTick { + select { + case <-periodicAppended: + case <-time.After(2 * updateStatusInSeconds * time.Second): + t.Error("periodic blob appends were not attempted") + } + } + return tc.scriptError, tc.scriptExitCode + } + var logs bytes.Buffer + env := types.HandlerEnvironment{} + env.HandlerEnvironment.EventsFolder = eventsDir + stdout, stderr, err, exitCode, cleanup := enable( + log.NewContext(log.NewLogfmtLogger(&logs)), env, &types.RunCommandInstanceView{}, + types.RCMetadata{DownloadPath: dataDir}, types.Cmd{}) + + require.Equal(t, tc.scriptError, err) + require.Equal(t, tc.scriptExitCode, exitCode) + require.Equal(t, "stdout", stdout) + require.Equal(t, "stderr", stderr) + require.NotNil(t, cleanup) + expectedAttempts := 1 + if tc.waitForTick { + expectedAttempts = 2 + } + require.Equal(t, 2*expectedAttempts, readCount) + if !tc.failRead { + require.Equal(t, expectedAttempts, outputAttempts) + require.Equal(t, expectedAttempts, errorAttempts) + } + expectedFailures := 0 + if tc.failOutput || tc.failError || tc.failRead { + expectedFailures = 1 + } + require.Equal(t, expectedFailures, strings.Count(logs.String(), "Failed to append command output")) + events, eventErr := os.ReadDir(eventsDir) + require.NoError(t, eventErr) + require.Len(t, events, expectedFailures) + for _, event := range events { + content, readErr := os.ReadFile(filepath.Join(eventsDir, event.Name())) + require.NoError(t, readErr) + var recorded struct { + TaskName string + EventLevel string + Message string + } + require.NoError(t, json.Unmarshal(content, &recorded)) + require.Equal(t, "enable", recorded.TaskName) + require.Equal(t, "Error", recorded.EventLevel) + require.Contains(t, recorded.Message, "injected append failure") + } + }) + } +} diff --git a/internal/cmds/cmds.go b/internal/cmds/cmds.go index c3f5592..950d63c 100755 --- a/internal/cmds/cmds.go +++ b/internal/cmds/cmds.go @@ -7,7 +7,6 @@ import ( "container/list" "context" "encoding/base64" - "encoding/json" "fmt" "io" "os" @@ -41,6 +40,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/telemetry" "github.com/Azure/run-command-handler-linux/internal/types" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" seqnum "github.com/Azure/run-command-handler-linux/pkg/seqnumutil" "github.com/Azure/run-command-handler-linux/pkg/versionutil" "github.com/go-kit/kit/log" @@ -59,6 +59,36 @@ const ( ) var ( + immediateInstall = immediatecmds.Install + immediateUninstall = immediatecmds.Uninstall + immediateEnable = immediatecmds.Enable + readCommandSettings = handlersettings.GetHandlerSettings + readOutputFromPosition = files.GetFileFromPosition + processDownloadedScript = files.DownloadAndProcessScript + validateDownloadedScript = func(path string, policy *extensionpolicysettingsrc.RCv2ExtensionPolicySettings) error { + return extensionpolicysettings.ValidateFileHashInAllowlist(path, policy.DownloadedScriptsAllowlist, hashutils.HashTypeSHA256) + } + commandMkdirAll = os.MkdirAll + commandMkdir = os.Mkdir + commandRemove = os.Remove + commandRemoveAll = os.RemoveAll + commandReadFile = os.ReadFile + commandWriteFile = os.WriteFile + commandStat = os.Stat + commandCopy = io.Copy + commandOpen = func(path string) (commandFile, error) { return os.Open(path) } + commandCreate = func(path string) (commandFile, error) { return os.Create(path) } + newManagedIdentityCredential = azidentity.NewManagedIdentityCredential + newAppendBlobClient = appendblob.NewClient + createManagedAppendBlob = func(client *appendblob.Client) error { + _, err := client.Create(context.Background(), nil) + return err + } + appendSASBlock = func(blob *storage.Blob, data []byte) error { return blob.AppendBlock(data, nil) } + appendManagedBlock = func(client *appendblob.Client, data []byte) error { + _, err := client.AppendBlock(context.Background(), streaming.NopCloser(bytes.NewReader(data)), nil) + return err + } cmdDefaultReportStatusFunc = status.ReportStatusToLocalFile cmdDefaultCleanupFunc = cleanup.RunCommandCleanup telemetryResult = telemetry.SendTelemetry(telemetry.NewTelemetryEventSender(), fullName, versionutil.Version) @@ -86,9 +116,14 @@ var ( // Used by unit tests to mock out the immediate run command update step immediateUpdate = immediatecmds.Update - ErrAlreadyProcessed = errors.New("the script configuration has already been processed, will not run again") + ErrAlreadyProcessed = types.ErrAlreadyProcessed ) +type commandFile interface { + io.ReadWriteCloser + ReadDir(int) ([]os.DirEntry, error) +} + func update(ctx *log.Context, h types.HandlerEnvironment, report *types.RunCommandInstanceView, metadata types.RCMetadata, c types.Cmd) (string, string, error, int, *types.CleanupRequest) { extensionEvents := createExtensionEventManager(ctx, h) @@ -143,15 +178,15 @@ func disable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComm func install(ctx *log.Context, h types.HandlerEnvironment, report *types.RunCommandInstanceView, metadata types.RCMetadata, c types.Cmd) (string, string, error, int, *types.CleanupRequest) { extensionEvents := createExtensionEventManager(ctx, h) - exitCode, err := immediatecmds.Install() + exitCode, err := immediateInstall() if err != nil { return "", "", err, exitCode, nil } - if err := os.MkdirAll(DataDir, 0755); err != nil { + if err := commandMkdirAll(DataDir, 0755); err != nil { errMessage := fmt.Sprintf("Failed to create data dir: %v due to: %v", DataDir, err) extensionEvents.LogErrorEvent("install", errMessage) - return "", "", errors.Wrap(err, errMessage), constants.ExitCode_CreateDataDirectoryFailed, nil + return "", "", errorutil.EnsureClarification(errorutil.Os_FailedToCreateDataDir, errors.Wrap(err, errMessage)), constants.ExitCode_CreateDataDirectoryFailed, nil } ctx.Log("event", "created data dir", "path", DataDir) @@ -162,7 +197,7 @@ func install(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComm func uninstall(ctx *log.Context, h types.HandlerEnvironment, report *types.RunCommandInstanceView, metadata types.RCMetadata, c types.Cmd) (string, string, error, int, *types.CleanupRequest) { extensionEvents := createExtensionEventManager(ctx, h) - exitCode, err := immediatecmds.Uninstall(ctx, h, metadata.ExtName, metadata.SeqNum, extensionEvents) + exitCode, err := immediateUninstall(ctx, h, metadata.ExtName, metadata.SeqNum, extensionEvents) if err != nil { return "", "", err, exitCode, nil } @@ -170,10 +205,10 @@ func uninstall(ctx *log.Context, h types.HandlerEnvironment, report *types.RunCo { // a new context scope with path ctx = ctx.With("path", DataDir) ctx.Log("event", "removing data dir", "path", DataDir) - if err := os.RemoveAll(DataDir); err != nil { + if err := commandRemoveAll(DataDir); err != nil { errMessage := fmt.Sprintf("Failed to delete data directory: %v due to: %v", DataDir, err) extensionEvents.LogErrorEvent("uninstall", errMessage) - return "", "", errors.Wrap(err, errMessage), constants.ExitCode_RemoveDataDirectoryFailed, nil + return "", "", errorutil.EnsureClarification(errorutil.Os_FailedToDeleteDataDir, errors.Wrap(err, errMessage)), constants.ExitCode_RemoveDataDirectoryFailed, nil } ctx.Log("event", "removed data dir") extensionEvents.LogInformationalEvent("uninstall", fmt.Sprintf("removed data dir %v", DataDir)) @@ -204,14 +239,14 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma extensionEvents := createExtensionEventManager(ctx, h) // parse the extension handler settings (not available prior to 'enable') - cfg, err1 := handlersettings.GetHandlerSettings(h.HandlerEnvironment.ConfigFolder, metadata.ExtName, metadata.SeqNum, ctx) + cfg, err1 := readCommandSettings(h.HandlerEnvironment.ConfigFolder, metadata.ExtName, metadata.SeqNum, ctx) if err1 != nil { errMessage := fmt.Sprintf("Failed to get configuration: %v", err1) extensionEvents.LogErrorEvent("enable", errMessage) return "", "", errors.Wrap(err1, "failed to get configuration"), constants.ExitCode_GetHandlerSettingsFailed, nil } - exitCode, err := immediatecmds.Enable(ctx, h, metadata.ExtName, metadata.SeqNum, cfg, extensionEvents) + exitCode, err := immediateEnable(ctx, h, metadata.ExtName, metadata.SeqNum, cfg, extensionEvents) // If there is an error or the customer requested to install the script as a service, return the error and exit code immediately. if err != nil || cfg.InstallAsService() { @@ -224,7 +259,7 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma policyPath := filepath.Join(h.HandlerEnvironment.ConfigFolder, constants.PolicyFileName) var rceps *extensionpolicysettingsrc.RCv2ExtensionPolicySettings - if _, err := os.Stat(policyPath); err == nil { + if _, err := commandStat(policyPath); err == nil { extensionPolicyManagerPtr, rceps, err, exitCode = extensionpolicysettingsrc.InitializeExtensionPolicySettings(ctx, policyPath) if err != nil { // TODO: In the future, fail the command if the policy file was invalid and return the exitCode. @@ -238,7 +273,7 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma ctx.Log("message", "extension policy settings file does not exist. No policy applied.", "error", err) extensionPolicyManagerPtr = nil } else { - return "", "", errors.Wrap(err, "failed to stat extension policy settings file"), constants.ExitCode_LoadExtensionPolicySettingsFailed, nil + return "", "", errorutil.EnsureClarification(errorutil.Os_FailedToStatRuntimePolicy, errors.Wrap(err, "failed to stat extension policy settings file")), constants.ExitCode_LoadExtensionPolicySettingsFailed, nil } // Validate handler settings against policy settings. @@ -290,7 +325,7 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma return "", "", errors.Wrap(outputBlobAppendCreateOrReplaceError, fmt.Sprintf(blobCreateOrReplaceError, cfg.OutputBlobURI)), - constants.ExitCode_BlobCreateOrReplaceFailed, nil + constants.ExitCode_OutputBlobCreateOrReplaceFailed, nil } } @@ -308,7 +343,7 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma return "", "", errors.Wrap(errorBlobAppendCreateOrReplaceError, fmt.Sprintf(blobCreateOrReplaceError, cfg.ErrorBlobURI)), - constants.ExitCode_BlobCreateOrReplaceFailed, nil + constants.ExitCode_ErrorBlobCreateOrReplaceFailed, nil } } @@ -323,6 +358,18 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma stdoutF, stderrF := exec.LogPaths(dir) + appendFailureLogged := false + appendOutput := func(sourceFilePath string, blobRef *storage.Blob, blobClient *appendblob.Client, position int64) int64 { + newPosition, appendErr := appendToBlob(sourceFilePath, blobRef, blobClient, position) + if appendErr != nil && !appendFailureLogged { + appendFailureLogged = true + message := fmt.Sprintf("Failed to append command output from '%s' to blob; blob appends are best effort and do not affect the command result: %v", sourceFilePath, appendErr) + ctx.Log("message", message, "error", appendErr) + extensionEvents.LogErrorEvent("enable", message) + } + return newPosition + } + // Implement ticker to update extension status periodically ticker := time.NewTicker(updateStatusInSeconds * time.Second) done := make(chan bool) @@ -337,8 +384,8 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma report.Output = stdoutTail report.Error = stderrTail instanceview.ReportInstanceView(ctx, h, metadata, statusToReport, c, report) - outputFilePosition, err = appendToBlob(stdoutF, outputBlobSASRef, outputBlobAppendClient, outputFilePosition, ctx) - errorFilePosition, err = appendToBlob(stderrF, errorBlobSASRef, errorBlobAppendClient, errorFilePosition, ctx) + outputFilePosition = appendOutput(stdoutF, outputBlobSASRef, outputBlobAppendClient, outputFilePosition) + errorFilePosition = appendOutput(stderrF, errorBlobSASRef, errorBlobAppendClient, errorFilePosition) } } }() @@ -362,8 +409,8 @@ func enable(ctx *log.Context, h types.HandlerEnvironment, report *types.RunComma } // Report the output streams to blobs - outputFilePosition, err = appendToBlob(stdoutF, outputBlobSASRef, outputBlobAppendClient, outputFilePosition, ctx) - errorFilePosition, err = appendToBlob(stderrF, errorBlobSASRef, errorBlobAppendClient, errorFilePosition, ctx) + outputFilePosition = appendOutput(stdoutF, outputBlobSASRef, outputBlobAppendClient, outputFilePosition) + errorFilePosition = appendOutput(stderrF, errorBlobSASRef, errorBlobAppendClient, errorFilePosition) return stdoutTail, stderrTail, runErr, exitCode, &types.CleanupRequest{RunAsUser: cfg.PublicSettings.RunAsUser} } @@ -377,45 +424,40 @@ func createExtensionEventManager(ctx *log.Context, hEnv types.HandlerEnvironment // Converts from the local HandlerEnvironment to the azure-extension-platform HandlerEnvironment func convertToPlatformHandlerEnv(myEnv types.HandlerEnvironment) *handlerenv.HandlerEnvironment { - data, err := json.Marshal(myEnv.HandlerEnvironment) - if err != nil { - return nil - } - - var result handlerenv.HandlerEnvironment - err = json.Unmarshal(data, &result) - if err != nil { - return nil + env := myEnv.HandlerEnvironment + return &handlerenv.HandlerEnvironment{ + HeartbeatFile: env.HeartbeatFile, + StatusFolder: env.StatusFolder, + ConfigFolder: env.ConfigFolder, + LogFolder: env.LogFolder, + EventsFolder: env.EventsFolder, + DeploymentID: env.DeploymentID, + RoleName: env.RoleName, + Instance: env.Instance, + HostResolverAddress: env.HostResolverAddress, } - - return &result } // appendToBlob saves a file (from seeking position to the end of the file) to AppendBlob. Returns the new position (end of the file) -func appendToBlob(sourceFilePath string, appendBlobRef *storage.Blob, appendBlobClient *appendblob.Client, outputFilePosition int64, ctx *log.Context) (int64, error) { +func appendToBlob(sourceFilePath string, appendBlobRef *storage.Blob, appendBlobClient *appendblob.Client, outputFilePosition int64) (int64, error) { var err error var newOutput []byte if appendBlobRef != nil || appendBlobClient != nil { // Save to blob - newOutput, err = files.GetFileFromPosition(sourceFilePath, outputFilePosition) + newOutput, err = readOutputFromPosition(sourceFilePath, outputFilePosition) if err == nil { newOutputSize := len(newOutput) if newOutputSize > 0 { if appendBlobRef != nil { - err = appendBlobRef.AppendBlock(newOutput, nil) + err = errorutil.EnsureClarification(errorutil.Storage_failedToAppendOutputWithSas, appendSASBlock(appendBlobRef, newOutput)) } else if appendBlobClient != nil { - ctx.Log("message", fmt.Sprintf("inside appendBlobClient. Output is '%s'", newOutput)) - _, err = appendBlobClient.AppendBlock(context.Background(), streaming.NopCloser(bytes.NewReader(newOutput)), nil) + err = errorutil.EnsureClarification(errorutil.Storage_failedToAppendOutputWithManagedIdentity, appendManagedBlock(appendBlobClient, newOutput)) } if err == nil { outputFilePosition += int64(newOutputSize) - } else { - ctx.Log("message", "AppendToBlob failed", "error", err) } } - } else { - ctx.Log("message", "AppendToBlob - GetFileFromPosition failed.", "error", err) } } @@ -463,7 +505,9 @@ func resetSeqNum(ctx log.Logger, mrseqPath string, extensionEvents *extensioneve message := fmt.Sprintf("Resetting seqnum by deleting file '%s'", mrseqPath) ctx.Log("message", message) extensionEvents.LogInformationalEvent("resetseqnum", message) - os.Remove(mrseqPath) + if err := commandRemove(mrseqPath); err != nil && !os.IsNotExist(err) { + ctx.Log("error", errorutil.EnsureClarification(errorutil.Os_FailedToResetSequenceNumber, errors.Wrap(err, "failed to reset sequence number")), "clarificationCode", errorutil.Os_FailedToResetSequenceNumber) + } } // Copy state of the extension from old version to new version during update (.mrseq files, .status files) @@ -587,11 +631,11 @@ func padTo(in []int, size int) []int { func rehydrateMrSeqFilesForProblematicUpgrades(ctx *log.Context, updateFromVersionDirectory string, updateToVersionDirectory string, extensionEvents *extensionevents.ExtensionEventManager) error { oldExtensionStatusDirectory := filepath.Join(updateFromVersionDirectory, constants.StatusFileDirectory) - extensionStatusDirectoryFDRef, err := os.Open(oldExtensionStatusDirectory) + extensionStatusDirectoryFDRef, err := commandOpen(oldExtensionStatusDirectory) if err != nil { errMessage := fmt.Sprintf("Failed to open status directory '%s' due to '%v'", oldExtensionStatusDirectory, err) extensionEvents.LogErrorEvent("rehydratemrseq", errMessage) - return errors.Wrap(err, errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToOpenRehydrationStatusDirectory, errors.Wrap(err, errMessage)) } defer extensionStatusDirectoryFDRef.Close() @@ -603,7 +647,7 @@ func rehydrateMrSeqFilesForProblematicUpgrades(ctx *log.Context, updateFromVersi errMessage := fmt.Sprintf("could not read directory entries from status directory %s", updateFromVersionDirectory) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("rehydratemrseq", errMessage) - return errors.Wrap(err, errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToReadRehydrationStatusDirectory, errors.Wrap(err, errMessage)) } for _, statusFile := range statusFiles { @@ -616,22 +660,25 @@ func rehydrateMrSeqFilesForProblematicUpgrades(ctx *log.Context, updateFromVersi } else { extensionName := parts[0] seqNo := parts[1] - seqNoAsInt, _ := strconv.Atoi(seqNo) + seqNoAsInt, parseErr := strconv.Atoi(seqNo) + if parseErr != nil { + return errorutil.EnsureClarification(errorutil.Internal_invalidStatusFileSequenceNumber, errors.Wrap(parseErr, "invalid sequence number in status filename")) + } mrSeqFileName := extensionName + constants.MrSeqFileExtension mrSeqFilePath := filepath.Join(updateToVersionDirectory, mrSeqFileName) - _, err = os.Stat(mrSeqFilePath) + _, err = commandStat(mrSeqFilePath) if err != nil { if errors.Is(err, os.ErrNotExist) { message := fmt.Sprintf("Rehydrating mrseq file for '%s' because it was mistakenly deleted during disable", extensionName) ctx.Log("message", message) extensionEvents.LogInformationalEvent("rehydratemrseq", message) - err = os.WriteFile(mrSeqFilePath, []byte(seqNo), os.FileMode(0600)) + err = commandWriteFile(mrSeqFilePath, []byte(seqNo), os.FileMode(0600)) if err != nil { errMessage := fmt.Sprintf("Could not write file '%s'", mrSeqFilePath) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("rehydratemrseq", errMessage) - return errors.Wrap(err, errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToRehydrateSequenceNumber, errors.Wrap(err, errMessage)) } message = fmt.Sprintf("Successfully rehydrated mrseq file for '%s' with seqNo '%s'. File location '%s'", extensionName, seqNo, mrSeqFilePath) ctx.Log("message", message) @@ -640,26 +687,29 @@ func rehydrateMrSeqFilesForProblematicUpgrades(ctx *log.Context, updateFromVersi errMessage := fmt.Sprintf("Could not access file '%s' even though it exists", mrSeqFilePath) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("rehydratemrseq", errMessage) - return errors.Wrap(err, errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToStatRehydrationSequenceFile, errors.Wrap(err, errMessage)) } } else { // Overwrite mrseq this seqNo is greater than the one it contains (simple solution for multiple status files) - contents, err := os.ReadFile(mrSeqFilePath) + contents, err := commandReadFile(mrSeqFilePath) if err != nil { errMessage := fmt.Sprintf("Could not read file '%s'", mrSeqFilePath) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("rehydratemrseq", errMessage) - return errors.Wrap(err, errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToReadRehydrationSequenceFile, errors.Wrap(err, errMessage)) } - currentSeqNo, _ := strconv.Atoi(string(contents)) + currentSeqNo, parseErr := strconv.Atoi(string(contents)) + if parseErr != nil { + return errorutil.EnsureClarification(errorutil.Internal_invalidMrseqFileSequenceNumber, errors.Wrap(parseErr, "invalid sequence number in mrseq file")) + } if seqNoAsInt > currentSeqNo { - err = os.WriteFile(mrSeqFilePath, []byte(seqNo), os.FileMode(0600)) + err = commandWriteFile(mrSeqFilePath, []byte(seqNo), os.FileMode(0600)) if err != nil { errMessage := fmt.Sprintf("Could not write file '%s'", mrSeqFilePath) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("rehydratemrseq", errMessage) - return errors.Wrap(err, errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToUpdateRehydratedSequenceFile, errors.Wrap(err, errMessage)) } message := fmt.Sprintf("Updated mrseq file for '%s' with seqNo '%s'. File location '%s'", extensionName, seqNo, mrSeqFilePath) ctx.Log("message", message) @@ -686,38 +736,41 @@ func copyFiles(ctx log.Logger, fileExtensionSuffix string, extensionSubdirectory upgradeFromVersionDirectory = filepath.Join(upgradeFromVersionDirectory, extensionSubdirectory) // Create subdirectory like "status" directory if it does not exist - _, err := os.Open(upgradeToVersionDirectory) + directory, err := commandOpen(upgradeToVersionDirectory) if err != nil { - errr := os.Mkdir(upgradeToVersionDirectory, 0700) + errr := commandMkdir(upgradeToVersionDirectory, 0700) if errr != nil { errMessage := fmt.Sprintf("Failed to create directory '%s'", upgradeToVersionDirectory) extensionEvents.LogErrorEvent("copyfiles", errMessage) - return nil, errors.Wrap(errr, errMessage) + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToCreateUpgradeStatusDirectory, errors.Wrap(errr, errMessage)) } + } else { + directory.Close() } } if upgradeFromVersionDirectory == "" || upgradeToVersionDirectory == "" { errMessage := "oldExtesionDirectory or newExtensionDirectory is empty" extensionEvents.LogErrorEvent("copyfiles", errMessage) - return nil, errors.New(errMessage) + return nil, errorutil.EnsureClarification(errorutil.Internal_emptyUpgradeDirectories, errors.New(errMessage)) } // Check if the directory exists - sourceDirectoryFDRef, err := os.Open(upgradeFromVersionDirectory) + sourceDirectoryFDRef, err := commandOpen(upgradeFromVersionDirectory) if err != nil { errMessage := fmt.Sprintf("could not open sourceDirectory %s", upgradeFromVersionDirectory) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("copyfiles", errMessage) - return nil, errors.Wrap(err, errMessage) + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToOpenUpgradeSourceDirectory, errors.Wrap(err, errMessage)) } + defer sourceDirectoryFDRef.Close() directoryEntries, err := sourceDirectoryFDRef.ReadDir(0) if err != nil { errMessage := fmt.Sprintf("could not read directory entries from sourceDirectory %s", upgradeFromVersionDirectory) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("copyfiles", errMessage) - return nil, errors.Wrap(err, errMessage) + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToReadUpgradeSourceDirectory, errors.Wrap(err, errMessage)) } numberOfFilesMigrated := 0 @@ -730,31 +783,31 @@ func copyFiles(ctx log.Logger, fileExtensionSuffix string, extensionSubdirectory sourceFileFullPath := filepath.Join(upgradeFromVersionDirectory, fileName) destinationFileFullPath := filepath.Join(upgradeToVersionDirectory, fileName) - sourceFile, sourceFileOpenError := os.Open(sourceFileFullPath) + sourceFile, sourceFileOpenError := commandOpen(sourceFileFullPath) if sourceFileOpenError != nil { errMessage := "Failed to open '%s' file '%s' for reading. Contact ICM team AzureRT\\Extensions for this service error." ctx.Log("message", fmt.Sprintf(errMessage, fileExtensionSuffix, sourceFileFullPath)) extensionEvents.LogErrorEvent("copyfiles", errMessage) - return fileNamesMigrated, errors.Wrap(sourceFileOpenError, errMessage) + return fileNamesMigrated, errorutil.EnsureClarification(errorutil.Os_FailedToOpenUpgradeSourceFile, errors.Wrap(sourceFileOpenError, errMessage)) } defer sourceFile.Close() - destFile, destFileCreateError := os.Create(destinationFileFullPath) + destFile, destFileCreateError := commandCreate(destinationFileFullPath) if destFileCreateError != nil { errMessage := "Failed to create '%s' file '%s'. Contact ICM team AzureRT\\Extensions for this service error." ctx.Log("message", fmt.Sprintf(errMessage, fileExtensionSuffix, destinationFileFullPath)) extensionEvents.LogErrorEvent("copyfiles", errMessage) - return fileNamesMigrated, errors.Wrap(destFileCreateError, errMessage) + return fileNamesMigrated, errorutil.EnsureClarification(errorutil.Os_FailedToCreateUpgradeDestinationFile, errors.Wrap(destFileCreateError, errMessage)) } defer destFile.Close() - _, copyError := io.Copy(destFile, sourceFile) + _, copyError := commandCopy(destFile, sourceFile) if copyError != nil { errMessage := fmt.Sprintf("Failed to copy '%s' file '%s' to path '%s'. Contact ICM team AzureRT\\Extensions for this service error.", fileExtensionSuffix, sourceFileFullPath, destinationFileFullPath) ctx.Log("message", errMessage) extensionEvents.LogErrorEvent("copyfiles", errMessage) - return fileNamesMigrated, errors.Wrap(copyError, errMessage) + return fileNamesMigrated, errorutil.EnsureClarification(errorutil.Os_FailedToCopyUpgradeStateFile, errors.Wrap(copyError, errMessage)) } else { message := fmt.Sprintf("File '%s' was copied successfully to '%s'", sourceFileFullPath, destinationFileFullPath) ctx.Log("message", message) @@ -792,19 +845,21 @@ func createDummyStatusFilesIfNeeded(ctx log.Logger, mrseqFilesNameList *list.Lis var instanceViewMessage string var err error var content []byte - var allErr error = errors.New("Refer to all error messages above.") + var allErr error for mreSeqFileNameElement := mrseqFilesNameList.Front(); mreSeqFileNameElement != nil; mreSeqFileNameElement = mreSeqFileNameElement.Next() { mrSeqFileName = (mreSeqFileNameElement.Value).(string) // Read the most recently executed sequence number from the .mrseq file mrSeqFileFullPath = filepath.Join(newExtensionDirectory, mrSeqFileName) - content, err = os.ReadFile(mrSeqFileFullPath) + content, err = commandReadFile(mrSeqFileFullPath) if err != nil { errorMessage = fmt.Sprintf("Reading mrseq (Most Recently executed Sequence number) from file '%s' failed with error '%s'", mrSeqFileFullPath, err.Error()) ctx.Log("error", errorMessage) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errorutil.EnsureClarification(errorutil.Os_FailedToReadDummyStatusSequenceFile, errors.Wrap(err, errorMessage)) + } continue } @@ -816,14 +871,18 @@ func createDummyStatusFilesIfNeeded(ctx log.Logger, mrseqFilesNameList *list.Lis errorMessage = fmt.Sprintf("mrseqNumberString to mrseqNumber conversion (string to int) of '%s' failed with error '%s'", mrseqNumberString, err.Error()) ctx.Log("error", errorMessage) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errorutil.EnsureClarification(errorutil.Internal_invalidDummyStatusSequenceNumber, errors.Wrap(err, errorMessage)) + } continue } } else { errorMessage = fmt.Sprintf("Empty .mrseq file content. No sequence number was found inside file '%s' ", mrSeqFileFullPath) ctx.Log("error", errorMessage) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errorutil.EnsureClarification(errorutil.Internal_emptyDummyStatusSequenceFile, errors.New(errorMessage)) + } continue } @@ -833,7 +892,9 @@ func createDummyStatusFilesIfNeeded(ctx log.Logger, mrseqFilesNameList *list.Lis errorMessage = fmt.Sprintf("Invalid mrseq file '%s'", mrSeqFileName) ctx.Log("error", errorMessage) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errorutil.EnsureClarification(errorutil.Internal_invalidDummyStatusSequenceFileName, errors.New(errorMessage)) + } continue } extensionName = mrSeqFileName[0:mrSeqFileExtensionIndex] @@ -860,7 +921,9 @@ func createDummyStatusFilesIfNeeded(ctx log.Logger, mrseqFilesNameList *list.Lis if err != nil { errorMessage = fmt.Sprintf("Failed to serialize unknown instanceView, error is '%s'", err.Error()) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errors.Wrap(err, errorMessage) + } continue } @@ -869,7 +932,9 @@ func createDummyStatusFilesIfNeeded(ctx log.Logger, mrseqFilesNameList *list.Lis if err != nil { errorMessage = fmt.Sprintf("failed to marshal status report into json for status file '%s' with error '%s'", statusFilePath, err.Error()) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errors.Wrap(err, errorMessage) + } continue } @@ -877,7 +942,9 @@ func createDummyStatusFilesIfNeeded(ctx log.Logger, mrseqFilesNameList *list.Lis if err != nil { errorMessage = fmt.Sprintf("Failed to create a dummy status file '%s' as it was not existing for .mrseq file '%s' with error '%s'", statusFilePath, mrSeqFileFullPath, err.Error()) extensionEvents.LogErrorEvent("createdummystatusfiles", errorMessage) - allErr = errors.Wrap(allErr, errorMessage) + if allErr == nil { + allErr = errors.Wrap(err, errorMessage) + } continue } } @@ -891,8 +958,8 @@ func downloadScript(ctx *log.Context, dir string, cfg *handlersettings.HandlerSe // - prepare the output directory for files and the command output // - create the directory if missing ctx.Log("event", "creating output directory", "path", dir) - if err := os.MkdirAll(dir, 0700); err != nil { - return "", errors.Wrap(err, "failed to prepare output directory") + if err := commandMkdirAll(dir, 0700); err != nil { + return "", errorutil.EnsureClarification(errorutil.FileDownload_unableToCreateDownloadDirectory, errors.Wrap(err, "failed to prepare output directory")) } ctx.Log("event", "created output directory") @@ -905,7 +972,7 @@ func downloadScript(ctx *log.Context, dir string, cfg *handlersettings.HandlerSe if scriptURI != "" { telemetryResult("scenario", fmt.Sprintf("source.scriptUri;dos2unix=%d", dos2unix), true, 0*time.Millisecond) ctx.Log("event", "download start") - file, err := files.DownloadAndProcessScript(ctx, scriptURI, dir, cfg) + file, err := processDownloadedScript(ctx, scriptURI, dir, cfg) if err != nil { ctx.Log("event", "download failed", "error", err) return "", errors.Wrapf(err, "failed to download file %s. ", scriptURI) @@ -915,15 +982,15 @@ func downloadScript(ctx *log.Context, dir string, cfg *handlersettings.HandlerSe if rceps != nil && cfg.ScriptType() != handlersettings.GalleryScript { // Assume the downloaded script TYPE is already allowed, since this was already validated earlier in enable(). - err = extensionpolicysettings.ValidateFileHashInAllowlist(scriptFilePath, rceps.DownloadedScriptsAllowlist, hashutils.HashTypeSHA256) + err = validateDownloadedScript(scriptFilePath, rceps) if err != nil { ctx.Log("message", "downloaded script file is not in the allowlist, attempting to delete", "scriptFilePath", scriptFilePath) - if delErr := os.Remove(scriptFilePath); delErr != nil { - ctx.Log("message", "failed to delete downloaded script file", "scriptFilePath", scriptFilePath, "error", delErr) + if delErr := commandRemove(scriptFilePath); delErr != nil { + ctx.Log("message", "failed to delete downloaded script file", "scriptFilePath", scriptFilePath, "error", errorutil.EnsureClarification(errorutil.Os_FailedToDeleteBlockedDownloadedScript, delErr), "clarificationCode", errorutil.Os_FailedToDeleteBlockedDownloadedScript) } else { ctx.Log("message", "successfully deleted downloaded script file", "scriptFilePath", scriptFilePath) } - return scriptFilePath, errors.Wrapf(err, "file %s blocked by policy", scriptFilePath) + return scriptFilePath, errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_downloadedScriptNotAllowed, errors.Wrapf(err, "file %s blocked by policy", scriptFilePath)) } } } @@ -1005,7 +1072,7 @@ func decodeScript(script string) (string, string, error) { // scripts must be base64 encoded s, err := base64.StdEncoding.DecodeString(script) if err != nil { - return "", "", errors.Wrap(err, "failed to decode script") + return "", "", errorutil.EnsureClarification(errorutil.CustomerInput_invalidEmbeddedScriptBase64, errors.Wrap(err, "failed to decode script")) } // scripts may be gzip'ed @@ -1019,7 +1086,7 @@ func decodeScript(script string) (string, string, error) { n, err := io.Copy(w, r) if err != nil { - return "", "", errors.Wrap(err, "failed to decompress script") + return "", "", errorutil.EnsureClarification(errorutil.CustomerInput_invalidCompressedScript, errors.Wrap(err, "failed to decompress script")) } w.Flush() @@ -1035,32 +1102,32 @@ func createOrReplaceAppendBlobUsingManagedIdentity(blobUri string, managedIdenti if managedIdentity.ClientId != "" { ID = managedIdentity.ClientId } else if managedIdentity.ObjectId != "" { //ObjectId is not supported by azidentity.NewManagedIdentityCredential - return nil, errors.New("Managed identity's ObjectId is not supported. Use ClientId instead") + return nil, errorutil.EnsureClarification(errorutil.CustomerInput_outputIdentityObjectIdNotSupported, errors.New("Managed identity's ObjectId is not supported. Use ClientId instead")) } } if ID != "" { // Use user-assigned identity if clientId is provided miCredentialOptions := azidentity.ManagedIdentityCredentialOptions{ID: azidentity.ClientID(ID)} - miCred, miCredError = azidentity.NewManagedIdentityCredential(&miCredentialOptions) + miCred, miCredError = newManagedIdentityCredential(&miCredentialOptions) } else { // Use system-assigned identity if clientId not provided - miCred, miCredError = azidentity.NewManagedIdentityCredential(nil) + miCred, miCredError = newManagedIdentityCredential(nil) } var appendBlobClient *appendblob.Client var appendBlobNewClientError error if miCredError == nil { - appendBlobClient, appendBlobNewClientError = appendblob.NewClient(blobUri, miCred, nil) + appendBlobClient, appendBlobNewClientError = newAppendBlobClient(blobUri, miCred, nil) if appendBlobNewClientError != nil { - return nil, errors.Wrap(appendBlobNewClientError, fmt.Sprintf("Error Creating client to Append Blob '%s'. Make sure you are using Append blob. Other types of blob such as PageBlob, BlockBlob are not supported types.", download.GetUriForLogging(blobUri))) + return nil, errorutil.EnsureClarification(errorutil.Storage_failedToCreateManagedAppendBlobClient, errors.Wrap(appendBlobNewClientError, fmt.Sprintf("Error Creating client to Append Blob '%s'. Make sure you are using Append blob. Other types of blob such as PageBlob, BlockBlob are not supported types.", download.GetUriForLogging(blobUri)))) } else { // Create or Replace Append blob. If AppendBlob already exists, blob gets cleared. - _, createAppendBlobError := appendBlobClient.Create(context.Background(), nil) + createAppendBlobError := createManagedAppendBlob(appendBlobClient) if createAppendBlobError != nil { - return nil, errors.Wrap(createAppendBlobError, fmt.Sprintf("Error creating or replacing the Append blob '%s'. Make sure you are using Append blob. Other types of blob such as PageBlob, BlockBlob are not supported types.", download.GetUriForLogging(blobUri))) + return nil, errorutil.EnsureClarification(errorutil.Storage_failedToCreateManagedAppendBlob, errors.Wrap(createAppendBlobError, fmt.Sprintf("Error creating or replacing the Append blob '%s'. Make sure you are using Append blob. Other types of blob such as PageBlob, BlockBlob are not supported types.", download.GetUriForLogging(blobUri)))) } } } else { - return nil, errors.Wrap(miCredError, "Error while retrieving managed identity credential") + return nil, errorutil.EnsureClarification(errorutil.Imds_failedToCreateManagedIdentityCredential, errors.Wrap(miCredError, "Error while retrieving managed identity credential")) } return appendBlobClient, nil diff --git a/internal/cmds/cmds_test.go b/internal/cmds/cmds_test.go index bb58079..f32bf77 100755 --- a/internal/cmds/cmds_test.go +++ b/internal/cmds/cmds_test.go @@ -25,42 +25,30 @@ import ( "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/internal/settings" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetb/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) func Test_CopyMrseqFiles_MrseqFilesAreCopied(t *testing.T) { - currentExtensionVersionDirectory := "Microsoft.CPlat.Core.RunCommandHandlerLinux-1.3.8" - os.Setenv(constants.ExtensionPathEnvName, currentExtensionVersionDirectory) - os.Setenv(constants.ExtensionVersionUpdatingFromEnvName, "1.3.7") - os.Setenv(constants.VersionEnvName, "1.3.8") - - currentVersion := os.Getenv(constants.VersionEnvName) - previousVersion := os.Getenv(constants.ExtensionVersionUpdatingFromEnvName) - previousExtensionVersionDirectory := strings.ReplaceAll(currentExtensionVersionDirectory, currentVersion, previousVersion) - - // Remove currentExtensionVersionDirectory if it exists - _, err := os.Stat(currentExtensionVersionDirectory) - if err == nil { - os.RemoveAll(currentExtensionVersionDirectory) - } - - // Remove previousExtensionVersionDirectory if it exists - _, err = os.Stat(previousExtensionVersionDirectory) - if err == nil { - os.RemoveAll(previousExtensionVersionDirectory) - } + root := t.TempDir() + currentExtensionVersionDirectory := filepath.Join(root, "Microsoft.CPlat.Core.RunCommandHandlerLinux-1.3.8") + previousExtensionVersionDirectory := filepath.Join(root, "Microsoft.CPlat.Core.RunCommandHandlerLinux-1.3.7") + t.Setenv(constants.ExtensionPathEnvName, currentExtensionVersionDirectory) + t.Setenv(constants.ExtensionVersionUpdatingFromEnvName, "1.3.7") + t.Setenv(constants.VersionEnvName, "1.3.8") // Create previousExtensionVersionDirectory - err = os.Mkdir(previousExtensionVersionDirectory, 0777) + err := os.Mkdir(previousExtensionVersionDirectory, 0777) require.Nil(t, err) // Create currentExtensionVersionDirectory err = os.Mkdir(currentExtensionVersionDirectory, 0777) require.Nil(t, err) - files, _ := ioutil.ReadDir(currentExtensionVersionDirectory) + files, err := ioutil.ReadDir(currentExtensionVersionDirectory) + require.NoError(t, err) require.Equal(t, 0, len(files)) createMrseqFile(filepath.Join(previousExtensionVersionDirectory, "1.mrseq"), "0", t) @@ -68,7 +56,7 @@ func Test_CopyMrseqFiles_MrseqFilesAreCopied(t *testing.T) { createMrseqFile(filepath.Join(previousExtensionVersionDirectory, "2345.mrseq"), "0", t) createMrseqFile(filepath.Join(previousExtensionVersionDirectory, "RC0804_0.mrseq"), "5", t) createMrseqFile(filepath.Join(previousExtensionVersionDirectory, "asdfsad.mrseq"), "20", t) - os.Create(filepath.Join(previousExtensionVersionDirectory, "abc.txt")) // this should not be copied to currentExtensionVersionDirectory + require.NoError(t, os.WriteFile(filepath.Join(previousExtensionVersionDirectory, "abc.txt"), nil, 0644)) // this should not be copied to currentExtensionVersionDirectory statusSubdirectory := "status" previousStatusDirectory := filepath.Join(previousExtensionVersionDirectory, statusSubdirectory) @@ -78,14 +66,12 @@ func Test_CopyMrseqFiles_MrseqFilesAreCopied(t *testing.T) { // Only two status files are available. The rest of the 3 status files should be created during Update operation which would have dummy status. // Dummy status files would be created to prevent poll status timeouts for already executed Run Commands after upgrade. - os.Create(filepath.Join(previousStatusDirectory, "1.0.status")) - os.Create(filepath.Join(previousStatusDirectory, "ABCD.1.status")) - os.Create(filepath.Join(previousStatusDirectory, "abc.cs")) // this should not be copied to currentExtensionVersionDirectory + require.NoError(t, os.WriteFile(filepath.Join(previousStatusDirectory, "1.0.status"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(previousStatusDirectory, "ABCD.1.status"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(previousStatusDirectory, "abc.cs"), nil, 0644)) // this should not be copied to currentExtensionVersionDirectory - tempDir, _ := os.MkdirTemp("", "deletecmd") - defer os.RemoveAll(tempDir) handlerEnvironment := handlerenv.HandlerEnvironment{ - EventsFolder: tempDir, + EventsFolder: t.TempDir(), } extensionLogger := logging.New(nil) @@ -93,7 +79,8 @@ func Test_CopyMrseqFiles_MrseqFilesAreCopied(t *testing.T) { err = CopyStateForUpdate(log.NewContext(log.NewNopLogger()), previousExtensionVersionDirectory, currentExtensionVersionDirectory, extensionEventManager) require.Nil(t, err) - files, _ = ioutil.ReadDir(currentExtensionVersionDirectory) + files, err = ioutil.ReadDir(currentExtensionVersionDirectory) + require.NoError(t, err) require.Equal(t, 6, len(files)) statusDirectoryCount := 0 mrseqFileCount := 0 @@ -110,19 +97,17 @@ func Test_CopyMrseqFiles_MrseqFilesAreCopied(t *testing.T) { require.Equal(t, 5, mrseqFileCount) currentStatusDirectory := filepath.Join(currentExtensionVersionDirectory, statusSubdirectory) - files, _ = ioutil.ReadDir(currentStatusDirectory) + files, err = ioutil.ReadDir(currentStatusDirectory) + require.NoError(t, err) require.Equal(t, 5, len(files)) for _, file := range files { require.True(t, strings.HasSuffix(file.Name(), ".status")) } - // Clean up - os.RemoveAll(currentExtensionVersionDirectory) - os.RemoveAll(previousExtensionVersionDirectory) } func createMrseqFile(mrseqFilePath string, mrseqNum string, t *testing.T) { - os.Create(mrseqFilePath) + t.Helper() err := os.WriteFile(mrseqFilePath, []byte(mrseqNum), 0644) require.Nil(t, err) } @@ -155,6 +140,7 @@ func Test_checkAndSaveSeqNum_fails(t *testing.T) { // pass in invalid seqnum format _, err := checkAndSaveSeqNum(log.NewNopLogger(), 0, "/non/existing/dir") require.NotNil(t, err) + requireCommandCode(t, errorutil.Os_FailedToWriteSequenceNumber, err) require.Contains(t, err.Error(), `failed to save sequence number`) } @@ -343,8 +329,9 @@ func Test_update_immediateRunCommandFails_traditionalStillUpdates(t *testing.T) // Simulate the immediate run command update failing. originalImmediateUpdate := immediateUpdate + updateFailure := errorutil.EnsureClarification(errorutil.Os_FailedToMakeServiceBinaryExecutable, errors.New("simulated immediate run command update failure")) immediateUpdate = func(ctx *log.Context, h types.HandlerEnvironment, name string, seqNum int, extensionEvents *extensionevents.ExtensionEventManager) (int, error) { - return constants.ExitCode_UpgradeInstalledServiceFailed, errors.New("simulated immediate run command update failure") + return constants.ExitCode_UpgradeInstalledServiceFailed, updateFailure } defer func() { immediateUpdate = originalImmediateUpdate }() @@ -355,6 +342,8 @@ func Test_update_immediateRunCommandFails_traditionalStillUpdates(t *testing.T) // The immediate run command update failed, so update() surfaces that error... require.Error(t, updateErr, "immediate run command update was expected to fail") + require.ErrorIs(t, updateErr, updateFailure) + requireCommandCode(t, errorutil.Os_FailedToMakeServiceBinaryExecutable, updateErr) // ...but the traditional run command state should STILL have been migrated to the new version. require.FileExists(t, filepath.Join(newVersionDirectory, extName+constants.MrSeqFileExtension), @@ -599,7 +588,11 @@ func enable_extension(t *testing.T, fakeEnv types.HandlerEnvironment, tempDir st // Verify whether we should have run the script require.Equal(t, shouldExecuteScript, wasCalled) } else { + // Already processed is a control-flow sentinel, not a classified failure. require.Equal(t, ErrAlreadyProcessed, err) + code, classified := errorutil.FindClarification(err) + require.False(t, classified, "already-processed sentinel must remain unclassified") + require.Zero(t, code) } // Report status to create a status file @@ -658,8 +651,9 @@ func Test_runCmd_fail(t *testing.T) { defer os.RemoveAll(dir) // Ensure that the script fails + executionFailure := errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("the chipmunks have risen in revolt")) ExecCmdInDir = func(ctx *log.Context, scriptFilePath, workdir string, cfg *handlersettings.HandlerSettings) (error, int) { - return errors.New("the chipmunks have risen in revolt"), 42 + return executionFailure, 42 } metadata := types.NewRCMetadata("extName", 0, constants.DownloadFolder, DataDir) @@ -667,6 +661,8 @@ func Test_runCmd_fail(t *testing.T) { PublicSettings: handlersettings.PublicSettings{Source: &handlersettings.ScriptSource{Script: "non-existing-cmd"}}, }, metadata) require.NotNil(t, err, "command terminated with exit status") + require.ErrorIs(t, err, executionFailure) + requireCommandCode(t, errorutil.CommandExecution_failureExitCode, err) require.Contains(t, err.Error(), "failed to execute command") require.NotEqual(t, constants.ExitCode_Okay, exitCode) } @@ -723,6 +719,7 @@ func Test_downloadArtifacts_Invalid(t *testing.T) { }) require.NotNil(t, err) + requireCommandCode(t, errorutil.Internal_invalidArtifactSpecification, err) require.Contains(t, err.Error(), "RunCommand artifact download failed. Reason: Invalid artifact specification. This is a product bug.") // ArtifactIds don't match @@ -749,6 +746,7 @@ func Test_downloadArtifacts_Invalid(t *testing.T) { }) require.NotNil(t, err) + requireCommandCode(t, errorutil.Internal_inconsistentArtifactSpecification, err) require.Contains(t, err.Error(), "RunCommand artifact download failed. Reason: Invalid artifact specification. This is a product bug.") } @@ -783,6 +781,7 @@ func Test_downloadArtifactsFail(t *testing.T) { }) require.NotNil(t, err) + requireCommandCode(t, errorutil.FileDownload_doesNotExist, err) require.Contains(t, err.Error(), "failed to download artifact") } @@ -894,8 +893,9 @@ func Test_TreatFailureAsDeploymentFailureIsTrue_Fails(t *testing.T) { defer os.RemoveAll(dir) // Ensure that the script fails + executionFailure := errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("the chipmunks do not like the script")) ExecCmdInDir = func(ctx *log.Context, scriptFilePath, workdir string, cfg *handlersettings.HandlerSettings) (error, int) { - return errors.New("the chipmunks do not like the script"), 127 + return executionFailure, 127 } metadata := types.NewRCMetadata("extName", 0, constants.DownloadFolder, DataDir) @@ -903,6 +903,8 @@ func Test_TreatFailureAsDeploymentFailureIsTrue_Fails(t *testing.T) { PublicSettings: handlersettings.PublicSettings{Source: &handlersettings.ScriptSource{Script: script}, TreatFailureAsDeploymentFailure: true}, }, metadata) require.NotNil(t, err) + require.ErrorIs(t, err, executionFailure) + requireCommandCode(t, errorutil.CommandExecution_failureExitCode, err) require.Contains(t, err.Error(), "failed to execute command: the chipmunks do not like the script") require.NotEqual(t, constants.ExitCode_Okay, exitCode) } @@ -1339,6 +1341,7 @@ func Test_rehydrateMrSeqFiles_OpenStatusDirFails(t *testing.T) { err := rehydrateMrSeqFilesForProblematicUpgrades(ctx, from, to, events) require.NotNil(t, err, "expected error when opening missing status dir") + requireCommandCode(t, errorutil.Os_FailedToOpenRehydrationStatusDirectory, err) require.True(t, strings.Contains(err.Error(), "Failed to open status directory"), "unexpected error message: %s", err.Error()) } @@ -1364,6 +1367,7 @@ func Test_rehydrateMrSeqFiles_ReadDirFails(t *testing.T) { err := rehydrateMrSeqFilesForProblematicUpgrades(ctx, from, to, events) require.NotNil(t, err, "expected error when ReadDir fails") + requireCommandCode(t, errorutil.Os_FailedToReadRehydrationStatusDirectory, err) require.True(t, strings.Contains(err.Error(), "could not read directory entries"), "unexpected error message: %s", err.Error()) } @@ -1560,6 +1564,7 @@ func Test_rehydrateMrSeqFiles_ReadExistingMrseqFails(t *testing.T) { err := rehydrateMrSeqFilesForProblematicUpgrades(ctx, from, to, events) require.NotNil(t, err, "expected error due to unreadable mrseq") + requireCommandCode(t, errorutil.Os_FailedToReadRehydrationSequenceFile, err) require.True(t, strings.Contains(err.Error(), "Could not read file"), "Unexpected error: %v", err.Error()) } @@ -1608,6 +1613,7 @@ func Test_downloadScript_BlockedByAllowlist(t *testing.T) { policy, ) require.Error(t, err) + requireCommandCode(t, errorutil.ExtensionPolicySettings_downloadedScriptNotAllowed, err) require.Contains(t, err.Error(), "blocked by policy") require.Contains(t, err.Error(), "item is not in the allowlist") } @@ -1826,10 +1832,11 @@ func Test_enable_e2e_extension_policy_settings_block_statussuccess(t *testing.T) } err = commandProcessor.ProcessHandlerCommandWithDetails(ctx, CmdEnable, fakeEnv, extName, seqNum, constants.DownloadFolder, dataDir) - require.Nil(t, err, "enable command should succeed") + require.NoError(t, err, "policy rejection must not fail the handler when deployment failure is disabled") require.False(t, scriptWasExecuted, "disallowed script should not be executed") report := readStatusReport(t, fakeEnv, extName, seqNum) // verify status report exists and is valid + requireStatusClarification(t, errorutil.ExtensionPolicySettings_galleryScriptNotAllowed, report) require.Equal(t, types.StatusSuccess, report[0].Status.Status, "status report should indicate success") require.True(t, strings.Contains(report[0].Status.FormattedMessage.Message, "executionState\":\"Failed\",\"executionMessage\":\"Execution failed"), "execution message should indicate failure") } @@ -1862,10 +1869,11 @@ func Test_enable_e2e_extension_policy_settings_block_statusfail(t *testing.T) { } err = commandProcessor.ProcessHandlerCommandWithDetails(ctx, CmdEnable, fakeEnv, extName, seqNum, constants.DownloadFolder, dataDir) - require.Nil(t, err, "enable command should succeed") + require.Error(t, err, "policy rejection must propagate to the caller") require.False(t, scriptWasExecuted, "disallowed script should not be executed") report := readStatusReport(t, fakeEnv, extName, seqNum) // verify status report exists and is valid + requireReportedClarification(t, errorutil.ExtensionPolicySettings_galleryScriptNotAllowed, err, report) require.Equal(t, types.StatusError, report[0].Status.Status, "status report should indicate failure") require.True(t, strings.Contains(report[0].Status.FormattedMessage.Message, "executionState\":\"Failed\",\"executionMessage\":\"Execution failed"), "execution message should indicate failure") } @@ -1946,10 +1954,11 @@ func Test_enable_e2e_extension_policy_settings_block_statussuccess_disableOutput } err = commandProcessor.ProcessHandlerCommandWithDetails(ctx, CmdEnable, fakeEnv, extName, seqNum, constants.DownloadFolder, dataDir) - require.Nil(t, err, "enable command should succeed") + require.NoError(t, err, "policy rejection must not fail the handler when deployment failure is disabled") require.False(t, scriptWasExecuted, "script should not be executed because output blobs are disabled") report := readStatusReport(t, fakeEnv, extName, seqNum) // verify status report exists and is valid + requireStatusClarification(t, errorutil.ExtensionPolicySettings_outputBlobsNotAllowed, report) require.Equal(t, types.StatusSuccess, report[0].Status.Status, "status report should indicate success") require.True(t, strings.Contains(report[0].Status.FormattedMessage.Message, "executionState\":\"Failed\",\"executionMessage\":\"Execution failed"), "execution message should indicate failure") require.True(t, strings.Contains(report[0].Status.FormattedMessage.Message, "output blobs are disabled in policy, but settings specify"), "execution message should indicate failure") @@ -1994,10 +2003,11 @@ func Test_enable_e2e_runtimePolicyFile_deletedAfterExecution_thenRunsWithoutPoli } err = commandProcessor.ProcessHandlerCommandWithDetails(ctx, CmdEnable, fakeEnv, extName, blockedSeqNum, constants.DownloadFolder, dataDir) - require.Nil(t, err, "enable command should succeed even when the script is blocked") + require.NoError(t, err, "policy rejection must not fail the handler when deployment failure is disabled") require.False(t, scriptWasExecuted, "non-allowlisted script must be blocked by policy") report := readStatusReport(t, fakeEnv, extName, blockedSeqNum) + requireStatusClarification(t, errorutil.ExtensionPolicySettings_downloadedScriptNotAllowed, report) require.Equal(t, types.StatusSuccess, report[0].Status.Status, "status report should indicate success") require.True(t, strings.Contains(report[0].Status.FormattedMessage.Message, "executionState\":\"Failed\",\"executionMessage\":\"Execution failed"), "blocked script should report failed execution") diff --git a/internal/commandProcessor/clarification_test.go b/internal/commandProcessor/clarification_test.go new file mode 100644 index 0000000..c9dde9a --- /dev/null +++ b/internal/commandProcessor/clarification_test.go @@ -0,0 +1,231 @@ +package commandProcessor + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "strconv" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/internal/status" + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func mockProcessor(t *testing.T) { + t.Helper() + read, write, marshal, remove := readHandlerSettings, writeSettingsFile, marshalSettings, removeRuntimePolicyFile + t.Cleanup(func() { + readHandlerSettings, writeSettingsFile, marshalSettings, removeRuntimePolicyFile = read, write, marshal, remove + }) + removeRuntimePolicyFile = func(string) error { return nil } +} + +func TestCommandFailurePreservesClarificationAndDeploymentBehavior(t *testing.T) { + for _, treatAsFailure := range []bool{false, true} { + for _, settingsFailure := range []bool{false, true} { + t.Run(strconv.FormatBool(treatAsFailure)+strconv.FormatBool(settingsFailure), func(t *testing.T) { + mockProcessor(t) + cause := errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("original script error")) + readHandlerSettings = func(string, string, int, *log.Context) (handlersettings.HandlerSettings, error) { + cfg := handlersettings.HandlerSettings{} + cfg.PublicSettings.TreatFailureAsDeploymentFailure = treatAsFailure + if settingsFailure { + return cfg, errors.New("settings read failed") + } + return cfg, nil + } + var states []types.StatusType + var reports []types.StatusReport + var reported error + cmd := types.CmdEnableTemplate.InitializeFunctions(types.CmdFunctions{ + Invoke: func(*log.Context, types.HandlerEnvironment, *types.RunCommandInstanceView, types.RCMetadata, types.Cmd) (string, string, error, int, *types.CleanupRequest) { + return "stdout", "stderr", cause, 12, nil + }, + ReportStatus: func(ctx *log.Context, _ types.HandlerEnvironment, m types.RCMetadata, state types.StatusType, cmd types.Cmd, message string) error { + states = append(states, state) + reported = m.ClarificationError + reports = append(reports, processorStatusReport(t, ctx, m, state, cmd, message)) + return nil + }, + }) + err := ProcessHandlerCommandWithDetails(log.NewContext(log.NewNopLogger()), cmd, types.HandlerEnvironment{}, "command", 1, "download", "/mock") + require.Equal(t, cause, reported) + expected := types.StatusSuccess + if treatAsFailure && !settingsFailure { + expected = types.StatusError + require.ErrorIs(t, err, cause) + requireProcessorCode(t, err, errorutil.CommandExecution_failureExitCode) + } else { + require.NoError(t, err) + } + require.Equal(t, []types.StatusType{types.StatusTransitioning, expected}, states) + require.Len(t, reports, 2) + require.Empty(t, reports[0][0].Status.Substatuses, "transitioning status must not contain a failure") + requireProcessorStatusCode(t, reports[1], errorutil.CommandExecution_failureExitCode) + var view types.RunCommandInstanceView + require.NoError(t, json.Unmarshal([]byte(reports[1][0].Status.FormattedMessage.Message), &view)) + require.Equal(t, types.ExecutionState(types.Failed), view.ExecutionState) + require.Equal(t, "Execution failed: "+cause.Error(), view.ExecutionMessage) + require.Equal(t, 12, view.ExitCode) + require.Equal(t, "stdout", view.Output) + require.Equal(t, "stderr", view.Error) + require.NotEmpty(t, view.EndTime) + }) + } + } +} + +func TestProcessorClarificationErrors(t *testing.T) { + for _, code := range []int{errorutil.Internal_unclassifiedCommandFailure, errorutil.Internal_invalidSequenceEnvironmentVariable, errorutil.Internal_failedToMarshalLocalSettings, errorutil.Os_FailedToWriteLocalSettings, errorutil.Os_FailedToDeleteRuntimePolicy} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockProcessor(t) + failure := errors.New("injected failure") + ctx := log.NewContext(log.NewNopLogger()) + var err error + switch code { + case errorutil.Internal_unclassifiedCommandFailure: + readHandlerSettings = func(string, string, int, *log.Context) (handlersettings.HandlerSettings, error) { + cfg := handlersettings.HandlerSettings{} + cfg.PublicSettings.TreatFailureAsDeploymentFailure = true + return cfg, nil + } + cmd := types.CmdEnableTemplate.InitializeFunctions(types.CmdFunctions{ + Invoke: func(*log.Context, types.HandlerEnvironment, *types.RunCommandInstanceView, types.RCMetadata, types.Cmd) (string, string, error, int, *types.CleanupRequest) { + return "", "", failure, 1, nil + }, + ReportStatus: func(ctx *log.Context, _ types.HandlerEnvironment, metadata types.RCMetadata, state types.StatusType, cmd types.Cmd, message string) error { + report := processorStatusReport(t, ctx, metadata, state, cmd, message) + if state != types.StatusTransitioning { + requireProcessorStatusCode(t, report, errorutil.Internal_unclassifiedCommandFailure) + } + return nil + }, + }) + err = ProcessHandlerCommandWithDetails(ctx, cmd, types.HandlerEnvironment{}, "command", 1, "download", "/mock") + case errorutil.Internal_invalidSequenceEnvironmentVariable: + t.Setenv(constants.ConfigSequenceNumberEnvName, "invalid") + _, err = getSeqNum(&ctx, types.HandlerEnvironment{}, "command") + case errorutil.Internal_failedToMarshalLocalSettings: + marshalSettings = func(interface{}) ([]byte, error) { return nil, failure } + err = storeConfigSettingsFileForLocalExecution(ctx, handlersettings.HandlerSettingsFile{}, types.HandlerEnvironment{}, "command", 1) + case errorutil.Os_FailedToWriteLocalSettings: + writeSettingsFile = func(string, []byte, os.FileMode) error { return failure } + err = storeConfigSettingsFileForLocalExecution(ctx, handlersettings.HandlerSettingsFile{}, types.HandlerEnvironment{}, "command", 1) + case errorutil.Os_FailedToDeleteRuntimePolicy: + var output bytes.Buffer + ctx = log.NewContext(log.NewLogfmtLogger(&output)) + removeRuntimePolicyFile = func(string) error { return failure } + deleteRuntimePolicySettingsFile(ctx, types.HandlerEnvironment{}) + require.Contains(t, output.String(), "clarificationCode="+strconv.Itoa(errorutil.Os_FailedToDeleteRuntimePolicy)) + return + } + actual, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, code, actual) + }) + } +} + +func TestReportingFailureDoesNotReplaceCommandError(t *testing.T) { + mockProcessor(t) + cause := errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("script failed")) + readHandlerSettings = func(string, string, int, *log.Context) (handlersettings.HandlerSettings, error) { + cfg := handlersettings.HandlerSettings{} + cfg.PublicSettings.TreatFailureAsDeploymentFailure = true + return cfg, nil + } + + calls := 0 + cmd := types.CmdEnableTemplate.InitializeFunctions(types.CmdFunctions{ + Invoke: func(*log.Context, types.HandlerEnvironment, *types.RunCommandInstanceView, types.RCMetadata, types.Cmd) (string, string, error, int, *types.CleanupRequest) { + return "", "", cause, 1, nil + }, + ReportStatus: func(ctx *log.Context, _ types.HandlerEnvironment, metadata types.RCMetadata, state types.StatusType, cmd types.Cmd, message string) error { + calls++ + if calls == 2 { + report := processorStatusReport(t, ctx, metadata, state, cmd, message) + requireProcessorStatusCode(t, report, errorutil.CommandExecution_failureExitCode) + return errorutil.EnsureClarification(errorutil.Os_FailedToPublishStatusFile, errors.New("status failed")) + } + return nil + }, + }) + err := ProcessHandlerCommandWithDetails(log.NewContext(log.NewNopLogger()), cmd, types.HandlerEnvironment{}, "command", 1, "download", "/mock") + require.ErrorIs(t, err, cause) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.CommandExecution_failureExitCode, code) +} + +func requireProcessorCode(t *testing.T, err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "expected classified error: %v", err) + require.Equal(t, want, code) +} + +func TestPrecheckClarificationStatus(t *testing.T) { + for _, tc := range []struct { + original error + code int + }{ + {errors.New("unclassified precheck failure"), errorutil.Internal_unclassifiedPrecheckFailure}, + {errorutil.EnsureClarification(errorutil.CustomerInput_invalidSettingsJson, errors.New("classified precheck failure")), errorutil.CustomerInput_invalidSettingsJson}, + // Already processed is intentional control flow, not a production failure. + {types.ErrAlreadyProcessed, 0}, + } { + original := tc.original + t.Run(original.Error(), func(t *testing.T) { + reported := false + cmd := types.CmdEnableTemplate.InitializeFunctions(types.CmdFunctions{ + Pre: func(*log.Context, types.HandlerEnvironment, types.RCMetadata, types.Cmd) error { return original }, + ReportStatus: func(_ *log.Context, _ types.HandlerEnvironment, metadata types.RCMetadata, state types.StatusType, _ types.Cmd, _ string) error { + reported = true + require.Equal(t, types.StatusError, state) + code, ok := errorutil.FindClarification(metadata.ClarificationError) + require.True(t, ok) + require.Equal(t, tc.code, code) + report := processorStatusReport(t, log.NewContext(log.NewNopLogger()), metadata, state, types.CmdEnableTemplate, "precheck failed") + requireProcessorStatusCode(t, report, tc.code) + return nil + }, + }) + err := executePreSteps(log.NewContext(log.NewNopLogger()), cmd, types.HandlerEnvironment{}, "command", 1, "download") + require.ErrorIs(t, err, original) + if tc.code != 0 { + requireProcessorCode(t, err, tc.code) + } else { + _, classified := errorutil.FindClarification(err) + require.False(t, classified, "already-processed control flow must remain unclassified") + } + require.Equal(t, !errors.Is(original, types.ErrAlreadyProcessed), reported) + }) + } +} + +func processorStatusReport(t *testing.T, ctx *log.Context, metadata types.RCMetadata, state types.StatusType, cmd types.Cmd, message string) types.StatusReport { + t.Helper() + item, err := status.GetSingleStatusItem(ctx, state, cmd, message, metadata.ExtName, metadata.ClarificationError) + require.NoError(t, err) + payload, err := status.MarshalStatusReportIntoJson(types.StatusReport{item}, false) + require.NoError(t, err) + var report types.StatusReport + require.NoError(t, json.Unmarshal(payload, &report)) + require.Len(t, report, 1) + return report +} + +func requireProcessorStatusCode(t *testing.T, report types.StatusReport, want errorutil.ErrorClarificationCode) { + t.Helper() + require.Len(t, report, 1) + require.Len(t, report[0].Status.Substatuses, 1) + require.Equal(t, "ErrorClarification", report[0].Status.Substatuses[0].Name) + require.Equal(t, string(types.StatusError), report[0].Status.Substatuses[0].Status) + require.Equal(t, want, report[0].Status.Substatuses[0].Code) +} diff --git a/internal/commandProcessor/commandProcessor.go b/internal/commandProcessor/commandProcessor.go index 0d70e75..e1dc9db 100644 --- a/internal/commandProcessor/commandProcessor.go +++ b/internal/commandProcessor/commandProcessor.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/internal/instanceview" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/seqnumutil" "github.com/Azure/run-command-handler-linux/pkg/versionutil" "github.com/go-kit/kit/log" @@ -24,6 +25,10 @@ import ( var ( handlerEnvironmentGetter func(name, version string) (he *handlerenv.HandlerEnvironment, _ error) = handlerenv.GetHandlerEnvironment + readHandlerSettings = handlersettings.GetHandlerSettings + writeSettingsFile = os.WriteFile + marshalSettings = json.Marshal + removeRuntimePolicyFile = os.Remove ) func ProcessImmediateHandlerCommand(cmd types.Cmd, hs handlersettings.HandlerSettingsFile, extensionName string, seqNum int) error { @@ -86,7 +91,9 @@ func processHandlerCommandWithDetails(ctx *log.Context, cmd types.Cmd, hEnv type } metadata := types.NewRCMetadata(extensionName, seqNum, downloadFolder, dataDir) - instanceview.ReportInstanceView(ctx, hEnv, metadata, types.StatusTransitioning, cmd, &instView) + if err := instanceview.ReportInstanceView(ctx, hEnv, metadata, types.StatusTransitioning, cmd, &instView); err != nil { + ctx.Log("event", "failed to report initial instance view", "error", err) + } // execute the subcommand stdout, stderr, cmdInvokeError, exitCode, cleanup := cmd.Functions.Invoke(ctx, hEnv, &instView, metadata, cmd) @@ -105,21 +112,28 @@ func processHandlerCommandWithDetails(ctx *log.Context, cmd types.Cmd, hEnv type instView.Output = stdout instView.Error = stderr if cmdInvokeError != nil { + cmdInvokeError = errorutil.EnsureClarification(errorutil.Internal_unclassifiedCommandFailure, cmdInvokeError) ctx.Log("event", "failed to handle", "error", cmdInvokeError) instView.ExecutionMessage = "Execution failed: " + cmdInvokeError.Error() instView.ExecutionState = types.Failed instView.EndTime = time.Now().UTC().Format(time.RFC3339) instView.ExitCode = exitCode + instView.ClarificationError = cmdInvokeError statusToReport := types.StatusSuccess - // If TreatFailureAsDeploymentFailure is set to true and the exit code is non-zero, set extension status to error - cfg, err := handlersettings.GetHandlerSettings(hEnv.HandlerEnvironment.ConfigFolder, extensionName, seqNum, ctx) + // Fail the extension status and handler invocation only when deployment failure is enabled and the command has a non-zero failure exit code. + cfg, err := readHandlerSettings(hEnv.HandlerEnvironment.ConfigFolder, extensionName, seqNum, ctx) if err == nil && cfg.PublicSettings.TreatFailureAsDeploymentFailure && cmd.FailExitCode != 0 { statusToReport = types.StatusError } - instanceview.ReportInstanceView(ctx, hEnv, metadata, statusToReport, cmd, &instView) - return errors.Wrapf(err, "command execution failed") + if reportErr := instanceview.ReportInstanceView(ctx, hEnv, metadata, statusToReport, cmd, &instView); reportErr != nil { + ctx.Log("event", "failed to report execution error", "error", reportErr) + } + if statusToReport == types.StatusError { + return errors.Wrapf(cmdInvokeError, "command execution failed") + } + return nil } else { // No error. Succeeded instView.ExecutionMessage = "Execution completed" instView.ExecutionState = types.Succeeded @@ -127,7 +141,9 @@ func processHandlerCommandWithDetails(ctx *log.Context, cmd types.Cmd, hEnv type instView.ExitCode = constants.ExitCode_Okay } - instanceview.ReportInstanceView(ctx, hEnv, metadata, types.StatusSuccess, cmd, &instView) + if err := instanceview.ReportInstanceView(ctx, hEnv, metadata, types.StatusSuccess, cmd, &instView); err != nil { + return errors.Wrap(err, "failed to report final instance view") + } ctx.Log("event", "end") return nil @@ -158,6 +174,19 @@ func executePreSteps(ctx *log.Context, cmd types.Cmd, hEnv types.HandlerEnvironm metadata := types.NewRCMetadata(extensionName, seqNum, downloadFolder, constants.DataDir) if err := cmd.Functions.Pre(ctx, hEnv, metadata, cmd); err != nil { ctx.Log("event", "pre-check failed", "error", err) + if !errors.Is(err, types.ErrAlreadyProcessed) { + err = errorutil.EnsureClarification(errorutil.Internal_unclassifiedPrecheckFailure, err) + view := types.RunCommandInstanceView{ + ExecutionState: types.Failed, + ExecutionMessage: "Pre-check failed: " + err.Error(), + ExitCode: cmd.FailExitCode, + EndTime: time.Now().UTC().Format(time.RFC3339), + ClarificationError: err, + } + if reportErr := instanceview.ReportInstanceView(ctx, hEnv, metadata, types.StatusError, cmd, &view); reportErr != nil { + ctx.Log("event", "failed to report pre-check error", "error", reportErr) + } + } return errors.Wrapf(err, "pre-check step failed") } } @@ -199,7 +228,7 @@ func getSeqNum(ctx **log.Context, hEnv types.HandlerEnvironment, extensionName s if err != nil { msg := fmt.Sprintf("failed to parse env variable ConfigSequenceNumber: %v", seqNumVariable) (*ctx).Log("message", msg, "error", err) - return seqNum, errors.Wrap(err, msg) + return seqNum, errorutil.EnsureClarification(errorutil.Internal_invalidSequenceEnvironmentVariable, errors.Wrap(err, msg)) } } @@ -208,6 +237,7 @@ func getSeqNum(ctx **log.Context, hEnv types.HandlerEnvironment, extensionName s seqNum, err = seqnumutil.FindSequenceNumberFromConfig(hEnv.HandlerEnvironment.ConfigFolder, constants.ConfigFileExtension, extensionName) if err != nil { (*ctx).Log("FindSequenceNumberFromConfig", "failed to find sequence number from config folder.", "error", err) + return seqNum, err } else { (*ctx).Log("FindSequenceNumberFromConfig", fmt.Sprintf("Sequence number determined from config folder: %d", seqNum)) } @@ -223,14 +253,14 @@ func storeConfigSettingsFileForLocalExecution(ctx *log.Context, hs handlersettin configFilePath := handlersettings.GetConfigFilePath(configFolder, seqNum, extensionName) ctx.Log("message", fmt.Sprintf("saving handler settings in file: %v", configFilePath)) - content, err := json.Marshal(hs) + content, err := marshalSettings(hs) if err != nil { - return errors.Wrap(err, "could not marshal handler settings file") + return errorutil.EnsureClarification(errorutil.Internal_failedToMarshalLocalSettings, errors.Wrap(err, "could not marshal handler settings file")) } - err = os.WriteFile(configFilePath, content, 0644) + err = writeSettingsFile(configFilePath, content, 0644) if err != nil { - return errors.Wrap(err, "could not store handler settings file locally to run the command") + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteLocalSettings, errors.Wrap(err, "could not store handler settings file locally to run the command")) } return nil @@ -239,8 +269,8 @@ func storeConfigSettingsFileForLocalExecution(ctx *log.Context, hs handlersettin func deleteRuntimePolicySettingsFile(ctx *log.Context, hEnv types.HandlerEnvironment) { policyFilePath := filepath.Join(hEnv.HandlerEnvironment.ConfigFolder, constants.PolicyFileName) ctx.Log("message", "removing runtime policy settings file if it exists") - err := os.Remove(policyFilePath) + err := removeRuntimePolicyFile(policyFilePath) if err != nil && !os.IsNotExist(err) { - ctx.Log("warning", "failed to remove runtime policy settings file", "error", err) + ctx.Log("warning", "failed to remove runtime policy settings file", "error", errorutil.EnsureClarification(errorutil.Os_FailedToDeleteRuntimePolicy, err), "clarificationCode", errorutil.Os_FailedToDeleteRuntimePolicy) } } diff --git a/internal/commandProcessor/commandProcessor_test.go b/internal/commandProcessor/commandProcessor_test.go index 4ae3c90..addc7cc 100644 --- a/internal/commandProcessor/commandProcessor_test.go +++ b/internal/commandProcessor/commandProcessor_test.go @@ -18,6 +18,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/settings" "github.com/Azure/run-command-handler-linux/internal/status" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -125,7 +126,14 @@ func Test_ProcessImmediateHandlerCommand_CleanupAfterFinalStatus(t *testing.T) { require.NoError(t, os.Remove(configPath)) }, }) - require.NoError(t, ProcessImmediateHandlerCommand(cmd, cleanupTestSettings(test.policy), name, seq)) + err := ProcessImmediateHandlerCommand(cmd, cleanupTestSettings(test.policy), name, seq) + if test.scriptFails && test.policy { + require.ErrorContains(t, err, "script failed") + } else if !test.scriptFails && test.reportFails { + require.ErrorContains(t, err, "status reporting failed") + } else { + require.NoError(t, err) + } require.Equal(t, []string{"progress", "invoke", "invoke returned", "final status", "cleanup"}, events) require.Equal(t, 1, cleanupCalls) require.NoFileExists(t, configPath) @@ -185,13 +193,13 @@ func Test_ProcessImmediateHandlerCommand_PrecheckCleanupRemainsImmediate(t *test Pre: func(ctx *log.Context, hEnv types.HandlerEnvironment, metadata types.RCMetadata, c types.Cmd) error { c.Functions.Cleanup(ctx, metadata, hEnv, "") require.True(t, cleaned) - return errors.New("already processed") + return types.ErrAlreadyProcessed }, Cleanup: func(_ *log.Context, _ types.RCMetadata, _ types.HandlerEnvironment, _ string) { cleaned = true }, }) - require.ErrorContains(t, ProcessImmediateHandlerCommand(cmd, cleanupTestSettings(false), "cleanup-test", 1), "already processed") + require.ErrorIs(t, ProcessImmediateHandlerCommand(cmd, cleanupTestSettings(false), "cleanup-test", 1), types.ErrAlreadyProcessed) require.True(t, cleaned) } @@ -259,10 +267,19 @@ func Test_ExecutePreStepsAndFailed(t *testing.T) { ctx := initializeLogger(cmd) extName, seqNum := "testExtension", 5 fakeEnv := types.HandlerEnvironment{} + fakeEnv.HandlerEnvironment.StatusFolder = t.TempDir() err := executePreSteps(ctx, cmd, fakeEnv, extName, seqNum, constants.DownloadFolder) require.ErrorContains(t, err, "pre-check step failed") + requireProcessorCode(t, err, errorutil.Internal_unclassifiedPrecheckFailure) require.ErrorContains(t, err, "expected error") + payload, err := os.ReadFile(filepath.Join(fakeEnv.HandlerEnvironment.StatusFolder, "testExtension.5.status")) + require.NoError(t, err) + var report types.StatusReport + require.NoError(t, json.Unmarshal(payload, &report)) + require.Len(t, report, 1) + require.Equal(t, types.StatusError, report[0].Status.Status) + requireProcessorStatusCode(t, report, errorutil.Internal_unclassifiedPrecheckFailure) } func Test_SaveConfigurationFileInConfigFolderSuccessfully(t *testing.T) { @@ -362,14 +379,17 @@ func Test_GetSeqNumFromEnvVariableFailedToParse(t *testing.T) { fakeEnv := types.HandlerEnvironment{} _, err := getSeqNum(&ctx, fakeEnv, extName) require.ErrorContains(t, err, "failed to parse env variable ConfigSequenceNumber") + requireProcessorCode(t, err, errorutil.Internal_invalidSequenceEnvironmentVariable) os.Setenv(constants.ConfigSequenceNumberEnvName, "-") _, err = getSeqNum(&ctx, fakeEnv, extName) require.ErrorContains(t, err, "failed to parse env variable ConfigSequenceNumber") + requireProcessorCode(t, err, errorutil.Internal_invalidSequenceEnvironmentVariable) os.Setenv(constants.ConfigSequenceNumberEnvName, "+") _, err = getSeqNum(&ctx, fakeEnv, extName) require.ErrorContains(t, err, "failed to parse env variable ConfigSequenceNumber") + requireProcessorCode(t, err, errorutil.Internal_invalidSequenceEnvironmentVariable) } func Test_GetSeqNumFromFailedToFind(t *testing.T) { @@ -387,6 +407,8 @@ func Test_GetSeqNumFromFailedToFind(t *testing.T) { os.Setenv(constants.ConfigSequenceNumberEnvName, "") actualSeqNum, err := getSeqNum(&ctx, fakeEnv, extName) - require.Nil(t, err) + require.Error(t, err) + requireProcessorCode(t, err, errorutil.Internal_noSequenceConfigurationFiles) + require.Contains(t, err.Error(), "can't find out seqnum") require.Equal(t, 0, actualSeqNum) } diff --git a/internal/constants/exitcodes.go b/internal/constants/exitcodes.go index 5eb12fc..a7dc3a1 100755 --- a/internal/constants/exitcodes.go +++ b/internal/constants/exitcodes.go @@ -13,6 +13,8 @@ const ( ExitCode_RunAsUserNotAllowedByExtensionPolicy = -105 ExitCode_DownloadedScriptBlockedByExtensionPolicy = -106 ExitCode_OutputBlobSpecifiedButNotAllowedByExtensionPolicy = -107 + ExitCode_OutputBlobCreateOrReplaceFailed = -108 + ExitCode_ErrorBlobCreateOrReplaceFailed = -109 // Service Errors (-200s): ExitCode_CreateDataDirectoryFailed = -200 diff --git a/internal/exec/clarification_test.go b/internal/exec/clarification_test.go new file mode 100644 index 0000000..26d024e --- /dev/null +++ b/internal/exec/clarification_test.go @@ -0,0 +1,221 @@ +package exec + +import ( + "errors" + "io" + "os" + osexec "os/exec" + "os/user" + "strconv" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func fmtCode(code int) string { return strconv.Itoa(code) } + +type clarificationFile struct { + readErr, writeErr, chownErr, chmodErr, closeErr error + closed bool +} + +func (f *clarificationFile) Read([]byte) (int, error) { return 0, f.readErr } +func (f *clarificationFile) Write(p []byte) (int, error) { return len(p), f.writeErr } +func (f *clarificationFile) Close() error { f.closed = true; return f.closeErr } +func (f *clarificationFile) Chown(int, int) error { return f.chownErr } +func (f *clarificationFile) Chmod(os.FileMode) error { return f.chmodErr } + +func mockExecution(t *testing.T) { + t.Helper() + open, mkdir, remove, lookup := openScriptFile, makeScriptDirectory, removeScriptFile, lookupRunAsUser + setenv, run, copy, status := setEnvironment, runCommand, copyScript, commandExitStatus + t.Cleanup(func() { + openScriptFile, makeScriptDirectory, removeScriptFile, lookupRunAsUser = open, mkdir, remove, lookup + setEnvironment, runCommand, copyScript, commandExitStatus = setenv, run, copy, status + }) + openScriptFile = func(string, int, os.FileMode) (scriptFile, error) { return &clarificationFile{}, nil } + makeScriptDirectory = func(string, os.FileMode) error { return nil } + removeScriptFile = func(string) error { return nil } + lookupRunAsUser = func(string) (*user.User, error) { return &user.User{Uid: "1000"}, nil } + setEnvironment = func(string, string) error { return nil } + runCommand = func(*osexec.Cmd) error { return nil } + copyScript = func(io.Writer, io.Reader) (int64, error) { return 0, nil } + commandExitStatus = func(error) (int, bool, bool) { return 0, false, false } +} + +func TestExecutionClarificationErrors(t *testing.T) { + failure := errors.New("injected machine failure") + tests := []struct { + code int + setup func(*handlersettings.HandlerSettings, *clarificationFile, *clarificationFile) + }{ + {errorutil.Internal_incorrectRunAsScriptPath, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + }}, + {errorutil.Os_FailedToCreateRunAsDirectory, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + makeScriptDirectory = func(string, os.FileMode) error { return failure } + }}, + {errorutil.Os_FailedToOpenRunAsSourceScript, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + openScriptFile = func(string, int, os.FileMode) (scriptFile, error) { return nil, failure } + }}, + {errorutil.Os_FailedToCopyRunAsScript, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + copyScript = func(io.Writer, io.Reader) (int64, error) { return 0, failure } + }}, + {errorutil.CommandExecution_runAsUserNotFound, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + lookupRunAsUser = func(string) (*user.User, error) { return nil, failure } + }}, + {errorutil.Os_InvalidRunAsUserId, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + lookupRunAsUser = func(string) (*user.User, error) { return &user.User{Uid: "invalid"}, nil } + }}, + {errorutil.Os_FailedToChangeRunAsScriptOwner, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + openScriptFile = func(string, int, os.FileMode) (scriptFile, error) { return &clarificationFile{chownErr: failure}, nil } + }}, + {errorutil.Os_FailedToChangeRunAsScriptPermissions, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + openScriptFile = func(string, int, os.FileMode) (scriptFile, error) { return &clarificationFile{chmodErr: failure}, nil } + }}, + {errorutil.Os_FailedToRemoveRunAsScript, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + removeScriptFile = func(string) error { return failure } + }}, + {errorutil.Os_FailedToCreateRunAsScript, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + openScriptFile = func(_ string, flag int, _ os.FileMode) (scriptFile, error) { + if flag&os.O_CREATE != 0 { + return nil, failure + } + return &clarificationFile{}, nil + } + }}, + {errorutil.CustomerInput_invalidParameterEnvironmentVariable, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.Parameters = []handlersettings.ParameterDefinition{{Name: "parameter", Value: "value"}, {Name: "next", Value: "value"}} + calls := 0 + setEnvironment = func(string, string) error { calls++; require.Equal(t, 1, calls); return failure } + }}, + {errorutil.CommandExecution_failedUnknownError, func(_ *handlersettings.HandlerSettings, _, _ *clarificationFile) { + runCommand = func(*osexec.Cmd) error { return failure } + }}, + {errorutil.CommandExecution_failureExitCode, func(_ *handlersettings.HandlerSettings, _, _ *clarificationFile) { + runCommand = func(*osexec.Cmd) error { return failure } + commandExitStatus = func(error) (int, bool, bool) { return 12, false, true } + }}, + {errorutil.CommandExecution_interruptedByVmShutdown, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.TimeoutInSeconds = 10 + runCommand = func(*osexec.Cmd) error { return failure } + commandExitStatus = func(error) (int, bool, bool) { return -1, true, true } + }}, + {errorutil.Os_FailedToCloseStdOut, func(_ *handlersettings.HandlerSettings, out, _ *clarificationFile) { out.closeErr = failure }}, + {errorutil.Os_FailedToCloseStdErr, func(_ *handlersettings.HandlerSettings, _, stderr *clarificationFile) { stderr.closeErr = failure }}, + {errorutil.Os_FailedToCloseRunAsSourceScript, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + openScriptFile = func(_ string, flag int, _ os.FileMode) (scriptFile, error) { + file := &clarificationFile{} + if flag&os.O_CREATE == 0 { + file.closeErr = failure + } + return file, nil + } + }}, + {errorutil.Os_FailedToCloseRunAsScript, func(cfg *handlersettings.HandlerSettings, _, _ *clarificationFile) { + cfg.PublicSettings.RunAsUser = "user" + openScriptFile = func(_ string, flag int, _ os.FileMode) (scriptFile, error) { + file := &clarificationFile{} + if flag&os.O_CREATE != 0 { + file.closeErr = failure + } + return file, nil + } + }}, + } + for _, test := range tests { + t.Run(fmtCode(test.code), func(t *testing.T) { + mockExecution(t) + cfg := handlersettings.HandlerSettings{} + out, stderr := &clarificationFile{}, &clarificationFile{} + test.setup(&cfg, out, stderr) + path := constants.DataDir + "/download/script.sh" + if test.code == errorutil.Internal_incorrectRunAsScriptPath { + path = "/unexpected/script.sh" + } + _, err := Exec(testContext, path, "/mock/workdir", out, stderr, &cfg) + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "error was not classified: %v", err) + require.Equal(t, test.code, code) + require.True(t, out.closed) + require.True(t, stderr.closed) + }) + } +} + +func TestOutputFileClarificationErrors(t *testing.T) { + for _, code := range []int{errorutil.Os_FailedToOpenStdOut, errorutil.Os_FailedToOpenStdErr} { + t.Run(fmtCode(code), func(t *testing.T) { + mockExecution(t) + out := &clarificationFile{} + calls := 0 + openScriptFile = func(string, int, os.FileMode) (scriptFile, error) { + calls++ + if code == errorutil.Os_FailedToOpenStdOut || calls == 2 { + return nil, errors.New("open failed") + } + return out, nil + } + err, _ := ExecCmdInDir(testContext, "unused", "/mock", &handlersettings.HandlerSettings{}) + actual, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, code, actual) + if code == errorutil.Os_FailedToOpenStdErr { + require.True(t, out.closed) + } + }) + } +} + +func TestExecutionPreservesLowerClarification(t *testing.T) { + mockExecution(t) + cause := errorutil.EnsureClarification(errorutil.CustomerInput_invalidSettingsJson, errors.New("lower failure")) + runCommand = func(*osexec.Cmd) error { return cause } + _, err := Exec(testContext, "unused", "/mock", &clarificationFile{}, &clarificationFile{}, &handlersettings.HandlerSettings{}) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.CustomerInput_invalidSettingsJson, code) + require.ErrorIs(t, err, cause) +} + +func TestExecutionInjectedSuccess(t *testing.T) { + for _, runAs := range []string{"", "customer"} { + t.Run(runAs, func(t *testing.T) { + mockExecution(t) + cfg := handlersettings.HandlerSettings{} + cfg.PublicSettings.RunAsUser = runAs + cfg.PublicSettings.TimeoutInSeconds = 30 + cfg.PublicSettings.Parameters = []handlersettings.ParameterDefinition{{Name: "name", Value: "value"}, {Value: "argument"}} + var environment map[string]string = make(map[string]string) + setEnvironment = func(name, value string) error { environment[name] = value; return nil } + runCommand = func(command *osexec.Cmd) error { + require.Equal(t, "/mock/workdir", command.Dir) + require.Contains(t, command.Args[2], " argument") + if runAs != "" { + require.Contains(t, command.Args[2], "sudo -n -u customer") + } + return nil + } + out, stderr := &clarificationFile{}, &clarificationFile{} + code, err := Exec(testContext, constants.DataDir+"/download/script.sh", "/mock/workdir", out, stderr, &cfg) + require.NoError(t, err) + require.Zero(t, code) + require.Equal(t, map[string]string{"name": "value"}, environment) + require.True(t, out.closed) + require.True(t, stderr.closed) + }) + } +} diff --git a/internal/exec/exec.go b/internal/exec/exec.go index d66c876..51abe7b 100755 --- a/internal/exec/exec.go +++ b/internal/exec/exec.go @@ -15,26 +15,66 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) +type scriptFile interface { + io.ReadWriteCloser + Chown(int, int) error + Chmod(os.FileMode) error +} + +var ( + openScriptFile = func(path string, flag int, mode os.FileMode) (scriptFile, error) { + return os.OpenFile(path, flag, mode) + } + makeScriptDirectory = os.MkdirAll + removeScriptFile = os.Remove + lookupRunAsUser = user.Lookup + setEnvironment = os.Setenv + runCommand = func(command *exec.Cmd) error { return command.Run() } + copyScript = io.Copy + commandExitStatus = func(err error) (int, bool, bool) { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return 0, false, false + } + status, ok := exitErr.Sys().(syscall.WaitStatus) + return status.ExitStatus(), status.Signaled(), ok + } +) + // Exec runs the given cmd in /bin/sh, saves its stdout/stderr streams to // the specified files. It waits until the execution terminates. // // On error, an exit code may be returned if it is an exit code error. // Given stdout and stderr will be closed upon returning. -func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, cfg *handlersettings.HandlerSettings) (int, error) { - defer stdout.Close() - defer stderr.Close() +func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, cfg *handlersettings.HandlerSettings) (exitCode int, resultErr error) { + defer func() { + if err := stdout.Close(); err != nil && resultErr == nil { + resultErr = errorutil.EnsureClarification(errorutil.Os_FailedToCloseStdOut, errors.Wrap(err, "failed to close stdout")) + exitCode = constants.ExitCode_OpenStdOutFileFailed + } + }() + defer func() { + if err := stderr.Close(); err != nil && resultErr == nil { + resultErr = errorutil.EnsureClarification(errorutil.Os_FailedToCloseStdErr, errors.Wrap(err, "failed to close stderr")) + exitCode = constants.ExitCode_OpenStdErrFileFailed + } + }() scriptPath := cmd commandArgs, err := SetEnvironmentVariables(cfg) + if err != nil { + return constants.ExitCode_CommandExecutionFailed, err + } // Add command args if any. Unnamed arguments go in 'commandArgs'. Named arguments are set as environment variables so the'd be available within the script. cmd = cmd + commandArgs - exitCode := constants.ExitCode_Okay + exitCode = constants.ExitCode_Okay if cfg.PublicSettings.RunAsUser != "" { ctx.Log("message", "RunAsUser is "+cfg.PublicSettings.RunAsUser) @@ -43,7 +83,7 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, if !strings.HasPrefix(scriptPath, constants.DataDir) { errMessage := "Failed to determine RunAs script path. Contact ICM team AzureRT\\Extensions for this service error." ctx.Log("message", errMessage) - return constants.ExitCode_RunAsIncorrectScriptPath, errors.New(errMessage) + return constants.ExitCode_RunAsIncorrectScriptPath, errorutil.EnsureClarification(errorutil.Internal_incorrectRunAsScriptPath, errors.New(errMessage)) } // Gets suffix "download//0/script.sh" @@ -53,15 +93,17 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, runAsScriptDirectoryPath := filepath.Dir(runAsScriptFilePath) // Get directory of runAsScript that doesn't exist yet // Create runAsScriptDirectoryPath and its intermediate directories if they do not exist - os.MkdirAll(runAsScriptDirectoryPath, 0777) + if err := makeScriptDirectory(runAsScriptDirectoryPath, 0777); err != nil { + return constants.ExitCode_RunAsCreateRunAsScriptFileFailed, errorutil.EnsureClarification(errorutil.Os_FailedToCreateRunAsDirectory, errors.Wrap(err, "failed to create RunAs script directory")) + } /// Copy source script at scriptPath to runAsScriptDirectoryPath // Get reference to source script by opening it - sourceScriptFile, sourceScriptFileOpenError := os.OpenFile(scriptPath, os.O_RDONLY, 0400) + sourceScriptFile, sourceScriptFileOpenError := openScriptFile(scriptPath, os.O_RDONLY, 0400) if sourceScriptFileOpenError != nil { errMessage := "Failed to open source script. Contact ICM team AzureRT\\Extensions for this service error." ctx.Log("message", errMessage+fmt.Sprintf(" Source script file is '%s'", scriptPath)) - return constants.ExitCode_RunAsOpenSourceScriptFileFailed, errors.Wrap(sourceScriptFileOpenError, errMessage) + return constants.ExitCode_RunAsOpenSourceScriptFileFailed, errorutil.EnsureClarification(errorutil.Os_FailedToOpenRunAsSourceScript, errors.Wrap(sourceScriptFileOpenError, errMessage)) } destScriptFile, destScriptCreateError := createRunAsScriptFile(runAsScriptFilePath) @@ -71,23 +113,26 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, ctx.Log("message", errMessage+fmt.Sprintf(" Destination runAs script file is '%s'", runAsScriptFilePath)) return constants.ExitCode_RunAsCreateRunAsScriptFileFailed, errors.Wrap(destScriptCreateError, errMessage) } - _, runAsScriptCopyError := io.Copy(destScriptFile, sourceScriptFile) + _, runAsScriptCopyError := copyScript(destScriptFile, sourceScriptFile) if runAsScriptCopyError != nil { sourceScriptFile.Close() destScriptFile.Close() errMessage := fmt.Sprintf("Failed to copy script file '%s' to Run As path '%s'. Contact ICM team AzureRT\\Extensions for this service error.", scriptPath, runAsScriptFilePath) ctx.Log("message", errMessage) - return constants.ExitCode_RunAsCopySourceScriptToRunAsScriptFileFailed, errors.Wrap(runAsScriptCopyError, errMessage) + return constants.ExitCode_RunAsCopySourceScriptToRunAsScriptFileFailed, errorutil.EnsureClarification(errorutil.Os_FailedToCopyRunAsScript, errors.Wrap(runAsScriptCopyError, errMessage)) + } + if err := sourceScriptFile.Close(); err != nil { + destScriptFile.Close() + return constants.ExitCode_RunAsCopySourceScriptToRunAsScriptFileFailed, errorutil.EnsureClarification(errorutil.Os_FailedToCloseRunAsSourceScript, errors.Wrap(err, "failed to close source script")) } - sourceScriptFile.Close() // Provide read and execute permissions to RunAsUser on .sh file at runAsScriptFilePath - lookedUpUser, lookupUserError := user.Lookup(cfg.PublicSettings.RunAsUser) + lookedUpUser, lookupUserError := lookupRunAsUser(cfg.PublicSettings.RunAsUser) if lookupUserError != nil { destScriptFile.Close() errMessage := fmt.Sprintf("Failed to lookup RunAs user '%s'. Looks like user does not exist. For RunAs to work properly, contact admin of VM and make sure RunAs user is added on the VM and user has access to resources accessed by the Run Command (Directories, Files, Network etc.). Refer: https://aka.ms/RunCommandManagedLinux", cfg.PublicSettings.RunAsUser) ctx.Log("message", errMessage) - return constants.ExitCode_RunAsLookupUserFailed, errors.Wrap(lookupUserError, errMessage) + return constants.ExitCode_RunAsLookupUserFailed, errorutil.EnsureClarification(errorutil.CommandExecution_runAsUserNotFound, errors.Wrap(lookupUserError, errMessage)) } lookedUpUserUid, lookedUpUserUidErr := strconv.Atoi(lookedUpUser.Uid) @@ -95,7 +140,7 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, destScriptFile.Close() errMessage := "Failed to determine RunAs user's Uid and Guid . Contact ICM team AzureRT\\Extensions for this service error." ctx.Log("message", errMessage) - return constants.ExitCode_RunAsLookupUserUidFailed, errors.Wrap(lookedUpUserUidErr, errMessage) + return constants.ExitCode_RunAsLookupUserUidFailed, errorutil.EnsureClarification(errorutil.Os_InvalidRunAsUserId, errors.Wrap(lookedUpUserUidErr, errMessage)) } runAsScriptChownError := destScriptFile.Chown(lookedUpUserUid, os.Getegid()) @@ -103,7 +148,7 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, destScriptFile.Close() errMessage := fmt.Sprintf("Failed to change owner of file '%s' to RunAs user '%s'. Contact ICM team AzureRT\\Extensions for this service error.", runAsScriptFilePath, cfg.PublicSettings.RunAsUser) ctx.Log("message", errMessage) - return constants.ExitCode_RunAsScriptFileChangeOwnerFailed, errors.Wrap(runAsScriptChownError, errMessage) + return constants.ExitCode_RunAsScriptFileChangeOwnerFailed, errorutil.EnsureClarification(errorutil.Os_FailedToChangeRunAsScriptOwner, errors.Wrap(runAsScriptChownError, errMessage)) } runAsScriptChmodError := destScriptFile.Chmod(0550) @@ -111,9 +156,11 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, destScriptFile.Close() errMessage := fmt.Sprintf("Failed to change permissions to execute for file '%s' for RunAs user '%s'. Contact ICM team AzureRT\\Extensions for this service error.", runAsScriptFilePath, cfg.PublicSettings.RunAsUser) ctx.Log("message", errMessage) - return constants.ExitCode_RunAsScriptFileChangePermissionsFailed, errors.Wrap(runAsScriptChmodError, errMessage) + return constants.ExitCode_RunAsScriptFileChangePermissionsFailed, errorutil.EnsureClarification(errorutil.Os_FailedToChangeRunAsScriptPermissions, errors.Wrap(runAsScriptChmodError, errMessage)) + } + if err := destScriptFile.Close(); err != nil { + return constants.ExitCode_RunAsCreateRunAsScriptFileFailed, errorutil.EnsureClarification(errorutil.Os_FailedToCloseRunAsScript, errors.Wrap(err, "failed to close RunAs script")) } - destScriptFile.Close() // Execute the command as RunAsUser. -n for non-interactive mode (do not prompt for password) // sudo -n -u @@ -134,35 +181,33 @@ func Exec(ctx *log.Context, cmd, workdir string, stdout, stderr io.WriteCloser, command.Dir = workdir command.Stdout = stdout command.Stderr = stderr - err = command.Run() + err = runCommand(command) if err != nil { - exitErr, ok := err.(*exec.ExitError) - if ok { - if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { - exitCode = status.ExitStatus() - if status.Signaled() { // Timed out - ctx.Log("message", "Timeout:"+err.Error()) - } - return exitCode, fmt.Errorf("command terminated with exit status=%d", exitCode) + if code, signaled, ok := commandExitStatus(err); ok { + exitCode = code + if signaled { + ctx.Log("message", "Timeout:"+err.Error()) + return exitCode, errorutil.EnsureClarification(errorutil.CommandExecution_interruptedByVmShutdown, fmt.Errorf("command terminated with exit status=%d", exitCode)) } + return exitCode, errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, fmt.Errorf("command terminated with exit status=%d", exitCode)) } } - return exitCode, errors.Wrap(err, "failed to execute command") + return exitCode, errorutil.EnsureClarification(errorutil.CommandExecution_failedUnknownError, errors.Wrap(err, "failed to execute command")) } -func createRunAsScriptFile(path string) (*os.File, error) { +func createRunAsScriptFile(path string) (scriptFile, error) { // os.Remove does not follow symlinks, so it is safe to use here. - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - return nil, err + if err := removeScriptFile(path); err != nil && !os.IsNotExist(err) { + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToRemoveRunAsScript, err) } // do not follow symlinks to avoid symlink attacks. - return os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0500) + file, err := openScriptFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL|runAsNoFollow, 0500) + return file, errorutil.EnsureClarification(errorutil.Os_FailedToCreateRunAsScript, err) } func SetEnvironmentVariables(cfg *handlersettings.HandlerSettings) (string, error) { - var err error commandArgs := "" parameters := []handlersettings.ParameterDefinition{} if cfg.PublicSettings.Parameters != nil && len(cfg.PublicSettings.Parameters) > 0 { @@ -177,14 +222,16 @@ func SetEnvironmentVariables(cfg *handlersettings.HandlerSettings) (string, erro value := parameters[i].Value if value != "" { if name != "" { // Named parameters are set as environmental setting - err = os.Setenv(name, value) + if err := setEnvironment(name, value); err != nil { + return commandArgs, errorutil.EnsureClarification(errorutil.CustomerInput_invalidParameterEnvironmentVariable, errors.Wrap(err, "failed to set parameter environment variable")) + } } else { // Unnamed parameters go to command args commandArgs += " " + value } } } - return commandArgs, err // Return command args and the last error if any + return commandArgs, nil } // ExecCmdInDir executes the given command in given directory and saves output @@ -197,13 +244,14 @@ func ExecCmdInDir(ctx *log.Context, scriptFilePath, workdir string, cfg *handler stdoutFileName, stderrFileName := LogPaths(workdir) - outF, err := os.OpenFile(stdoutFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + outF, err := openScriptFile(stdoutFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if err != nil { - return errors.Wrapf(err, "failed to open stdout file"), constants.ExitCode_OpenStdOutFileFailed + return errorutil.EnsureClarification(errorutil.Os_FailedToOpenStdOut, errors.Wrapf(err, "failed to open stdout file")), constants.ExitCode_OpenStdOutFileFailed } - errF, err := os.OpenFile(stderrFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + errF, err := openScriptFile(stderrFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if err != nil { - return errors.Wrapf(err, "failed to open stderr file"), constants.ExitCode_OpenStdErrFileFailed + outF.Close() + return errorutil.EnsureClarification(errorutil.Os_FailedToOpenStdErr, errors.Wrapf(err, "failed to open stderr file")), constants.ExitCode_OpenStdErrFileFailed } exitCode, err := Exec(ctx, scriptFilePath, workdir, outF, errF, cfg) diff --git a/internal/exec/exec_linux.go b/internal/exec/exec_linux.go new file mode 100644 index 0000000..ed2df25 --- /dev/null +++ b/internal/exec/exec_linux.go @@ -0,0 +1,5 @@ +package exec + +import "syscall" + +const runAsNoFollow = syscall.O_NOFOLLOW diff --git a/internal/exec/exec_test.go b/internal/exec/exec_test.go index 9f1ad15..a400fe3 100644 --- a/internal/exec/exec_test.go +++ b/internal/exec/exec_test.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -19,6 +20,13 @@ var ( testContext = log.NewContext(log.NewNopLogger()) ) +func requireExecCode(t *testing.T, err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "expected classified error: %v", err) + require.Equal(t, want, code) +} + func TestExec_success(t *testing.T) { v := new(mockFile) ec, err := Exec(testContext, "date", "/", v, v, &testHandlerSettings) @@ -43,6 +51,7 @@ func TestExec_failure_exitError(t *testing.T) { ec, err := Exec(testContext, "exit 12", "/", new(mockFile), new(mockFile), &testHandlerSettings) require.NotNil(t, err) require.EqualError(t, err, "command terminated with exit status=12") // error is customized + requireExecCode(t, err, errorutil.CommandExecution_failureExitCode) require.EqualValues(t, 12, ec) } @@ -52,6 +61,7 @@ func TestExec_failure_timeout(t *testing.T) { testHandlerSettings.PublicSettings.TimeoutInSeconds = 0 require.NotNil(t, err) require.EqualError(t, err, "command terminated with exit status=-1") // error is customized + requireExecCode(t, err, errorutil.CommandExecution_interruptedByVmShutdown) require.EqualValues(t, -1, ec) } @@ -133,6 +143,7 @@ func TestExec_failure_genericError(t *testing.T) { _, err := Exec(testContext, "date", "/non-existing-path", new(mockFile), new(mockFile), &testHandlerSettings) require.NotNil(t, err) require.Contains(t, err.Error(), "failed to execute command:") // error is wrapped + requireExecCode(t, err, errorutil.CommandExecution_failedUnknownError) } func TestExec_failure_fdClosed(t *testing.T) { @@ -142,6 +153,7 @@ func TestExec_failure_fdClosed(t *testing.T) { _, err := Exec(testContext, "date", "/", out, out, &testHandlerSettings) require.NotNil(t, err) require.Contains(t, err.Error(), "file closed") // error is wrapped + requireExecCode(t, err, errorutil.CommandExecution_failedUnknownError) } func TestExec_failure_redirectsStdStreams_closesFds(t *testing.T) { @@ -151,6 +163,7 @@ func TestExec_failure_redirectsStdStreams_closesFds(t *testing.T) { _, err := Exec(testContext, `/bin/echo 'I am stdout!'>&1; /bin/echo 'I am stderr!'>&2; exit 12`, "/", o, e, &testHandlerSettings) require.NotNil(t, err) + requireExecCode(t, err, errorutil.CommandExecution_failureExitCode) require.Equal(t, "I am stdout!\n", string(o.b.Bytes())) require.Equal(t, "I am stderr!\n", string(e.b.Bytes())) require.True(t, o.closed, "stdout closed") @@ -179,6 +192,7 @@ func TestExecCmdInDir(t *testing.T) { func TestExecCmdInDir_cantOpenError(t *testing.T) { err, exitCode := ExecCmdInDir(testContext, "/bin/echo 'Hello world'", "/non-existing-dir", &testHandlerSettings) + requireExecCode(t, err, errorutil.Os_FailedToOpenStdOut) require.Contains(t, err.Error(), "failed to open stdout file") require.NotEqual(t, constants.ExitCode_Okay, exitCode) } @@ -231,6 +245,7 @@ func (m *mockFile) Close() error { } func fileExists(t *testing.T, path string) bool { + // This raw os.Stat probe checks fixtures, not a production error boundary. _, err := os.Stat(path) if err == nil { return true diff --git a/internal/exec/exec_windows.go b/internal/exec/exec_windows.go new file mode 100644 index 0000000..3185bc2 --- /dev/null +++ b/internal/exec/exec_windows.go @@ -0,0 +1,4 @@ +package exec + +// O_CREATE|O_EXCL already rejects an existing final path, including a symlink. +const runAsNoFollow = 0 diff --git a/internal/extensionpolicysettingsrc/errorclarification_test.go b/internal/extensionpolicysettingsrc/errorclarification_test.go new file mode 100644 index 0000000..db9aab4 --- /dev/null +++ b/internal/extensionpolicysettingsrc/errorclarification_test.go @@ -0,0 +1,190 @@ +package extensionpolicysettingsrc + +import ( + "errors" + "fmt" + "testing" + + "github.com/Azure/azure-extension-platform/vmextension" + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func requirePolicyCode(t *testing.T, err error, want int) { + t.Helper() + require.Error(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "missing clarification: %v", err) + require.Equal(t, want, code) +} + +func mockPolicyBoundaries(t *testing.T) { + t.Helper() + newManager, load, get, allow := newPolicyManager, loadPolicySettings, getPolicySettings, validatePolicyAllowlist + t.Cleanup(func() { + newPolicyManager, loadPolicySettings, getPolicySettings, validatePolicyAllowlist = newManager, load, get, allow + }) + newPolicyManager = func(string) (*policyManager, error) { return new(policyManager), nil } + loadPolicySettings = func(*policyManager) error { return nil } + getPolicySettings = func(*policyManager) (*RCv2ExtensionPolicySettings, error) { + return &RCv2ExtensionPolicySettings{}, nil + } +} + +func TestPolicyFormatClarifications(t *testing.T) { + t.Run("invalid script type", func(t *testing.T) { + _, err := StringToAllowedScriptTypeFlag("unknown") + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_invalidPolicyFileFormat) + requirePolicyCode(t, (RCv2ExtensionPolicySettings{LimitScripts: "unknown"}).ValidateFormat(), errorutil.ExtensionPolicySettings_invalidPolicyFileFormat) + requirePolicyCode(t, ValidateScriptTypeAgainstPolicy(nopCtx(), handlersettings.InlineScript, "unknown"), errorutil.ExtensionPolicySettings_invalidPolicyFileFormat) + }) + t.Run("download allowlist 2501", func(t *testing.T) { + err := (RCv2ExtensionPolicySettings{LimitScripts: "inline", DownloadedScriptsAllowlist: []string{"hash"}}).ValidateFormat() + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_inconsistentDownloadedScriptAllowlist) + }) + t.Run("command allowlist 2502", func(t *testing.T) { + err := (RCv2ExtensionPolicySettings{LimitScripts: "inline", CommandIdAllowlist: []string{"command"}}).ValidateFormat() + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_inconsistentCommandIdAllowlist) + }) +} + +func TestPolicyScriptClarifications(t *testing.T) { + for _, tc := range []struct { + script handlersettings.ScriptType + code int + }{ + {handlersettings.InlineScript, errorutil.ExtensionPolicySettings_scriptNotAllowed}, + {handlersettings.DownloadedScript, errorutil.ExtensionPolicySettings_downloadedScriptTypeNotAllowed}, + {handlersettings.GalleryScript, errorutil.ExtensionPolicySettings_galleryScriptNotAllowed}, + {handlersettings.DiagnosticScript, errorutil.ExtensionPolicySettings_diagnosticScriptNotAllowed}, + {handlersettings.CommandIdScript, errorutil.ExtensionPolicySettings_commandIdScriptNotAllowed}, + {handlersettings.ScriptType("unknown"), errorutil.ExtensionPolicySettings_unknownScriptType}, + } { + t.Run(string(tc.script), func(t *testing.T) { + requirePolicyCode(t, CompareScriptTypeToAllowedScriptType(tc.script, AllowedScriptNone), tc.code) + err := ValidateScriptTypeAgainstPolicy(nopCtx(), tc.script, "") + requirePolicyCode(t, err, tc.code) + err, exit := ValidateHandlerSettingsAgainstPolicy(nopCtx(), makeSettings(tc.script, "", "", "", ""), &RCv2ExtensionPolicySettings{}) + requirePolicyCode(t, err, tc.code) + require.Equal(t, constants.ExitCode_ScriptTypeNotAllowedByExtensionPolicy, exit) + }) + } +} + +func TestPolicyInitializationClarifications(t *testing.T) { + t.Run("empty path -2510", func(t *testing.T) { + mockPolicyBoundaries(t) + _, _, err, exit := InitializeExtensionPolicySettings(nopCtx(), "") + requirePolicyCode(t, err, errorutil.Internal_emptyPolicyPath) + require.Equal(t, constants.ExitCode_InitializeCalledWithNoPolicyPath, exit) + }) + for _, tc := range []struct { + name string + code int + exit int + }{ + {"create", errorutil.Internal_failedToCreatePolicyManager, constants.ExitCode_FailedToCreateExtensionPolicySettingsManager}, + {"load", errorutil.ExtensionPolicySettings_policyLoadFailed, constants.ExitCode_LoadExtensionPolicySettingsFailed}, + {"get", errorutil.Internal_failedToGetPolicySettings, constants.ExitCode_FailedToGetExtensionPolicySettings}, + } { + t.Run(tc.name, func(t *testing.T) { + mockPolicyBoundaries(t) + cause := errors.New("boundary failed") + switch tc.name { + case "create": + newPolicyManager = func(string) (*policyManager, error) { return nil, cause } + case "load": + loadPolicySettings = func(*policyManager) error { return cause } + case "get": + getPolicySettings = func(*policyManager) (*RCv2ExtensionPolicySettings, error) { return nil, cause } + } + manager, policy, err, exit := InitializeExtensionPolicySettings(nopCtx(), "policy.json") + require.Nil(t, manager) + require.Nil(t, policy) + requirePolicyCode(t, err, tc.code) + require.ErrorIs(t, err, cause) + require.Equal(t, tc.exit, exit) + }) + } + t.Run("success", func(t *testing.T) { + mockPolicyBoundaries(t) + manager, policy, err, exit := InitializeExtensionPolicySettings(nopCtx(), "policy.json") + require.NotNil(t, manager) + require.NotNil(t, policy) + require.NoError(t, err) + require.Zero(t, exit) + }) +} + +func TestPolicySettingsClarifications(t *testing.T) { + t.Run("nil policy -2514", func(t *testing.T) { + err, exit := ValidateHandlerSettingsAgainstPolicy(nopCtx(), makeSettings(handlersettings.InlineScript, "", "", "", ""), nil) + requirePolicyCode(t, err, errorutil.Internal_nilPolicySettings) + require.Equal(t, constants.ExitCode_ValidateCalledWithNilPolicy, exit) + }) + t.Run("command 2515", func(t *testing.T) { + mockPolicyBoundaries(t) + cause := errors.New("not in allowlist") + validatePolicyAllowlist = func(string, []string) error { return cause } + policy := &RCv2ExtensionPolicySettings{LimitScripts: "allowall", CommandIdAllowlist: []string{"safe"}} + err, exit := ValidateHandlerSettingsAgainstPolicy(nopCtx(), makeSettings(handlersettings.CommandIdScript, "unsafe", "", "", ""), policy) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_commandToExecuteNotAllowed) + require.ErrorIs(t, err, cause) + require.Equal(t, constants.ExitCode_CommandIdNotAllowedByExtensionPolicy, exit) + }) + t.Run("user 2516", func(t *testing.T) { + policy := &RCv2ExtensionPolicySettings{LimitScripts: "allowall", RunAsUser: "alice"} + err, exit := ValidateHandlerSettingsAgainstPolicy(nopCtx(), makeSettings(handlersettings.InlineScript, "", "bob", "", ""), policy) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_runAsUserNotAllowed) + require.Equal(t, constants.ExitCode_RunAsUserNotAllowedByExtensionPolicy, exit) + }) + for _, output := range []bool{true, false} { + t.Run(fmt.Sprintf("output blobs 2517 output=%t", output), func(t *testing.T) { + policy := &RCv2ExtensionPolicySettings{LimitScripts: "allowall", DisableOutputBlobs: true} + settings := makeSettings(handlersettings.InlineScript, "", "", "", "") + if output { + settings.OutputBlobURI = "https://example.test/blob?sig=secret" + } else { + settings.ErrorBlobURI = "https://example.test/blob?sig=secret" + } + err, exit := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, policy) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_outputBlobsNotAllowed) + require.NotContains(t, err.Error(), "secret") + require.Equal(t, constants.ExitCode_OutputBlobSpecifiedButNotAllowedByExtensionPolicy, exit) + }) + } +} + +func TestPolicyPreservesLowerClarifications(t *testing.T) { + for _, lower := range []error{ + vmextension.NewErrorWithClarificationPtr(errorutil.CustomerInput_invalidSettingsJson, errors.New("pointer")), + vmextension.NewErrorWithClarification(errorutil.CustomerInput_invalidSettingsJson, errors.New("value")), + } { + for _, boundary := range []string{"create", "load", "get", "allowlist"} { + t.Run(fmt.Sprintf("%s/%v", boundary, lower), func(t *testing.T) { + mockPolicyBoundaries(t) + wrapped := fmt.Errorf("boundary: %w", lower) + switch boundary { + case "create": + newPolicyManager = func(string) (*policyManager, error) { return nil, wrapped } + case "load": + loadPolicySettings = func(*policyManager) error { return wrapped } + case "get": + getPolicySettings = func(*policyManager) (*RCv2ExtensionPolicySettings, error) { return nil, wrapped } + case "allowlist": + validatePolicyAllowlist = func(string, []string) error { return wrapped } + } + var err error + if boundary == "allowlist" { + err = ValidateCommandId(nopCtx(), makeSettings(handlersettings.CommandIdScript, "command", "", "", ""), &RCv2ExtensionPolicySettings{CommandIdAllowlist: []string{"safe"}}) + } else { + _, _, err, _ = InitializeExtensionPolicySettings(nopCtx(), "policy.json") + } + requirePolicyCode(t, err, errorutil.CustomerInput_invalidSettingsJson) + require.ErrorIs(t, err, lower) + }) + } + } +} diff --git a/internal/extensionpolicysettingsrc/extensionpolicysettingsrc.go b/internal/extensionpolicysettingsrc/extensionpolicysettingsrc.go index df2f900..e42b404 100644 --- a/internal/extensionpolicysettingsrc/extensionpolicysettingsrc.go +++ b/internal/extensionpolicysettingsrc/extensionpolicysettingsrc.go @@ -8,34 +8,44 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) +type policyManager = extensionpolicysettings.ExtensionPolicySettingsManager[RCv2ExtensionPolicySettings] + +var ( + newPolicyManager = extensionpolicysettings.NewExtensionPolicySettingsManager[RCv2ExtensionPolicySettings] + loadPolicySettings = (*policyManager).LoadExtensionPolicySettings + getPolicySettings = (*policyManager).GetSettings + validatePolicyAllowlist = extensionpolicysettings.ValidateValueInAllowlist +) + func InitializeExtensionPolicySettings(ctx *log.Context, policyPath string) (*extensionpolicysettings.ExtensionPolicySettingsManager[RCv2ExtensionPolicySettings], *RCv2ExtensionPolicySettings, error, int) { if policyPath == "" { - err := fmt.Errorf("policy path to initialize extension policy settings is empty") + err := errorutil.EnsureClarification(errorutil.Internal_emptyPolicyPath, fmt.Errorf("policy path to initialize extension policy settings is empty")) ctx.Log("message", "policy path is empty. "+constants.ContactICMForServiceErrorsMessage, "error", err) return nil, nil, err, constants.ExitCode_InitializeCalledWithNoPolicyPath } - extensionPolicyManager, err := extensionpolicysettings.NewExtensionPolicySettingsManager[RCv2ExtensionPolicySettings](policyPath) + extensionPolicyManager, err := newPolicyManager(policyPath) if err != nil { // Manager only fails to be created if policy path is empty, so this shouldn't fail. - err = errors.Wrap(err, "failed to create extension policy settings manager. Ensure the policy path is valid") + err = errorutil.EnsureClarification(errorutil.Internal_failedToCreatePolicyManager, errors.Wrap(err, "failed to create extension policy settings manager. Ensure the policy path is valid")) ctx.Log("message", "failed to create extension policy settings manager. "+constants.ContactICMForServiceErrorsMessage, "error", err, "policyPath", policyPath) return nil, nil, err, constants.ExitCode_FailedToCreateExtensionPolicySettingsManager } - err = extensionPolicyManager.LoadExtensionPolicySettings() + err = loadPolicySettings(extensionPolicyManager) if err != nil { - err = errors.Wrap(err, "failed to load extension policy settings from file. Ensure the policy format is valid and the file is accessible") + err = errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_policyLoadFailed, errors.Wrap(err, "failed to load extension policy settings from file. Ensure the policy format is valid and the file is accessible")) ctx.Log("message", "failed to load extension policy settings. "+constants.ContactICMForServiceErrorsMessage, "error", err, "policyPath", policyPath) return nil, nil, err, constants.ExitCode_LoadExtensionPolicySettingsFailed } - rceps, err := extensionPolicyManager.GetSettings() //rceps is the pointer to the actual policy struct + rceps, err := getPolicySettings(extensionPolicyManager) //rceps is the pointer to the actual policy struct if err != nil { - err = errors.Wrap(err, "failed to get extension policy settings after loading") + err = errorutil.EnsureClarification(errorutil.Internal_failedToGetPolicySettings, errors.Wrap(err, "failed to get extension policy settings after loading")) ctx.Log("message", "failed to get extension policy settings. "+constants.ContactICMForServiceErrorsMessage, "error", err, "policyPath", policyPath) return nil, nil, err, constants.ExitCode_FailedToGetExtensionPolicySettings } @@ -45,7 +55,7 @@ func InitializeExtensionPolicySettings(ctx *log.Context, policyPath string) (*ex func ValidateHandlerSettingsAgainstPolicy(ctx *log.Context, settings *handlersettings.HandlerSettings, policy *RCv2ExtensionPolicySettings) (error, int) { if policy == nil { ctx.Log("message", "no policy provided for extension policy settings") - return fmt.Errorf("no policy provided to validate handler settings"), constants.ExitCode_ValidateCalledWithNilPolicy + return errorutil.EnsureClarification(errorutil.Internal_nilPolicySettings, fmt.Errorf("no policy provided to validate handler settings")), constants.ExitCode_ValidateCalledWithNilPolicy } if err := ValidateScriptTypeAgainstPolicy(ctx, settings.ScriptType(), policy.LimitScripts); err != nil { return err, constants.ExitCode_ScriptTypeNotAllowedByExtensionPolicy @@ -72,7 +82,10 @@ func ValidateHandlerSettingsAgainstPolicy(ctx *log.Context, settings *handlerset } func ValidateScriptTypeAgainstPolicy(ctx *log.Context, scriptType handlersettings.ScriptType, allowedScriptTypesString string) error { - allowedScriptTypes, _ := StringToAllowedScriptTypeFlag(allowedScriptTypesString) + allowedScriptTypes, parseErr := StringToAllowedScriptTypeFlag(allowedScriptTypesString) + if parseErr != nil && allowedScriptTypesString != "" { + return fmt.Errorf("script type %s is not allowed by policy: invalid allowed script types: %w", scriptType, parseErr) + } // Compare the script type of the command with the allowed script types in the policy. err := CompareScriptTypeToAllowedScriptType(scriptType, allowedScriptTypes) if err != nil { @@ -91,10 +104,10 @@ func ValidateCommandId(ctx *log.Context, settings *handlersettings.HandlerSettin ctx.Log("message", "allowedCommandID list empty, allowing all commands") return nil } - err := extensionpolicysettings.ValidateValueInAllowlist(settingsCommandId, allowedCommandIds) + err := validatePolicyAllowlist(settingsCommandId, allowedCommandIds) if err != nil { ctx.Log("message", "command ID is not allowed by policy", "error", err, "commandId", settingsCommandId) - return errors.Wrapf(err, "command ID %s is not allowed by policy", settingsCommandId) + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_commandToExecuteNotAllowed, errors.Wrapf(err, "command ID %s is not allowed by policy", settingsCommandId)) } return nil } @@ -104,7 +117,7 @@ func ValidateRunAsUser(ctx *log.Context, settings *handlersettings.HandlerSettin policyRunAsUser := strings.ToLower(strings.TrimSpace(policy.RunAsUser)) if strings.Compare(settingsRunAsUser, policyRunAsUser) != 0 { - err := fmt.Errorf("runAsUser '%s' in settings does not match runAsUser '%s' in policy", settingsRunAsUser, policyRunAsUser) + err := errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_runAsUserNotAllowed, fmt.Errorf("runAsUser '%s' in settings does not match runAsUser '%s' in policy", settingsRunAsUser, policyRunAsUser)) ctx.Log("message", "runAsUser settings does not match runAsUser in policy", "error", err, "settingsRunAsUser", settingsRunAsUser, "policyRunAsUser", policyRunAsUser) return err } @@ -117,7 +130,7 @@ func ValidateDisableOutputBlobs(ctx *log.Context, settings *handlersettings.Hand trimmedErrorBlobURI := strings.TrimSpace(settings.ErrorBlobURI) if trimmedOutputBlobURI != "" || trimmedErrorBlobURI != "" { - err := fmt.Errorf("output blobs are disabled in policy, but settings specify outputBlobURI '%s' or errorBlobURI '%s'", download.GetUriForLogging(settings.OutputBlobURI), download.GetUriForLogging(settings.ErrorBlobURI)) + err := errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_outputBlobsNotAllowed, fmt.Errorf("output blobs are disabled in policy, but settings specify outputBlobURI '%s' or errorBlobURI '%s'", download.GetUriForLogging(settings.OutputBlobURI), download.GetUriForLogging(settings.ErrorBlobURI))) ctx.Log("message", "output blobs are disabled in policy, but settings specify output or error blob URIs", "error", err, "outputBlobURI", download.GetUriForLogging(settings.OutputBlobURI), "errorBlobURI", download.GetUriForLogging(settings.ErrorBlobURI)) return err } diff --git a/internal/extensionpolicysettingsrc/extensionpolicysettingsrc_test.go b/internal/extensionpolicysettingsrc/extensionpolicysettingsrc_test.go index bc59cbd..7f1e350 100644 --- a/internal/extensionpolicysettingsrc/extensionpolicysettingsrc_test.go +++ b/internal/extensionpolicysettingsrc/extensionpolicysettingsrc_test.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -30,12 +31,14 @@ func makeSettings(scriptType handlersettings.ScriptType, commandID string, runAs func TestInitializeExtensionPolicySettings_EmptyPath_ReturnsError(t *testing.T) { _, _, err, exitCode := InitializeExtensionPolicySettings(nopCtx(), "") require.Error(t, err) + requirePolicyCode(t, err, errorutil.Internal_emptyPolicyPath) require.Contains(t, err.Error(), "policy path to initialize extension policy settings is empty") require.Equal(t, constants.ExitCode_InitializeCalledWithNoPolicyPath, exitCode) } func TestInitializeExtensionPolicySettings_InvalidPath_ReturnsError(t *testing.T) { _, _, err, exitCode := InitializeExtensionPolicySettings(nopCtx(), "/definitely/not/found/policy.json") require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_policyLoadFailed) require.Contains(t, err.Error(), "failed to load extension policy settings from file. Ensure the policy format is valid and the file is accessible") require.Equal(t, constants.ExitCode_LoadExtensionPolicySettingsFailed, exitCode) } @@ -50,6 +53,7 @@ func TestInitializeExtensionPolicySettings_InvalidPolicyFails(t *testing.T) { _, _, err, exitCode := InitializeExtensionPolicySettings(nopCtx(), policyPath) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_policyLoadFailed) require.Contains(t, err.Error(), "failed to load extension policy settings from file. Ensure the policy format is valid and the file is accessible") require.Equal(t, constants.ExitCode_LoadExtensionPolicySettingsFailed, exitCode) } @@ -91,6 +95,7 @@ func TestValidateHandlerSettingsAgainstPolicy(t *testing.T) { settings := makeSettings(handlersettings.InlineScript, "", "", "", "") err, exitCode := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, nil) require.Error(t, err) + requirePolicyCode(t, err, errorutil.Internal_nilPolicySettings) require.Contains(t, err.Error(), "no policy provided to validate handler settings") require.Equal(t, constants.ExitCode_ValidateCalledWithNilPolicy, exitCode) }) @@ -105,6 +110,7 @@ func TestValidateHandlerSettingsAgainstPolicy(t *testing.T) { err, exitCode := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_scriptNotAllowed) require.Contains(t, err.Error(), "script type Inline is not allowed by policy") require.Equal(t, constants.ExitCode_ScriptTypeNotAllowedByExtensionPolicy, exitCode) }) @@ -120,6 +126,7 @@ func TestValidateHandlerSettingsAgainstPolicy(t *testing.T) { err, exitCode := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_commandToExecuteNotAllowed) require.Equal(t, constants.ExitCode_CommandIdNotAllowedByExtensionPolicy, exitCode) }) @@ -132,6 +139,7 @@ func TestValidateHandlerSettingsAgainstPolicy(t *testing.T) { err, exitCode := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_runAsUserNotAllowed) require.Contains(t, err.Error(), "does not match") require.Equal(t, constants.ExitCode_RunAsUserNotAllowedByExtensionPolicy, exitCode) }) @@ -146,6 +154,7 @@ func TestValidateHandlerSettingsAgainstPolicy(t *testing.T) { } err, exitCode := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, policy) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_commandIdScriptNotAllowed) require.Contains(t, err.Error(), "script type CommandId is not allowed by policy") require.Equal(t, constants.ExitCode_ScriptTypeNotAllowedByExtensionPolicy, exitCode) }) @@ -161,6 +170,7 @@ func TestValidateHandlerSettingsAgainstPolicy(t *testing.T) { err, exitCode := ValidateHandlerSettingsAgainstPolicy(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_outputBlobsNotAllowed) require.Contains(t, err.Error(), fmt.Sprintf("output blobs are disabled in policy, but settings specify outputBlobURI '%s' or errorBlobURI '%s'", settings.OutputBlobURI, settings.ErrorBlobURI)) require.Equal(t, constants.ExitCode_OutputBlobSpecifiedButNotAllowedByExtensionPolicy, exitCode) }) @@ -218,6 +228,7 @@ func TestValidateScriptTypeAgainstPolicy(t *testing.T) { t.Run("blocked", func(t *testing.T) { err := ValidateScriptTypeAgainstPolicy(nopCtx(), handlersettings.GalleryScript, "inline") require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_galleryScriptNotAllowed) require.Contains(t, err.Error(), "script type Gallery is not allowed by policy") }) @@ -225,6 +236,7 @@ func TestValidateScriptTypeAgainstPolicy(t *testing.T) { t.Run("invalid policy token is treated as blocked", func(t *testing.T) { err := ValidateScriptTypeAgainstPolicy(nopCtx(), handlersettings.InlineScript, "notARealScriptType") require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_invalidPolicyFileFormat) require.Contains(t, err.Error(), "script type Inline is not allowed by policy") }) } @@ -254,6 +266,7 @@ func TestValidateCommandId(t *testing.T) { CommandIdAllowlist: []string{"safeCommand", "other"}, } err := ValidateCommandId(nopCtx(), settings, policy) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_commandToExecuteNotAllowed) require.Contains(t, err.Error(), "command ID restartVM is not allowed by policy") require.Contains(t, err.Error(), "item is not in the allowlist") }) @@ -276,6 +289,7 @@ func TestValidateRunAsUser(t *testing.T) { } err := ValidateRunAsUser(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_runAsUserNotAllowed) require.Contains(t, err.Error(), "runAsUser 'bob' in settings does not match runAsUser 'alice' in policy") }) } @@ -288,6 +302,7 @@ func TestValidateDisableOutputBlobs(t *testing.T) { } err := ValidateDisableOutputBlobs(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_outputBlobsNotAllowed) require.Contains(t, err.Error(), fmt.Sprintf("output blobs are disabled in policy, but settings specify outputBlobURI '%s' or errorBlobURI '%s'", settings.OutputBlobURI, settings.ErrorBlobURI)) }) @@ -298,6 +313,7 @@ func TestValidateDisableOutputBlobs(t *testing.T) { } err := ValidateDisableOutputBlobs(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_outputBlobsNotAllowed) require.Contains(t, err.Error(), fmt.Sprintf("output blobs are disabled in policy, but settings specify outputBlobURI '%s' or errorBlobURI '%s'", settings.OutputBlobURI, settings.ErrorBlobURI)) }) @@ -308,6 +324,7 @@ func TestValidateDisableOutputBlobs(t *testing.T) { } err := ValidateDisableOutputBlobs(nopCtx(), settings, policy) require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_outputBlobsNotAllowed) require.Contains(t, err.Error(), fmt.Sprintf("output blobs are disabled in policy, but settings specify outputBlobURI '%s' or errorBlobURI '%s'", settings.OutputBlobURI, settings.ErrorBlobURI)) }) diff --git a/internal/extensionpolicysettingsrc/types.go b/internal/extensionpolicysettingsrc/types.go index f6eea20..e170009 100644 --- a/internal/extensionpolicysettingsrc/types.go +++ b/internal/extensionpolicysettingsrc/types.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" ) // This refers *specifically* to file types that require signature verification @@ -57,7 +58,7 @@ func StringToAllowedScriptTypeFlag(s string) (AllowedScriptTypeFlag, error) { flag |= AllowAll // TO-DO: consider the case where 'none' scripts are allowed to run. default: - return 0, fmt.Errorf("Unknown script type in policy: %s", part) + return 0, errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_invalidPolicyFileFormat, fmt.Errorf("Unknown script type in policy: %s", part)) } } return flag, nil @@ -90,19 +91,19 @@ func (rceps RCv2ExtensionPolicySettings) ValidateFormat() error { // 2. LimitScripts must be a valid AllowedScriptType value. so map/check the value to the AllowedScriptTypeFlag bitmask. if rceps.LimitScripts != "" { if err != nil { - return fmt.Errorf("at least one of the values in LimitScripts is not a valid script type: %v", rceps.LimitScripts) + return fmt.Errorf("at least one of the values in LimitScripts is not a valid script type: %s: %w", rceps.LimitScripts, err) } } // 3. If DownloadedScriptsAllowlist is not empty, limit scripts must allow "downloaded" scripts. if len(rceps.DownloadedScriptsAllowlist) > 0 { if (flag & AllowedDownloaded) == 0 { - return fmt.Errorf("DownloadedScriptsAllowlist not empty, but LimitScripts does not allow 'downloaded' scripts") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_inconsistentDownloadedScriptAllowlist, fmt.Errorf("DownloadedScriptsAllowlist not empty, but LimitScripts does not allow 'downloaded' scripts")) } } // 4. If CommandIdAllowlist is not empty, limit scripts must allow "commandId" scripts. if len(rceps.CommandIdAllowlist) > 0 { if (flag & AllowedCommandId) == 0 { - return fmt.Errorf("CommandIdAllowlist not empty, but LimitScripts does not allow 'commandId' scripts") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_inconsistentCommandIdAllowlist, fmt.Errorf("CommandIdAllowlist not empty, but LimitScripts does not allow 'commandId' scripts")) } } return nil @@ -116,26 +117,26 @@ func CompareScriptTypeToAllowedScriptType(scriptType handlersettings.ScriptType, switch scriptType { case handlersettings.InlineScript: if (allowedScriptTypes & Inline) == 0 { - return fmt.Errorf("inline scripts are not allowed by policy") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_scriptNotAllowed, fmt.Errorf("inline scripts are not allowed by policy")) } case handlersettings.DownloadedScript: if (allowedScriptTypes & AllowedDownloaded) == 0 { - return fmt.Errorf("downloaded scripts are not allowed by policy") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_downloadedScriptTypeNotAllowed, fmt.Errorf("downloaded scripts are not allowed by policy")) } case handlersettings.GalleryScript: if (allowedScriptTypes & Gallery) == 0 { - return fmt.Errorf("gallery scripts are not allowed by policy") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_galleryScriptNotAllowed, fmt.Errorf("gallery scripts are not allowed by policy")) } case handlersettings.DiagnosticScript: if (allowedScriptTypes & Diagnostic) == 0 { - return fmt.Errorf("diagnostic scripts are not allowed by policy") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_diagnosticScriptNotAllowed, fmt.Errorf("diagnostic scripts are not allowed by policy")) } case handlersettings.CommandIdScript: if (allowedScriptTypes & AllowedCommandId) == 0 { - return fmt.Errorf("commandId scripts are not allowed by policy") + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_commandIdScriptNotAllowed, fmt.Errorf("commandId scripts are not allowed by policy")) } default: - return fmt.Errorf("unknown script type: %v", scriptType) + return errorutil.EnsureClarification(errorutil.ExtensionPolicySettings_unknownScriptType, fmt.Errorf("unknown script type: %v", scriptType)) } return nil } diff --git a/internal/extensionpolicysettingsrc/types_test.go b/internal/extensionpolicysettingsrc/types_test.go index ba02dc3..cbac663 100644 --- a/internal/extensionpolicysettingsrc/types_test.go +++ b/internal/extensionpolicysettingsrc/types_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/stretchr/testify/require" ) @@ -108,6 +109,7 @@ func TestStringToAllowedScriptTypeFlag(t *testing.T) { if tt.wantErr != "" { require.Error(t, err) + requirePolicyCode(t, err, errorutil.ExtensionPolicySettings_invalidPolicyFileFormat) require.Equal(t, tt.wantErr, err.Error()) return } @@ -120,9 +122,10 @@ func TestStringToAllowedScriptTypeFlag(t *testing.T) { func TestValidateFormat(t *testing.T) { tests := []struct { - name string - input RCv2ExtensionPolicySettings - wantErr string + name string + input RCv2ExtensionPolicySettings + wantErr string + wantCode int }{ { name: "valid policy", @@ -139,7 +142,8 @@ func TestValidateFormat(t *testing.T) { input: RCv2ExtensionPolicySettings{ LimitScripts: "inline,notARealType", }, - wantErr: "at least one of the values in LimitScripts is not a valid script type: inline,notARealType", + wantErr: "at least one of the values in LimitScripts is not a valid script type: inline,notARealType: Unknown script type in policy: notarealtype", + wantCode: errorutil.ExtensionPolicySettings_invalidPolicyFileFormat, }, { name: "downloaded allowlist present, but downloaded scripts are blocked", @@ -147,7 +151,8 @@ func TestValidateFormat(t *testing.T) { LimitScripts: "inline,gallery", DownloadedScriptsAllowlist: []string{"hash1"}, }, - wantErr: "DownloadedScriptsAllowlist not empty, but LimitScripts does not allow 'downloaded' scripts", + wantErr: "DownloadedScriptsAllowlist not empty, but LimitScripts does not allow 'downloaded' scripts", + wantCode: errorutil.ExtensionPolicySettings_inconsistentDownloadedScriptAllowlist, }, { name: "command ID allowlist present, but command IDs are blocked", @@ -155,7 +160,8 @@ func TestValidateFormat(t *testing.T) { LimitScripts: "inline,gallery", CommandIdAllowlist: []string{"cmd1"}, }, - wantErr: "CommandIdAllowlist not empty, but LimitScripts does not allow 'commandId' scripts", + wantErr: "CommandIdAllowlist not empty, but LimitScripts does not allow 'commandId' scripts", + wantCode: errorutil.ExtensionPolicySettings_inconsistentCommandIdAllowlist, }, } @@ -165,6 +171,7 @@ func TestValidateFormat(t *testing.T) { if tt.wantErr != "" { require.Error(t, err) + requirePolicyCode(t, err, tt.wantCode) require.Equal(t, tt.wantErr, err.Error()) return } @@ -180,12 +187,14 @@ func TestCompareScriptTypeToAllowedScriptType(t *testing.T) { scriptType handlersettings.ScriptType allowed AllowedScriptTypeFlag wantErr string + wantCode int }{ { name: "none allowed, gallery denied", scriptType: handlersettings.GalleryScript, allowed: AllowedScriptNone, wantErr: "gallery scripts are not allowed by policy", + wantCode: errorutil.ExtensionPolicySettings_galleryScriptNotAllowed, }, { name: "allow all, allow inline", @@ -217,6 +226,7 @@ func TestCompareScriptTypeToAllowedScriptType(t *testing.T) { scriptType: handlersettings.InlineScript, allowed: Diagnostic, wantErr: "inline scripts are not allowed by policy", + wantCode: errorutil.ExtensionPolicySettings_scriptNotAllowed, }, { name: "allowed downloaded, allow downloaded", @@ -228,12 +238,14 @@ func TestCompareScriptTypeToAllowedScriptType(t *testing.T) { scriptType: handlersettings.ScriptType("made-up"), allowed: AllowAll, wantErr: "unknown script type: made-up", + wantCode: errorutil.ExtensionPolicySettings_unknownScriptType, }, { name: "'none' script currently treated as unknown", scriptType: handlersettings.NoneScript, allowed: AllowAll, wantErr: "unknown script type: None", + wantCode: errorutil.ExtensionPolicySettings_unknownScriptType, }, } @@ -243,6 +255,7 @@ func TestCompareScriptTypeToAllowedScriptType(t *testing.T) { if tt.wantErr != "" { require.Error(t, err) + requirePolicyCode(t, err, tt.wantCode) require.Equal(t, tt.wantErr, err.Error()) return } diff --git a/internal/files/clarification_test.go b/internal/files/clarification_test.go new file mode 100644 index 0000000..60beda7 --- /dev/null +++ b/internal/files/clarification_test.go @@ -0,0 +1,196 @@ +package files + +import ( + "errors" + "io" + "os" + "testing" + "time" + + "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type fileErrorLogger struct{ err error } + +func (l *fileErrorLogger) Log(values ...interface{}) error { + for _, value := range values { + if err, ok := value.(error); ok { + l.err = err + } + } + return nil +} + +type fakeScriptWriter struct{ writeErr, closeErr error } + +func (w fakeScriptWriter) WriteString(s string) (int, error) { return len(s), w.writeErr } +func (w fakeScriptWriter) Close() error { return w.closeErr } + +func assertFileCode(t *testing.T, err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) +} + +func TestScriptClarifications(t *testing.T) { + oldDownload, oldSave := downloadSASBlob, saveDownload + oldText, oldRead, oldWrite, oldOpen := isScriptText, readScript, writeScript, openScriptFile + oldMode := setScriptMode + t.Cleanup(func() { + downloadSASBlob, saveDownload = oldDownload, oldSave + isScriptText, readScript, writeScript, openScriptFile = oldText, oldRead, oldWrite, oldOpen + setScriptMode = oldMode + }) + cause := errors.New("file failure") + setScriptMode = func(string, os.FileMode) error { return nil } + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + _, err := downloadAndProcessURL(ctx, "://", ".", "script", "", nil) + assertFileCode(t, err, errorutil.CustomerInput_invalidScriptUrl) + downloadSASBlob = func(string, string, string) (string, error) { return "", cause } + saveDownload = func(*log.Context, []download.Downloader, string, os.FileMode) (int64, error) { return 0, nil } + isScriptText = func(string) (bool, error) { return false, nil } + logger := &fileErrorLogger{} + _, err = downloadAndProcessURL(log.NewContext(logger), "https://example.invalid/script", ".", "script", "sas", nil) + require.NoError(t, err, "SAS failure still falls back to public download") + assertFileCode(t, logger.err, errorutil.FileDownload_scriptSasDownloadFailed) + require.ErrorIs(t, logger.err, cause) + saveDownload = func(*log.Context, []download.Downloader, string, os.FileMode) (int64, error) { return 0, cause } + _, err = downloadAndProcessURL(ctx, "https://example.invalid/script", ".", "script", "", nil) + assertFileCode(t, err, errorutil.Os_FailedToMakeDownloadedFileExecutable) + require.ErrorIs(t, err, cause) + _, err = getDownloaders("", nil, download.ProdMsiDownloader{}) + assertFileCode(t, err, errorutil.CustomerInput_emptyDownloadUrl) + _, err = getDownloaders("https://account.blob.core.windows.net/container/script", &handlersettings.RunCommandManagedIdentity{ClientId: "client", ObjectId: "object"}, download.ProdMsiDownloader{}) + assertFileCode(t, err, errorutil.CustomerInput_clientIdObjectIdBothSpecified) + _, err = UrlToFileName("%") + assertFileCode(t, err, errorutil.CustomerInput_invalidDownloadFileUrl) + _, err = UrlToFileName("https://example.invalid/") + assertFileCode(t, err, errorutil.CustomerInput_missingDownloadFileName) + isScriptText = func(string) (bool, error) { return false, cause } + assertFileCode(t, PostProcessFile("script"), errorutil.Os_FailedToDetermineScriptFileType) + isScriptText = func(string) (bool, error) { return true, nil } + readScript = func(string) ([]byte, error) { return nil, cause } + assertFileCode(t, PostProcessFile("script"), errorutil.Os_FailedToReadScriptForPostprocessing) + readScript = func(string) ([]byte, error) { return []byte("script\r\n"), nil } + writeScript = func(string, []byte, os.FileMode) error { return cause } + err = PostProcessFile("script") + assertFileCode(t, err, errorutil.Os_FailedToWritePostprocessedScript) + require.ErrorIs(t, err, cause, "write errors must not be overwritten by rename") + for _, want := range []int{errorutil.Os_FailedToOpenEmbeddedScript, errorutil.Os_FailedToWriteEmbeddedScript, errorutil.Os_FailedToCloseEmbeddedScript} { + openScriptFile = func(string, int, os.FileMode) (scriptWriter, error) { + if want == errorutil.Os_FailedToOpenEmbeddedScript { + return nil, cause + } + if want == errorutil.Os_FailedToWriteEmbeddedScript { + return fakeScriptWriter{writeErr: cause}, nil + } + return fakeScriptWriter{closeErr: cause}, nil + } + err := SaveScriptFile("script", "content") + assertFileCode(t, err, want) + require.ErrorIs(t, err, cause) + } +} + +func TestDownloadedScriptPermissionClarifications(t *testing.T) { + oldDownload, oldText, oldMode := downloadSASBlob, isScriptText, setScriptMode + t.Cleanup(func() { downloadSASBlob, isScriptText, setScriptMode = oldDownload, oldText, oldMode }) + cause := errors.New("permission failure") + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + downloadSASBlob = func(string, string, string) (string, error) { return "script", nil } + isScriptText = func(string) (bool, error) { return false, nil } + for _, want := range []int{errorutil.Os_FailedToMakeDownloadedFileWritable, errorutil.Os_FailedToRestoreDownloadedFilePermissions} { + var modes []os.FileMode + setScriptMode = func(_ string, mode os.FileMode) error { + modes = append(modes, mode) + if want == errorutil.Os_FailedToMakeDownloadedFileWritable || len(modes) == 2 { + return cause + } + return nil + } + _, err := downloadAndProcessURL(ctx, "https://example.invalid/script", ".", "script", "sas", nil) + assertFileCode(t, err, want) + require.ErrorIs(t, err, cause) + if want == errorutil.Os_FailedToRestoreDownloadedFilePermissions { + require.Equal(t, []os.FileMode{0700, 0500}, modes) + } + } + var modes []os.FileMode + setScriptMode = func(_ string, mode os.FileMode) error { + modes = append(modes, mode) + return nil + } + isScriptText = func(string) (bool, error) { return false, cause } + _, err := downloadAndProcessURL(ctx, "https://example.invalid/script", ".", "script", "sas", nil) + assertFileCode(t, err, errorutil.Os_FailedToDetermineScriptFileType) + require.ErrorIs(t, err, cause) + require.Equal(t, []os.FileMode{0700, 0500}, modes, "permissions are restored after post-processing errors") +} + +type tailFileInfo struct{} + +func (tailFileInfo) Name() string { return "file" } +func (tailFileInfo) Size() int64 { return 10 } +func (tailFileInfo) Mode() os.FileMode { return 0600 } +func (tailFileInfo) ModTime() time.Time { return time.Time{} } +func (tailFileInfo) IsDir() bool { return false } +func (tailFileInfo) Sys() interface{} { return nil } + +type failingReadableFile struct { + stage string + err error +} + +func (f failingReadableFile) Stat() (os.FileInfo, error) { + if f.stage == "stat" { + return nil, f.err + } + return tailFileInfo{}, nil +} +func (f failingReadableFile) Seek(int64, int) (int64, error) { + if f.stage == "seek" { + return 0, f.err + } + return 0, nil +} +func (f failingReadableFile) Read([]byte) (int, error) { return 0, f.err } +func (f failingReadableFile) Close() error { return nil } + +func TestTailClarifications(t *testing.T) { + oldOpen := openReadableFile + t.Cleanup(func() { openReadableFile = oldOpen }) + cause := errors.New("read failure") + for _, tc := range []struct { + stage string + tailCode int + positionCode int + }{ + {"open", errorutil.Os_FailedToOpenTailFile, errorutil.Os_FailedToOpenOutputFile}, + {"stat", errorutil.Os_FailedToStatTailFile, errorutil.Os_FailedToStatOutputFile}, + {"seek", errorutil.Os_FailedToSeekTailFile, errorutil.Os_FailedToSeekOutputFile}, + {"read", errorutil.Os_FailedToReadTailFile, errorutil.Os_FailedToReadOutputFile}, + } { + openReadableFile = func(string) (readableFile, error) { + if tc.stage == "open" { + return nil, cause + } + return failingReadableFile{stage: tc.stage, err: cause}, nil + } + _, err := TailFile("file", 1) + assertFileCode(t, err, tc.tailCode) + require.ErrorIs(t, err, cause) + _, err = GetFileFromPosition("file", 0) + assertFileCode(t, err, tc.positionCode) + require.ErrorIs(t, err, cause) + } + openReadableFile = func(string) (readableFile, error) { return nil, os.ErrNotExist } + _, err := TailFile("missing", 1) + require.NoError(t, err) + _, err = GetFileFromPosition("missing", 0) + require.NoError(t, err) +} diff --git a/internal/files/files.go b/internal/files/files.go index 18f9965..741768f 100644 --- a/internal/files/files.go +++ b/internal/files/files.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/preprocess" "github.com/Azure/run-command-handler-linux/pkg/urlutil" "github.com/go-kit/kit/log" @@ -19,6 +20,24 @@ import ( var UseMockSASDownloadFailure bool = false +var ( + downloadSASBlob = download.GetSASBlob + saveDownload = download.SaveTo + isScriptText = preprocess.IsTextFile + readScript = ioutil.ReadFile + writeScript = ioutil.WriteFile + setScriptMode = os.Chmod +) + +type scriptWriter interface { + WriteString(string) (int, error) + Close() error +} + +var openScriptFile = func(name string, flag int, mode os.FileMode) (scriptWriter, error) { + return os.OpenFile(name, flag, mode) +} + func DownloadAndProcessArtifact(ctx *log.Context, downloadDir string, artifact *handlersettings.UnifiedArtifact) (string, error) { fileName := artifact.FileName if fileName == "" { @@ -48,7 +67,7 @@ func DownloadAndProcessScript(ctx *log.Context, url, downloadDir string, cfg *ha func downloadAndProcessURL(ctx *log.Context, url, downloadDir string, fileName string, scriptSAS string, sourceManagedIdentity *handlersettings.RunCommandManagedIdentity) (string, error) { var err error if !urlutil.IsValidUrl(url) { - return "", fmt.Errorf("%s is not a valid url", url) // url does not contain SAS to se can log it + return "", errorutil.EnsureClarification(errorutil.CustomerInput_invalidScriptUrl, fmt.Errorf("%s is not a valid url", url)) // url does not contain SAS to se can log it } targetFilePath := filepath.Join(downloadDir, fileName) @@ -59,8 +78,9 @@ func downloadAndProcessURL(ctx *log.Context, url, downloadDir string, fileName s if UseMockSASDownloadFailure { scriptSASDownloadErr = errors.New("Downloading script using SAS token failed.") } else { - downloadedFilePath, scriptSASDownloadErr = download.GetSASBlob(url, scriptSAS, downloadDir) + downloadedFilePath, scriptSASDownloadErr = downloadSASBlob(url, scriptSAS, downloadDir) } + scriptSASDownloadErr = errorutil.EnsureClarification(errorutil.FileDownload_scriptSasDownloadFailed, scriptSASDownloadErr) // Download was successful using SAS. So use downloadedFilePath if scriptSASDownloadErr == nil && downloadedFilePath != "" { targetFilePath = downloadedFilePath @@ -69,24 +89,38 @@ func downloadAndProcessURL(ctx *log.Context, url, downloadDir string, fileName s //If there was an error downloading using SAS URI or SAS was not provided, download using managedIdentity or publicly. if scriptSASDownloadErr != nil || scriptSAS == "" { - ctx.Log("info", fmt.Sprintf("Downloading script using SAS token failed: %v. Attempting download using managed identity or public access.", scriptSASDownloadErr)) + if scriptSASDownloadErr != nil { + ctx.Log("info", "Downloading script using SAS token failed. Attempting download using managed identity or public access.", "error", scriptSASDownloadErr) + } downloaders, getDownloadersError := getDownloaders(url, sourceManagedIdentity, download.ProdMsiDownloader{}) if getDownloadersError == nil { const mode = 0500 // we assume users download scripts to execute - _, err = download.SaveTo(ctx, downloaders, targetFilePath, mode) + _, err = saveDownload(ctx, downloaders, targetFilePath, mode) } else { return "", getDownloadersError } } if err != nil { - return "", err + return "", errorutil.EnsureClarification(errorutil.Os_FailedToMakeDownloadedFileExecutable, err) } + // Downloads are executable but read-only; allow the owner to normalize the + // script, then restore the original execution permissions even on failure. + if err := setScriptMode(targetFilePath, 0700); err != nil { + return "", errorutil.EnsureClarification(errorutil.Os_FailedToMakeDownloadedFileWritable, errors.Wrap(err, "failed to make downloaded file writable for post-processing")) + } err = PostProcessFile(targetFilePath) + restoreErr := errorutil.EnsureClarification(errorutil.Os_FailedToRestoreDownloadedFilePermissions, errors.Wrap(setScriptMode(targetFilePath, 0500), "failed to restore downloaded file permissions")) if err != nil { + if restoreErr != nil { + return "", fmt.Errorf("failed to post-process %q: %w; failed to restore permissions: %w", fileName, err, restoreErr) + } return "", errors.Wrapf(err, "failed to post-process '%s'", fileName) } + if restoreErr != nil { + return "", restoreErr + } return targetFilePath, nil } @@ -97,7 +131,7 @@ func downloadAndProcessURL(ctx *log.Context, url, downloadDir string, fileName s func getDownloaders(fileURL string, managedIdentity *handlersettings.RunCommandManagedIdentity, msiDownloader download.MsiDownloader) ([]download.Downloader, error) { if fileURL == "" { - return nil, fmt.Errorf("fileURL is empty") + return nil, errorutil.EnsureClarification(errorutil.CustomerInput_emptyDownloadUrl, fmt.Errorf("fileURL is empty")) } if download.IsAzureStorageBlobUri(fileURL) { @@ -116,7 +150,7 @@ func getDownloaders(fileURL string, managedIdentity *handlersettings.RunCommandM // uses user-managed identity msiProvider = msiDownloader.GetMsiProviderByObjectId(fileURL, managedIdentity.ObjectId) default: - return nil, fmt.Errorf("use either ClientId or ObjectId for managed identity. Not both") + return nil, errorutil.EnsureClarification(errorutil.CustomerInput_clientIdObjectIdBothSpecified, fmt.Errorf("use either ClientId or ObjectId for managed identity. Not both")) } _, msiError := msiProvider() @@ -144,7 +178,7 @@ func getDownloaders(fileURL string, managedIdentity *handlersettings.RunCommandM func UrlToFileName(fileURL string) (string, error) { u, err := url.Parse(fileURL) if err != nil { - return "", errors.Wrapf(err, "unable to parse URL: %q", fileURL) + return "", errorutil.EnsureClarification(errorutil.CustomerInput_invalidDownloadFileUrl, errors.Wrapf(err, "unable to parse URL: %q", fileURL)) } s := strings.Split(u.Path, "/") @@ -154,39 +188,45 @@ func UrlToFileName(fileURL string) (string, error) { return fn, nil } } - return "", fmt.Errorf("cannot extract file name from URL: %q", fileURL) + return "", errorutil.EnsureClarification(errorutil.CustomerInput_missingDownloadFileName, fmt.Errorf("cannot extract file name from URL: %q", fileURL)) } // postProcessFile determines if path is a script file based on heuristics // and makes in-place changes to the file with some post-processing such as BOM // and DOS-line endings fixes to make the script POSIX-friendly. func PostProcessFile(path string) error { - ok, err := preprocess.IsTextFile(path) + ok, err := isScriptText(path) if err != nil { - return errors.Wrapf(err, "error determining if script is a text file") + return errorutil.EnsureClarification(errorutil.Os_FailedToDetermineScriptFileType, errors.Wrapf(err, "error determining if script is a text file")) } if !ok { return nil } - b, err := ioutil.ReadFile(path) // read the file into memory for processing + b, err := readScript(path) // read the file into memory for processing if err != nil { - return errors.Wrapf(err, "error reading file") + return errorutil.EnsureClarification(errorutil.Os_FailedToReadScriptForPostprocessing, errors.Wrapf(err, "error reading file")) } b = preprocess.RemoveBOM(b) b = preprocess.Dos2Unix(b) - err = ioutil.WriteFile(path, b, 0) - return errors.Wrap(os.Rename(path, path), "error writing file") + err = writeScript(path, b, 0) + if err != nil { + return errorutil.EnsureClarification(errorutil.Os_FailedToWritePostprocessedScript, errors.Wrap(err, "error writing file")) + } + return nil } func SaveScriptFile(filePath string, content string) error { const mode = 0500 // scripts should have execute permissions - file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, mode) + file, err := openScriptFile(filePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, mode) if err != nil { - return errors.Wrap(err, "failed to open file for writing: "+filePath) + return errorutil.EnsureClarification(errorutil.Os_FailedToOpenEmbeddedScript, errors.Wrap(err, "failed to open file for writing: "+filePath)) } _, err = file.WriteString(content) - file.Close() - return errors.Wrap(err, "failed to write to the file: "+filePath) + closeErr := file.Close() + if err != nil { + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteEmbeddedScript, errors.Wrap(err, "failed to write to the file: "+filePath)) + } + return errorutil.EnsureClarification(errorutil.Os_FailedToCloseEmbeddedScript, errors.Wrap(closeErr, "failed to close the file: "+filePath)) } diff --git a/internal/files/files_test.go b/internal/files/files_test.go index 2466a1e..6de88c4 100644 --- a/internal/files/files_test.go +++ b/internal/files/files_test.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetalpbalkan/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -61,6 +62,7 @@ func Test_getDownloaders_SystemIdentityVersusByClientIdOrObjectId(t *testing.T) d, err := getDownloaders("http://acct.blob.core.windows.net/", &mockManagedIdentityBoth, mockMsiDownloder) fmt.Println(err.Error()) require.NotNil(t, err) + assertFileCode(t, err, errorutil.CustomerInput_clientIdObjectIdBothSpecified) require.Equal(t, err.Error(), "use either ClientId or ObjectId for managed identity. Not both") download.MockReturnErrorForMockMsiDownloader = false @@ -98,6 +100,7 @@ func Test_urlToFileName_badURL(t *testing.T) { _, err := UrlToFileName("http://192.168.0.%31/") require.NotNil(t, err) require.Contains(t, err.Error(), `unable to parse URL: "http://192.168.0.%31/"`) + assertFileCode(t, err, errorutil.CustomerInput_invalidDownloadFileUrl) } func Test_urlToFileName_noFileName(t *testing.T) { @@ -116,6 +119,7 @@ func Test_urlToFileName_noFileName(t *testing.T) { for _, c := range cases { _, err := UrlToFileName(c) require.NotNil(t, err, "not failed: %s", "url=%s", c) + assertFileCode(t, err, errorutil.CustomerInput_missingDownloadFileName) require.Contains(t, err.Error(), "cannot extract file name from URL", "url=%s", c) } } @@ -136,7 +140,9 @@ func Test_urlToFileName(t *testing.T) { } func Test_postProcessFile_fail(t *testing.T) { - require.NotNil(t, PostProcessFile("/non/existing/path")) + err := PostProcessFile("/non/existing/path") + require.NotNil(t, err) + assertFileCode(t, err, errorutil.Os_FailedToOpenTextFile) } func Test_postProcessFile(t *testing.T) { diff --git a/internal/files/logtail.go b/internal/files/logtail.go index ceb28b9..e0efad9 100644 --- a/internal/files/logtail.go +++ b/internal/files/logtail.go @@ -4,24 +4,34 @@ import ( "io" "os" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) +type readableFile interface { + io.Reader + io.Seeker + Stat() (os.FileInfo, error) + Close() error +} + +var openReadableFile = func(path string) (readableFile, error) { return os.Open(path) } + // TailFile returns the last max bytes (or the entire file if the file size is // smaller than max) from the file at path. If the file does not exist, it // returns a nil slice and no error. func TailFile(path string, max int64) ([]byte, error) { - f, err := os.Open(path) + f, err := openReadableFile(path) if err != nil && os.IsNotExist(err) { return nil, nil } else if err != nil { - return nil, errors.Wrap(err, "error opening file") + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToOpenTailFile, errors.Wrap(err, "error opening file")) } defer f.Close() fi, err := f.Stat() if err != nil { - return nil, errors.Wrap(err, "error retrieving file info") + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToStatTailFile, errors.Wrap(err, "error retrieving file info")) } size := fi.Size() @@ -31,25 +41,25 @@ func TailFile(path string, max int64) ([]byte, error) { } _, err = f.Seek(-n, io.SeekEnd) if err != nil { - return nil, errors.Wrapf(err, "error seeking file: offset=%d whence=%v", n, io.SeekEnd) + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToSeekTailFile, errors.Wrapf(err, "error seeking file: offset=%d whence=%v", n, io.SeekEnd)) } b, err := io.ReadAll(io.LimitReader(f, max)) - return b, errors.Wrap(err, "error reading from file") + return b, errorutil.EnsureClarification(errorutil.Os_FailedToReadTailFile, errors.Wrap(err, "error reading from file")) } func GetFileFromPosition(path string, position int64) ([]byte, error) { - f, err := os.Open(path) + f, err := openReadableFile(path) if err != nil && os.IsNotExist(err) { return nil, nil } else if err != nil { - return nil, errors.Wrap(err, "error opening file") + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToOpenOutputFile, errors.Wrap(err, "error opening file")) } defer f.Close() fi, err := f.Stat() if err != nil { - return nil, errors.Wrap(err, "error retrieving file info") + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToStatOutputFile, errors.Wrap(err, "error retrieving file info")) } size := fi.Size() if position >= size { @@ -59,9 +69,9 @@ func GetFileFromPosition(path string, position int64) ([]byte, error) { _, err = f.Seek(position, io.SeekStart) if err != nil { - return nil, errors.Wrapf(err, "error seeking file: %s, offset=%d", path, position) + return nil, errorutil.EnsureClarification(errorutil.Os_FailedToSeekOutputFile, errors.Wrapf(err, "error seeking file: %s, offset=%d", path, position)) } b, err := io.ReadAll(io.LimitReader(f, size-position)) - return b, errors.Wrap(err, "error reading from file: "+path) + return b, errorutil.EnsureClarification(errorutil.Os_FailedToReadOutputFile, errors.Wrap(err, "error reading from file: "+path)) } diff --git a/internal/files/logtail_test.go b/internal/files/logtail_test.go index 18119fa..8a738d6 100644 --- a/internal/files/logtail_test.go +++ b/internal/files/logtail_test.go @@ -2,6 +2,7 @@ package files import ( "bytes" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "os" "testing" @@ -21,6 +22,7 @@ func Test_tailFile_openError(t *testing.T) { require.Nil(t, os.Chmod(tf, 0333)) // no read _, err := TailFile(tf, 1024) require.NotNil(t, err) + assertFileCode(t, err, errorutil.Os_FailedToOpenTailFile) require.Regexp(t, `^error opening file:`, err.Error()) } diff --git a/internal/goalstate/clarification_test.go b/internal/goalstate/clarification_test.go new file mode 100644 index 0000000..808e0f7 --- /dev/null +++ b/internal/goalstate/clarification_test.go @@ -0,0 +1,265 @@ +package goalstate + +import ( + "encoding/json" + "errors" + "io" + "os" + "strconv" + "testing" + "time" + + "github.com/Azure/run-command-handler-linux/internal/commandProcessor" + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/handlersettings" + "github.com/Azure/run-command-handler-linux/internal/hostgacommunicator" + "github.com/Azure/run-command-handler-linux/internal/observer" + "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type failingCommunicator struct{ err error } + +func (c failingCommunicator) GetImmediateVMSettings(*log.Context, string) (*hostgacommunicator.ResponseData, error) { + return nil, c.err +} + +func TestGoalStateClarifications(t *testing.T) { + oldExecute, oldBuild, oldTimeout := executeImmediateCommand, buildStatusItem, goalStateTimeout + t.Cleanup(func() { executeImmediateCommand, buildStatusItem, goalStateTimeout = oldExecute, oldBuild, oldTimeout }) + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("goal state failure") + assertCode := func(err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + } + n := &observer.Notifier{} + key := types.GoalStateKey{RuntimeSettingsState: "enabled"} + view := &types.RunCommandInstanceView{} + assertCode(ReportFinalStatusForImmediateGoalState(ctx, nil, key, types.StatusError, view), errorutil.Internal_nilGoalStateNotifier) + assertCode(ReportFinalStatusForImmediateGoalState(ctx, n, key, types.StatusError, nil), errorutil.Internal_nilGoalStateInstanceView) + invalidKey := types.GoalStateKey{RuntimeSettingsState: "invalid"} + assertCode(ReportFinalStatusForImmediateGoalState(ctx, n, invalidKey, types.StatusError, view), errorutil.Internal_unknownGoalStateCommand) + // errorutil.Internal_failedToMarshalGoalStateInstanceView cannot fail with today's instance view: all serialized fields have + // JSON-supported scalar types, and ClarificationError is excluded from JSON. + buildStatusItem = func(*log.Context, types.StatusType, types.Cmd, string, string, ...error) (types.StatusItem, error) { + return types.StatusItem{}, cause + } + err := ReportFinalStatusForImmediateGoalState(ctx, n, key, types.StatusError, view) + assertCode(err, errorutil.Internal_failedToCreateGoalStateStatus) + require.ErrorIs(t, err, cause) + name, state, seq := "test", "enabled", 1 + setting := settings.SettingsCommon{ExtensionName: &name, ExtensionState: &state, SeqNo: &seq} + runAsync := func(s settings.SettingsCommon, n *observer.Notifier) error { + t.Helper() + done, failed := make(chan bool, 1), make(chan error, 1) + startAsync(ctx, constants.RunCommandTestExtensionName, s, n, done, failed) + select { + case err := <-failed: + return err + default: + t.Fatal("expected an asynchronous failure") + return nil + } + } + assertCode(runAsync(setting, nil), errorutil.Internal_nilImmediateCommandNotifier) + assertCode(runAsync(settings.SettingsCommon{}, n), errorutil.Internal_incompleteImmediateCommandGoalState) + invalidState := "invalid" + invalidSetting := setting + invalidSetting.ExtensionState = &invalidState + assertCode(runAsync(invalidSetting, n), errorutil.Internal_unknownImmediateCommand) + executeImmediateCommand = func(cmd types.Cmd, _ handlersettings.HandlerSettingsFile, _ string, _ int) error { + return cmd.Functions.ReportStatus(ctx, types.HandlerEnvironment{}, types.RCMetadata{}, types.StatusError, cmd, "{}") + } + assertCode(runAsync(setting, n), errorutil.Internal_failedToCreateImmediateCommandStatus) + executeImmediateCommand = func(cmd types.Cmd, _ handlersettings.HandlerSettingsFile, _ string, _ int) error { + return cmd.Functions.ReportStatus(ctx, types.HandlerEnvironment{}, types.RCMetadata{}, types.StatusError, cmd, "invalid json") + } + assertCode(runAsync(setting, n), errorutil.Internal_failedToParseImmediateInstanceView) + executeImmediateCommand = func(types.Cmd, handlersettings.HandlerSettingsFile, string, int) error { return cause } + assertCode(runAsync(setting, n), errorutil.Internal_failedToProcessImmediateCommand) + _, _, err = GetImmediateRunCommandGoalStates(ctx, nil, "etag") + assertCode(err, errorutil.Internal_nilGoalStateCommunicator) + _, etag, err := GetImmediateRunCommandGoalStates(ctx, failingCommunicator{cause}, "etag") + assertCode(err, errorutil.HostGA_failedToRetrieveVmSettings) + require.Equal(t, "etag", etag) + require.ErrorIs(t, err, cause) + buildStatusItem = oldBuild + release, finished := make(chan struct{}), make(chan struct{}) + executeImmediateCommand = func(types.Cmd, handlersettings.HandlerSettingsFile, string, int) error { + <-release + close(finished) + return nil + } + goalStateTimeout = time.Nanosecond + _, err = HandleImmediateGoalState(ctx, constants.RunCommandTestExtensionName, setting, n) + close(release) + <-finished + assertCode(err, errorutil.CommandExecution_immediateGoalStateTimedOut) +} + +func TestGoalStateStatusCarriesClarification(t *testing.T) { + oldExecute, oldBuild := executeImmediateCommand, buildStatusItem + t.Cleanup(func() { executeImmediateCommand, buildStatusItem = oldExecute, oldBuild }) + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + lower := errorutil.EnsureClarification(errorutil.CustomerInput_invalidSettingsJson, errors.New("customer error")) + calls := 0 + buildStatusItem = func(ctx *log.Context, state types.StatusType, cmd types.Cmd, message string, name string, clarifications ...error) (types.StatusItem, error) { + require.Equal(t, []error{lower}, clarifications) + calls++ + return oldBuild(ctx, state, cmd, message, name, clarifications...) + } + n := &observer.Notifier{} + recorded := &goalStateStatusRecorder{} + n.Register(recorded) + require.NoError(t, ReportFinalStatusForImmediateGoalState(ctx, n, types.GoalStateKey{RuntimeSettingsState: "enabled"}, types.StatusError, &types.RunCommandInstanceView{ClarificationError: lower})) + executeImmediateCommand = func(cmd types.Cmd, _ handlersettings.HandlerSettingsFile, _ string, _ int) error { + return cmd.Functions.ReportStatus(ctx, types.HandlerEnvironment{}, types.RCMetadata{ClarificationError: lower}, types.StatusError, cmd, "{}") + } + name, state, seq := "test", "enabled", 1 + done, failed := make(chan bool, 1), make(chan error, 1) + startAsync(ctx, constants.RunCommandTestExtensionName, settings.SettingsCommon{ExtensionName: &name, ExtensionState: &state, SeqNo: &seq}, n, done, failed) + require.Len(t, failed, 1) + var executionErr *ImmediateCommandExecutionError + require.ErrorAs(t, <-failed, &executionErr) + require.True(t, executionErr.Reported) + require.ErrorIs(t, executionErr, lower) + require.Equal(t, 2, calls) + require.Len(t, recorded.items, 2) + for _, item := range recorded.items { + require.Equal(t, types.StatusError, item.Status.Status) + requireGoalStateStatusCode(t, item.Status, errorutil.CustomerInput_invalidSettingsJson) + } +} + +func TestImmediateScriptFailureRetainsExecutionResult(t *testing.T) { + for _, deploymentFailure := range []bool{false, true} { + for _, reportingFailure := range []bool{false, true} { + t.Run(strconv.FormatBool(deploymentFailure)+"/"+strconv.FormatBool(reportingFailure), func(t *testing.T) { + oldExecute := executeImmediateCommand + t.Cleanup(func() { executeImmediateCommand = oldExecute }) + ctx := log.NewContext(log.NewNopLogger()) + name, state, seq := "script", "enabled", 1 + setting := settings.SettingsCommon{ + ExtensionName: &name, ExtensionState: &state, SeqNo: &seq, + PublicSettings: map[string]interface{}{ + "source": map[string]interface{}{"script": "exit 12"}, + "treatFailureAsDeploymentFailure": deploymentFailure, + }, + } + env := types.HandlerEnvironment{} + env.HandlerEnvironment.ConfigFolder = t.TempDir() + scriptErr := errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("script exited with code 12")) + executeImmediateCommand = func(cmd types.Cmd, hs handlersettings.HandlerSettingsFile, extension string, sequence int) error { + payload, err := json.Marshal(hs) + if err != nil { + return err + } + if err := os.WriteFile(handlersettings.GetConfigFilePath(env.HandlerEnvironment.ConfigFolder, sequence, extension), payload, 0600); err != nil { + return err + } + cmd.Functions.Invoke = func(*log.Context, types.HandlerEnvironment, *types.RunCommandInstanceView, types.RCMetadata, types.Cmd) (string, string, error, int, *types.CleanupRequest) { + return "script output", "script stderr", scriptErr, 12, nil + } + return commandProcessor.ProcessHandlerCommandWithDetails(ctx, cmd, env, extension, sequence, "download", "data") + } + recorder := &goalStateStatusRecorder{} + if reportingFailure { + recorder.terminalError = errors.New("notification failed") + } + notifier := &observer.Notifier{} + notifier.Register(recorder) + exitCode, err := HandleImmediateGoalState(ctx, constants.RunCommandTestExtensionName, setting, notifier) + require.Equal(t, 12, exitCode) + require.ErrorIs(t, err, scriptErr) + var executionErr *ImmediateCommandExecutionError + require.ErrorAs(t, err, &executionErr) + require.Equal(t, !reportingFailure, executionErr.Reported) + require.ErrorIs(t, executionErr.InstanceView.ClarificationError, scriptErr) + require.Equal(t, "script output", executionErr.InstanceView.Output) + require.Equal(t, "script stderr", executionErr.InstanceView.Error) + require.Equal(t, 12, executionErr.InstanceView.ExitCode) + require.Equal(t, types.ExecutionState(types.Failed), executionErr.InstanceView.ExecutionState) + require.NotEmpty(t, executionErr.InstanceView.StartTime) + require.NotEmpty(t, executionErr.InstanceView.EndTime) + expectedStatus := types.StatusSuccess + if deploymentFailure { + expectedStatus = types.StatusError + } + require.Equal(t, expectedStatus, executionErr.Status) + require.Len(t, recorder.items, 2, "only one transitioning and one terminal result should be produced") + terminal := recorder.items[1].Status + require.Equal(t, expectedStatus, terminal.Status) + requireGoalStateStatusCode(t, terminal, errorutil.CommandExecution_failureExitCode) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.CommandExecution_failureExitCode, code) + require.Positive(t, code, "script failure is a customer error") + payload, marshalErr := executionErr.InstanceView.Marshal() + require.NoError(t, marshalErr) + require.JSONEq(t, string(payload), terminal.FormattedMessage.Message) + }) + } + } +} + +func TestImmediateSuccessfulCommandReturnsSuccess(t *testing.T) { + oldExecute := executeImmediateCommand + t.Cleanup(func() { executeImmediateCommand = oldExecute }) + ctx := log.NewContext(log.NewNopLogger()) + name, state, seq := "script", "enabled", 1 + setting := settings.SettingsCommon{ExtensionName: &name, ExtensionState: &state, SeqNo: &seq} + recorder := &goalStateStatusRecorder{} + notifier := &observer.Notifier{} + notifier.Register(recorder) + executeImmediateCommand = func(cmd types.Cmd, _ handlersettings.HandlerSettingsFile, _ string, _ int) error { + view := types.RunCommandInstanceView{ExecutionState: types.Succeeded, ExecutionMessage: "Execution completed", ExitCode: 0, Output: "stdout"} + message, err := view.Marshal() + if err != nil { + return err + } + return cmd.Functions.ReportStatus(ctx, types.HandlerEnvironment{}, types.NewRCMetadata(name, seq, "download", "data"), types.StatusSuccess, cmd, string(message)) + } + exitCode, err := HandleImmediateGoalState(ctx, constants.RunCommandTestExtensionName, setting, notifier) + require.NoError(t, err) + require.Zero(t, exitCode) + require.Len(t, recorder.items, 1) + require.Equal(t, types.StatusSuccess, recorder.items[0].Status.Status) + require.Empty(t, recorder.items[0].Status.Substatuses) +} + +type goalStateStatusRecorder struct { + items []types.StatusItem + terminalError error +} + +func (r *goalStateStatusRecorder) OnDemandNotify() error { return nil } + +func (r *goalStateStatusRecorder) OnNotify(event types.StatusEventArgs) error { + payload, err := json.Marshal(event.TopLevelStatus) + if err != nil { + return err + } + var item types.StatusItem + if err := json.Unmarshal(payload, &item); err != nil { + return err + } + r.items = append(r.items, item) + if item.Status.Status != types.StatusTransitioning { + return r.terminalError + } + return nil +} + +func requireGoalStateStatusCode(t *testing.T, reported types.Status, want errorutil.ErrorClarificationCode) { + t.Helper() + require.Len(t, reported.Substatuses, 1) + require.Equal(t, "ErrorClarification", reported.Substatuses[0].Name) + require.Equal(t, string(types.StatusError), reported.Substatuses[0].Status) + require.Equal(t, want, reported.Substatuses[0].Code) +} diff --git a/internal/goalstate/goalstate.go b/internal/goalstate/goalstate.go index 8ff2fc7..66aa904 100644 --- a/internal/goalstate/goalstate.go +++ b/internal/goalstate/goalstate.go @@ -1,6 +1,7 @@ package goalstate import ( + "encoding/json" "fmt" "time" @@ -13,6 +14,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/settings" "github.com/Azure/run-command-handler-linux/internal/status" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -28,6 +30,21 @@ var statusToCommandMap = map[string]string{ "disabled": disableCommand, } +var goalStateTimeout = time.Minute * time.Duration(maxExecutionTimeInMinutes) +var executeImmediateCommand = commandProcessor.ProcessImmediateHandlerCommand +var buildStatusItem = status.GetSingleStatusItem + +// ImmediateCommandExecutionError retains the terminal execution result for fallback reporting. +type ImmediateCommandExecutionError struct { + InstanceView types.RunCommandInstanceView + Status types.StatusType + Reported bool + Err error +} + +func (e *ImmediateCommandExecutionError) Error() string { return e.Err.Error() } +func (e *ImmediateCommandExecutionError) Unwrap() error { return e.Err } + // HandleImmediateGoalState handles the immediate goal state by executing the command and waiting for it to finish. // ctx: The logger context. // handlerName: The handler identity from the immediate goal state. @@ -35,19 +52,23 @@ var statusToCommandMap = map[string]string{ // notifier: The notifier to send the status to the HGAP. This is a notifier that must have been initialized and a status observer must have been added to it. // Returns the exit code and an error if there was an issue executing the goal state. func HandleImmediateGoalState(ctx *log.Context, handlerName string, setting settings.SettingsCommon, notifier *observer.Notifier) (int, error) { - done := make(chan bool) - err := make(chan error) + done := make(chan bool, 1) + err := make(chan error, 1) go startAsync(ctx, handlerName, setting, notifier, done, err) select { case e := <-err: ctx.Log("error", fmt.Sprintf("error when trying to execute goal state: %v", e)) + var executionErr *ImmediateCommandExecutionError + if errors.As(e, &executionErr) { + return executionErr.InstanceView.ExitCode, errors.Wrap(e, "error when trying to execute goal state") + } return constants.ExitCode_ImmediateTaskFailed, errors.Wrapf(e, "error when trying to execute goal state") case <-done: ctx.Log("message", "goal state successfully finished") return constants.ExitCode_Okay, nil - case <-time.After(time.Minute * time.Duration(maxExecutionTimeInMinutes)): + case <-time.After(goalStateTimeout): ctx.Log("message", "timeout when trying to execute goal state") - return constants.ExitCode_ImmediateTaskTimeout, errors.New("timeout when trying to execute goal state") + return constants.ExitCode_ImmediateTaskTimeout, errorutil.EnsureClarification(errorutil.CommandExecution_immediateGoalStateTimedOut, errors.New("timeout when trying to execute goal state")) } } @@ -58,14 +79,17 @@ func HandleImmediateGoalState(ctx *log.Context, handlerName string, setting sett // The instance view is normally added as a message to the status item. func ReportFinalStatusForImmediateGoalState(ctx *log.Context, notifier *observer.Notifier, goalStateKey types.GoalStateKey, statusType types.StatusType, instanceview *types.RunCommandInstanceView) error { if notifier == nil { - return errors.New("notifier is nil. Cannot report status to HGAP") + return errorutil.EnsureClarification(errorutil.Internal_nilGoalStateNotifier, errors.New("notifier is nil. Cannot report status to HGAP")) + } + if instanceview == nil { + return errorutil.EnsureClarification(errorutil.Internal_nilGoalStateInstanceView, errors.New("instance view is nil. Cannot report status to HGAP")) } extensionState := goalStateKey.RuntimeSettingsState cmdToReport := statusToCommandMap[extensionState] cmd, ok := commands.Cmds[cmdToReport] if !ok { - return errors.New(fmt.Sprintf("missing command %v", extensionState)) + return errorutil.EnsureClarification(errorutil.Internal_unknownGoalStateCommand, errors.New(fmt.Sprintf("missing command %v", extensionState))) } if !cmd.ShouldReportStatus { @@ -75,12 +99,12 @@ func ReportFinalStatusForImmediateGoalState(ctx *log.Context, notifier *observer msg, err := instanceview.Marshal() if err != nil { - return errors.Wrapf(err, "failed to marshal instance view") + return errorutil.EnsureClarification(errorutil.Internal_failedToMarshalGoalStateInstanceView, errors.Wrapf(err, "failed to marshal instance view")) } - statusItem, err := status.GetSingleStatusItem(ctx, statusType, cmd, string(msg), goalStateKey.ExtensionName) + statusItem, err := buildStatusItem(ctx, statusType, cmd, string(msg), goalStateKey.ExtensionName, instanceview.ClarificationError) if err != nil { - return errors.Wrap(err, "failed to get status item") + return errorutil.EnsureClarification(errorutil.Internal_failedToCreateGoalStateStatus, errors.Wrap(err, "failed to get status item")) } ctx.Log("message", "reporting status of skipped goal state by notifying the observer to then send to HGAP") @@ -95,7 +119,11 @@ func ReportFinalStatusForImmediateGoalState(ctx *log.Context, notifier *observer // The function to cleanup the command is overwritten to cleanup the command immediately after it has been executed. func startAsync(ctx *log.Context, handlerName string, setting settings.SettingsCommon, notifier *observer.Notifier, done chan bool, err chan error) { if notifier == nil { - err <- errors.New("notifier is nil. Cannot report status to HGAP") + err <- errorutil.EnsureClarification(errorutil.Internal_nilImmediateCommandNotifier, errors.New("notifier is nil. Cannot report status to HGAP")) + return + } + if setting.ExtensionState == nil || setting.ExtensionName == nil || setting.SeqNo == nil { + err <- errorutil.EnsureClarification(errorutil.Internal_incompleteImmediateCommandGoalState, errors.New("goal state is missing extension state, extension name or sequence number")) return } @@ -105,10 +133,12 @@ func startAsync(ctx *log.Context, handlerName string, setting settings.SettingsC cmdToReport := statusToCommandMap[extensionState] cmd, ok := commands.Cmds[cmdToReport] if !ok { - err <- errors.New(fmt.Sprintf("missing command %v", extensionState)) + err <- errorutil.EnsureClarification(errorutil.Internal_unknownImmediateCommand, errors.New(fmt.Sprintf("missing command %v", extensionState))) return } + var terminalResult *ImmediateCommandExecutionError + var terminalReportErr error // Overwrite function to report status to HGAP. This function prepares the status to be sent to the HGAP and then calls the notifier to send it. cmd.Functions.ReportStatus = func(ctx *log.Context, _ types.HandlerEnvironment, metadata types.RCMetadata, statusType types.StatusType, c types.Cmd, msg string) error { if !c.ShouldReportStatus { @@ -116,13 +146,27 @@ func startAsync(ctx *log.Context, handlerName string, setting settings.SettingsC return nil } - statusItem, err := status.GetSingleStatusItem(ctx, statusType, c, msg, metadata.ExtName) + if statusType != types.StatusTransitioning { + var view types.RunCommandInstanceView + if err := json.Unmarshal([]byte(msg), &view); err != nil { + terminalReportErr = errorutil.EnsureClarification(errorutil.Internal_failedToParseImmediateInstanceView, errors.Wrap(err, "failed to unmarshal immediate command instance view")) + return terminalReportErr + } + view.ClarificationError = metadata.ClarificationError + terminalResult = &ImmediateCommandExecutionError{InstanceView: view, Status: statusType} + } + + statusItem, err := buildStatusItem(ctx, statusType, c, msg, metadata.ExtName, metadata.ClarificationError) if err != nil { - return errors.Wrap(err, "failed to get status item") + reportErr := errorutil.EnsureClarification(errorutil.Internal_failedToCreateImmediateCommandStatus, errors.Wrap(err, "failed to get status item")) + if terminalResult != nil { + terminalReportErr = reportErr + } + return reportErr } ctx.Log("message", fmt.Sprintf("reporting status by notifying the observer to then send to HGAP for extension name %v and seq number %v", metadata.ExtName, metadata.SeqNum)) - return notifier.Notify(types.StatusEventArgs{ + err = notifier.Notify(types.StatusEventArgs{ StatusKey: types.GoalStateKey{ HandlerName: handlerName, ExtensionName: metadata.ExtName, @@ -131,6 +175,11 @@ func startAsync(ctx *log.Context, handlerName string, setting settings.SettingsC }, TopLevelStatus: statusItem, }) + if terminalResult != nil { + terminalResult.Reported = err == nil + terminalReportErr = err + } + return err } // Overwrite function to cleanup the command. This function is called after the command has been executed. @@ -140,6 +189,32 @@ func startAsync(ctx *log.Context, handlerName string, setting settings.SettingsC var runtimeSettings []handlersettings.RunTimeSettingsFile hs.RuntimeSettings = append(runtimeSettings, handlersettings.RunTimeSettingsFile{HandlerSettings: setting}) ctx.Log("message", "executing immediate goal state") - commandProcessor.ProcessImmediateHandlerCommand(cmd, hs, *setting.ExtensionName, *setting.SeqNo) + commandErr := executeImmediateCommand(cmd, hs, *setting.ExtensionName, *setting.SeqNo) + if errors.Is(commandErr, types.ErrAlreadyProcessed) { + done <- true + return + } + if terminalResult != nil { + cause := terminalResult.InstanceView.ClarificationError + if cause == nil { + cause = commandErr + } + if cause == nil { + cause = terminalReportErr + } + if cause != nil { + terminalResult.Err = cause + err <- terminalResult + return + } + } + if commandErr != nil { + err <- errorutil.EnsureClarification(errorutil.Internal_failedToProcessImmediateCommand, errors.Wrap(commandErr, "failed to process immediate handler command")) + return + } + if terminalReportErr != nil { + err <- terminalReportErr + return + } done <- true } diff --git a/internal/goalstate/goalstate_test.go b/internal/goalstate/goalstate_test.go index 27d07a4..8570700 100644 --- a/internal/goalstate/goalstate_test.go +++ b/internal/goalstate/goalstate_test.go @@ -2,6 +2,7 @@ package goalstate import ( "encoding/json" + "errors" "os" "path/filepath" "testing" @@ -14,6 +15,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/settings" "github.com/Azure/run-command-handler-linux/internal/status" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/statusreporter" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -105,7 +107,7 @@ func Test_handleSkippedImmediateGoalState_NotifyObserver(t *testing.T) { err = os.Mkdir(tempDir+"/status", 0755) require.NoError(t, err, "Failed to create status directory") - os.Setenv(constants.ExtensionPathEnvName, tempDir) + t.Setenv(constants.ExtensionPathEnvName, tempDir) ctx.Log("msg", "Creating status observer") obs := &status.StatusObserver{} @@ -125,13 +127,14 @@ func Test_handleSkippedImmediateGoalState_NotifyObserver(t *testing.T) { errorMsg := "Test error message" instView := types.RunCommandInstanceView{ - ExecutionState: types.Failed, - ExecutionMessage: "Execution failed", - ExitCode: constants.ExitCode_SkippedImmediateGoalState, - Output: "", - Error: errorMsg, - StartTime: time.Now().UTC().Format(time.RFC3339), - EndTime: time.Now().UTC().Format(time.RFC3339), + ExecutionState: types.Failed, + ExecutionMessage: "Execution failed", + ExitCode: constants.ExitCode_SkippedImmediateGoalState, + Output: "", + Error: errorMsg, + ClarificationError: errorutil.EnsureClarification(errorutil.CommandExecution_concurrentGoalStateLimitExceeded, errors.New(errorMsg)), + StartTime: time.Now().UTC().Format(time.RFC3339), + EndTime: time.Now().UTC().Format(time.RFC3339), } err = ReportFinalStatusForImmediateGoalState(ctx, notifier, goalStateKey, types.StatusSkipped, &instView) @@ -153,6 +156,7 @@ func Test_handleSkippedImmediateGoalState_NotifyObserver(t *testing.T) { require.Equal(t, goalStateKey.HandlerName, latestStatus.HandlerName) require.Equal(t, "Enable", latestStatus.Status.Operation, "Operation should be equal") require.Equal(t, types.StatusSkipped, latestStatus.Status.Status, "Status should be equal") + requireGoalStateStatusCode(t, latestStatus.Status, errorutil.CommandExecution_concurrentGoalStateLimitExceeded) json, err := json.Marshal(instView) require.Nil(t, err, "Marshal should not return an error") diff --git a/internal/goalstate/goalstatefromvmsettings.go b/internal/goalstate/goalstatefromvmsettings.go index 8c30147..8ec9b2f 100644 --- a/internal/goalstate/goalstatefromvmsettings.go +++ b/internal/goalstate/goalstatefromvmsettings.go @@ -6,18 +6,19 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/hostgacommunicator" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) func GetImmediateRunCommandGoalStates(ctx *log.Context, communicator hostgacommunicator.IHostGACommunicator, lastProcessedETag string) ([]hostgacommunicator.ImmediateExtensionGoalState, string, error) { if communicator == nil { - return nil, lastProcessedETag, errors.New("communicator cannot be nil") + return nil, lastProcessedETag, errorutil.EnsureClarification(errorutil.Internal_nilGoalStateCommunicator, errors.New("communicator cannot be nil")) } responseData, err := communicator.GetImmediateVMSettings(ctx, lastProcessedETag) if err != nil { - return nil, lastProcessedETag, errors.Wrapf(err, "failed to retrieve immediate VMSettings") + return nil, lastProcessedETag, errorutil.EnsureClarification(errorutil.HostGA_failedToRetrieveVmSettings, errors.Wrapf(err, "failed to retrieve immediate VMSettings")) } if responseData != nil && responseData.Modified { diff --git a/internal/goalstate/goalstatefromvmsettings_test.go b/internal/goalstate/goalstatefromvmsettings_test.go index bc0b956..6d861a7 100644 --- a/internal/goalstate/goalstatefromvmsettings_test.go +++ b/internal/goalstate/goalstatefromvmsettings_test.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/goalstate" "github.com/Azure/run-command-handler-linux/internal/hostgacommunicator" "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -111,6 +112,9 @@ func Test_GetFilteredImmediateVMSettingsFailedToRetrieve(t *testing.T) { badCommunicator := new(BadCommunicator) _, _, err := goalstate.GetImmediateRunCommandGoalStates(ctx, badCommunicator, "") require.ErrorContains(t, err, "failed to retrieve immediate VMSettings") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.HostGA_failedToRetrieveVmSettings, code) require.ErrorContains(t, err, "http expected failure") } diff --git a/internal/handlersettings/errorclarification_test.go b/internal/handlersettings/errorclarification_test.go new file mode 100644 index 0000000..c59df75 --- /dev/null +++ b/internal/handlersettings/errorclarification_test.go @@ -0,0 +1,240 @@ +package handlersettings + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/Azure/azure-extension-platform/vmextension" + "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func requireSettingsCode(t *testing.T, err error, want int) { + t.Helper() + require.Error(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "missing clarification: %v", err) + require.Equal(t, want, code) +} + +func mockSettingsMachine(t *testing.T) { + t.Helper() + read, abs, decrypt := readSettingsFile, settingsAbsPath, decryptSettings + t.Cleanup(func() { + readSettingsFile, settingsAbsPath, decryptSettings = read, abs, decrypt + }) + settingsAbsPath = func(string) (string, error) { return filepath.Join("extension", "bin", "handler"), nil } + readSettingsFile = func(string) ([]byte, error) { return nil, os.ErrNotExist } + decryptSettings = func(string, string, string, []byte) ([]byte, string, error) { + t.Fatal("unexpected external process") + return nil, "", nil + } +} + +func TestHandlerEnvironmentClarifications(t *testing.T) { + t.Run("absolute path 2400", func(t *testing.T) { + mockSettingsMachine(t) + cause := errors.New("working directory unavailable") + settingsAbsPath = func(string) (string, error) { return "", cause } + _, err := GetHandlerEnv() + requireSettingsCode(t, err, errorutil.Os_FailedToGetWorkingDirectory) + require.ErrorIs(t, err, cause) + }) + t.Run("read 2401", func(t *testing.T) { + mockSettingsMachine(t) + readSettingsFile = func(string) ([]byte, error) { return nil, os.ErrPermission } + _, err := GetHandlerEnv() + requireSettingsCode(t, err, errorutil.Os_FailedToExamineHandlerEnvironment) + require.ErrorIs(t, err, os.ErrPermission) + }) + t.Run("missing 2402", func(t *testing.T) { + mockSettingsMachine(t) + _, err := GetHandlerEnv() + requireSettingsCode(t, err, errorutil.Os_CouldNotFindHandlerEnvironment) + }) + t.Run("json 2403", func(t *testing.T) { + _, err := ParseHandlerEnv([]byte("{")) + requireSettingsCode(t, err, errorutil.Os_InvalidHandlerEnvironmentJson) + var syntax *json.SyntaxError + require.ErrorAs(t, err, &syntax) + }) + t.Run("count 2404", func(t *testing.T) { + _, err := ParseHandlerEnv([]byte("[]")) + requireSettingsCode(t, err, errorutil.Os_UnexpectedHandlerEnvironmentCount) + }) + t.Run("parent path and success", func(t *testing.T) { + mockSettingsMachine(t) + calls := 0 + readSettingsFile = func(string) ([]byte, error) { + calls++ + if calls == 1 { + return nil, os.ErrNotExist + } + return []byte("[{}]"), nil + } + _, err := GetHandlerEnv() + require.NoError(t, err) + require.Equal(t, 2, calls) + }) +} + +func TestSettingsJSONClarifications(t *testing.T) { + t.Run("marshal -2410", func(t *testing.T) { + err := unmarshalSettings(make(chan int), new(interface{})) + requireSettingsCode(t, err, errorutil.Internal_failedToMarshalHandlerSettings) + var unsupported *json.UnsupportedTypeError + require.ErrorAs(t, err, &unsupported) + }) + t.Run("unmarshal 2411", func(t *testing.T) { + err := UnmarshalHandlerSettings(map[string]interface{}{"timeoutInSeconds": "bad"}, nil, new(PublicSettings), new(ProtectedSettings)) + requireSettingsCode(t, err, errorutil.CustomerInput_invalidHandlerSettingsJson) + var mismatch *json.UnmarshalTypeError + require.ErrorAs(t, err, &mismatch) + err = UnmarshalHandlerSettings(nil, map[string]interface{}{"runAsPassword": 12}, new(PublicSettings), new(ProtectedSettings)) + requireSettingsCode(t, err, errorutil.CustomerInput_invalidHandlerSettingsJson) + }) + for _, tc := range []struct { + name string + data string + readErr error + code int + }{ + {"read 2412", "", os.ErrPermission, errorutil.Os_FailedToReadHandlerSettings}, + {"parse config", "{", nil, errorutil.Internal_badConfig}, + {"count 2414", "{}", nil, errorutil.CustomerInput_unexpectedRuntimeSettingsCount}, + } { + t.Run(tc.name, func(t *testing.T) { + mockSettingsMachine(t) + readSettingsFile = func(string) ([]byte, error) { return []byte(tc.data), tc.readErr } + _, _, err := ReadSettings("config.settings") + requireSettingsCode(t, err, tc.code) + if tc.readErr != nil { + require.ErrorIs(t, err, tc.readErr) + } + }) + } + t.Run("source 2420 through validation", func(t *testing.T) { + mockSettingsMachine(t) + readSettingsFile = func(string) ([]byte, error) { + return []byte(`{"runtimeSettings":[{"handlerSettings":{}}]}`), nil + } + _, err := ParseAndValidateSettings(log.NewContext(log.NewNopLogger()), "config.settings") + requireSettingsCode(t, err, errorutil.CustomerInput_commandToExecuteAndScriptNotSpecified) + require.ErrorIs(t, err, errSourceNotSpecified) + requireSettingsCode(t, HandlerSettings{}.validate(), errorutil.CustomerInput_commandToExecuteAndScriptNotSpecified) + require.NoError(t, (HandlerSettings{PublicSettings: PublicSettings{InstallAsService: true}}).validate()) + }) +} + +func TestProtectedSettingsClarifications(t *testing.T) { + encrypted := settings.SettingsCommon{ + ProtectedSettingsBase64: base64.StdEncoding.EncodeToString([]byte("encrypted")), + SettingsCertThumbprint: "thumb", + } + t.Run("thumbprint 2415", func(t *testing.T) { + hs := encrypted + hs.SettingsCertThumbprint = "" + requireSettingsCode(t, unmarshalProtectedSettings("config", hs, new(interface{})), errorutil.CustomerInput_missingCertificateThumbprint) + }) + t.Run("base64 2416", func(t *testing.T) { + hs := encrypted + hs.ProtectedSettingsBase64 = "!" + err := unmarshalProtectedSettings("config", hs, new(interface{})) + requireSettingsCode(t, err, errorutil.CustomerInput_invalidProtectedSettingsBase64) + var cause base64.CorruptInputError + require.ErrorAs(t, err, &cause) + }) + t.Run("cms 2417 and smime 2418 preserve both causes", func(t *testing.T) { + mockSettingsMachine(t) + cmsCause, smimeCause := errors.New("cms unavailable"), errors.New("smime unavailable") + methods := []string{} + decryptSettings = func(method, crt, prv string, input []byte) ([]byte, string, error) { + methods = append(methods, method) + require.Equal(t, filepath.Join("config", "..", "..", "thumb.crt"), crt) + require.Equal(t, filepath.Join("config", "..", "..", "thumb.prv"), prv) + require.Equal(t, []byte("encrypted"), input) + if method == "cms" { + return nil, "cms stderr", cmsCause + } + return nil, "smime stderr", smimeCause + } + err := unmarshalProtectedSettings("config", encrypted, new(interface{})) + requireSettingsCode(t, err, errorutil.Os_FailedToDecryptSettingsWithSmime) + require.ErrorIs(t, err, cmsCause) + require.ErrorIs(t, err, smimeCause) + joined, ok := err.(interface{ Unwrap() []error }) + require.True(t, ok) + requireSettingsCode(t, joined.Unwrap()[1], errorutil.Os_FailedToDecryptSettingsWithCms) + require.Equal(t, []string{"cms", "smime"}, methods) + require.Contains(t, err.Error(), "cms stderr") + require.Contains(t, err.Error(), "smime stderr") + }) + t.Run("decrypted json 2419", func(t *testing.T) { + mockSettingsMachine(t) + decryptSettings = func(string, string, string, []byte) ([]byte, string, error) { + return []byte("{"), "", nil + } + err := unmarshalProtectedSettings("config", encrypted, new(interface{})) + requireSettingsCode(t, err, errorutil.CustomerInput_invalidDecryptedSettingsJson) + var syntax *json.SyntaxError + require.ErrorAs(t, err, &syntax) + }) + t.Run("fallback succeeds", func(t *testing.T) { + mockSettingsMachine(t) + decryptSettings = func(method, _, _ string, _ []byte) ([]byte, string, error) { + if method == "cms" { + return nil, "", errors.New("unsupported") + } + return []byte(`{"runAsPassword":"secret"}`), "", nil + } + var output ProtectedSettings + require.NoError(t, unmarshalProtectedSettings("config", encrypted, &output)) + require.Equal(t, "secret", output.RunAsPassword) + }) + t.Run("empty protected settings", func(t *testing.T) { + mockSettingsMachine(t) + require.NoError(t, unmarshalProtectedSettings("", settings.SettingsCommon{}, new(interface{}))) + }) +} + +func TestArtifactClarifications(t *testing.T) { + for _, tc := range []struct { + name string + protected []ProtectedArtifactSource + code int + }{ + {"count -2421", nil, errorutil.Internal_invalidArtifactSpecification}, + {"id -2422", []ProtectedArtifactSource{{ArtifactId: 2}}, errorutil.Internal_inconsistentArtifactSpecification}, + } { + t.Run(tc.name, func(t *testing.T) { + hs := HandlerSettings{ + PublicSettings: PublicSettings{Artifacts: []PublicArtifactSource{{ArtifactId: 1}}}, + ProtectedSettings: ProtectedSettings{Artifacts: tc.protected}, + } + _, err := hs.ReadArtifacts() + requireSettingsCode(t, err, tc.code) + }) + } +} + +func TestSettingsPreserveLowerClarification(t *testing.T) { + for _, lower := range []error{ + vmextension.NewErrorWithClarificationPtr(errorutil.CustomerInput_invalidSettingsJson, errors.New("lower pointer")), + vmextension.NewErrorWithClarification(errorutil.CustomerInput_invalidSettingsJson, errors.New("lower value")), + } { + t.Run(fmt.Sprint(lower), func(t *testing.T) { + mockSettingsMachine(t) + readSettingsFile = func(string) ([]byte, error) { return nil, fmt.Errorf("file boundary: %w", lower) } + _, _, err := readSettings("config.settings") + requireSettingsCode(t, err, errorutil.CustomerInput_invalidSettingsJson) + require.ErrorIs(t, err, lower) + }) + } +} diff --git a/internal/handlersettings/handlerenv.go b/internal/handlersettings/handlerenv.go index e6f6489..afca6d9 100644 --- a/internal/handlersettings/handlerenv.go +++ b/internal/handlersettings/handlerenv.go @@ -8,6 +8,12 @@ import ( "strings" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" +) + +var ( + readSettingsFile = os.ReadFile + settingsAbsPath = filepath.Abs ) // HandlerEnvFileName is the file name of the Handler Environment as placed by the @@ -20,7 +26,7 @@ const HandlerEnvFileName = "HandlerEnvironment.json" func GetHandlerEnv() (he types.HandlerEnvironment, _ error) { dir, err := scriptDir() if err != nil { - return he, fmt.Errorf("vmextension: cannot find base directory of the running process: %v", err) + return he, fmt.Errorf("vmextension: cannot find base directory of the running process: %w", err) } paths := []string{ filepath.Join(dir, HandlerEnvFileName), // this level (i.e. executable is in [EXT_NAME]/.) @@ -28,25 +34,25 @@ func GetHandlerEnv() (he types.HandlerEnvironment, _ error) { } var b []byte for _, p := range paths { - o, err := os.ReadFile(p) + o, err := readSettingsFile(p) if err != nil && !os.IsNotExist(err) { - return he, fmt.Errorf("vmextension: error examining HandlerEnvironment at '%s': %v", p, err) + return he, errorutil.EnsureClarification(errorutil.Os_FailedToExamineHandlerEnvironment, fmt.Errorf("vmextension: error examining HandlerEnvironment at '%s': %w", p, err)) } else if err == nil { b = o break } } if b == nil { - return he, fmt.Errorf("vmextension: Cannot find HandlerEnvironment at paths: %s", strings.Join(paths, ", ")) + return he, errorutil.EnsureClarification(errorutil.Os_CouldNotFindHandlerEnvironment, fmt.Errorf("vmextension: Cannot find HandlerEnvironment at paths: %s", strings.Join(paths, ", "))) } return ParseHandlerEnv(b) } // scriptDir returns the absolute path of the running process. func scriptDir() (string, error) { - p, err := filepath.Abs(os.Args[0]) + p, err := settingsAbsPath(os.Args[0]) if err != nil { - return "", err + return "", errorutil.EnsureClarification(errorutil.Os_FailedToGetWorkingDirectory, err) } return filepath.Dir(p), nil } @@ -57,10 +63,10 @@ func ParseHandlerEnv(b []byte) (he types.HandlerEnvironment, _ error) { var hf []types.HandlerEnvironment if err := json.Unmarshal(b, &hf); err != nil { - return he, fmt.Errorf("vmextension: failed to parse handler env: %v", err) + return he, errorutil.EnsureClarification(errorutil.Os_InvalidHandlerEnvironmentJson, fmt.Errorf("vmextension: failed to parse handler env: %w", err)) } if len(hf) != 1 { - return he, fmt.Errorf("vmextension: expected 1 config in parsed HandlerEnvironment, found: %v", len(hf)) + return he, errorutil.EnsureClarification(errorutil.Os_UnexpectedHandlerEnvironmentCount, fmt.Errorf("vmextension: expected 1 config in parsed HandlerEnvironment, found: %v", len(hf))) } return hf[0], nil } diff --git a/internal/handlersettings/handlersettings.go b/internal/handlersettings/handlersettings.go index dfecd8f..4bf47aa 100644 --- a/internal/handlersettings/handlersettings.go +++ b/internal/handlersettings/handlersettings.go @@ -1,12 +1,13 @@ package handlersettings import ( + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) var ( - errSourceNotSpecified = errors.New("Either 'source.script' or 'source.scriptUri' has to be specified") + errSourceNotSpecified = errorutil.EnsureClarification(errorutil.CustomerInput_commandToExecuteAndScriptNotSpecified, errors.New("Either 'source.script' or 'source.scriptUri' has to be specified")) ) // parseAndValidateSettings reads configuration from configFolder, decrypts it, diff --git a/internal/handlersettings/handlersettingscommon.go b/internal/handlersettings/handlersettingscommon.go index 50f6e51..8d1ea00 100644 --- a/internal/handlersettings/handlersettingscommon.go +++ b/internal/handlersettings/handlersettingscommon.go @@ -5,11 +5,11 @@ import ( "encoding/base64" "encoding/json" "fmt" - "os" "os/exec" "path/filepath" "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) @@ -21,23 +21,29 @@ type RunTimeSettingsFile struct { HandlerSettings settings.SettingsCommon `json:"handlerSettings"` } +var decryptSettings = func(method, crt, prv string, input []byte) ([]byte, string, error) { + cmd := exec.Command("openssl", method, "-inform", "DER", "-decrypt", "-recip", crt, "-inkey", prv) + var stdout, stderr bytes.Buffer + cmd.Stdin = bytes.NewReader(input) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.Bytes(), stderr.String(), err +} + // ReadSettings locates the .settings file and returns public settings // JSON, and protected settings JSON (by decrypting it with the keys in // configFolder). func ReadSettings(configFilePath string) (public, protected map[string]interface{}, _ error) { - // cf, err := settingsPath(configFolder) - // if err != nil { - // return nil, nil, fmt.Errorf("canot locate settings file: %v", err) - // } hs, err := parseHandlerSettingsFile(configFilePath) if err != nil { - return nil, nil, fmt.Errorf("error parsing settings file: %v", err) + return nil, nil, fmt.Errorf("error parsing settings file: %w", err) } public = hs.PublicSettings configFolder := filepath.Dir(configFilePath) if err := unmarshalProtectedSettings(configFolder, hs, &protected); err != nil { - return nil, nil, fmt.Errorf("failed to parse protected settings: %v", err) + return nil, nil, fmt.Errorf("failed to parse protected settings: %w", err) } return public, protected, nil } @@ -47,10 +53,10 @@ func ReadSettings(configFilePath string) (public, protected map[string]interface // (of struct types that contain structured fields for settings). func UnmarshalHandlerSettings(publicSettings, protectedSettings map[string]interface{}, publicV, protectedV interface{}) error { if err := unmarshalSettings(publicSettings, &publicV); err != nil { - return fmt.Errorf("failed to unmarshal public settings: %v", err) + return fmt.Errorf("failed to unmarshal public settings: %w", err) } if err := unmarshalSettings(protectedSettings, &protectedV); err != nil { - return fmt.Errorf("failed to unmarshal protected settings: %v", err) + return fmt.Errorf("failed to unmarshal protected settings: %w", err) } return nil } @@ -60,10 +66,10 @@ func UnmarshalHandlerSettings(publicSettings, protectedSettings map[string]inter func unmarshalSettings(in interface{}, v interface{}) error { s, err := json.Marshal(in) if err != nil { - return fmt.Errorf("failed to marshal into json: %v", err) + return errorutil.EnsureClarification(errorutil.Internal_failedToMarshalHandlerSettings, fmt.Errorf("failed to marshal into json: %w", err)) } if err := json.Unmarshal(s, &v); err != nil { - return fmt.Errorf("failed to unmarshal json: %v", err) + return errorutil.EnsureClarification(errorutil.CustomerInput_invalidHandlerSettingsJson, fmt.Errorf("failed to unmarshal json: %w", err)) } return nil } @@ -71,9 +77,9 @@ func unmarshalSettings(in interface{}, v interface{}) error { // parseHandlerSettings parses a handler settings file (e.g. 0.settings) and // returns it as a structured object. func parseHandlerSettingsFile(path string) (h settings.SettingsCommon, _ error) { - b, err := os.ReadFile(path) + b, err := readSettingsFile(path) if err != nil { - return h, fmt.Errorf("error reading %s: %v", path, err) + return h, errorutil.EnsureClarification(errorutil.Os_FailedToReadHandlerSettings, fmt.Errorf("error reading %s: %w", path, err)) } if len(b) == 0 { // if no config is specified, we get an empty file return h, nil @@ -81,10 +87,10 @@ func parseHandlerSettingsFile(path string) (h settings.SettingsCommon, _ error) var f HandlerSettingsFile if err := json.Unmarshal(b, &f); err != nil { - return h, fmt.Errorf("error parsing json: %v", err) + return h, errorutil.EnsureClarification(errorutil.Internal_badConfig, fmt.Errorf("error parsing json: %w", err)) } if len(f.RuntimeSettings) != 1 { - return h, fmt.Errorf("wrong runtimeSettings count. expected:1, got:%d", len(f.RuntimeSettings)) + return h, errorutil.EnsureClarification(errorutil.CustomerInput_unexpectedRuntimeSettingsCount, fmt.Errorf("wrong runtimeSettings count. expected:1, got:%d", len(f.RuntimeSettings))) } return f.RuntimeSettings[0].HandlerSettings, nil } @@ -97,47 +103,32 @@ func unmarshalProtectedSettings(configFolder string, hs settings.SettingsCommon, return nil } if hs.SettingsCertThumbprint == "" { - return errors.New("HandlerSettings has protected settings but no cert thumbprint") + return errorutil.EnsureClarification(errorutil.CustomerInput_missingCertificateThumbprint, errors.New("HandlerSettings has protected settings but no cert thumbprint")) } decoded, err := base64.StdEncoding.DecodeString(hs.ProtectedSettingsBase64) if err != nil { - return fmt.Errorf("failed to decode base64: %v", err) + return errorutil.EnsureClarification(errorutil.CustomerInput_invalidProtectedSettingsBase64, fmt.Errorf("failed to decode base64: %w", err)) } // go two levels up where certs are placed (/var/lib/waagent) crt := filepath.Join(configFolder, "..", "..", fmt.Sprintf("%s.crt", hs.SettingsCertThumbprint)) prv := filepath.Join(configFolder, "..", "..", fmt.Sprintf("%s.prv", hs.SettingsCertThumbprint)) - // we use os/exec instead of azure-docker-extension/pkg/executil here as - // other extension handlers depend on this package for parsing handler - // settings. - - //using cms command to support for FIPS 140-3 - cmd := exec.Command("openssl", "cms", "-inform", "DER", "-decrypt", "-recip", crt, "-inkey", prv) - var bOut, bErr bytes.Buffer - var errMsg error - cmd.Stdin = bytes.NewReader(decoded) - cmd.Stdout = &bOut - cmd.Stderr = &bErr - - //back up smime command in case cms fails - if err := cmd.Run(); err != nil { - errMsg = fmt.Errorf("decrypting protected settings with cms command failed: error=%v stderr=%s \n now decrypting with smime command", err, bErr.String()) - cmd = exec.Command("openssl", "smime", "-inform", "DER", "-decrypt", "-recip", crt, "-inkey", prv) - cmd.Stdin = bytes.NewReader(decoded) - bOut.Reset() - bErr.Reset() - cmd.Stdout = &bOut - cmd.Stderr = &bErr - if err := cmd.Run(); err != nil { - return errors.Wrapf(errMsg, "decrypting protected settings with smime command failed: error=%v stderr=%s", err, bErr.String()) + // Prefer cms for FIPS 140-3, falling back to smime on older installations. + output, stderr, err := decryptSettings("cms", crt, prv, decoded) + if err != nil { + cmsErr := errorutil.EnsureClarification(errorutil.Os_FailedToDecryptSettingsWithCms, fmt.Errorf("decrypting protected settings with cms command failed: error=%w stderr=%s", err, stderr)) + output, stderr, err = decryptSettings("smime", crt, prv, decoded) + if err != nil { + smimeErr := errorutil.EnsureClarification(errorutil.Os_FailedToDecryptSettingsWithSmime, fmt.Errorf("decrypting protected settings with smime command failed: error=%w stderr=%s", err, stderr)) + return fmt.Errorf("%w; previous attempt: %w", smimeErr, cmsErr) } } // decrypted: json object for protected settings - if err := json.Unmarshal(bOut.Bytes(), &v); err != nil { - return fmt.Errorf("failed to unmarshal decrypted settings json: %v", err) + if err := json.Unmarshal(output, &v); err != nil { + return errorutil.EnsureClarification(errorutil.CustomerInput_invalidDecryptedSettingsJson, fmt.Errorf("failed to unmarshal decrypted settings json: %w", err)) } return nil } diff --git a/internal/handlersettings/types.go b/internal/handlersettings/types.go index 9eb1a74..18cb6b6 100644 --- a/internal/handlersettings/types.go +++ b/internal/handlersettings/types.go @@ -1,6 +1,7 @@ package handlersettings import ( + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) @@ -60,7 +61,7 @@ func (s HandlerSettings) ReadArtifacts() ([]UnifiedArtifact, error) { } if len(s.ProtectedSettings.Artifacts) != len(s.PublicSettings.Artifacts) { - return nil, errors.New(("RunCommand artifact download failed. Reason: Invalid artifact specification. This is a product bug.")) + return nil, errorutil.EnsureClarification(errorutil.Internal_invalidArtifactSpecification, errors.New("RunCommand artifact download failed. Reason: Invalid artifact specification. This is a product bug.")) } artifacts := make([]UnifiedArtifact, len(s.PublicSettings.Artifacts)) @@ -84,7 +85,7 @@ func (s HandlerSettings) ReadArtifacts() ([]UnifiedArtifact, error) { } if !found { - return nil, errors.New(("RunCommand artifact download failed. Reason: Invalid artifact specification. This is a product bug.")) + return nil, errorutil.EnsureClarification(errorutil.Internal_inconsistentArtifactSpecification, errors.New("RunCommand artifact download failed. Reason: Invalid artifact specification. This is a product bug.")) } } diff --git a/internal/hostgacommunicator/clarification_test.go b/internal/hostgacommunicator/clarification_test.go new file mode 100644 index 0000000..e01e89b --- /dev/null +++ b/internal/hostgacommunicator/clarification_test.go @@ -0,0 +1,79 @@ +package hostgacommunicator + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/Azure/run-command-handler-linux/internal/requesthelper" + "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type clarificationRequestManager struct { + manager *requesthelper.RequestManager + err error +} + +func (m clarificationRequestManager) GetVMSettingsRequestManager(*log.Context) (*requesthelper.RequestManager, error) { + return m.manager, m.err +} + +func TestHostGAClarifications(t *testing.T) { + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("host failure") + assertCode := func(err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + } + c := NewHostGACommunicator(clarificationRequestManager{err: cause}) + _, err := c.GetImmediateVMSettings(ctx, "") + assertCode(err, errorutil.HostGA_failedToCreateRequestManager) + require.ErrorIs(t, err, cause) + for _, want := range []int{errorutil.HostGA_failedToReadVmSettingsResponse, errorutil.HostGA_failedToParseVmSettingsResponse, errorutil.HostGA_missingVmSettingsETag} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch want { + case errorutil.HostGA_failedToReadVmSettingsResponse: + w.Header().Set("Content-Length", "100") + io.WriteString(w, "{}") + case errorutil.HostGA_failedToParseVmSettingsResponse: + io.WriteString(w, "{") + case errorutil.HostGA_missingVmSettingsETag: + io.WriteString(w, "{}") + } + })) + manager := requesthelper.GetRequestManager(requestFactory{url: server.URL}, time.Second) + c := NewHostGACommunicator(clarificationRequestManager{manager: manager}) + _, err := c.GetImmediateVMSettings(ctx, "") + server.Close() + assertCode(err, want) + } + // errorutil.Internal_invalidWireServerFallbackAddress cannot be triggered: WireServerFallbackAddress is a valid URL constant. + _, err = (requestFactory{url: "://"}).GetRequest(ctx, "") + assertCode(err, errorutil.HostGA_failedToCreateVmSettings) + oldEnv, oldStat := readHandlerEnv, statCertificate + t.Cleanup(func() { readHandlerEnv, statCertificate = oldEnv, oldStat }) + readHandlerEnv = func() (types.HandlerEnvironment, error) { return types.HandlerEnvironment{}, cause } + state := ImmediateExtensionGoalState{} + _, err = state.ValidateSignature() + assertCode(err, errorutil.HostGA_failedToParseHandlerEnvironment) + require.ErrorIs(t, err, cause) + readHandlerEnv = func() (types.HandlerEnvironment, error) { return types.HandlerEnvironment{}, nil } + state.Settings = []settings.SettingsCommon{{ProtectedSettingsBase64: "protected"}} + _, err = state.ValidateSignature() + assertCode(err, errorutil.HostGA_missingSettingsCertificateThumbprint) + state.Settings[0].SettingsCertThumbprint = "thumbprint" + statCertificate = func(string) (os.FileInfo, error) { return nil, cause } + _, err = state.ValidateSignature() + assertCode(err, errorutil.HostGA_failedToDecryptProtectedSettings) + require.ErrorIs(t, err, cause) +} diff --git a/internal/hostgacommunicator/hostgacommunicator.go b/internal/hostgacommunicator/hostgacommunicator.go index 5d51dcd..70363fa 100644 --- a/internal/hostgacommunicator/hostgacommunicator.go +++ b/internal/hostgacommunicator/hostgacommunicator.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/requesthelper" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -45,7 +46,7 @@ type IVMSettingsRequestManager interface { func (c *HostGACommunicator) GetImmediateVMSettings(ctx *log.Context, eTag string) (*ResponseData, error) { requestManager, err := c.vmRequestManager.GetVMSettingsRequestManager(ctx) if err != nil { - return nil, errors.Wrapf(err, "could not create the request manager to get immediate VMsettings") + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToCreateRequestManager, errors.Wrapf(err, "could not create the request manager to get immediate VMsettings")) } resp, err := requesthelper.WithRetries(ctx, requestManager, requesthelper.ActualSleep, eTag) @@ -61,17 +62,17 @@ func (c *HostGACommunicator) GetImmediateVMSettings(ctx *log.Context, eTag strin defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - panic(err) + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToReadVmSettingsResponse, errors.Wrap(err, "failed to read immediate VMSettings response")) } var vmSettings VMImmediateExtensionsGoalState if err := json.Unmarshal(body, &vmSettings); err != nil { - return nil, errors.Wrapf(err, "failed to parse immediate VMSettings json") + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToParseVmSettingsResponse, errors.Wrapf(err, "failed to parse immediate VMSettings json")) } newETag := resp.Header.Get(constants.ETagHeaderName) if newETag == "" { - return nil, errors.New("ETag not found in response header when retrieving immediate VMSettings") + return nil, errorutil.EnsureClarification(errorutil.HostGA_missingVmSettingsETag, errors.New("ETag not found in response header when retrieving immediate VMSettings")) } return &ResponseData{VMSettings: &vmSettings, ETag: newETag, Modified: eTag != newETag}, nil @@ -83,7 +84,7 @@ func getOperationUri(ctx *log.Context, operationName string) (string, error) { // and decide if we want to add that wire protocol address as a potential endpoint to use when provided uri, err := url.Parse(WireServerFallbackAddress) if err != nil { - return "", errors.Wrap(err, "could not parse address "+WireServerFallbackAddress) + return "", errorutil.EnsureClarification(errorutil.Internal_invalidWireServerFallbackAddress, errors.Wrap(err, "could not parse address "+WireServerFallbackAddress)) } uri.Path = operationName return uri.String(), nil diff --git a/internal/hostgacommunicator/vmsettings.go b/internal/hostgacommunicator/vmsettings.go index 8d5af31..4ad1b7d 100644 --- a/internal/hostgacommunicator/vmsettings.go +++ b/internal/hostgacommunicator/vmsettings.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/handlersettings" requesthelper "github.com/Azure/run-command-handler-linux/internal/requesthelper" "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -19,6 +20,9 @@ const ( vmSettingsOperation = "immediateGoalState" ) +var readHandlerEnv = handlersettings.GetHandlerEnv +var statCertificate = os.Stat + const ( vmSettingsRequestTimeout = 30 * time.Second ) @@ -62,7 +66,7 @@ func (u requestFactory) GetRequest(ctx *log.Context, eTag string) (*http.Request request, err := http.NewRequest("GET", u.url, nil) if err != nil { errMsg := fmt.Sprintf("failed to create request to %v", u.url) - return nil, errors.Wrap(err, errMsg) + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToCreateVmSettings, errors.Wrap(err, errMsg)) } if eTag != "" { @@ -72,9 +76,9 @@ func (u requestFactory) GetRequest(ctx *log.Context, eTag string) (*http.Request } func (goalState *ImmediateExtensionGoalState) ValidateSignature() (bool, error) { - he, err := handlersettings.GetHandlerEnv() + he, err := readHandlerEnv() if err != nil { - return false, errors.Wrap(err, "failed to parse handlerenv") + return false, errorutil.EnsureClarification(errorutil.HostGA_failedToParseHandlerEnvironment, errors.Wrap(err, "failed to parse handlerenv")) } configFolder := he.HandlerEnvironment.ConfigFolder @@ -86,16 +90,18 @@ func (goalState *ImmediateExtensionGoalState) ValidateSignature() (bool, error) } if s.SettingsCertThumbprint == "" { - return false, errors.New("HandlerSettings has protected settings but no cert thumbprint") + return false, errorutil.EnsureClarification(errorutil.HostGA_missingSettingsCertificateThumbprint, errors.New("HandlerSettings has protected settings but no cert thumbprint")) } // go two levels up where certs are placed (/var/lib/waagent) crt := filepath.Join(configFolder, "..", "..", fmt.Sprintf("%s.crt", s.SettingsCertThumbprint)) prv := filepath.Join(configFolder, "..", "..", fmt.Sprintf("%s.prv", s.SettingsCertThumbprint)) - if !fileExists(crt) || !fileExists(prv) { - message := fmt.Sprintf("Certificate %v needed by %v is missing from the goal state", s.SettingsCertThumbprint, s.ExtensionName) - return false, errors.New(message) + for _, certificate := range []string{crt, prv} { + if _, err := statCertificate(certificate); err != nil { + message := fmt.Sprintf("Certificate %v needed by %v is unavailable in the goal state", s.SettingsCertThumbprint, s.ExtensionName) + return false, errorutil.EnsureClarification(errorutil.HostGA_failedToDecryptProtectedSettings, errors.Wrap(err, message)) + } } } diff --git a/internal/hostgacommunicator/vmsettings_test.go b/internal/hostgacommunicator/vmsettings_test.go index 8a8570f..e6f166d 100644 --- a/internal/hostgacommunicator/vmsettings_test.go +++ b/internal/hostgacommunicator/vmsettings_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/Azure/run-command-handler-linux/internal/requesthelper" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetb/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -44,6 +45,9 @@ func Test_GetImmediateVMSettingsFailedToParseJson(t *testing.T) { _, err := communicator.GetImmediateVMSettings(ctx, "") require.NotNil(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.HostGA_failedToParseVmSettingsResponse, code) require.ErrorContains(t, err, "failed to parse immediate VMSettings json") } diff --git a/internal/immediatecmds/immediatecmds.go b/internal/immediatecmds/immediatecmds.go index 378eb7f..52fcdea 100644 --- a/internal/immediatecmds/immediatecmds.go +++ b/internal/immediatecmds/immediatecmds.go @@ -82,9 +82,9 @@ func Uninstall(ctx *log.Context, h types.HandlerEnvironment, extName string, seq } if isInstalled { - error := service.DeRegister(ctx, extensionEvents) - if error != nil { - errMessage := fmt.Sprintf("Failed to uninstall run command service: %v", error) + err := service.DeRegister(ctx, extensionEvents) + if err != nil { + errMessage := fmt.Sprintf("Failed to uninstall run command service: %v", err) extensionEvents.LogErrorEvent("immediatedisable", errMessage) return constants.ExitCode_UninstallInstalledServiceFailed, errors.Wrap(err, "failed to uninstall run command service") } diff --git a/internal/immediateruncommand/clarification_test.go b/internal/immediateruncommand/clarification_test.go new file mode 100644 index 0000000..11b3796 --- /dev/null +++ b/internal/immediateruncommand/clarification_test.go @@ -0,0 +1,115 @@ +package immediateruncommand + +import ( + "encoding/json" + "errors" + "fmt" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/goalstate" + "github.com/Azure/run-command-handler-linux/internal/hostgacommunicator" + "github.com/Azure/run-command-handler-linux/internal/observer" + "github.com/Azure/run-command-handler-linux/internal/settings" + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func TestImmediateRunCommandClarifications(t *testing.T) { + name, state, seq := "test", "enabled", 1 + valid := settings.SettingsCommon{ExtensionName: &name, ExtensionState: &state, SeqNo: &seq} + require.NoError(t, validateGoalStateSettings(valid)) + for _, s := range []settings.SettingsCommon{ + {}, + {ExtensionName: &name, SeqNo: &seq}, + {ExtensionName: &name, ExtensionState: &state}, + {ExtensionState: &state, SeqNo: &seq}, + } { + _, _, err := getGoalStatesToProcess([]hostgacommunicator.ImmediateExtensionGoalState{{Settings: []settings.SettingsCommon{s}}}, 1) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Internal_incompleteFetchedGoalState, code) + } + + err := newConcurrencyLimitError(5, valid) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.CommandExecution_concurrentGoalStateLimitExceeded, code) + require.Contains(t, err.Error(), "Allowed new goal state count: 5") +} + +type immediateStatusRecorder struct{ events []types.StatusEventArgs } + +func (r *immediateStatusRecorder) OnDemandNotify() error { return nil } +func (r *immediateStatusRecorder) OnNotify(event types.StatusEventArgs) error { + r.events = append(r.events, event) + return nil +} + +func TestExecutionFailureFallbackPreservesOriginalResult(t *testing.T) { + for _, statusType := range []types.StatusType{types.StatusSuccess, types.StatusError} { + for _, reported := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%t", statusType, reported), func(t *testing.T) { + recorder := &immediateStatusRecorder{} + notifier := &observer.Notifier{} + notifier.Register(recorder) + scriptErr := errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("script failed")) + view := types.RunCommandInstanceView{ + ExecutionState: types.Failed, ExecutionMessage: "Execution failed: script failed", + ExitCode: 12, Output: "stdout", Error: "stderr", + StartTime: "original start", EndTime: "original end", ClarificationError: scriptErr, + } + executionErr := &goalstate.ImmediateCommandExecutionError{ + InstanceView: view, Status: statusType, Reported: reported, Err: scriptErr, + } + key := types.GoalStateKey{ExtensionName: "script", SeqNumber: 1, RuntimeSettingsState: "enabled"} + err := reportImmediateGoalStateFailure(log.NewContext(log.NewNopLogger()), notifier, key, "fallback start", constants.ExitCode_ImmediateTaskFailed, fmt.Errorf("wrapped: %w", executionErr)) + require.NoError(t, err) + if reported { + require.Empty(t, recorder.events, "an already-reported execution must not produce another terminal result") + return + } + require.Len(t, recorder.events, 1) + require.Equal(t, key, recorder.events[0].StatusKey) + payload, err := json.Marshal(recorder.events[0].TopLevelStatus) + require.NoError(t, err) + var item types.StatusItem + require.NoError(t, json.Unmarshal(payload, &item)) + require.Equal(t, statusType, item.Status.Status) + expected, err := view.Marshal() + require.NoError(t, err) + require.JSONEq(t, string(expected), item.Status.FormattedMessage.Message) + require.Len(t, item.Status.Substatuses, 1) + require.Equal(t, "ErrorClarification", item.Status.Substatuses[0].Name) + require.Equal(t, string(types.StatusError), item.Status.Substatuses[0].Status) + require.Equal(t, errorutil.CommandExecution_failureExitCode, item.Status.Substatuses[0].Code) + }) + } + } +} + +func TestSetupFailureStillReportsFallback(t *testing.T) { + recorder := &immediateStatusRecorder{} + notifier := &observer.Notifier{} + notifier.Register(recorder) + cause := errorutil.EnsureClarification(errorutil.Internal_failedToProcessImmediateCommand, errors.New("setup failed")) + key := types.GoalStateKey{ExtensionName: "script", SeqNumber: 1, RuntimeSettingsState: "enabled"} + require.NoError(t, reportImmediateGoalStateFailure(log.NewContext(log.NewNopLogger()), notifier, key, "start", constants.ExitCode_ImmediateTaskFailed, cause)) + require.Len(t, recorder.events, 1) + payload, err := json.Marshal(recorder.events[0].TopLevelStatus) + require.NoError(t, err) + var item types.StatusItem + require.NoError(t, json.Unmarshal(payload, &item)) + require.Equal(t, types.StatusError, item.Status.Status) + var view types.RunCommandInstanceView + require.NoError(t, json.Unmarshal([]byte(item.Status.FormattedMessage.Message), &view)) + require.Equal(t, constants.ExitCode_ImmediateTaskFailed, view.ExitCode) + require.Equal(t, "start", view.StartTime) + require.Equal(t, cause.Error(), view.Error) + require.Len(t, item.Status.Substatuses, 1) + require.Equal(t, "ErrorClarification", item.Status.Substatuses[0].Name) + require.Equal(t, string(types.StatusError), item.Status.Substatuses[0].Status) + require.Equal(t, errorutil.Internal_failedToProcessImmediateCommand, item.Status.Substatuses[0].Code) +} diff --git a/internal/immediateruncommand/immediateruncommand.go b/internal/immediateruncommand/immediateruncommand.go index de5d637..e942fe6 100644 --- a/internal/immediateruncommand/immediateruncommand.go +++ b/internal/immediateruncommand/immediateruncommand.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/status" "github.com/Azure/run-command-handler-linux/internal/types" "github.com/Azure/run-command-handler-linux/pkg/counterutil" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -92,6 +93,9 @@ func processImmediateRunCommandGoalStates(ctx *log.Context, communicator hostgac var goalStateKeys []types.GoalStateKey for _, s := range goalStates { for _, setting := range s.Settings { + if err := validateGoalStateSettings(setting); err != nil { + return newEtag, err + } if setting.ExtensionState != nil { goalStateKeys = append(goalStateKeys, types.GoalStateKey{HandlerName: s.Name, ExtensionName: *setting.ExtensionName, SeqNumber: *setting.SeqNo, RuntimeSettingsState: *setting.ExtensionState}) } @@ -118,7 +122,9 @@ func processImmediateRunCommandGoalStates(ctx *log.Context, communicator hostgac notifier := &observer.Notifier{} notifier.Register(&goalStateEventObserver) - notifier.Notify(status) + if err := notifier.Notify(status); err != nil { + ctx.Log("error", "failed to notify initial goal state status", "message", err) + } startTime := time.Now().UTC().Format(time.RFC3339) exitCode, err := goalstate.HandleImmediateGoalState(ctx, state.handlerName, state.SettingsCommon, notifier) @@ -129,16 +135,9 @@ func processImmediateRunCommandGoalStates(ctx *log.Context, communicator hostgac // For successful goal states, the status is reported by the usual workflow if err != nil { ctx.Log("error", "failed to execute goal state", "message", err) - instView := types.RunCommandInstanceView{ - ExecutionState: types.Failed, - ExecutionMessage: "Execution failed", - ExitCode: exitCode, - Output: "", - Error: err.Error(), - StartTime: startTime, - EndTime: time.Now().UTC().Format(time.RFC3339), + if reportErr := reportImmediateGoalStateFailure(ctx, notifier, statusKey, startTime, exitCode, err); reportErr != nil { + ctx.Log("error", "failed to report final goal state status", "message", reportErr) } - goalstate.ReportFinalStatusForImmediateGoalState(ctx, notifier, statusKey, types.StatusError, &instView) } }(newGoalStates[idx]) @@ -156,17 +155,20 @@ func processImmediateRunCommandGoalStates(ctx *log.Context, communicator hostgac notifier := &observer.Notifier{} notifier.Register(&goalStateEventObserver) - errorMsg := fmt.Sprintf("Exceeded concurrent goal state processing limit. Allowed new goal state count: %d. Extension: %s, SeqNumber: %d", maxTasksToFetch, *skippedGoalState.ExtensionName, *skippedGoalState.SeqNo) + skippedErr := newConcurrencyLimitError(maxTasksToFetch, skippedGoalState.SettingsCommon) instView := types.RunCommandInstanceView{ - ExecutionState: types.Failed, - ExecutionMessage: "Execution was skipped due to reaching the maximum concurrent tasks", - ExitCode: constants.ExitCode_SkippedImmediateGoalState, - Output: "", - Error: errorMsg, - StartTime: time.Now().UTC().Format(time.RFC3339), - EndTime: time.Now().UTC().Format(time.RFC3339), + ExecutionState: types.Failed, + ExecutionMessage: "Execution was skipped due to reaching the maximum concurrent tasks", + ExitCode: constants.ExitCode_SkippedImmediateGoalState, + Output: "", + Error: skippedErr.Error(), + ClarificationError: skippedErr, + StartTime: time.Now().UTC().Format(time.RFC3339), + EndTime: time.Now().UTC().Format(time.RFC3339), + } + if err := goalstate.ReportFinalStatusForImmediateGoalState(ctx, notifier, statusKey, types.StatusSkipped, &instView); err != nil { + ctx.Log("error", "failed to report skipped goal state status", "message", err) } - goalstate.ReportFinalStatusForImmediateGoalState(ctx, notifier, statusKey, types.StatusSkipped, &instView) } } else { ctx.Log("message", "no goal states were skipped") @@ -175,11 +177,47 @@ func processImmediateRunCommandGoalStates(ctx *log.Context, communicator hostgac return newEtag, nil } +func reportImmediateGoalStateFailure(ctx *log.Context, notifier *observer.Notifier, statusKey types.GoalStateKey, startTime string, exitCode int, err error) error { + var executionErr *goalstate.ImmediateCommandExecutionError + if errors.As(err, &executionErr) { + if executionErr.Reported { + return nil + } + return goalstate.ReportFinalStatusForImmediateGoalState(ctx, notifier, statusKey, executionErr.Status, &executionErr.InstanceView) + } + instView := types.RunCommandInstanceView{ + ExecutionState: types.Failed, + ExecutionMessage: "Execution failed", + ExitCode: exitCode, + Error: err.Error(), + StartTime: startTime, + EndTime: time.Now().UTC().Format(time.RFC3339), + ClarificationError: err, + } + return goalstate.ReportFinalStatusForImmediateGoalState(ctx, notifier, statusKey, types.StatusError, &instView) +} + +func validateGoalStateSettings(s settings.SettingsCommon) error { + if s.ExtensionName == nil || s.SeqNo == nil || s.ExtensionState == nil { + return errorutil.EnsureClarification(errorutil.Internal_incompleteFetchedGoalState, errors.New("goal state is missing extension name, sequence number or state")) + } + return nil +} + +func newConcurrencyLimitError(maxTasksToFetch int, state settings.SettingsCommon) error { + return errorutil.EnsureClarification(errorutil.CommandExecution_concurrentGoalStateLimitExceeded, fmt.Errorf("Exceeded concurrent goal state processing limit. Allowed new goal state count: %d. Extension: %s, SeqNumber: %d", maxTasksToFetch, *state.ExtensionName, *state.SeqNo)) +} + // Get the goal states that have not been processed yet func getGoalStatesToProcess(goalStates []hostgacommunicator.ImmediateExtensionGoalState, maxTasksToFetch int) ([]immediateGoalState, []immediateGoalState, error) { var newGoalStates []immediateGoalState var skippedGoalStates []immediateGoalState for _, el := range goalStates { + for _, s := range el.Settings { + if err := validateGoalStateSettings(s); err != nil { + return nil, nil, err + } + } validSignature, err := el.ValidateSignature() if err != nil { return nil, nil, errors.Wrap(err, "failed to validate goal state signature") diff --git a/internal/instanceview/instanceview.go b/internal/instanceview/instanceview.go index 109c2d3..b57e403 100755 --- a/internal/instanceview/instanceview.go +++ b/internal/instanceview/instanceview.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" ) @@ -23,13 +24,14 @@ func ReportInstanceView(ctx *log.Context, hEnv types.HandlerEnvironment, metadat return err } + metadata.ClarificationError = instanceview.ClarificationError return c.Functions.ReportStatus(ctx, hEnv, metadata, t, c, msg) } func SerializeInstanceView(instanceview *types.RunCommandInstanceView) (string, error) { bytes, err := instanceview.Marshal() if err != nil { - return "", fmt.Errorf("status: failed to marshal into json: %v", err) + return "", errorutil.EnsureClarification(errorutil.Internal_failedToMarshalInstanceView, fmt.Errorf("status: failed to marshal into json: %w", err)) } return string(bytes), err } diff --git a/internal/observer/clarification_test.go b/internal/observer/clarification_test.go new file mode 100644 index 0000000..ecfb8de --- /dev/null +++ b/internal/observer/clarification_test.go @@ -0,0 +1,35 @@ +package observer + +import ( + "errors" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +type failingObserver struct{ err error } + +func (o failingObserver) OnNotify(types.StatusEventArgs) error { return o.err } +func (o failingObserver) OnDemandNotify() error { return o.err } + +func TestObserverClarification(t *testing.T) { + cause := errors.New("observer failed") + n := Notifier{} + n.Register(failingObserver{cause}) + err := n.Notify(types.StatusEventArgs{}) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Internal_failedToNotifyObserver, code) + require.ErrorIs(t, err, cause) + lower := errorutil.EnsureClarification(errorutil.HostGA_failedToReportImmediateStatus, cause) + n.Register(failingObserver{lower}) + err = n.Notify(types.StatusEventArgs{}) + require.Same(t, lower, err) + code, ok = errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.HostGA_failedToReportImmediateStatus, code) + n.Unregister() + require.NoError(t, n.Notify(types.StatusEventArgs{})) +} diff --git a/internal/observer/notifier.go b/internal/observer/notifier.go index 22b7db6..aa94235 100644 --- a/internal/observer/notifier.go +++ b/internal/observer/notifier.go @@ -4,6 +4,7 @@ package observer import ( "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" ) type Notifier struct { @@ -21,7 +22,7 @@ func (n *Notifier) Unregister() { func (n *Notifier) Notify(status types.StatusEventArgs) error { tempObserver := n.observer if tempObserver != nil { - return tempObserver.OnNotify(status) + return errorutil.EnsureClarification(errorutil.Internal_failedToNotifyObserver, tempObserver.OnNotify(status)) } return nil diff --git a/internal/pid/clarification_test.go b/internal/pid/clarification_test.go new file mode 100644 index 0000000..40ebffd --- /dev/null +++ b/internal/pid/clarification_test.go @@ -0,0 +1,65 @@ +package pid + +import ( + "errors" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type pidErrorLogger struct{ err error } + +func (l *pidErrorLogger) Log(values ...interface{}) error { + for _, value := range values { + if err, ok := value.(error); ok { + l.err = err + } + } + return nil +} + +func TestPIDClarifications(t *testing.T) { + oldStart, oldKill := processStartTime, killProcessGroup + t.Cleanup(func() { processStartTime, killProcessGroup = oldStart, oldKill }) + cause := errors.New("machine failure") + assertCode := func(err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + } + processStartTime = func(string) ([]byte, error) { return nil, cause } + _, err := GetProcessStartTime(123) + assertCode(err, errorutil.Os_FailedToGetProcessStartTime) + require.ErrorIs(t, err, cause) + processStartTime = func(string) ([]byte, error) { return []byte("start"), nil } + dir, err := os.MkdirTemp(".", ".clarification-") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(dir) }) + assertCode(SaveCurrentPidAndStartTime(dir), errorutil.Os_FailedToWriteProcessRecord) + file := filepath.Join(dir, "pid") + assertCode(DeleteCurrentPidAndStartTime(file), errorutil.Os_FailedToDeleteProcessRecord) + _, _, err = ReadPidAndStartTime(dir) + assertCode(err, errorutil.Os_FailedToReadProcessRecord) + require.NoError(t, os.WriteFile(file, []byte("bad"), 0600)) + _, _, err = ReadPidAndStartTime(file) + assertCode(err, errorutil.Os_InvalidProcessRecordFormat) + require.NoError(t, os.WriteFile(file, []byte("bad\tstart"), 0600)) + _, _, err = ReadPidAndStartTime(file) + assertCode(err, errorutil.Os_InvalidProcessRecordId) + require.NoError(t, os.WriteFile(file, []byte("123\tstart"), 0600)) + killProcessGroup = func(pid int, signal syscall.Signal) error { + require.Equal(t, -123, pid) + require.Equal(t, syscall.SIGKILL, signal) + return cause + } + logger := &pidErrorLogger{} + KillPreviousExtension(log.NewContext(logger), file) + assertCode(logger.err, errorutil.Os_FailedToKillPreviousProcessGroup) + require.ErrorIs(t, logger.err, cause) +} diff --git a/internal/pid/pid.go b/internal/pid/pid.go index bf46afe..19cfa33 100644 --- a/internal/pid/pid.go +++ b/internal/pid/pid.go @@ -9,6 +9,7 @@ import ( "strings" "syscall" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -18,12 +19,16 @@ const ( chmod = os.FileMode(0600) ) +var processStartTime = func(pid string) ([]byte, error) { + return exec.Command("bash", "-c", "ps -o lstart= -p "+pid).Output() +} + // GetProcessStartTime returns the start time of the active process if still active func GetProcessStartTime(pid int) (string, error) { pidString := fmt.Sprintf("%d", pid) - startTime, err := exec.Command("bash", "-c", "ps -o lstart= -p "+pidString).Output() + startTime, err := processStartTime(pidString) if err != nil { - return "", errors.Wrap(err, "failed to execute bash ps command") + return "", errorutil.EnsureClarification(errorutil.Os_FailedToGetProcessStartTime, errors.Wrap(err, "failed to execute bash ps command")) } return string(startTime), nil } @@ -39,12 +44,12 @@ func SaveCurrentPidAndStartTime(path string) error { } b := []byte(fmt.Sprintf("%s\t%s", pidString, startTime)) - return errors.Wrap(os.WriteFile(path, b, chmod), "extName.pid: failed to write") + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteProcessRecord, errors.Wrap(os.WriteFile(path, b, chmod), "extName.pid: failed to write")) } // DeleteCurrentPidAndStartTime delete the file created by SaveCurrentPidAndStartTime func DeleteCurrentPidAndStartTime(path string) error { - return errors.Wrap(os.Remove(path), "failed to delete "+path) + return errorutil.EnsureClarification(errorutil.Os_FailedToDeleteProcessRecord, errors.Wrap(os.Remove(path), "failed to delete "+path)) } // ReadPidAndStartTime reads the stored pid and process start time from a file extName.pid @@ -55,16 +60,16 @@ func ReadPidAndStartTime(path string) (int, string, error) { if os.IsNotExist(err) { return 0, "", nil } - return 0, "", errors.Wrap(err, "extName.pid: failed to read:"+path) + return 0, "", errorutil.EnsureClarification(errorutil.Os_FailedToReadProcessRecord, errors.Wrap(err, "extName.pid: failed to read:"+path)) } data := strings.Split(string(b), "\t") if len(data) != 2 { - return 0, "", errors.Wrap(err, "unexpected format in extName.pid:"+string(b)) + return 0, "", errorutil.EnsureClarification(errorutil.Os_InvalidProcessRecordFormat, fmt.Errorf("unexpected format in extName.pid:%s", b)) } pid, err := strconv.Atoi(data[0]) if err != nil { - return 0, "", errors.Wrap(err, "failed to convert pid:"+data[0]) + return 0, "", errorutil.EnsureClarification(errorutil.Os_InvalidProcessRecordId, errors.Wrap(err, "failed to convert pid:"+data[0])) } return pid, data[1], nil } @@ -94,7 +99,12 @@ func KillPreviousExtension(ctx *log.Context, pidFilePath string) { if ctx != nil { ctx.Log("event", "check process", "Active previous execution found. Killing pid ", previousPid) } - syscall.Kill(-previousPid, syscall.SIGKILL) // Negative pid means kill the whole process group - DeleteCurrentPidAndStartTime(pidFilePath) + killErr := errorutil.EnsureClarification(errorutil.Os_FailedToKillPreviousProcessGroup, killProcessGroup(-previousPid, syscall.SIGKILL)) // Negative pid means kill the whole process group + if killErr != nil && ctx != nil { + ctx.Log("error", killErr) + } + if err := DeleteCurrentPidAndStartTime(pidFilePath); err != nil && ctx != nil { + ctx.Log("error", err) + } } } diff --git a/internal/pid/pid_linux.go b/internal/pid/pid_linux.go new file mode 100644 index 0000000..daf79d0 --- /dev/null +++ b/internal/pid/pid_linux.go @@ -0,0 +1,5 @@ +package pid + +import "syscall" + +var killProcessGroup = syscall.Kill diff --git a/internal/pid/pid_test.go b/internal/pid/pid_test.go index 83d05c5..dd2b576 100644 --- a/internal/pid/pid_test.go +++ b/internal/pid/pid_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/stretchr/testify/require" ) @@ -53,5 +54,8 @@ func Test_GetProcessStartTime(t *testing.T) { startTime, err = GetProcessStartTime(123456) require.Empty(t, startTime) require.NotNil(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Os_FailedToGetProcessStartTime, code) require.Contains(t, err.Error(), "failed to execute bash ps command") } diff --git a/internal/pid/pid_windows.go b/internal/pid/pid_windows.go new file mode 100644 index 0000000..5a7e633 --- /dev/null +++ b/internal/pid/pid_windows.go @@ -0,0 +1,11 @@ +package pid + +import ( + "errors" + "syscall" +) + +// Windows builds support mocked tests, not Linux process-group operations. +var killProcessGroup = func(int, syscall.Signal) error { + return errors.ErrUnsupported +} diff --git a/internal/pid/pid_windows_test.go b/internal/pid/pid_windows_test.go new file mode 100644 index 0000000..52b3994 --- /dev/null +++ b/internal/pid/pid_windows_test.go @@ -0,0 +1,18 @@ +package pid + +import ( + "errors" + "syscall" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func TestWindowsProcessGroupUnsupported(t *testing.T) { + // The platform shim returns a raw unsupported-operation sentinel; KillPreviousExtensionCommand classifies it. + err := killProcessGroup(-123, syscall.SIGKILL) + require.ErrorIs(t, err, errors.ErrUnsupported) + _, classified := errorutil.FindClarification(err) + require.False(t, classified) +} diff --git a/internal/requesthelper/clarification_test.go b/internal/requesthelper/clarification_test.go new file mode 100644 index 0000000..7b01eb9 --- /dev/null +++ b/internal/requesthelper/clarification_test.go @@ -0,0 +1,80 @@ +package requesthelper + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type clarificationFactory struct{ err error } + +func (f clarificationFactory) GetRequest(*log.Context, string) (*http.Request, error) { + if f.err != nil { + return nil, f.err + } + return http.NewRequest(http.MethodGet, "http://example.invalid", nil) +} + +type clarificationTransport func(*http.Request) (*http.Response, error) + +func (f clarificationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type temporaryClarificationError struct{} + +func (temporaryClarificationError) Error() string { return "temporary failure" } +func (temporaryClarificationError) Temporary() bool { return true } +func (temporaryClarificationError) Timeout() bool { return true } + +func TestRequestClarifications(t *testing.T) { + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("network failure") + for _, want := range []int{errorutil.HostGA_failedToCreateRequest, errorutil.HostGA_failedToSendRequest, errorutil.HostGA_requestReturnedErrorStatus} { + factory := clarificationFactory{} + if want == errorutil.HostGA_failedToCreateRequest { + factory.err = cause + } + rm := GetRequestManager(factory, time.Second) + rm.httpClient.Transport = clarificationTransport(func(*http.Request) (*http.Response, error) { + if want == errorutil.HostGA_failedToSendRequest { + return nil, cause + } + return &http.Response{StatusCode: http.StatusBadRequest, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(""))}, nil + }) + resp, err := rm.MakeRequest(ctx, "") + if resp != nil { + resp.Body.Close() + } + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, want, code) + if want != errorutil.HostGA_requestReturnedErrorStatus { + require.ErrorIs(t, err, cause) + } + } +} + +func TestRequestRetriesThroughClarification(t *testing.T) { + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + rm := GetRequestManager(clarificationFactory{}, time.Second) + attempts, sleeps := 0, 0 + rm.httpClient.Transport = clarificationTransport(func(*http.Request) (*http.Response, error) { + attempts++ + return nil, temporaryClarificationError{} + }) + _, err := WithRetries(ctx, rm, func(time.Duration) { sleeps++ }, "") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.HostGA_failedToSendRequest, code) + require.Equal(t, expRetryN, attempts) + require.Equal(t, expRetryN-1, sleeps) + var temporary interface{ Temporary() bool } + require.ErrorAs(t, err, &temporary) + require.True(t, temporary.Temporary()) +} diff --git a/internal/requesthelper/request.go b/internal/requesthelper/request.go index b3ef9e3..d4c6179 100644 --- a/internal/requesthelper/request.go +++ b/internal/requesthelper/request.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" ) @@ -35,12 +36,12 @@ func GetRequestManager(rf RequestFactory, timeout time.Duration) *RequestManager func (rm *RequestManager) MakeRequest(ctx *log.Context, eTag string) (*http.Response, error) { req, err := rm.requestFactory.GetRequest(ctx, eTag) if err != nil { - return nil, err + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToCreateRequest, err) } resp, err := rm.httpClient.Do(req) if err != nil { - return resp, err + return resp, errorutil.EnsureClarification(errorutil.HostGA_failedToSendRequest, err) } if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusPartialContent || resp.StatusCode == http.StatusNotModified { @@ -48,7 +49,7 @@ func (rm *RequestManager) MakeRequest(ctx *log.Context, eTag string) (*http.Resp } err = fmt.Errorf("unexpected status code: actual=%d expected=%d", resp.StatusCode, http.StatusOK) - return resp, err + return resp, errorutil.EnsureClarification(errorutil.HostGA_requestReturnedErrorStatus, err) } // httpClient is the default client to be used. http.Get() uses a client without timeouts (http.DefaultClient) diff --git a/internal/requesthelper/request_test.go b/internal/requesthelper/request_test.go index ca9437e..c398a71 100644 --- a/internal/requesthelper/request_test.go +++ b/internal/requesthelper/request_test.go @@ -11,11 +11,19 @@ import ( "time" "github.com/Azure/run-command-handler-linux/internal/requesthelper" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetalpbalkan/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) +func requireRequestCode(t *testing.T, err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "expected classified error: %v", err) + require.Equal(t, want, code) +} + const ( testRequestTimeout = 2 * time.Second badRequestorErrorMsg = "expected error from bad request" @@ -85,6 +93,7 @@ func TestMakeRequest_WrapsGetRequestError(t *testing.T) { require.Equal(t, badRequestor.calls, 1) require.NotNil(t, err) require.EqualError(t, err, badRequestorErrorMsg) + requireRequestCode(t, err, errorutil.HostGA_failedToCreateRequest) } func TestMakeRequest_WrapsHttpBadUrlError(t *testing.T) { @@ -95,6 +104,7 @@ func TestMakeRequest_WrapsHttpBadUrlError(t *testing.T) { require.Equal(t, testUrlRequest.calls, 1) require.NotNil(t, err) require.Contains(t, err.Error(), "unsupported protocol scheme") + requireRequestCode(t, err, errorutil.HostGA_failedToSendRequest) } func TestMakeRequest_RequestTimeout(t *testing.T) { @@ -109,6 +119,7 @@ func TestMakeRequest_RequestTimeout(t *testing.T) { require.Equal(t, timeoutTestUrl.calls, 1) require.NotNil(t, err) require.Contains(t, err.Error(), "Timeout exceeded") + requireRequestCode(t, err, errorutil.HostGA_failedToSendRequest) testUrl := NewTestURLRequest(fmt.Sprintf("%s/delay/%d", srv.URL, 1)) rm = requesthelper.GetRequestManager(testUrl, testRequestTimeout) @@ -138,6 +149,7 @@ func TestMakeRequest_BadStatusCodeFails(t *testing.T) { _, err := rm.MakeRequest(ctx, "") require.Equal(t, badTestCodeUrl.calls, 1) require.NotNil(t, err, "not failed for code: %d", code) + requireRequestCode(t, err, errorutil.HostGA_requestReturnedErrorStatus) require.Contains(t, err.Error(), "unexpected status code", "actual=%d", code) } } diff --git a/internal/requesthelper/retry.go b/internal/requesthelper/retry.go index c0e5d3b..c4f4c6d 100644 --- a/internal/requesthelper/retry.go +++ b/internal/requesthelper/retry.go @@ -1,6 +1,7 @@ package requesthelper import ( + "errors" "fmt" "math" "net/http" @@ -55,12 +56,10 @@ func WithRetries(ctx *log.Context, rm *RequestManager, sf SleepFunc, eTag string // status == -1 means that there wasn't any http request if status == -1 { - te, haste := lastErr.(interface { - Temporary() bool - }) - to, hasto := lastErr.(interface { - Timeout() bool - }) + var te interface{ Temporary() bool } + var to interface{ Timeout() bool } + haste := errors.As(lastErr, &te) + hasto := errors.As(lastErr, &to) if haste || hasto { if haste && te.Temporary() { diff --git a/internal/requesthelper/retry_test.go b/internal/requesthelper/retry_test.go index ffd9eb7..0db227d 100644 --- a/internal/requesthelper/retry_test.go +++ b/internal/requesthelper/retry_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Azure/run-command-handler-linux/internal/requesthelper" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetalpbalkan/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -60,6 +61,7 @@ func TestWithRetries_noRecovery(t *testing.T) { sr := new(sleepRecorder) resp, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.NotNil(t, err, "should have failed") + requireRequestCode(t, err, errorutil.HostGA_requestReturnedErrorStatus) require.Nil(t, resp, "response exists") require.Equal(t, []time.Duration(nil), []time.Duration(*sr), "sleep should not be called") } @@ -79,6 +81,7 @@ func TestWithRetries_noResponse(t *testing.T) { sr := new(sleepRecorder) resp, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.NotNil(t, err, "should have failed") + requireRequestCode(t, err, errorutil.HostGA_failedToSendRequest) require.Nil(t, resp, "response exists") require.Equal(t, []time.Duration(nil), []time.Duration(*sr), "sleep should not be called") } @@ -94,6 +97,7 @@ func TestWithRetries_failing_validateNumberOfCalls(t *testing.T) { sr := new(sleepRecorder) _, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.EqualError(t, err, "unexpected status code: actual=429 expected=200") + requireRequestCode(t, err, errorutil.HostGA_requestReturnedErrorStatus) require.EqualValues(t, 7, d.calls, "calls exactly expRetryN times") } @@ -105,6 +109,7 @@ func TestWithRetries_failedCreateRequest(t *testing.T) { sr := new(sleepRecorder) _, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.EqualError(t, err, badRequestorErrorMsg) + requireRequestCode(t, err, errorutil.HostGA_failedToCreateRequest) require.EqualValues(t, 1, bd.calls, "called exactly one time") } @@ -116,6 +121,7 @@ func TestWithRetries_requestFailedTimeout(t *testing.T) { sr := new(sleepRecorder) _, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.EqualError(t, err, requestErrorMsg) + requireRequestCode(t, err, errorutil.HostGA_failedToCreateRequest) require.EqualValues(t, 7, er.calls, "calls exactly expRetryN times") } @@ -127,6 +133,7 @@ func TestWithRetries_requestFailedTemporary(t *testing.T) { sr := new(sleepRecorder) _, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.EqualError(t, err, requestErrorMsg) + requireRequestCode(t, err, errorutil.HostGA_failedToCreateRequest) require.EqualValues(t, 7, er.calls, "calls exactly expRetryN times") } @@ -138,6 +145,7 @@ func TestWithRetries_requestFailedOther(t *testing.T) { sr := new(sleepRecorder) _, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.EqualError(t, err, requestErrorMsg) + requireRequestCode(t, err, errorutil.HostGA_failedToCreateRequest) require.EqualValues(t, 1, er.calls, "called exactly one time") } @@ -152,6 +160,7 @@ func TestWithRetries_failingBadStatusCode_validateSleeps(t *testing.T) { sr := new(sleepRecorder) _, err := requesthelper.WithRetries(ctx, rm, sr.Sleep, "") require.EqualError(t, err, "unexpected status code: actual=429 expected=200") + requireRequestCode(t, err, errorutil.HostGA_requestReturnedErrorStatus) require.Equal(t, sleepSchedule, []time.Duration(*sr)) } diff --git a/internal/service/clarification_test.go b/internal/service/clarification_test.go new file mode 100644 index 0000000..ef095ca --- /dev/null +++ b/internal/service/clarification_test.go @@ -0,0 +1,43 @@ +package service + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/Azure/azure-extension-platform/pkg/extensionevents" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/Azure/run-command-handler-linux/pkg/systemd" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func TestServiceClarifications(t *testing.T) { + oldPresent, oldChmod, oldReport := systemdPresent, markExecutable, reportServiceError + oldPath := systemd.GetUnitConfigurationFilePath + t.Cleanup(func() { + systemdPresent, markExecutable, reportServiceError = oldPresent, oldChmod, oldReport + systemd.GetUnitConfigurationFilePath = oldPath + }) + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + reportServiceError = func(*extensionevents.ExtensionEventManager, string, string) {} + systemdPresent = func() bool { return false } + err := Register(ctx, nil) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Os_SystemdNotSupported, code) + dir, err := os.MkdirTemp(".", ".clarification-") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(dir) }) + systemd.GetUnitConfigurationFilePath = func(string, *log.Context) (string, error) { return filepath.Join(dir, "missing"), nil } + systemdPresent = func() bool { return true } + cause := errors.New("permission denied") + markExecutable = func(string, os.FileMode) error { return cause } + err = Register(ctx, nil) + code, ok = errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Os_FailedToMakeServiceBinaryExecutable, code) + require.ErrorIs(t, err, cause) +} diff --git a/internal/service/serviceinstall.go b/internal/service/serviceinstall.go index bd14aeb..c5b8032 100644 --- a/internal/service/serviceinstall.go +++ b/internal/service/serviceinstall.go @@ -7,12 +7,21 @@ import ( "github.com/Azure/azure-extension-platform/pkg/extensionevents" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/servicehandler" "github.com/Azure/run-command-handler-linux/pkg/systemd" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) +var ( + systemdPresent = systemd.IsSystemDPresent + markExecutable = os.Chmod + reportServiceError = func(events *extensionevents.ExtensionEventManager, operation, message string) { + events.LogErrorEvent(operation, message) + } +) + const ( systemdUnitName = "managedruncommand.service" systemdUnitConfigurationPath = "misc/managedruncommand.service" @@ -36,8 +45,8 @@ WantedBy=multi-user.target` func Register(ctx *log.Context, extensionEvents *extensionevents.ExtensionEventManager) error { if !isSystemdSupported(ctx) { - extensionEvents.LogErrorEvent("register", "Systemd not supported. Failed to register service") - return errors.New("Systemd not supported. Failed to register service") + reportServiceError(extensionEvents, "register", "Systemd not supported. Failed to register service") + return errorutil.EnsureClarification(errorutil.Os_SystemdNotSupported, errors.New("Systemd not supported. Failed to register service")) } targetVersion := os.Getenv(constants.VersionEnvName) ctx.Log("message", "trying to register extension with version: "+targetVersion) @@ -68,11 +77,11 @@ func Register(ctx *log.Context, extensionEvents *extensionevents.ExtensionEventM ctx.Log("message", "Making immediate-run-command-handler executable") execDirectory := os.Getenv(constants.ExtensionPathEnvName) + "/bin/immediate-run-command-handler" - err = os.Chmod(execDirectory, 0744) + err = markExecutable(execDirectory, 0744) if err != nil { errMessage := fmt.Sprintf("Error while marking the immediate run command binary as executable: %v", err) - extensionEvents.LogErrorEvent("register", errMessage) - return errors.Wrap(err, "error while marking the immediate run command binary as executable") + reportServiceError(extensionEvents, "register", errMessage) + return errorutil.EnsureClarification(errorutil.Os_FailedToMakeServiceBinaryExecutable, errors.Wrap(err, "error while marking the immediate run command binary as executable")) } err = serviceHandler.Register(ctx, systemdUnitContent) @@ -222,7 +231,7 @@ func IsInstalled(ctx *log.Context) (bool, error) { return false, nil } -func getSystemdHandler(ctx *log.Context) *servicehandler.Handler { +var getSystemdHandler = func(ctx *log.Context) *servicehandler.Handler { ctx.Log("message", "Getting service handler for "+systemdUnitName) config := servicehandler.NewConfiguration(systemdUnitName) handler := servicehandler.NewHandler(systemd.NewUnitManager(), config, ctx) @@ -239,7 +248,7 @@ func generateServiceConfigurationContent(ctx *log.Context) string { func isSystemdSupported(ctx *log.Context) bool { ctx.Log("message", "Check if systemd is present on the system before applying next operation") - result := systemd.IsSystemDPresent() + result := systemdPresent() if result { ctx.Log("message", "systemd was found on the system") diff --git a/internal/settings/clarification_test.go b/internal/settings/clarification_test.go new file mode 100644 index 0000000..2f27707 --- /dev/null +++ b/internal/settings/clarification_test.go @@ -0,0 +1,39 @@ +package settings + +import ( + "encoding/json" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func TestSettingsClarifications(t *testing.T) { + for _, tc := range []struct { + input string + code int + }{ + {`{"seqNo":"invalid"}`, errorutil.CustomerInput_invalidSettingsJson}, + {`{"publicSettings":"{"}`, errorutil.CustomerInput_invalidPublicSettingsJson}, + {`{"publicSettings":true}`, errorutil.CustomerInput_publicSettingsNotObject}, + {`{"publicSettings":[]}`, errorutil.CustomerInput_publicSettingsNotObject}, + } { + t.Run(tc.input, func(t *testing.T) { + var settings SettingsCommon + err := json.Unmarshal([]byte(tc.input), &settings) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) + }) + } + _, err := (SettingsCommon{PublicSettings: map[string]interface{}{"bad": make(chan int)}}).MarshalJSON() + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Internal_failedToMarshalPublicSettings, code) + for _, input := range []string{`{}`, `{"publicSettings":null}`, `{"publicSettings":{}}`, `{"publicSettings":"{}"}`} { + var settings SettingsCommon + require.NoError(t, json.Unmarshal([]byte(input), &settings)) + } + // errorutil.Internal_failedToMarshalSettings cannot currently fail: the only interface field in the helper is + // replaced with a string after the independently tested errorutil.Internal_failedToMarshalPublicSettings marshal. +} diff --git a/internal/settings/settings.go b/internal/settings/settings.go index ce99f79..7d7dff8 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -2,7 +2,9 @@ package settings import ( "encoding/json" + "fmt" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) @@ -20,7 +22,7 @@ func (li *SettingsCommon) UnmarshalJSON(data []byte) error { type localItem SettingsCommon var loc localItem if err := json.Unmarshal(data, &loc); err != nil { - return err + return errorutil.EnsureClarification(errorutil.CustomerInput_invalidSettingsJson, err) } *li = SettingsCommon(loc) @@ -32,14 +34,17 @@ func (li *SettingsCommon) UnmarshalJSON(data []byte) error { var publicSettings map[string]interface{} if publicSettingsRawString != "" { if err := json.Unmarshal([]byte(publicSettingsRawString), &publicSettings); err != nil { - return errors.Wrapf(err, "failed to parse public settings from json") + return errorutil.EnsureClarification(errorutil.CustomerInput_invalidPublicSettingsJson, errors.Wrapf(err, "failed to parse public settings from json")) } } li.PublicSettings = publicSettings - case interface{}: + case map[string]interface{}: // This covers the scenario to parse the settings for a normal run command request li.PublicSettings = li.PublicSettingsRaw.(map[string]interface{}) + case nil: + default: + return errorutil.EnsureClarification(errorutil.CustomerInput_publicSettingsNotObject, fmt.Errorf("public settings must be an object or JSON object string")) } return nil @@ -50,8 +55,9 @@ func (li SettingsCommon) MarshalJSON() ([]byte, error) { s := SettingsCommonHelper(li) publicSettingsRaw, err := json.Marshal(s.PublicSettings) if err != nil { - return []byte{}, errors.Wrap(err, "could not marshal public settings") + return []byte{}, errorutil.EnsureClarification(errorutil.Internal_failedToMarshalPublicSettings, errors.Wrap(err, "could not marshal public settings")) } s.PublicSettingsRaw = string(publicSettingsRaw) - return json.Marshal(s) + b, err := json.Marshal(s) + return b, errorutil.EnsureClarification(errorutil.Internal_failedToMarshalSettings, errors.Wrap(err, "could not marshal settings")) } diff --git a/internal/settings/settings_test.go b/internal/settings/settings_test.go index d702130..692e9e1 100644 --- a/internal/settings/settings_test.go +++ b/internal/settings/settings_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/stretchr/testify/require" ) @@ -52,7 +53,10 @@ func Test_UnmarshalSettingsFromStandardRunCommand(t *testing.T) { func Test_FailedToUnmarshalSettingsFromVMSettingsAPI(t *testing.T) { var settings SettingsCommon err := json.Unmarshal([]byte(jsonBadFormatVMSettingsAPI), &settings) + // Invalid JSON syntax is rejected by encoding/json before the production unmarshaler runs. require.ErrorContains(t, err, "invalid character") + _, classified := errorutil.FindClarification(err) + require.False(t, classified) } func Test_MarshalVMSettingAPI(t *testing.T) { diff --git a/internal/status/clarification_test.go b/internal/status/clarification_test.go new file mode 100644 index 0000000..c11c46e --- /dev/null +++ b/internal/status/clarification_test.go @@ -0,0 +1,168 @@ +package status + +import ( + "encoding/json" + "errors" + "net/http" + "os" + "strconv" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/Azure/run-command-handler-linux/pkg/statusreporter" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +// errorutil.Internal_failedToMarshalTerminalGoalStates, errorutil.Internal_failedToMarshalStatusReport and errorutil.Internal_failedToMarshalImmediateStatus cannot be raised: their JSON payloads contain only +// strings, integers and slices of those structs (no custom marshalers). +// errorutil.Internal_unexpectedStatusItemCount cannot be raised: NewStatusReport always constructs exactly one item. +// errorutil.Internal_failedToMarshalInstanceView in instanceview.SerializeInstanceView likewise marshals only scalar +// fields; ClarificationError is explicitly excluded from JSON. + +type mockTemporaryStatus struct{ closeErr error } + +func (f mockTemporaryStatus) Name() string { return "/mock/status.tmp" } +func (f mockTemporaryStatus) Close() error { return f.closeErr } + +func mockStatusFilesystem(t *testing.T) { + t.Helper() + create, write, read, rename, stat, remove := statusCreateTemp, statusWriteFile, statusReadFile, statusRename, statusStat, statusRemove + t.Cleanup(func() { + statusCreateTemp, statusWriteFile, statusReadFile, statusRename, statusStat, statusRemove = create, write, read, rename, stat, remove + }) + statusCreateTemp = func(string, string) (temporaryStatusFile, error) { return mockTemporaryStatus{}, nil } + statusWriteFile = func(string, []byte, os.FileMode) error { return nil } + statusReadFile = func(string) ([]byte, error) { return []byte("[]"), nil } + statusRename = func(string, string) error { return nil } + statusStat = func(string) (os.FileInfo, error) { return nil, nil } + statusRemove = func(string) error { return nil } +} + +func TestStatusFilesystemClarifications(t *testing.T) { + failure := errors.New("mock disk failure") + ctx := log.NewContext(log.NewNopLogger()) + for _, code := range []int{errorutil.Os_FailedToCreateTemporaryStatusFile, errorutil.Os_FailedToCloseTemporaryStatusFile, errorutil.Os_FailedToWriteTemporaryStatusFile, errorutil.Os_FailedToPublishStatusFile, errorutil.Os_FailedToWriteTerminalGoalStates, errorutil.Os_FailedToPublishTerminalGoalStates, errorutil.Os_FailedToReadTerminalGoalStates, errorutil.Os_InvalidTerminalGoalStatesJson, errorutil.Os_FailedToStatTerminalGoalStates, errorutil.Internal_invalidGoalStateRemovalState} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + mockStatusFilesystem(t) + var err error + switch code { + case errorutil.Os_FailedToCreateTemporaryStatusFile: + statusCreateTemp = func(string, string) (temporaryStatusFile, error) { return nil, failure } + case errorutil.Os_FailedToCloseTemporaryStatusFile: + statusCreateTemp = func(string, string) (temporaryStatusFile, error) { return mockTemporaryStatus{failure}, nil } + case errorutil.Os_FailedToWriteTemporaryStatusFile, errorutil.Os_FailedToWriteTerminalGoalStates: + statusWriteFile = func(string, []byte, os.FileMode) error { return failure } + case errorutil.Os_FailedToPublishStatusFile, errorutil.Os_FailedToPublishTerminalGoalStates: + statusRename = func(string, string) error { return failure } + case errorutil.Os_FailedToReadTerminalGoalStates: + statusReadFile = func(string) ([]byte, error) { return nil, failure } + case errorutil.Os_InvalidTerminalGoalStatesJson: + statusReadFile = func(string) ([]byte, error) { return []byte("{invalid"), nil } + case errorutil.Os_FailedToStatTerminalGoalStates: + statusStat = func(string) (os.FileInfo, error) { return nil, failure } + } + switch code { + case errorutil.Os_FailedToCreateTemporaryStatusFile, errorutil.Os_FailedToCloseTemporaryStatusFile, errorutil.Os_FailedToWriteTemporaryStatusFile, errorutil.Os_FailedToPublishStatusFile: + err = SaveStatusReport("/mock", "command", 1, []byte("[]")) + case errorutil.Os_FailedToWriteTerminalGoalStates, errorutil.Os_FailedToPublishTerminalGoalStates: + err = SaveGoalStatesInTerminalStatus(ctx, nil) + case errorutil.Internal_invalidGoalStateRemovalState: + err = RemoveDisabledAndUpdatedGoalStatesInLocalStatusFile(ctx, []types.GoalStateKey{{RuntimeSettingsState: "invalid"}}) + default: + _, err = GetGoalStatesInTerminalStatus(ctx) + } + actual, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, code, actual) + }) + } +} + +func TestClarificationsInManagedAndImmediateStatus(t *testing.T) { + for _, topLevel := range []types.StatusType{types.StatusSuccess, types.StatusError} { + for _, code := range []int{errorutil.CommandExecution_failureExitCode, errorutil.Internal_unclassifiedCommandFailure} { + t.Run(string(topLevel)+strconv.Itoa(code), func(t *testing.T) { + mockStatusFilesystem(t) + ctx := log.NewContext(log.NewNopLogger()) + cause := errorutil.EnsureClarification(code, errors.New("original execution error")) + metadata := types.NewRCMetadata("command", 9, "download", "/mock") + metadata.ClarificationError = cause + env := types.HandlerEnvironment{} + env.HandlerEnvironment.StatusFolder = "/mock/status" + var written []byte + statusWriteFile = func(_ string, data []byte, _ os.FileMode) error { written = data; return nil } + require.NoError(t, ReportStatusToLocalFile(ctx, env, metadata, topLevel, types.CmdEnableTemplate, "instance view JSON")) + var managed types.StatusReport + require.NoError(t, json.Unmarshal(written, &managed)) + immediate, err := GetSingleStatusItem(ctx, topLevel, types.CmdEnableTemplate, "instance view JSON", "command", cause) + require.NoError(t, err) + var uploaded string + statusObserver := &StatusObserver{} + statusObserver.Initialize(ctx) + statusObserver.Reporter = mockStatusReporter{ + response: &http.Response{StatusCode: http.StatusOK, Status: "200 OK"}, + payload: &uploaded, + } + require.NoError(t, statusObserver.OnNotify(types.StatusEventArgs{ + StatusKey: types.GoalStateKey{HandlerName: constants.RunCommandTestExtensionName, ExtensionName: "command", SeqNumber: 9, RuntimeSettingsState: "enabled"}, + TopLevelStatus: immediate, + })) + var aggregate ImmediateTopLevelStatus + require.NoError(t, json.Unmarshal([]byte(uploaded), &aggregate)) + require.Len(t, aggregate.AggregateHandlerImmediateStatus, 1) + require.Len(t, aggregate.AggregateHandlerImmediateStatus[0].AggregateImmediateStatus, 1) + wireStatus := aggregate.AggregateHandlerImmediateStatus[0].AggregateImmediateStatus[0].Status + for _, item := range []types.StatusItem{managed[0], immediate, {Status: wireStatus}} { + require.Equal(t, topLevel, item.Status.Status) + require.Equal(t, "instance view JSON", item.Status.FormattedMessage.Message) + requireStatusClarification(t, item.Status, code) + } + }) + } + } + normal := types.NewStatusReport(types.StatusSuccess, "Enable", "success", "command") + require.Empty(t, normal[0].Status.Substatuses) +} + +func requireStatusClarification(t *testing.T, reported types.Status, want errorutil.ErrorClarificationCode) { + t.Helper() + require.Len(t, reported.Substatuses, 1) + require.Equal(t, "ErrorClarification", reported.Substatuses[0].Name) + require.Equal(t, string(types.StatusError), reported.Substatuses[0].Status) + require.Equal(t, want, reported.Substatuses[0].Code) +} + +type mockStatusReporter struct { + statusreporter.IGuestInformationServiceClient + err error + response *http.Response + payload *string +} + +func (r mockStatusReporter) GetPutStatusUri() string { return "http://mock/status" } +func (r mockStatusReporter) ReportStatus(_ *log.Context, payload string) (*http.Response, error) { + if r.payload != nil { + *r.payload = payload + } + return r.response, r.err +} + +func TestImmediateStatusReportingErrors(t *testing.T) { + for _, code := range []int{errorutil.HostGA_failedToReportImmediateStatus, errorutil.HostGA_immediateStatusReturnedErrorStatus} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + reporter := mockStatusReporter{response: &http.Response{StatusCode: 500, Status: "500 Internal Server Error"}} + if code == errorutil.HostGA_failedToReportImmediateStatus { + reporter.response = nil + reporter.err = errors.New("transport failed") + } + observer := StatusObserver{ctx: log.NewContext(log.NewNopLogger()), Reporter: reporter} + err := observer.reportImmediateStatus(ImmediateTopLevelStatus{}) + actual, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, code, actual) + }) + } +} diff --git a/internal/status/immediatestatus.go b/internal/status/immediatestatus.go index df30a2c..f750b1a 100644 --- a/internal/status/immediatestatus.go +++ b/internal/status/immediatestatus.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/internal/hostgacommunicator" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/statusreporter" "github.com/go-kit/kit/log" "github.com/pkg/errors" @@ -64,7 +65,7 @@ func (o *StatusObserver) OnNotify(status types.StatusEventArgs) error { o.ctx.Log("message", fmt.Sprintf("Processing status event for goal state with key %v", status.StatusKey)) o.goalStateEventMap.Store(status.StatusKey, status.TopLevelStatus) // The local copy is for debugging only; HTTP reporting remains authoritative. - if status.TopLevelStatus != (types.StatusItem{}) { + if item := status.TopLevelStatus; item.Version != 0 || item.TimestampUTC != "" || item.Status.Name != "" || item.Status.Operation != "" || item.Status.Status != "" || item.Status.FormattedMessage != (types.FormattedMessage{}) || len(item.Status.Substatuses) > 0 { if err := saveImmediateStatusToLocalFile(status); err != nil { o.ctx.Log("message", "failed to save local immediate status debug copy; continuing HTTP reporting", "goalState", status.StatusKey, "error", err) @@ -100,7 +101,7 @@ func (o *StatusObserver) getImmediateTopLevelStatusToReport() ImmediateTopLevelS goalStateKey := key.(types.GoalStateKey) statusByHandler[goalStateKey.HandlerName] = []ImmediateStatus{} if goalStateKey.RuntimeSettingsState != "disabled" { - if value.(types.StatusItem) != (types.StatusItem{}) { + if item := value.(types.StatusItem); item.Version != 0 || item.TimestampUTC != "" || item.Status.Name != "" || item.Status.Operation != "" || item.Status.Status != "" || item.Status.FormattedMessage != (types.FormattedMessage{}) || len(item.Status.Substatuses) > 0 { o.ctx.Log("message", fmt.Sprintf("Goal state %v is not empty. Processing it.", goalStateKey)) statusItem := value.(types.StatusItem) immediateStatus := ImmediateStatus{ @@ -182,19 +183,19 @@ func (o *StatusObserver) reportImmediateStatus(immediateStatus ImmediateTopLevel o.ctx.Log("message", "Marshalling immediate status into json") rootStatusJson, err := json.Marshal(immediateStatus) if err != nil { - return fmt.Errorf("status: failed to marshal immediate status into json: %v", err) + return errorutil.EnsureClarification(errorutil.Internal_failedToMarshalImmediateStatus, fmt.Errorf("status: failed to marshal immediate status into json: %w", err)) } o.ctx.Log("message", "create request to upload status to: "+o.Reporter.GetPutStatusUri()) response, err := o.Reporter.ReportStatus(o.ctx, string(rootStatusJson)) - o.ctx.Log("message", fmt.Sprintf("Status received from request to %v: %v", response.Request.URL, response.Status)) if err != nil { - return errors.Wrap(err, "failed to report status to HGAP") + return errorutil.EnsureClarification(errorutil.HostGA_failedToReportImmediateStatus, errors.Wrap(err, "failed to report status to HGAP")) } + o.ctx.Log("message", fmt.Sprintf("Status received from request: %v", response.Status)) if response.StatusCode != 200 { - return errors.New("failed to report status with error code " + response.Status) + return errorutil.EnsureClarification(errorutil.HostGA_immediateStatusReturnedErrorStatus, errors.New("failed to report status with error code "+response.Status)) } return nil diff --git a/internal/status/immediatestatus_test.go b/internal/status/immediatestatus_test.go index 8df7ccd..a51e05f 100644 --- a/internal/status/immediatestatus_test.go +++ b/internal/status/immediatestatus_test.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/handlersettings" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/statusreporter" "github.com/ahmetb/go-httpbin" "github.com/go-kit/kit/log" @@ -305,7 +306,15 @@ func Test_getTerminalStatus_doesNotGuessLegacyHandlerName(t *testing.T) { require.Contains(t, logs.String(), "ignoring cached immediate status without a handler name") } +func isolateImmediateStatusPersistence(t *testing.T) { + t.Helper() + directory := t.TempDir() + t.Setenv(constants.ExtensionPathEnvName, directory) + require.NoError(t, os.Mkdir(filepath.Join(directory, constants.ImmediateStatusFileDirectory), 0755)) +} + func Test_onNotify_success(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) observer := StatusObserver{} observer.Initialize(ctx) @@ -338,6 +347,7 @@ func Test_onNotify_success(t *testing.T) { } func Test_onNotify_fails(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) srv := httptest.NewServer(httpbin.GetMux()) defer srv.Close() @@ -367,6 +377,7 @@ func Test_onNotify_fails(t *testing.T) { } err := observer.OnNotify(topStatus) require.NotNil(t, err, "OnNotify should return an error") + requireImmediateStatusCode(t, err, errorutil.HostGA_statusUploadReturnedErrorStatus) require.Contains(t, err.Error(), strconv.Itoa(http.StatusNotFound)) status, ok := observer.GetStatusForKey(types.GoalStateKey{HandlerName: constants.RunCommandTestExtensionName, SeqNumber: 1, ExtensionName: "testExtension"}) @@ -375,6 +386,7 @@ func Test_onNotify_fails(t *testing.T) { } func Test_getImmediateTopLevelStatusToReport_filterNone(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) observer := StatusObserver{} observer.Initialize(ctx) @@ -418,6 +430,7 @@ func Test_getImmediateTopLevelStatusToReport_filterNone(t *testing.T) { } func Test_getImmediateTopLevelStatusToReport_filterAllWithEmptyStatus(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) observer := StatusObserver{} observer.Initialize(ctx) @@ -444,6 +457,7 @@ func Test_getImmediateTopLevelStatusToReport_filterAllWithEmptyStatus(t *testing } func Test_getImmediateTopLevelStatusToReport_filterSomeWithEmptyStatus(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) observer := StatusObserver{} observer.Initialize(ctx) @@ -546,9 +560,17 @@ func Test_reportStatusToEndpointNotFound(t *testing.T) { err := observer.reportImmediateStatus(immediateStatus) require.ErrorContains(t, err, strconv.Itoa(http.StatusNotFound)) + requireImmediateStatusCode(t, err, errorutil.HostGA_statusUploadReturnedErrorStatus) require.ErrorContains(t, err, "Not Found") } +func requireImmediateStatusCode(t *testing.T, err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "expected classified error: %v", err) + require.Equal(t, want, code) +} + func Test_getStatusForKey_statusFound(t *testing.T) { ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) observer := StatusObserver{} @@ -583,6 +605,7 @@ func Test_getStatusForKey_statusNotFound(t *testing.T) { } func Test_removeProcessedGoalStates_RemoveAll(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) ctx.Log("msg", "Creating status observer") observer := StatusObserver{} @@ -604,6 +627,7 @@ func Test_removeProcessedGoalStates_RemoveAll(t *testing.T) { } func Test_removeProcessedGoalStates_RemoveNone(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) ctx.Log("msg", "Creating status observer") observer := StatusObserver{} @@ -629,6 +653,7 @@ func Test_removeProcessedGoalStates_RemoveNone(t *testing.T) { } func Test_removeProcessedGoalStates_RemoveOne(t *testing.T) { + isolateImmediateStatusPersistence(t) ctx := log.NewContext(log.NewSyncLogger(log.NewLogfmtLogger(os.Stdout))).With("time", log.DefaultTimestamp) ctx.Log("msg", "Creating status observer") observer := StatusObserver{} diff --git a/internal/status/status.go b/internal/status/status.go index b886666..f5724c9 100755 --- a/internal/status/status.go +++ b/internal/status/status.go @@ -9,12 +9,27 @@ import ( "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) var immediateGSInTerminalStatusLock = sync.Mutex{} +type temporaryStatusFile interface { + Name() string + Close() error +} + +var ( + statusCreateTemp = func(dir, pattern string) (temporaryStatusFile, error) { return os.CreateTemp(dir, pattern) } + statusWriteFile = os.WriteFile + statusReadFile = os.ReadFile + statusRename = os.Rename + statusStat = os.Stat + statusRemove = os.Remove +) + // ReportStatusToLocalFile saves operation status to the status file for the extension // handler with the optional given message, if the given cmd requires reporting // status. @@ -28,7 +43,7 @@ func ReportStatusToLocalFile(ctx *log.Context, hEnv types.HandlerEnvironment, me return nil } - rootStatusJson, err := getRootStatusJson(ctx, statusType, c, msg, true, metadata.ExtName) + rootStatusJson, err := getRootStatusJson(ctx, statusType, c, msg, true, metadata.ExtName, metadata.ClarificationError) if err != nil { return errors.Wrap(err, "failed to get json for status report") } @@ -55,18 +70,21 @@ func SaveStatusReport(statusFolder string, extName string, seqNo int, rootStatus } path := filepath.Join(statusFolder, fn) - tmpFile, err := os.CreateTemp(statusFolder, fn) + tmpFile, err := statusCreateTemp(statusFolder, fn) if err != nil { - return fmt.Errorf("status: failed to create temporary file: %v", err) + return errorutil.EnsureClarification(errorutil.Os_FailedToCreateTemporaryStatusFile, fmt.Errorf("status: failed to create temporary file: %w", err)) + } + defer statusRemove(tmpFile.Name()) + if err := tmpFile.Close(); err != nil { + return errorutil.EnsureClarification(errorutil.Os_FailedToCloseTemporaryStatusFile, fmt.Errorf("status: failed to close temporary file: %w", err)) } - tmpFile.Close() - if err := os.WriteFile(tmpFile.Name(), rootStatusJson, 0644); err != nil { - return fmt.Errorf("status: failed to path=%s error=%v", tmpFile.Name(), err) + if err := statusWriteFile(tmpFile.Name(), rootStatusJson, 0644); err != nil { + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteTemporaryStatusFile, fmt.Errorf("status: failed to path=%s error=%w", tmpFile.Name(), err)) } - if err := os.Rename(tmpFile.Name(), path); err != nil { - return fmt.Errorf("status: failed to move to path=%s error=%v", path, err) + if err := statusRename(tmpFile.Name(), path); err != nil { + return errorutil.EnsureClarification(errorutil.Os_FailedToPublishStatusFile, fmt.Errorf("status: failed to move to path=%s error=%w", path, err)) } return nil @@ -96,21 +114,21 @@ func SaveGoalStatesInTerminalStatus(ctx *log.Context, newStatusInTerminalState [ } rootStatusJson, err := json.MarshalIndent(cachedStatuses, "", "\t") if err != nil { - return fmt.Errorf("status: failed to marshal status report into json: %v", err) + return errorutil.EnsureClarification(errorutil.Internal_failedToMarshalTerminalGoalStates, fmt.Errorf("status: failed to marshal status report into json: %w", err)) } ctx.Log("message", "writing the content to the temporary status file") - err = os.WriteFile(tempStatusFile, rootStatusJson, 0644) + err = statusWriteFile(tempStatusFile, rootStatusJson, 0644) if err != nil { - return fmt.Errorf("status: failed to write status file: %v", err) + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteTerminalGoalStates, fmt.Errorf("status: failed to write status file: %w", err)) } ctx.Log("message", "Renaming the temporary status file to the final status file") - err = os.Rename(tempStatusFile, statusFile) + err = statusRename(tempStatusFile, statusFile) if err != nil { - return fmt.Errorf("status: failed to move status file: %v", err) + return errorutil.EnsureClarification(errorutil.Os_FailedToPublishTerminalGoalStates, fmt.Errorf("status: failed to move status file: %w", err)) } return nil @@ -130,17 +148,17 @@ func GetGoalStatesInTerminalStatus(ctx *log.Context) ([]ImmediateStatus, error) result := []ImmediateStatus{} - if _, err := os.Stat(statusFile); err == nil { + if _, err := statusStat(statusFile); err == nil { ctx.Log("message", "status file already exists. Reading the content") - fileContent, err := os.ReadFile(statusFile) + fileContent, err := statusReadFile(statusFile) if err != nil { - return result, fmt.Errorf("status: failed to read status file: %v", err) + return result, errorutil.EnsureClarification(errorutil.Os_FailedToReadTerminalGoalStates, fmt.Errorf("status: failed to read status file: %w", err)) } ctx.Log("message", "unmarshalling the content of the status file") var existingStatus []cachedImmediateStatus if err := json.Unmarshal(fileContent, &existingStatus); err != nil { - return result, fmt.Errorf("status: failed to unmarshal status file: %v", err) + return result, errorutil.EnsureClarification(errorutil.Os_InvalidTerminalGoalStatesJson, fmt.Errorf("status: failed to unmarshal status file: %w", err)) } ctx.Log("message", "merging the new status with the existing content") @@ -154,8 +172,10 @@ func GetGoalStatesInTerminalStatus(ctx *log.Context) ([]ImmediateStatus, error) result = append(result, cached.ImmediateStatus) } ctx.Log("message", fmt.Sprintf("Found %v goal states in terminal state", len(result))) - } else { + } else if os.IsNotExist(err) { ctx.Log("message", "status file does not exist. No goal states in terminal status") + } else { + return result, errorutil.EnsureClarification(errorutil.Os_FailedToStatTerminalGoalStates, fmt.Errorf("status: failed to stat status file: %w", err)) } return result, nil @@ -191,7 +211,7 @@ func RemoveDisabledAndUpdatedGoalStatesInLocalStatusFile(ctx *log.Context, goalS } } } else { - return errors.New(fmt.Sprintf("goal state %v is not disabled. Cannot remove it from the statusInTerminalState list", goalStateKey)) + return errorutil.EnsureClarification(errorutil.Internal_invalidGoalStateRemovalState, errors.New(fmt.Sprintf("goal state %v is not disabled. Cannot remove it from the statusInTerminalState list", goalStateKey))) } } @@ -206,9 +226,9 @@ func RemoveDisabledAndUpdatedGoalStatesInLocalStatusFile(ctx *log.Context, goalS return nil } -func getRootStatusJson(ctx *log.Context, statusType types.StatusType, c types.Cmd, msg string, indent bool, extName string) ([]byte, error) { +func getRootStatusJson(ctx *log.Context, statusType types.StatusType, c types.Cmd, msg string, indent bool, extName string, clarificationErrors ...error) ([]byte, error) { ctx.Log("message", "creating json to report status") - statusReport := types.NewStatusReport(statusType, c.Name, msg, extName) + statusReport := types.NewStatusReport(statusType, c.Name, msg, extName, clarificationErrors...) b, err := MarshalStatusReportIntoJson(statusReport, indent) if err != nil { @@ -220,11 +240,11 @@ func getRootStatusJson(ctx *log.Context, statusType types.StatusType, c types.Cm // getSingleStatusItem returns a single status item for the given status type, command, and message. // This is useful when only a single status item is needed for an immediate status report. -func GetSingleStatusItem(ctx *log.Context, statusType types.StatusType, c types.Cmd, msg string, extName string) (types.StatusItem, error) { +func GetSingleStatusItem(ctx *log.Context, statusType types.StatusType, c types.Cmd, msg string, extName string, clarificationErrors ...error) (types.StatusItem, error) { ctx.Log("message", "creating json to report status") - statusReport := types.NewStatusReport(statusType, c.Name, msg, extName) + statusReport := types.NewStatusReport(statusType, c.Name, msg, extName, clarificationErrors...) if len(statusReport) != 1 { - return types.StatusItem{}, errors.New("expected a single status item") + return types.StatusItem{}, errorutil.EnsureClarification(errorutil.Internal_unexpectedStatusItemCount, errors.New("expected a single status item")) } return statusReport[0], nil } @@ -238,5 +258,5 @@ func MarshalStatusReportIntoJson(statusReport types.StatusReport, indent bool) ( b, err = json.Marshal(statusReport) } - return b, err + return b, errorutil.EnsureClarification(errorutil.Internal_failedToMarshalStatusReport, err) } diff --git a/internal/status/status/immediateGoalStatesInTerminalStatusFile.status b/internal/status/status/immediateGoalStatesInTerminalStatusFile.status new file mode 100644 index 0000000..dbc778f --- /dev/null +++ b/internal/status/status/immediateGoalStatesInTerminalStatusFile.status @@ -0,0 +1,106 @@ +[ + { + "sequenceNumber": 1, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 3, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 2, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 3, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 3, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 2, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 1, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + }, + { + "sequenceNumber": 3, + "timestampUTC": "", + "status": { + "name": "", + "operation": "", + "status": "", + "formattedMessage": { + "lang": "", + "message": "" + } + } + } +] \ No newline at end of file diff --git a/internal/status/status_test.go b/internal/status/status_test.go index 55a46de..adddd91 100644 --- a/internal/status/status_test.go +++ b/internal/status/status_test.go @@ -1,12 +1,15 @@ package status import ( + "encoding/json" + "errors" "os" "path/filepath" "testing" "github.com/Azure/run-command-handler-linux/internal/constants" "github.com/Azure/run-command-handler-linux/internal/types" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -18,6 +21,9 @@ func Test_reportStatus_fails(t *testing.T) { metadata := types.NewRCMetadata("", 1, constants.DownloadFolder, constants.DataDir) err := ReportStatusToLocalFile(log.NewContext(log.NewNopLogger()), fakeEnv, metadata, types.StatusSuccess, types.CmdEnableTemplate, "") require.NotNil(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Os_FailedToCreateTemporaryStatusFile, code) require.Contains(t, err.Error(), "failed to save handler status") } @@ -31,12 +37,18 @@ func Test_reportStatus_fileExists(t *testing.T) { fakeEnv.HandlerEnvironment.StatusFolder = tmpDir metadata := types.NewRCMetadata(extName, 1, constants.DownloadFolder, constants.DataDir) + metadata.ClarificationError = errorutil.EnsureClarification(errorutil.CommandExecution_failureExitCode, errors.New("FOO ERROR")) require.Nil(t, ReportStatusToLocalFile(log.NewContext(log.NewNopLogger()), fakeEnv, metadata, types.StatusError, types.CmdEnableTemplate, "FOO ERROR")) path := filepath.Join(tmpDir, "first.1.status") b, err := os.ReadFile(path) require.Nil(t, err, ".status file exists") require.NotEqual(t, 0, len(b), ".status file not empty") + var report types.StatusReport + require.NoError(t, json.Unmarshal(b, &report)) + require.Len(t, report, 1) + require.Equal(t, types.StatusError, report[0].Status.Status) + requireStatusClarification(t, report[0].Status, errorutil.CommandExecution_failureExitCode) } func Test_reportStatus_checksIfShouldBeReported(t *testing.T) { @@ -52,6 +64,7 @@ func Test_reportStatus_checksIfShouldBeReported(t *testing.T) { require.Nil(t, ReportStatusToLocalFile(log.NewContext(log.NewNopLogger()), fakeEnv, metadata, types.StatusSuccess, c, "")) fp := filepath.Join(tmpDir, "first.2.status") + // os.Stat is a raw library probe of file presence, not a production error. _, err = os.Stat(fp) // check if the .status file is there if c.ShouldReportStatus && err != nil { t.Fatalf("cmd=%q should have reported status file=%q err=%v", c.Name, fp, err) diff --git a/internal/telemetry/clarification_test.go b/internal/telemetry/clarification_test.go new file mode 100644 index 0000000..2ebfbbc --- /dev/null +++ b/internal/telemetry/clarification_test.go @@ -0,0 +1,84 @@ +package telemetry + +import ( + "errors" + "io" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +type failingTelemetryWriter struct { + writeErr error + closeErr error + short bool +} + +func (w failingTelemetryWriter) Write(b []byte) (int, error) { + if w.short { + return 0, nil + } + return len(b), w.writeErr +} +func (w failingTelemetryWriter) Close() error { return w.closeErr } + +func TestTelemetryClarifications(t *testing.T) { + originalOpen, originalRename := openTelemetryFile, renameTelemetryFile + t.Cleanup(func() { openTelemetryFile, renameTelemetryFile = originalOpen, originalRename }) + cause := errors.New("machine failure") + for _, tc := range []struct { + code int + writer failingTelemetryWriter + open error + rename error + }{ + {code: errorutil.Os_FailedToOpenTelemetryFile, open: cause}, + {code: errorutil.Os_FailedToWriteTelemetryFile, writer: failingTelemetryWriter{writeErr: cause}}, + {code: errorutil.Os_ShortTelemetryFileWrite, writer: failingTelemetryWriter{short: true}}, + {code: errorutil.Os_FailedToPublishTelemetryFile, rename: cause}, + {code: errorutil.Os_FailedToCloseTelemetryFile, writer: failingTelemetryWriter{closeErr: cause}}, + } { + t.Run(string(rune(tc.code)), func(t *testing.T) { + openTelemetryFile = func(string) (io.WriteCloser, error) { return tc.writer, tc.open } + renameTelemetryFile = func(string, string) error { return tc.rename } + w := telemetryEventWriter{} + _, err := w.Write([]byte("event")) + if tc.code == errorutil.Os_FailedToCloseTelemetryFile { + require.NoError(t, err) + err = w.Close() + } + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) + if tc.code == errorutil.Os_ShortTelemetryFileWrite { + require.ErrorIs(t, err, io.ErrShortWrite) + } else { + require.ErrorIs(t, err, cause) + } + }) + } + for _, tc := range []struct { + code int + event telemetryEvent + writer failingTelemetryWriter + }{ + {code: errorutil.Os_FailedToCloseTelemetryWriter, writer: failingTelemetryWriter{closeErr: cause}}, + {code: errorutil.Internal_failedToMarshalTelemetryEvent, event: telemetryEvent{Parameters: []interface{}{make(chan int)}}}, + {code: errorutil.Os_FailedToWriteTelemetryEvent, writer: failingTelemetryWriter{writeErr: cause}}, + {code: errorutil.Os_ShortTelemetryEventWrite, writer: failingTelemetryWriter{short: true}}, + } { + s := newTelemetryEventSenderWithWriteCloser(tc.writer) + err := s.send(tc.event) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) + } + // The writer's primary error must not be overwritten by a close failure. + s := newTelemetryEventSenderWithWriteCloser(failingTelemetryWriter{writeErr: cause, closeErr: io.EOF}) + err := s.send(telemetryEvent{}) + require.ErrorIs(t, err, cause) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Os_FailedToWriteTelemetryEvent, code) +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index d8887da..24c80f5 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) @@ -38,28 +39,40 @@ type telemetryEvent struct { } type telemetryEventWriter struct { - fh *os.File + fh io.WriteCloser } +var openTelemetryFile = func(name string) (io.WriteCloser, error) { + return os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0400) +} + +var renameTelemetryFile = os.Rename + func (w *telemetryEventWriter) Write(bs []byte) (n int, err error) { fn := getTelemetryFileName() temp := fn + ".tmp" - fh, err := os.OpenFile(temp, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0400) + fh, err := openTelemetryFile(temp) if err != nil { - return 0, errors.Wrap(err, "failed to open telemetry file") + return 0, errorutil.EnsureClarification(errorutil.Os_FailedToOpenTelemetryFile, errors.Wrap(err, "failed to open telemetry file")) } w.fh = fh n, err = w.fh.Write(bs) - err = os.Rename(temp, fn) + if err != nil { + return n, errorutil.EnsureClarification(errorutil.Os_FailedToWriteTelemetryFile, errors.Wrap(err, "failed to write telemetry file")) + } + if n != len(bs) { + return n, errorutil.EnsureClarification(errorutil.Os_ShortTelemetryFileWrite, io.ErrShortWrite) + } + err = errorutil.EnsureClarification(errorutil.Os_FailedToPublishTelemetryFile, errors.Wrap(renameTelemetryFile(temp, fn), "failed to publish telemetry file")) return } func (w *telemetryEventWriter) Close() (err error) { if w.fh != nil { - err = w.fh.Close() + err = errorutil.EnsureClarification(errorutil.Os_FailedToCloseTelemetryFile, errors.Wrap(w.fh.Close(), "failed to close telemetry file")) w.fh = nil } return @@ -84,17 +97,24 @@ func newTelemetryEventSenderWithWriteCloser(writer io.WriteCloser) *telemetryEve return &telemetryEventSender{writer: writer} } -func (w *telemetryEventSender) send(e telemetryEvent) error { - defer w.writer.Close() +func (w *telemetryEventSender) send(e telemetryEvent) (result error) { + defer func() { + if closeErr := w.writer.Close(); result == nil { + result = errorutil.EnsureClarification(errorutil.Os_FailedToCloseTelemetryWriter, errors.Wrap(closeErr, "failed to close telemetry event writer")) + } + }() bs, err := json.Marshal(e) if err != nil { - return errors.Wrap(err, "failed to marhsal telemetry event") + return errorutil.EnsureClarification(errorutil.Internal_failedToMarshalTelemetryEvent, errors.Wrap(err, "failed to marshal telemetry event")) } - _, err = w.writer.Write(bs) + n, err := w.writer.Write(bs) if err != nil { - return errors.Wrap(err, "failed to write telemetry event") + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteTelemetryEvent, errors.Wrap(err, "failed to write telemetry event")) + } + if n != len(bs) { + return errorutil.EnsureClarification(errorutil.Os_ShortTelemetryEventWrite, io.ErrShortWrite) } return nil diff --git a/internal/types/commands.go b/internal/types/commands.go index f65b880..137a75c 100755 --- a/internal/types/commands.go +++ b/internal/types/commands.go @@ -1,9 +1,13 @@ package types import ( + "errors" + "github.com/go-kit/kit/log" ) +var ErrAlreadyProcessed = errors.New("the script configuration has already been processed, will not run again") + type cmdFunc func(ctx *log.Context, hEnv HandlerEnvironment, report *RunCommandInstanceView, metadata RCMetadata, c Cmd) (stdout string, stderr string, err error, exitCode int, cleanup *CleanupRequest) type reportStatusFunc func(ctx *log.Context, hEnv HandlerEnvironment, metadata RCMetadata, statusType StatusType, c Cmd, msg string) error type preFunc func(ctx *log.Context, hEnv HandlerEnvironment, metadata RCMetadata, c Cmd) error diff --git a/internal/types/instanceview.go b/internal/types/instanceview.go index 31774fb..e31e709 100644 --- a/internal/types/instanceview.go +++ b/internal/types/instanceview.go @@ -30,13 +30,14 @@ const ( // RunCommandInstanceView reports script execution status type RunCommandInstanceView struct { - ExecutionState ExecutionState `json:"executionState"` - ExecutionMessage string `json:"executionMessage"` - Output string `json:"output"` - Error string `json:"error"` - ExitCode int `json:"exitCode"` - StartTime string `json:"startTime"` - EndTime string `json:"endTime"` + ClarificationError error `json:"-"` + ExecutionState ExecutionState `json:"executionState"` + ExecutionMessage string `json:"executionMessage"` + Output string `json:"output"` + Error string `json:"error"` + ExitCode int `json:"exitCode"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` } func (instanceView RunCommandInstanceView) Marshal() ([]byte, error) { diff --git a/internal/types/runcommandmetadata.go b/internal/types/runcommandmetadata.go index 846a6d8..237e7e8 100644 --- a/internal/types/runcommandmetadata.go +++ b/internal/types/runcommandmetadata.go @@ -7,6 +7,8 @@ import ( ) type RCMetadata struct { + ClarificationError error + // Most recent sequence, which was previously traced by seqNumFile. This was // incorrect. The correct way is mrseq. This file is auto-preserved by the agent. MostRecentSequence string diff --git a/internal/types/status.go b/internal/types/status.go index 8745502..f88b598 100755 --- a/internal/types/status.go +++ b/internal/types/status.go @@ -1,12 +1,17 @@ package types -import "time" +import ( + "time" + + platformstatus "github.com/Azure/azure-extension-platform/pkg/status" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" +) // StatusReport contains one or more status items and is the parent object type StatusReport []StatusItem -func NewStatusReport(statusType StatusType, operation string, message string, extName string) StatusReport { - return []StatusItem{ +func NewStatusReport(statusType StatusType, operation string, message string, extName string, clarificationErrors ...error) StatusReport { + report := []StatusItem{ { Version: 1, // this is the protocol version do not change unless you are sure TimestampUTC: time.Now().UTC().Format(time.RFC3339), @@ -20,6 +25,17 @@ func NewStatusReport(statusType StatusType, operation string, message string, ex }, }, } + for _, err := range clarificationErrors { + if code, ok := errorutil.FindClarification(err); ok { + report[0].Status.Substatuses = []platformstatus.Substatus{{ + Name: platformstatus.ErrorClarificationSubStatusName, + Status: string(StatusError), + Code: code, + }} + break + } + } + return report } // StatusItem is used to serialize an individual part of the status read by the server @@ -48,10 +64,11 @@ const ( // Status is used for serializing status in a manner the server understands type Status struct { - Name string `json:"name"` - Operation string `json:"operation"` - Status StatusType `json:"status"` - FormattedMessage FormattedMessage `json:"formattedMessage"` + Substatuses []platformstatus.Substatus `json:"substatus,omitempty"` + Name string `json:"name"` + Operation string `json:"operation"` + Status StatusType `json:"status"` + FormattedMessage FormattedMessage `json:"formattedMessage"` } // FormattedMessage is a struct used for serializing status diff --git a/pkg/blobutil/clarification_test.go b/pkg/blobutil/clarification_test.go new file mode 100644 index 0000000..4c59b48 --- /dev/null +++ b/pkg/blobutil/clarification_test.go @@ -0,0 +1,29 @@ +package blobutil + +import ( + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func TestParseBlobURLClarifications(t *testing.T) { + for _, tc := range []struct { + url string + code int + }{ + {"https://%", errorutil.CustomerInput_invalidBlobUrl}, + {"ftp://a.blob.core.windows.net/c/b", errorutil.CustomerInput_unsupportedBlobUrlScheme}, + {"https://host/c/b", errorutil.CustomerInput_missingBlobUrlHost}, + {"https://a.file.core.windows.net/c/b", errorutil.CustomerInput_invalidBlobHost}, + {"https://a.blob./c/b", errorutil.CustomerInput_invalidStorageEndpoint}, + {"https://a.blob.core.windows.net", errorutil.CustomerInput_missingBlobContainerOrName}, + } { + t.Run(tc.url, func(t *testing.T) { + _, err := ParseBlobURL(tc.url) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) + }) + } +} diff --git a/pkg/blobutil/url.go b/pkg/blobutil/url.go index a912d98..980f355 100644 --- a/pkg/blobutil/url.go +++ b/pkg/blobutil/url.go @@ -5,6 +5,7 @@ import ( "net/url" "strings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) @@ -31,30 +32,30 @@ type AzureBlobRef struct { func ParseBlobURL(blobURL string) (v AzureBlobRef, err error) { u, err := url.Parse(blobURL) if err != nil { - return v, errors.Wrapf(err, "cannot parse URL: %q", blobURL) + return v, errorutil.EnsureClarification(errorutil.CustomerInput_invalidBlobUrl, errors.Wrapf(err, "cannot parse URL: %q", blobURL)) } v.Scheme = strings.ToLower(u.Scheme) if v.Scheme != "http" && v.Scheme != "https" { - return v, fmt.Errorf("unsupported scheme in URL: %q", blobURL) + return v, errorutil.EnsureClarification(errorutil.CustomerInput_unsupportedBlobUrlScheme, fmt.Errorf("unsupported scheme in URL: %q", blobURL)) } hostParts := strings.Split(u.Host, ".") // {account}.blob.{storageBase} if len(hostParts) < 3 { - return v, fmt.Errorf("cannot parse azure blob URL: %q", blobURL) + return v, errorutil.EnsureClarification(errorutil.CustomerInput_missingBlobUrlHost, fmt.Errorf("cannot parse azure blob URL: %q", blobURL)) } if strings.ToLower(hostParts[1]) != "blob" { - return v, fmt.Errorf("blob host not in *.blob.* format: %q", blobURL) + return v, errorutil.EnsureClarification(errorutil.CustomerInput_invalidBlobHost, fmt.Errorf("blob host not in *.blob.* format: %q", blobURL)) } v.StorageBase = strings.Join(hostParts[2:], ".") // glue them back if v.StorageBase == "" { - return v, fmt.Errorf("cannot parse azure storage endpoint in blob URL: %q", blobURL) + return v, errorutil.EnsureClarification(errorutil.CustomerInput_invalidStorageEndpoint, fmt.Errorf("cannot parse azure storage endpoint in blob URL: %q", blobURL)) } var ok bool v.Container, v.Blob, ok = parseAzureContainerBlobNameFromPath(u.Path) if !ok { - return v, fmt.Errorf("cannot extract Azure container/blob name from: %q", blobURL) + return v, errorutil.EnsureClarification(errorutil.CustomerInput_missingBlobContainerOrName, fmt.Errorf("cannot extract Azure container/blob name from: %q", blobURL)) } return v, nil diff --git a/pkg/blobutil/url_test.go b/pkg/blobutil/url_test.go index c5da105..debe99d 100644 --- a/pkg/blobutil/url_test.go +++ b/pkg/blobutil/url_test.go @@ -4,30 +4,43 @@ import ( "testing" "github.com/Azure/run-command-handler-linux/pkg/blobutil" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/stretchr/testify/require" ) func TestParseBlobURL_badURL(t *testing.T) { - for _, u := range []string{ - "", - " http://a.b.c/d/e", - "http://a.b.c/d/e ", + for _, tc := range []struct { + url string + code errorutil.ErrorClarificationCode + }{ + {"", errorutil.CustomerInput_unsupportedBlobUrlScheme}, + {" http://a.b.c/d/e", errorutil.CustomerInput_invalidBlobUrl}, + {"http://a.b.c/d/e ", errorutil.CustomerInput_invalidBlobHost}, } { - _, err := blobutil.ParseBlobURL(u) - require.NotNil(t, err, "invalid URL: %q", u) + _, err := blobutil.ParseBlobURL(tc.url) + require.NotNil(t, err, "invalid URL: %q", tc.url) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) } } func TestParseBlobURL_badHost(t *testing.T) { - for _, v := range []string{ // bad URL formats are already captured by json-schema - "http://1/c/blob.txt", - "http://1.blob/c/blob.txt", - "http://1.blob./c/blob.txt", - "http://1..blob./c/blob.txt", - "http://1.notblob.a/c/blob.txt", + for _, tc := range []struct { + url string + code errorutil.ErrorClarificationCode + }{ // bad URL formats are already captured by json-schema + {"http://1/c/blob.txt", errorutil.CustomerInput_missingBlobUrlHost}, + {"http://1.blob/c/blob.txt", errorutil.CustomerInput_missingBlobUrlHost}, + {"http://1.blob./c/blob.txt", errorutil.CustomerInput_invalidStorageEndpoint}, + {"http://1..blob./c/blob.txt", errorutil.CustomerInput_invalidBlobHost}, + {"http://1.notblob.a/c/blob.txt", errorutil.CustomerInput_invalidBlobHost}, } { - _, err := blobutil.ParseBlobURL(v) - require.NotNil(t, err, "bad host: %q", v) + _, err := blobutil.ParseBlobURL(tc.url) + require.NotNil(t, err, "bad host: %q", tc.url) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, tc.code, code) } } @@ -56,6 +69,9 @@ func TestParseBlobURL_unsupportedScheme(t *testing.T) { _, err := blobutil.ParseBlobURL(v) require.NotNil(t, err, "url: %q", v) require.Contains(t, err.Error(), "unsupported scheme in URL", "url: %q") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.CustomerInput_unsupportedBlobUrlScheme, code) } } @@ -73,6 +89,9 @@ func TestParseBlobURL_scheme(t *testing.T) { func TestParseBlobURL_containerMissing(t *testing.T) { _, err := blobutil.ParseBlobURL("http://acct.blob.core.windows.net/") require.NotNil(t, err, "missing blob/container name") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.CustomerInput_missingBlobContainerOrName, code) require.Contains(t, err.Error(), "cannot extract Azure container/blob name") } diff --git a/pkg/download/blob.go b/pkg/download/blob.go index b681ef9..3734d91 100755 --- a/pkg/download/blob.go +++ b/pkg/download/blob.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/run-command-handler-linux/pkg/blobutil" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -22,6 +23,16 @@ const ( blobSASDuration = time.Minute * 30 ) +var ( + newStorageClient = storage.NewClient + blobSASURI = func(blob *storage.Blob, options storage.BlobSASOptions) (string, error) { + return blob.GetSASURI(options) + } + sasHTTPGet = http.Get + containerFromSAS = storage.GetContainerReferenceFromSASURI + putAppendBlob = func(blob *storage.Blob) error { return blob.PutAppendBlob(nil) } +) + type blobDownload struct { accountName, accountKey string blob blobutil.AzureBlobRef @@ -36,16 +47,16 @@ func (b blobDownload) GetRequest() (*http.Request, error) { if req != nil { req.Header.Set(xMsClientRequestIdHeaderName, uuid.New().String()) } - return req, err + return req, errorutil.EnsureClarification(errorutil.FileDownload_failedToCreateBlobRequest, err) } // getURL returns publicly downloadable URL of the Azure Blob // by generating a URL with a temporary Shared Access Signature. func (b blobDownload) getURL() (string, error) { - client, err := storage.NewClient(b.accountName, b.accountKey, + client, err := newStorageClient(b.accountName, b.accountKey, b.blob.StorageBase, storage.DefaultAPIVersion, true) if err != nil { - return "", errors.Wrap(err, "failed to initialize azure storage client") + return "", errorutil.EnsureClarification(errorutil.Storage_failedToInitializeClient, errors.Wrap(err, "failed to initialize azure storage client")) } // get read-only @@ -58,10 +69,10 @@ func (b blobDownload) getURL() (string, error) { Expiry: time.Now().UTC().Add(blobSASDuration), }, } - sasURL, err := blob.GetSASURI(options) + sasURL, err := blobSASURI(blob, options) if err != nil { - return "", errors.Wrap(err, "failed to generate SAS key for blob") + return "", errorutil.EnsureClarification(errorutil.Storage_failedToGenerateSas, errors.Wrap(err, "failed to generate SAS key for blob")) } return sasURL, nil } @@ -78,39 +89,38 @@ func GetSASBlob(blobURI, blobSas, targetDir string) (string, error) { loggableBlobUri := GetUriForLogging(blobURI) - resp, err := http.Get(blobFullURL) + resp, err := sasHTTPGet(blobFullURL) if err != nil { - return "", errors.Wrapf(err, "Failed to download file: %q", loggableBlobUri) + return "", errorutil.EnsureClarification(errorutil.FileDownload_sasRequestFailed, errors.Wrapf(err, "Failed to download file: %q", loggableBlobUri)) } defer resp.Body.Close() // Ensure the response body is closed after we're done // Check if the HTTP status code indicates success (e.g., 200 OK) if resp.StatusCode != http.StatusOK { - return "", errors.Wrapf(err, "Failed to download file: %q, Http status code: %s", loggableBlobUri, resp.Status) + if resp.StatusCode >= http.StatusInternalServerError { + return "", errorutil.EnsureClarification(errorutil.Storage_sasDownloadInternalServerError, fmt.Errorf("Failed to download file: %q, Http status code: %s", loggableBlobUri, resp.Status)) + } + return "", errorutil.EnsureClarification(errorutil.FileDownload_sasDownloadErrorStatus, fmt.Errorf("Failed to download file: %q, Http status code: %s", loggableBlobUri, resp.Status)) } blobParsedurl, err := url.Parse(blobURI) if err != nil { - return "", errors.Wrapf(err, "unable to parse URL: %q", loggableBlobUri) + return "", errorutil.EnsureClarification(errorutil.CustomerInput_invalidSasBlobUrl, errors.Wrapf(err, "unable to parse URL: %q", loggableBlobUri)) } // Extract container name from Path of the url https:/// For ex. Path = "containerName/dir1/dir2/file.sh" trimmedPath := strings.Trim(blobParsedurl.Path, "/") splitStrings := strings.Split(trimmedPath, "/") - if len(splitStrings) == 0 { - return "", fmt.Errorf("cannot extract file name from URL: %q. Trimmed path was empty", loggableBlobUri); - } - fileName := splitStrings[len(splitStrings)-1] if fileName == "" { - return "", fmt.Errorf("cannot extract file name from URL: %q", loggableBlobUri) + return "", errorutil.EnsureClarification(errorutil.CustomerInput_missingSasBlobFileName, fmt.Errorf("cannot extract file name from URL: %q", loggableBlobUri)) } // Create the local file scriptFilePath := filepath.Join(targetDir, fileName) const mode = 0500 // scripts should have execute permissions - outFile, err := os.OpenFile(scriptFilePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, mode) + outFile, err := openDownloadFile(scriptFilePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, mode) if err != nil { - return "", errors.Wrapf(err, "Failed to open file '%s' for writing: ", scriptFilePath) + return "", errorutil.EnsureClarification(errorutil.FileDownload_failedToOpenSasDestination, errors.Wrapf(err, "Failed to open file '%s' for writing: ", scriptFilePath)) } defer outFile.Close() // Ensure the file is closed after we're done @@ -119,7 +129,7 @@ func GetSASBlob(blobURI, blobSas, targetDir string) (string, error) { // loading the entire file into memory. _, err = io.Copy(outFile, resp.Body) if err != nil { - return "", errors.Wrapf(err, "Failed to copy data to file '%s'", scriptFilePath) + return "", errorutil.EnsureClarification(errorutil.FileDownload_failedToWriteSasDestination, errors.Wrapf(err, "Failed to copy data to file '%s'", scriptFilePath)) } return scriptFilePath, nil @@ -130,23 +140,26 @@ func CreateOrReplaceAppendBlob(blobURI, blobSas string) (*storage.Blob, error) { bloburl, err := url.Parse(blobURI + blobSas) if err != nil { - return nil, err + return nil, errorutil.EnsureClarification(errorutil.CustomerInput_invalidAppendBlobSasUrl, err) } - containerRef, err := storage.GetContainerReferenceFromSASURI(*bloburl) + containerRef, err := containerFromSAS(*bloburl) if err != nil { - return nil, err + return nil, errorutil.EnsureClarification(errorutil.Storage_failedToGetSasContainer, err) } fileName, blobPathError := getBlobPathAfterContainerName(blobURI, containerRef.Name) if fileName == "" { + if blobPathError == nil { + blobPathError = errorutil.EnsureClarification(errorutil.CustomerInput_emptyAppendBlobPath, errors.New("blob path is empty")) + } return nil, errors.Wrapf(blobPathError, "cannot extract blob path name from URL: %q", GetUriForLogging(blobURI)) } blobref := containerRef.GetBlobReference(fileName) - err = blobref.PutAppendBlob(nil) // Create the append blob + err = putAppendBlob(blobref) // Create the append blob if err != nil { - return nil, err + return nil, errorutil.EnsureClarification(errorutil.Storage_failedToCreateAppendBlob, err) } return blobref, nil @@ -158,7 +171,7 @@ func CreateOrReplaceAppendBlob(blobURI, blobSas string) (*storage.Blob, error) { func getBlobPathAfterContainerName(blobURI string, containerName string) (string, error) { blobURL, err := url.Parse(blobURI) if err != nil { - return "", err + return "", errorutil.EnsureClarification(errorutil.CustomerInput_invalidBlobPathUrl, err) } containerNameSearchString := containerName + "/" @@ -167,6 +180,6 @@ func getBlobPathAfterContainerName(blobURI string, containerName string) (string if index >= 0 { return blobPathWithoutHost[index+len(containerNameSearchString):], nil } else { - return "", errors.New(fmt.Sprintf("Unable to find '%s' in blobURI '%s'. Unable to get blob path suffix after container name.", containerNameSearchString, GetUriForLogging(blobURI))) + return "", errorutil.EnsureClarification(errorutil.CustomerInput_missingBlobContainerPath, fmt.Errorf("Unable to find '%s' in blobURI '%s'. Unable to get blob path suffix after container name.", containerNameSearchString, GetUriForLogging(blobURI))) } } diff --git a/pkg/download/blob_test.go b/pkg/download/blob_test.go index 4f23143..e362f96 100644 --- a/pkg/download/blob_test.go +++ b/pkg/download/blob_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/run-command-handler-linux/pkg/blobutil" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -27,17 +28,20 @@ func Test_blobDownload_validateInputs(t *testing.T) { _, err := NewBlobDownload("", "", blobutil.AzureBlobRef{}).GetRequest() require.NotNil(t, err) + requireDownloadCode(t, errorutil.Storage_failedToInitializeClient, err) errorMessage := err.Error() require.Contains(t, errorMessage, "failed to initialize azure storage client") require.Contains(t, errorMessage, "azure: account name is not valid") _, err = NewBlobDownload("account", "", blobutil.AzureBlobRef{}).GetRequest() require.NotNil(t, err) + requireDownloadCode(t, errorutil.Storage_failedToInitializeClient, err) require.Contains(t, err.Error(), "failed to initialize azure storage client") require.Contains(t, err.Error(), "account key required") _, err = NewBlobDownload("account", "Zm9vCg==", blobutil.AzureBlobRef{}).GetRequest() require.NotNil(t, err) + requireDownloadCode(t, errorutil.Storage_failedToInitializeClient, err) require.Contains(t, err.Error(), "failed to initialize azure storage client") _, err = NewBlobDownload("account", "Zm9vCg==", blobutil.AzureBlobRef{ @@ -92,6 +96,7 @@ func Test_blobDownload_fails_badCreds(t *testing.T) { status, _, err := Download(testctx, d) require.NotNil(t, err) + requireDownloadCode(t, errorutil.FileDownload_accessDenied, err) require.Contains(t, err.Error(), "Please verify the machine has network connectivity") require.Contains(t, err.Error(), "403") require.Equal(t, status, http.StatusForbidden) @@ -120,12 +125,18 @@ func Test_blobDownload_fails_badRequest(t *testing.T) { status, _, err := Download(testctx, d) require.NotNil(t, err) + requireDownloadCode(t, errorutil.FileDownload_badRequest, err) require.Contains(t, err.Error(), "parts of the request were incorrectly formatted, missing, and/or invalid") require.Contains(t, err.Error(), "400") require.Equal(t, status, http.StatusBadRequest) } func Test_blobDownload_fails_urlNotFound(t *testing.T) { + old := MakeHttpRequest + t.Cleanup(func() { MakeHttpRequest = old }) + MakeHttpRequest = func(*http.Request) (*http.Response, error) { + return nil, errorutil.EnsureClarification(errorutil.FileDownload_networkingError, fmt.Errorf("host not found")) + } d := NewBlobDownload("accountname", "Zm9vCg==", blobutil.AzureBlobRef{ StorageBase: ".example.com", Blob: "fooBlob.txt", @@ -134,6 +145,7 @@ func Test_blobDownload_fails_urlNotFound(t *testing.T) { _, _, err := Download(testctx, d) require.NotNil(t, err) + requireDownloadCode(t, errorutil.FileDownload_networkingError, err) require.Contains(t, err.Error(), "http request failed:") } @@ -207,70 +219,70 @@ func Test_GetSASBlob_nestedDirectories_actualBlob(t *testing.T) { // Then, run the following commands prior to this test // export AZURE_STORAGE_ACCOUNT="" // export AZURE_STORAGE_ACCESS_KEY="" - acct := os.Getenv("AZURE_STORAGE_ACCOUNT") - key := os.Getenv("AZURE_STORAGE_ACCESS_KEY") - if acct == "" || key == "" { - t.Skipf("Skipping: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not specified to run this test") - } - base := storage.DefaultBaseURL - - // Create a blob in a nested directory structure - client, err := storage.NewClient(acct, key, base, storage.DefaultAPIVersion, true) - require.Nil(t, err) - blobStorageClient := client.GetBlobService() - - var ( - content = []byte("test script content\n") - name = "dir1/dir2/script.sh" // nested path - container = fmt.Sprintf("run-command-test-%d", rand.New(rand.NewSource(time.Now().UnixNano())).Int63()) - ) - - containerReference := blobStorageClient.GetContainerReference(container) - _, err = containerReference.DeleteIfExists(nil) - require.Nil(t, err) - _, err = containerReference.CreateIfNotExists(&storage.CreateContainerOptions{Access: storage.ContainerAccessTypePrivate}) - require.Nil(t, err) - defer containerReference.Delete(nil) - - blobReference := containerReference.GetBlobReference(name) - require.Nil(t, blobReference.PutAppendBlob(nil)) - require.Nil(t, blobReference.AppendBlock(content, nil)) - - // Generate SAS URL for the blob - d := NewBlobDownload(acct, key, blobutil.AzureBlobRef{ - Container: container, - Blob: name, - StorageBase: base, - }) - v, ok := d.(blobDownload) - require.True(t, ok) - sasURL, err := v.getURL() - require.Nil(t, err) - - // Parse the SAS URL to separate URI from SAS token - blobURI := fmt.Sprintf("https://%s.blob.%s/%s/%s", acct, base, container, name) - sasToken := sasURL[len(blobURI):] // Everything after the base URI is the SAS token - - // Download the blob using GetSASBlob - tmpDir, err := ioutil.TempDir("", "nested-test-") - require.Nil(t, err) - defer os.RemoveAll(tmpDir) - - scriptFilePath, err := GetSASBlob(blobURI, sasToken, tmpDir) - require.Nil(t, err) - - // Verify that the file was downloaded with correct content - result, err := ioutil.ReadFile(scriptFilePath) - require.Nil(t, err) - require.EqualValues(t, content, result, "downloaded content should match") - - // Verify that the file was created - require.Contains(t, scriptFilePath, tmpDir, "file path should be in temp directory") - - // Verify the file exists and is not a directory - fileInfo, err := os.Stat(scriptFilePath) - require.Nil(t, err) - require.False(t, fileInfo.IsDir(), "should be a file, not a directory") + acct := os.Getenv("AZURE_STORAGE_ACCOUNT") + key := os.Getenv("AZURE_STORAGE_ACCESS_KEY") + if acct == "" || key == "" { + t.Skipf("Skipping: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not specified to run this test") + } + base := storage.DefaultBaseURL + + // Create a blob in a nested directory structure + client, err := storage.NewClient(acct, key, base, storage.DefaultAPIVersion, true) + require.Nil(t, err) + blobStorageClient := client.GetBlobService() + + var ( + content = []byte("test script content\n") + name = "dir1/dir2/script.sh" // nested path + container = fmt.Sprintf("run-command-test-%d", rand.New(rand.NewSource(time.Now().UnixNano())).Int63()) + ) + + containerReference := blobStorageClient.GetContainerReference(container) + _, err = containerReference.DeleteIfExists(nil) + require.Nil(t, err) + _, err = containerReference.CreateIfNotExists(&storage.CreateContainerOptions{Access: storage.ContainerAccessTypePrivate}) + require.Nil(t, err) + defer containerReference.Delete(nil) + + blobReference := containerReference.GetBlobReference(name) + require.Nil(t, blobReference.PutAppendBlob(nil)) + require.Nil(t, blobReference.AppendBlock(content, nil)) + + // Generate SAS URL for the blob + d := NewBlobDownload(acct, key, blobutil.AzureBlobRef{ + Container: container, + Blob: name, + StorageBase: base, + }) + v, ok := d.(blobDownload) + require.True(t, ok) + sasURL, err := v.getURL() + require.Nil(t, err) + + // Parse the SAS URL to separate URI from SAS token + blobURI := fmt.Sprintf("https://%s.blob.%s/%s/%s", acct, base, container, name) + sasToken := sasURL[len(blobURI):] // Everything after the base URI is the SAS token + + // Download the blob using GetSASBlob + tmpDir, err := ioutil.TempDir("", "nested-test-") + require.Nil(t, err) + defer os.RemoveAll(tmpDir) + + scriptFilePath, err := GetSASBlob(blobURI, sasToken, tmpDir) + require.Nil(t, err) + + // Verify that the file was downloaded with correct content + result, err := ioutil.ReadFile(scriptFilePath) + require.Nil(t, err) + require.EqualValues(t, content, result, "downloaded content should match") + + // Verify that the file was created + require.Contains(t, scriptFilePath, tmpDir, "file path should be in temp directory") + + // Verify the file exists and is not a directory + fileInfo, err := os.Stat(scriptFilePath) + require.Nil(t, err) + require.False(t, fileInfo.IsDir(), "should be a file, not a directory") } func Test_blobAppend_actualBlob(t *testing.T) { diff --git a/pkg/download/blobwithmsitoken.go b/pkg/download/blobwithmsitoken.go index ede13af..82ba43d 100644 --- a/pkg/download/blobwithmsitoken.go +++ b/pkg/download/blobwithmsitoken.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/azure-extension-foundation/httputil" "github.com/Azure/azure-extension-foundation/msi" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -30,6 +31,17 @@ type blobWithMsiToken struct { type MsiProvider func() (msi.Msi, error) +type managedIdentityProvider interface { + GetMsiForResource(string) (msi.Msi, error) + GetMsiUsingClientId(string, string) (msi.Msi, error) + GetMsiUsingObjectId(string, string) (msi.Msi, error) +} + +var newMSIProvider = func() managedIdentityProvider { + provider := msi.NewMsiProvider(httputil.NewSecureHttpClient(httputil.DefaultRetryBehavior)) + return &provider +} + type MsiDownloader interface { GetMsiProvider(blobUri string) MsiProvider GetMsiProviderByClientId(blobUri, clientId string) MsiProvider @@ -45,15 +57,15 @@ var MockReturnErrorForMockMsiDownloader = false // Used only for test func (self *blobWithMsiToken) GetRequest() (*http.Request, error) { msi, err := self.msiProvider() if err != nil { - return nil, err + return nil, errorutil.EnsureClarification(errorutil.Msi_failedToGetDownloadToken, err) } if msi.AccessToken == "" { - return nil, errors.New("MSI token is empty") + return nil, errorutil.EnsureClarification(errorutil.Msi_emptyToken, errors.New("MSI token is empty")) } request, err := http.NewRequest(http.MethodGet, self.url, nil) if err != nil { - return nil, err + return nil, errorutil.EnsureClarification(errorutil.CustomerInput_invalidMsiDownloadUrl, err) } request.Header.Set(xMsClientRequestIdHeaderName, uuid.New().String()) @@ -70,13 +82,13 @@ func NewBlobWithMsiDownload(url string, msiProvider MsiProvider) Downloader { // Uses system identity to get Msi token func (prodMsiDownloader ProdMsiDownloader) GetMsiProvider(blobUri string) MsiProvider { - msiProvider := msi.NewMsiProvider(httputil.NewSecureHttpClient(httputil.DefaultRetryBehavior)) + msiProvider := newMSIProvider() return func() (msi.Msi, error) { msi, err := msiProvider.GetMsiForResource(GetResourceNameFromBlobUri(blobUri)) if err != nil { - return msi, errors.Wrapf(err, "Unable to get managed identity. "+ + return msi, errorutil.EnsureClarification(errorutil.Msi_failedToGetSystemIdentity, errors.Wrapf(err, "Unable to get managed identity. "+ "Please make sure that system assigned managed identity is enabled on the VM "+ - "or user assigned identity is added to the system.") + "or user assigned identity is added to the system.")) } return msi, nil } @@ -90,7 +102,7 @@ func (mockMsiDownloader MockMsiDownloader) GetMsiProvider(blobUri string) MsiPro Resource: "Msi by System Identity for blob " + blobUri, } if MockReturnErrorForMockMsiDownloader { - return mockMsi, errors.New("Error getting msi") + return mockMsi, errorutil.EnsureClarification(errorutil.Imds_mockSystemIdentityError, errors.New("Error getting msi")) } else { return mockMsi, nil } @@ -100,12 +112,12 @@ func (mockMsiDownloader MockMsiDownloader) GetMsiProvider(blobUri string) MsiPro // Get Msi token by clientId func (prodMsiDownloader ProdMsiDownloader) GetMsiProviderByClientId(blobUri, clientId string) MsiProvider { - msiProvider := msi.NewMsiProvider(httputil.NewSecureHttpClient(httputil.DefaultRetryBehavior)) + msiProvider := newMSIProvider() return func() (msi.Msi, error) { msi, err := msiProvider.GetMsiUsingClientId(clientId, GetResourceNameFromBlobUri(blobUri)) if err != nil { - return msi, errors.Wrapf(err, "Unable to get managed identity with client id %s. "+ - "Please make sure that the user assigned managed identity is added to the VM ", clientId) + return msi, errorutil.EnsureClarification(errorutil.Msi_failedToGetClientIdentity, errors.Wrapf(err, "Unable to get managed identity with client id %s. "+ + "Please make sure that the user assigned managed identity is added to the VM ", clientId)) } return msi, nil } @@ -119,7 +131,7 @@ func (mockMsiDownloader MockMsiDownloader) GetMsiProviderByClientId(blobUri stri Resource: "Msi by clientId for blob " + blobUri, } if MockReturnErrorForMockMsiDownloader { - return mockMsi, errors.New("Error getting msi") + return mockMsi, errorutil.EnsureClarification(errorutil.Imds_mockClientIdentityError, errors.New("Error getting msi")) } else { return mockMsi, nil } @@ -128,12 +140,12 @@ func (mockMsiDownloader MockMsiDownloader) GetMsiProviderByClientId(blobUri stri // Get Msi token by objectId func (prodMsiDownloader ProdMsiDownloader) GetMsiProviderByObjectId(blobUri, objectId string) MsiProvider { - msiProvider := msi.NewMsiProvider(httputil.NewSecureHttpClient(httputil.DefaultRetryBehavior)) + msiProvider := newMSIProvider() return func() (msi.Msi, error) { msi, err := msiProvider.GetMsiUsingObjectId(objectId, GetResourceNameFromBlobUri(blobUri)) if err != nil { - return msi, errors.Wrapf(err, "Unable to get managed identity with object id %s. "+ - "Please make sure that the user assigned managed identity is added to the VM ", objectId) + return msi, errorutil.EnsureClarification(errorutil.Msi_failedToGetObjectIdentity, errors.Wrapf(err, "Unable to get managed identity with object id %s. "+ + "Please make sure that the user assigned managed identity is added to the VM ", objectId)) } return msi, nil } @@ -147,7 +159,7 @@ func (mockMsiDownloader MockMsiDownloader) GetMsiProviderByObjectId(blobUri, obj Resource: "Msi by objectId for blob " + blobUri, } if MockReturnErrorForMockMsiDownloader { - return mockMsi, errors.New("Error getting msi") + return mockMsi, errorutil.EnsureClarification(errorutil.Imds_mockObjectIdentityError, errors.New("Error getting msi")) } else { return mockMsi, nil } diff --git a/pkg/download/clarification_test.go b/pkg/download/clarification_test.go new file mode 100644 index 0000000..1824495 --- /dev/null +++ b/pkg/download/clarification_test.go @@ -0,0 +1,486 @@ +package download + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/Azure/azure-extension-foundation/msi" + "github.com/Azure/azure-extension-platform/vmextension" + "github.com/Azure/azure-sdk-for-go/storage" + "github.com/Azure/run-command-handler-linux/pkg/blobutil" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type clarificationDownloader struct{ err error } + +func (d clarificationDownloader) GetRequest() (*http.Request, error) { return nil, d.err } + +type clarificationTransport func(*http.Request) (*http.Response, error) + +func (f clarificationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type clarificationReader struct { + err error + closed bool +} + +func (r *clarificationReader) Read([]byte) (int, error) { return 0, r.err } +func (r *clarificationReader) Close() error { r.closed = true; return nil } + +type clarificationWriter struct { + buffer bytes.Buffer + err error + closed bool +} + +func (w *clarificationWriter) Write(b []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + return w.buffer.Write(b) +} +func (w *clarificationWriter) String() string { return w.buffer.String() } +func (w *clarificationWriter) Close() error { w.closed = true; return nil } + +func requireDownloadCode(t *testing.T, expected int, err error) { + t.Helper() + require.Error(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "missing clarification: %v", err) + require.Equal(t, expected, code) +} + +func clarificationLog() *log.Context { return log.NewContext(log.NewNopLogger()) } + +func stubDownloadResponse(t *testing.T, status int, body io.ReadCloser) { + t.Helper() + old := MakeHttpRequest + t.Cleanup(func() { MakeHttpRequest = old }) + MakeHttpRequest = func(*http.Request) (*http.Response, error) { + header := make(http.Header) + header.Set(xMsServiceRequestIdHeaderName, "mock-request") + return &http.Response{StatusCode: status, Status: fmt.Sprint(status), Header: header, Body: body}, nil + } +} + +func stubDownloadFile(t *testing.T, writer io.WriteCloser, err error) { + t.Helper() + old := openDownloadFile + t.Cleanup(func() { openDownloadFile = old }) + openDownloadFile = func(string, int, os.FileMode) (io.WriteCloser, error) { return writer, err } +} + +func TestRequestAndTransportClarifications(t *testing.T) { + _, err := NewURLDownload("https://%").GetRequest() + requireDownloadCode(t, errorutil.FileDownload_invalidRequestUrl, err) + cause := errors.New("injected transport failure") + old := httpClient + t.Cleanup(func() { httpClient = old }) + httpClient = &http.Client{Transport: clarificationTransport(func(*http.Request) (*http.Response, error) { + return nil, cause + })} + request, err := http.NewRequest(http.MethodGet, "https://example.invalid", nil) + require.NoError(t, err) + _, err = HttpClientDo(request) + requireDownloadCode(t, errorutil.FileDownload_networkingError, err) + require.True(t, errors.Is(err, cause)) + timeout := &net.DNSError{IsTimeout: true} + httpClient.Transport = clarificationTransport(func(*http.Request) (*http.Response, error) { + return nil, timeout + }) + _, err = HttpClientDo(request) + requireDownloadCode(t, errorutil.FileDownload_exceededTimeout, err) + require.ErrorIs(t, err, timeout) + _, _, err = Download(clarificationLog(), clarificationDownloader{cause}) + requireDownloadCode(t, errorutil.Internal_failedToCreateDownloadRequest, err) + require.True(t, errors.Is(err, cause)) +} + +func TestDownloadNetworkAndStatusClarifications(t *testing.T) { + old := MakeHttpRequest + t.Cleanup(func() { MakeHttpRequest = old }) + cause := errors.New(`Get "https://example.invalid/script?sig=secret": failed`) + MakeHttpRequest = func(*http.Request) (*http.Response, error) { return nil, cause } + _, _, err := Download(clarificationLog(), NewURLDownload("https://example.invalid/script")) + requireDownloadCode(t, errorutil.FileDownload_httpRequestFailed, err) + require.NotContains(t, err.Error(), "secret") + require.True(t, errors.Is(err, cause)) + for _, tc := range []struct { + status int + direct, managed errorutil.ErrorClarificationCode + }{ + {400, errorutil.FileDownload_badRequest, errorutil.Msi_doesNotHaveRightPermissions}, + {401, errorutil.FileDownload_accessDenied, errorutil.Msi_doesNotHaveRightPermissions}, + {403, errorutil.FileDownload_accessDenied, errorutil.Msi_doesNotHaveRightPermissions}, + {404, errorutil.FileDownload_doesNotExist, errorutil.Msi_notFound}, + {409, errorutil.FileDownload_genericError, errorutil.Msi_doesNotHaveRightPermissions}, + {429, errorutil.FileDownload_genericError, errorutil.Msi_GenericRetrievalError}, + {500, errorutil.Storage_internalServerError, errorutil.Imds_internalMsiError}, + {502, errorutil.FileDownload_internalServerError, errorutil.Imds_downloadInternalServerError}, + {503, errorutil.FileDownload_internalServerError, errorutil.Imds_downloadInternalServerError}, + } { + for _, identity := range []bool{false, true} { + body := &clarificationReader{} + stubDownloadResponse(t, tc.status, body) + d := NewURLDownload("https://example.invalid/script") + if identity { + d = NewBlobWithMsiDownload("https://example.invalid/script", func() (msi.Msi, error) { + return msi.Msi{AccessToken: "fake"}, nil + }) + } + _, _, err := Download(clarificationLog(), d) + expected := tc.direct + if identity { + expected = tc.managed + } + requireDownloadCode(t, expected, err) + require.Contains(t, err.Error(), "Service request ID: mock-request") + require.True(t, body.closed) + } + } + stubDownloadResponse(t, http.StatusForbidden, nil) + status, body, err := Download(clarificationLog(), NewURLDownload("https://example.invalid/script")) + requireDownloadCode(t, errorutil.FileDownload_accessDenied, err) + require.Equal(t, http.StatusForbidden, status) + require.Nil(t, body) +} + +func TestRetryAndSaveClarifications(t *testing.T) { + _, err := WithRetries(clarificationLog(), nil, func(time.Duration) {}) + requireDownloadCode(t, errorutil.Internal_noDownloaders, err) + first := vmextension.NewErrorWithClarificationPtr(1997, errors.New("first failure")) + last := vmextension.NewErrorWithClarificationPtr(1998, errors.New("last failure")) + _, err = WithRetries(clarificationLog(), []Downloader{ + clarificationDownloader{first}, clarificationDownloader{last}, + }, func(time.Duration) {}) + requireDownloadCode(t, 1998, err) + require.True(t, errors.Is(err, last)) + require.True(t, errors.Is(err, first)) + require.Contains(t, err.Error(), "first failure") + _, err = WithRetries(clarificationLog(), []Downloader{ + clarificationDownloader{first}, clarificationDownloader{*last}, + }, func(time.Duration) {}) + requireDownloadCode(t, 1998, err) + require.True(t, errors.Is(err, *last)) + require.True(t, errors.Is(err, first)) + // errorutil.Internal_failedToAggregateDownloadAttempts is a defensive retry-wrapper fallback: Download always classifies its + // error first, so assigning it at runtime would overwrite a lower-level code. + + cause := errors.New("injected file failure") + stubDownloadFile(t, nil, cause) + _, err = SaveTo(clarificationLog(), nil, "script", 0500) + requireDownloadCode(t, errorutil.FileDownload_failedToOpenDestination, err) + require.True(t, errors.Is(err, cause)) + + writer := &clarificationWriter{} + stubDownloadFile(t, writer, nil) + _, err = SaveTo(clarificationLog(), []Downloader{clarificationDownloader{last}}, "script", 0500) + requireDownloadCode(t, 1998, err) + require.True(t, writer.closed) + // errorutil.Internal_failedToDownloadToFile is a defensive SaveTo wrapper fallback: WithRetries always returns + // a classified error (including the empty-downloader case), preserved here. + + reader := &clarificationReader{err: cause} + stubDownloadResponse(t, http.StatusOK, reader) + writer = &clarificationWriter{} + stubDownloadFile(t, writer, nil) + _, err = SaveTo(clarificationLog(), []Downloader{NewURLDownload("https://example.invalid")}, "script", 0500) + requireDownloadCode(t, errorutil.FileDownload_failedToWriteDestination, err) + require.True(t, errors.Is(err, cause)) + require.True(t, reader.closed) + require.True(t, writer.closed) + + stubDownloadResponse(t, http.StatusOK, io.NopCloser(strings.NewReader("script"))) + writer = &clarificationWriter{err: cause} + stubDownloadFile(t, writer, nil) + _, err = SaveTo(clarificationLog(), []Downloader{NewURLDownload("https://example.invalid")}, "script", 0500) + requireDownloadCode(t, errorutil.FileDownload_failedToWriteDestination, err) + require.True(t, errors.Is(err, cause)) + require.True(t, writer.closed) +} + +func TestBlobRequestClarifications(t *testing.T) { + blob := blobutil.AzureBlobRef{StorageBase: "core.windows.net", Container: "container", Blob: "script"} + _, err := NewBlobDownload("account", "invalid base64!", blob).GetRequest() + requireDownloadCode(t, errorutil.Storage_failedToInitializeClient, err) + old := blobSASURI + t.Cleanup(func() { blobSASURI = old }) + cause := errors.New("injected signing failure") + blobSASURI = func(*storage.Blob, storage.BlobSASOptions) (string, error) { return "", cause } + _, err = NewBlobDownload("account", "a2V5", blob).GetRequest() + requireDownloadCode(t, errorutil.Storage_failedToGenerateSas, err) + require.True(t, errors.Is(err, cause)) + blobSASURI = func(*storage.Blob, storage.BlobSASOptions) (string, error) { return "https://%", nil } + _, err = NewBlobDownload("account", "a2V5", blob).GetRequest() + requireDownloadCode(t, errorutil.FileDownload_failedToCreateBlobRequest, err) +} + +func stubSASResponse(t *testing.T, status int, body io.ReadCloser, err error) { + t.Helper() + old := sasHTTPGet + t.Cleanup(func() { sasHTTPGet = old }) + sasHTTPGet = func(string) (*http.Response, error) { + if err != nil { + return nil, err + } + return &http.Response{StatusCode: status, Status: fmt.Sprint(status), Body: body}, nil + } +} + +func TestSASDownloadClarifications(t *testing.T) { + cause := errors.New("injected network failure") + stubSASResponse(t, 0, nil, cause) + _, err := GetSASBlob("https://example.invalid/container/script", "", "unused") + requireDownloadCode(t, errorutil.FileDownload_sasRequestFailed, err) + require.True(t, errors.Is(err, cause)) + for _, status := range []int{404, 500} { + body := &clarificationReader{} + stubSASResponse(t, status, body, nil) + _, err = GetSASBlob("https://example.invalid/container/script", "", "unused") + expected := errorutil.FileDownload_sasDownloadErrorStatus + if status == 500 { + expected = errorutil.Storage_sasDownloadInternalServerError + } + requireDownloadCode(t, expected, err) + require.True(t, body.closed) + } + for _, tc := range []struct { + uri string + code int + }{ + {"https://%", errorutil.CustomerInput_invalidSasBlobUrl}, + {"https://example.invalid/", errorutil.CustomerInput_missingSasBlobFileName}, + } { + stubSASResponse(t, 200, io.NopCloser(strings.NewReader("script")), nil) + _, err = GetSASBlob(tc.uri, "", "unused") + requireDownloadCode(t, tc.code, err) + } + stubSASResponse(t, 200, io.NopCloser(strings.NewReader("script")), nil) + stubDownloadFile(t, nil, cause) + _, err = GetSASBlob("https://example.invalid/container/script", "", "unused") + requireDownloadCode(t, errorutil.FileDownload_failedToOpenSasDestination, err) + require.True(t, errors.Is(err, cause)) + for _, reading := range []bool{false, true} { + writer := &clarificationWriter{} + var reader io.ReadCloser = io.NopCloser(strings.NewReader("script")) + if reading { + reader = &clarificationReader{err: cause} + } else { + writer.err = cause + } + stubSASResponse(t, 200, reader, nil) + stubDownloadFile(t, writer, nil) + _, err = GetSASBlob("https://example.invalid/container/script", "", "unused") + requireDownloadCode(t, errorutil.FileDownload_failedToWriteSasDestination, err) + require.True(t, errors.Is(err, cause)) + require.True(t, writer.closed) + } +} + +func TestAppendBlobClarifications(t *testing.T) { + _, err := CreateOrReplaceAppendBlob("https://%", "") + requireDownloadCode(t, errorutil.CustomerInput_invalidAppendBlobSasUrl, err) + originalContainer := containerFromSAS + originalPut := putAppendBlob + t.Cleanup(func() { containerFromSAS = originalContainer; putAppendBlob = originalPut }) + cause := errors.New("injected container failure") + containerFromSAS = func(url.URL) (*storage.Container, error) { return nil, cause } + _, err = CreateOrReplaceAppendBlob("https://example.invalid/container/script", "") + requireDownloadCode(t, errorutil.Storage_failedToGetSasContainer, err) + require.True(t, errors.Is(err, cause)) + client, err := storage.NewClient("account", "a2V5", "core.windows.net", storage.DefaultAPIVersion, true) + require.NoError(t, err) + service := client.GetBlobService() + container := service.GetContainerReference("container") + containerFromSAS = func(url.URL) (*storage.Container, error) { return container, nil } + _, err = CreateOrReplaceAppendBlob("https://example.invalid/container/", "") + requireDownloadCode(t, errorutil.CustomerInput_emptyAppendBlobPath, err) + putAppendBlob = func(*storage.Blob) error { return cause } + _, err = CreateOrReplaceAppendBlob("https://example.invalid/container/script", "") + requireDownloadCode(t, errorutil.Storage_failedToCreateAppendBlob, err) + require.True(t, errors.Is(err, cause)) + _, err = getBlobPathAfterContainerName("https://%", "container") + requireDownloadCode(t, errorutil.CustomerInput_invalidBlobPathUrl, err) + _, err = getBlobPathAfterContainerName("https://example.invalid/other/script", "container") + requireDownloadCode(t, errorutil.CustomerInput_missingBlobContainerPath, err) +} + +type clarificationMSI struct{ err error } + +func (p clarificationMSI) GetMsiForResource(string) (msi.Msi, error) { + return msi.Msi{}, p.err +} +func (p clarificationMSI) GetMsiUsingClientId(string, string) (msi.Msi, error) { + return msi.Msi{}, p.err +} +func (p clarificationMSI) GetMsiUsingObjectId(string, string) (msi.Msi, error) { + return msi.Msi{}, p.err +} + +func TestMSIClarifications(t *testing.T) { + cause := errors.New("injected identity failure") + _, err := NewBlobWithMsiDownload("https://example.invalid", func() (msi.Msi, error) { + return msi.Msi{}, cause + }).GetRequest() + requireDownloadCode(t, errorutil.Msi_failedToGetDownloadToken, err) + require.True(t, errors.Is(err, cause)) + _, err = NewBlobWithMsiDownload("https://example.invalid", func() (msi.Msi, error) { + return msi.Msi{}, nil + }).GetRequest() + requireDownloadCode(t, errorutil.Msi_emptyToken, err) + _, err = NewBlobWithMsiDownload("https://%", func() (msi.Msi, error) { + return msi.Msi{AccessToken: "fake"}, nil + }).GetRequest() + requireDownloadCode(t, errorutil.CustomerInput_invalidMsiDownloadUrl, err) + + old := newMSIProvider + t.Cleanup(func() { newMSIProvider = old }) + newMSIProvider = func() managedIdentityProvider { return clarificationMSI{cause} } + prod := ProdMsiDownloader{} + for _, tc := range []struct { + provider MsiProvider + code int + }{ + {prod.GetMsiProvider("blob"), errorutil.Msi_failedToGetSystemIdentity}, + {prod.GetMsiProviderByClientId("blob", "client"), errorutil.Msi_failedToGetClientIdentity}, + {prod.GetMsiProviderByObjectId("blob", "object"), errorutil.Msi_failedToGetObjectIdentity}, + } { + _, err = tc.provider() + requireDownloadCode(t, tc.code, err) + require.True(t, errors.Is(err, cause)) + } + originalMock := MockReturnErrorForMockMsiDownloader + t.Cleanup(func() { MockReturnErrorForMockMsiDownloader = originalMock }) + MockReturnErrorForMockMsiDownloader = true + mock := MockMsiDownloader{} + for _, tc := range []struct { + provider MsiProvider + code int + }{ + {mock.GetMsiProvider("blob"), errorutil.Imds_mockSystemIdentityError}, + {mock.GetMsiProviderByClientId("blob", "client"), errorutil.Imds_mockClientIdentityError}, + {mock.GetMsiProviderByObjectId("blob", "object"), errorutil.Imds_mockObjectIdentityError}, + } { + _, err = tc.provider() + requireDownloadCode(t, tc.code, err) + } +} + +func TestDownloadBoundariesPreserveLowerClarifications(t *testing.T) { + pointer := vmextension.NewErrorWithClarificationPtr(1998, errors.New("injected classified failure")) + for _, cause := range []error{pointer, *pointer} { + t.Run(fmt.Sprintf("%T", cause), func(t *testing.T) { + oldHTTP, oldClient := MakeHttpRequest, httpClient + oldSAS, oldSigner, oldContainer, oldPut := sasHTTPGet, blobSASURI, containerFromSAS, putAppendBlob + oldStorage, oldMSI := newStorageClient, newMSIProvider + t.Cleanup(func() { + MakeHttpRequest, httpClient = oldHTTP, oldClient + sasHTTPGet, blobSASURI, containerFromSAS, putAppendBlob = oldSAS, oldSigner, oldContainer, oldPut + newStorageClient, newMSIProvider = oldStorage, oldMSI + }) + check := func(err error) { + t.Helper() + requireDownloadCode(t, 1998, err) + require.True(t, errors.Is(err, cause)) + } + _, _, err := Download(clarificationLog(), clarificationDownloader{cause}) + check(err) + MakeHttpRequest = func(*http.Request) (*http.Response, error) { return nil, cause } + _, _, err = Download(clarificationLog(), NewURLDownload("https://example.invalid")) + check(err) + httpClient = &http.Client{Transport: clarificationTransport(func(*http.Request) (*http.Response, error) { + return nil, cause + })} + request, err := http.NewRequest(http.MethodGet, "https://example.invalid", nil) + require.NoError(t, err) + _, err = HttpClientDo(request) + check(err) + + stubDownloadFile(t, nil, cause) + _, err = SaveTo(clarificationLog(), nil, "script", 0500) + check(err) + sasHTTPGet = func(string) (*http.Response, error) { return nil, cause } + _, err = GetSASBlob("https://example.invalid/container/script", "", "unused") + check(err) + stubSASResponse(t, 200, io.NopCloser(strings.NewReader("script")), nil) + _, err = GetSASBlob("https://example.invalid/container/script", "", "unused") + check(err) + + writer := &clarificationWriter{} + stubDownloadFile(t, writer, nil) + stubSASResponse(t, 200, &clarificationReader{err: cause}, nil) + _, err = GetSASBlob("https://example.invalid/container/script", "", "unused") + check(err) + stubDownloadResponse(t, 200, &clarificationReader{err: cause}) + _, err = SaveTo(clarificationLog(), []Downloader{NewURLDownload("https://example.invalid")}, "script", 0500) + check(err) + + newStorageClient = func(string, string, string, string, bool) (storage.Client, error) { + return storage.Client{}, cause + } + blob := blobutil.AzureBlobRef{StorageBase: "core.windows.net", Container: "container", Blob: "script"} + _, err = NewBlobDownload("account", "a2V5", blob).GetRequest() + check(err) + newStorageClient = oldStorage + blobSASURI = func(*storage.Blob, storage.BlobSASOptions) (string, error) { return "", cause } + _, err = NewBlobDownload("account", "a2V5", blob).GetRequest() + check(err) + + containerFromSAS = func(url.URL) (*storage.Container, error) { return nil, cause } + _, err = CreateOrReplaceAppendBlob("https://example.invalid/container/script", "") + check(err) + client, err := storage.NewClient("account", "a2V5", "core.windows.net", storage.DefaultAPIVersion, true) + require.NoError(t, err) + service := client.GetBlobService() + containerFromSAS = func(url.URL) (*storage.Container, error) { + return service.GetContainerReference("container"), nil + } + putAppendBlob = func(*storage.Blob) error { return cause } + _, err = CreateOrReplaceAppendBlob("https://example.invalid/container/script", "") + check(err) + _, err = NewBlobWithMsiDownload("https://example.invalid", func() (msi.Msi, error) { + return msi.Msi{}, cause + }).GetRequest() + check(err) + newMSIProvider = func() managedIdentityProvider { return clarificationMSI{cause} } + prod := ProdMsiDownloader{} + for _, provider := range []MsiProvider{ + prod.GetMsiProvider("blob"), prod.GetMsiProviderByClientId("blob", "client"), + prod.GetMsiProviderByObjectId("blob", "object"), + } { + _, err = provider() + check(err) + } + }) + } +} + +func TestDownloadInjectedSuccess(t *testing.T) { + stubDownloadResponse(t, 200, io.NopCloser(strings.NewReader("script"))) + writer := &clarificationWriter{} + stubDownloadFile(t, writer, nil) + n, err := SaveTo(clarificationLog(), []Downloader{NewURLDownload("https://example.invalid")}, "script", 0500) + require.NoError(t, err) + require.EqualValues(t, 6, n) + require.Equal(t, "script", writer.String()) + require.True(t, writer.closed) + stubSASResponse(t, 200, io.NopCloser(strings.NewReader("sas script")), nil) + writer = &clarificationWriter{} + stubDownloadFile(t, writer, nil) + path, err := GetSASBlob("https://example.invalid/container/script", "", "unused") + require.NoError(t, err) + require.NotEmpty(t, path) + require.Equal(t, "sas script", writer.String()) + require.True(t, writer.closed) +} diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 4664bd5..b441ee9 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/urlutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" @@ -46,7 +47,12 @@ var ( ) func HttpClientDo(request *http.Request) (*http.Response, error) { - return httpClient.Do(request) + response, err := httpClient.Do(request) + var networkError net.Error + if errors.As(err, &networkError) && networkError.Timeout() { + return response, errorutil.EnsureClarification(errorutil.FileDownload_exceededTimeout, err) + } + return response, errorutil.EnsureClarification(errorutil.FileDownload_networkingError, err) } // Download retrieves a response body and checks the response status code to see @@ -55,7 +61,7 @@ func HttpClientDo(request *http.Request) (*http.Response, error) { func Download(ctx *log.Context, downloader Downloader) (int, io.ReadCloser, error) { request, err := downloader.GetRequest() if err != nil { - return -1, nil, errors.Wrapf(err, "failed to create http request") + return -1, nil, errorutil.EnsureClarification(errorutil.Internal_failedToCreateDownloadRequest, errors.Wrapf(err, "failed to create http request")) } requestID := request.Header.Get(xMsClientRequestIdHeaderName) if len(requestID) > 0 { @@ -64,62 +70,80 @@ func Download(ctx *log.Context, downloader Downloader) (int, io.ReadCloser, erro response, err := MakeHttpRequest(request) if err != nil { - err = urlutil.RemoveUrlFromErr(err) + err = urlutil.RemoveUrlFromErr(errorutil.EnsureClarification(errorutil.FileDownload_httpRequestFailed, err)) return -1, nil, errors.Wrapf(err, "http request failed") } if response.StatusCode == http.StatusOK { return response.StatusCode, response.Body, nil } + if response.Body != nil { + defer response.Body.Close() + } errString := fmt.Sprintf("Status code %d while downloading blob '%s'. Use either a public script URI that points to .sh file, Azure storage blob SAS URI or storage blob accessible by a managed identity and retry. For more information, see https://aka.ms/RunCommandManagedLinux", response.StatusCode, request.URL.Opaque) requestId := response.Header.Get(xMsServiceRequestIdHeaderName) + requestIdSuffix := "" + if len(requestId) > 0 { + requestIdSuffix = fmt.Sprintf(" (Service request ID: %s)", requestId) + } switch downloader.(type) { case *blobWithMsiToken: switch response.StatusCode { case http.StatusNotFound: notFoundError := fmt.Sprintf("RunCommand failed to download the blob '%s' and received a response code '%s'. Make sure that the Azure blob and managed identity exist, and the identity has access to the storage blob's container with the 'Storage Blob Data Reader' role assignment. For a user-assigned identity, add it under the VM's identity. For more information, see https://aka.ms/RunCommandManagedLinux", request.URL.Opaque, response.Status) errString = fmt.Sprintf("%s: %s", MsiDownload404ErrorString, notFoundError) + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.Msi_notFound, errors.New(errString+requestIdSuffix)) case http.StatusForbidden, http.StatusUnauthorized, http.StatusBadRequest, http.StatusConflict: forbiddenError := fmt.Sprintf("RunCommand failed to download the blob '%s' and received a response code '%s'. Ensure that the managed identity has access to the storage blob's container with the 'Storage Blob Data Reader' role assignment. For a user-assigned identity, add it under the VM's identity. For more information, see https://aka.ms/RunCommandManagedLinux", request.URL.Opaque, response.Status) errString = fmt.Sprintf("%s: %s", MsiDownload403ErrorString, forbiddenError) + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.Msi_doesNotHaveRightPermissions, errors.New(errString+requestIdSuffix)) + case http.StatusInternalServerError: + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.Imds_internalMsiError, errors.New(errString+requestIdSuffix)) + default: + if response.StatusCode >= http.StatusInternalServerError { + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.Imds_downloadInternalServerError, errors.New(errString+requestIdSuffix)) + } + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.Msi_GenericRetrievalError, errors.New(errString+requestIdSuffix)) } - break default: hostname := request.URL.Host + errString = fmt.Sprintf("RunCommand failed to download the file from %s because the server returned a response code and message of %q Please verify the machine has network connectivity.", + hostname, + response.Status) switch response.StatusCode { - case http.StatusUnauthorized: - errString = fmt.Sprintf("RunCommand failed to download the file from %s because access was denied. Please fix the blob permissions and try again. The response code and message returned were: %q.", - hostname, - response.Status) - break + case http.StatusUnauthorized, http.StatusForbidden: + if response.StatusCode == http.StatusUnauthorized { + errString = fmt.Sprintf("RunCommand failed to download the file from %s because access was denied. Please fix the blob permissions and try again. The response code and message returned were: %q.", + hostname, + response.Status) + } + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.FileDownload_accessDenied, errors.New(errString+requestIdSuffix)) case http.StatusNotFound: errString = fmt.Sprintf("RunCommand failed to download the file from %s because it does not exist. Please create the blob and try again, the response code and message returned were: %q", hostname, response.Status) + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.FileDownload_doesNotExist, errors.New(errString+requestIdSuffix)) case http.StatusBadRequest: errString = fmt.Sprintf("RunCommand failed to download the file from %s because parts of the request were incorrectly formatted, missing, and/or invalid. The response code and message returned were: %q", hostname, response.Status) + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.FileDownload_badRequest, errors.New(errString+requestIdSuffix)) case http.StatusInternalServerError: errString = fmt.Sprintf("RunCommand failed to download the file from %s due to an issue with storage. The response code and message returned were: %q", hostname, response.Status) + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.Storage_internalServerError, errors.New(errString+requestIdSuffix)) default: - errString = fmt.Sprintf("RunCommand failed to download the file from %s because the server returned a response code and message of %q Please verify the machine has network connectivity.", - hostname, - response.Status) + if response.StatusCode >= http.StatusInternalServerError { + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.FileDownload_internalServerError, errors.New(errString+requestIdSuffix)) + } + return response.StatusCode, nil, errorutil.EnsureClarification(errorutil.FileDownload_genericError, errors.New(errString+requestIdSuffix)) } } - - if len(requestId) > 0 { - errString += fmt.Sprintf(" (Service request ID: %s)", requestId) - - } - return response.StatusCode, nil, errors.New(errString) } diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index f910667..1a1ab23 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-extension-foundation/msi" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetalpbalkan/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -18,6 +19,14 @@ import ( type badDownloader struct{ calls int } +func requireDownloadCode(t *testing.T, expected errorutil.ErrorClarificationCode, err error) { + t.Helper() + require.Error(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "missing clarification: %v", err) + require.Equal(t, expected, code) +} + var ( testctx = log.NewContext(log.NewNopLogger()) ) @@ -29,12 +38,14 @@ func (b *badDownloader) GetRequest() (*http.Request, error) { func TestDownload_wrapsGetRequestError(t *testing.T) { _, _, err := download.Download(testctx, new(badDownloader)) + requireDownloadCode(t, errorutil.Internal_failedToCreateDownloadRequest, err) require.NotNil(t, err) require.EqualError(t, err, "failed to create http request: expected error") } func TestDownload_wrapsHTTPError(t *testing.T) { _, _, err := download.Download(testctx, download.NewURLDownload("bad url")) + requireDownloadCode(t, errorutil.FileDownload_networkingError, err) require.NotNil(t, err) require.Contains(t, err.Error(), "http request failed:") } @@ -43,16 +54,21 @@ func TestDownload_wrapsCommonErrorCodes(t *testing.T) { srv := httptest.NewServer(httpbin.GetMux()) defer srv.Close() - for _, code := range []int{ - http.StatusNotFound, - http.StatusForbidden, - http.StatusInternalServerError, - http.StatusBadGateway, - http.StatusBadRequest, - http.StatusUnauthorized, + for _, tc := range []struct { + status int + clarification errorutil.ErrorClarificationCode + }{ + {http.StatusNotFound, errorutil.FileDownload_doesNotExist}, + {http.StatusForbidden, errorutil.FileDownload_accessDenied}, + {http.StatusInternalServerError, errorutil.Storage_internalServerError}, + {http.StatusBadGateway, errorutil.FileDownload_internalServerError}, + {http.StatusBadRequest, errorutil.FileDownload_badRequest}, + {http.StatusUnauthorized, errorutil.FileDownload_accessDenied}, } { + code := tc.status respCode, _, err := download.Download(testctx, download.NewURLDownload(fmt.Sprintf("%s/status/%d", srv.URL, code))) require.NotNil(t, err, "not failed for code:%d", code) + requireDownloadCode(t, tc.clarification, err) require.Equal(t, code, respCode) switch respCode { case http.StatusNotFound: @@ -89,6 +105,7 @@ func TestDowload_msiDownloaderErrorMessage(t *testing.T) { msiDownloader404 := download.NewBlobWithMsiDownload(srv.URL+"/status/404", mockMsiProvider) returnCode, body, err := download.Download(testctx, msiDownloader404) + requireDownloadCode(t, errorutil.Msi_notFound, err) require.True(t, strings.Contains(err.Error(), download.MsiDownload404ErrorString), "error string doesn't contain the correct message") require.Contains(t, err.Error(), "For more information, see https://aka.ms/RunCommandManagedLinux", "error string doesn't contain full message") require.Nil(t, body, "body is not nil for failed download") @@ -96,6 +113,7 @@ func TestDowload_msiDownloaderErrorMessage(t *testing.T) { msiDownloader403 := download.NewBlobWithMsiDownload(srv.URL+"/status/403", mockMsiProvider) returnCode, body, err = download.Download(testctx, msiDownloader403) + requireDownloadCode(t, errorutil.Msi_doesNotHaveRightPermissions, err) require.True(t, strings.Contains(err.Error(), download.MsiDownload403ErrorString), "error string doesn't contain the correct message") require.Contains(t, err.Error(), "For more information, see https://aka.ms/RunCommandManagedLinux", "error string doesn't contain full message") require.Nil(t, body, "body is not nil for failed download") @@ -104,6 +122,7 @@ func TestDowload_msiDownloaderErrorMessage(t *testing.T) { // Should use default error message for any error code other than 400, 401, 403, 404, and 409 msiDownloader500 := download.NewBlobWithMsiDownload(srv.URL+"/status/500", mockMsiProvider) returnCode, body, err = download.Download(testctx, msiDownloader500) + requireDownloadCode(t, errorutil.Imds_internalMsiError, err) fmt.Println(err.Error()) require.Contains(t, err.Error(), "Use either a public script URI that points to .sh file", "error string doesn't contain full message") require.Contains(t, err.Error(), "For more information, see https://aka.ms/RunCommandManagedLinux", "error string doesn't contain full message") diff --git a/pkg/download/retry.go b/pkg/download/retry.go index fd504da..1505057 100644 --- a/pkg/download/retry.go +++ b/pkg/download/retry.go @@ -1,12 +1,14 @@ package download import ( + stderrors "errors" "fmt" "io" "math" "net/http" "time" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -27,6 +29,20 @@ const ( expRetryM = 2 ) +type retryFailure struct { + attempt int + latest error + previous error +} + +func (e retryFailure) Error() string { + return fmt.Sprintf("attempt %d: %s; previous attempts: %s", e.attempt, e.latest, e.previous) +} + +func (e retryFailure) Unwrap() error { return e.latest } + +func (e retryFailure) Is(target error) bool { return stderrors.Is(e.previous, target) } + // WithRetries retrieves a response body using the specified downloader. Any // error returned from d will be retried (and retrieved response bodies will be // closed on failures). If the retries do not succeed, the last error is returned. @@ -43,7 +59,9 @@ func WithRetries(ctx *log.Context, downloaders []Downloader, sf SleepFunc) (io.R } if downloadErrors != nil { - downloadErrors = errors.Wrap(downloadErrors, fmt.Sprintf("Attempt %d: %s ", n+1, err.Error())) + // Unwrap only the latest cause so older pointer classifications + // cannot shadow its value classification. Is retains prior causes. + downloadErrors = errorutil.EnsureClarification(errorutil.Internal_failedToAggregateDownloadAttempts, retryFailure{n + 1, err, downloadErrors}) } else { downloadErrors = err } @@ -74,6 +92,9 @@ func WithRetries(ctx *log.Context, downloaders []Downloader, sf SleepFunc) (io.R } } } + if downloadErrors == nil { + return nil, errorutil.EnsureClarification(errorutil.Internal_noDownloaders, errors.New("no downloaders supplied")) + } return nil, downloadErrors } diff --git a/pkg/download/retry_test.go b/pkg/download/retry_test.go index 085b258..ad47dc7 100644 --- a/pkg/download/retry_test.go +++ b/pkg/download/retry_test.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-extension-foundation/msi" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetalpbalkan/go-httpbin" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" @@ -48,6 +49,7 @@ func TestWithRetries_failing_validateNumberOfCalls(t *testing.T) { bd := new(badDownloader) _, err := download.WithRetries(nopLog(), []download.Downloader{bd}, new(sleepRecorder).Sleep) + requireDownloadCode(t, errorutil.Internal_failedToCreateDownloadRequest, err) require.Contains(t, err.Error(), "expected error", "error is preserved") require.EqualValues(t, 3, bd.calls, "calls exactly expRetryN times") } @@ -60,6 +62,7 @@ func TestWithRetries_failingBadStatusCode_validateSleeps(t *testing.T) { sr := new(sleepRecorder) _, err := download.WithRetries(nopLog(), []download.Downloader{d}, sr.Sleep) + requireDownloadCode(t, errorutil.FileDownload_genericError, err) require.Contains(t, err.Error(), "429 Too Many Requests") require.Contains(t, err.Error(), "Please verify the machine has network connectivity") @@ -102,6 +105,7 @@ func TestRetriesWith_SwitchDownloaderThenFailWithCorrectErrorMessage(t *testing. d403 := mockDownloader{0, svr.URL + "/status/403"} msiDownloader403 := download.NewBlobWithMsiDownload(svr.URL+"/status/403", mockMsiProvider) resp, err := download.WithRetries(nopLog(), []download.Downloader{&d403, msiDownloader403}, func(d time.Duration) { return }) + requireDownloadCode(t, errorutil.Msi_doesNotHaveRightPermissions, err) require.NotNil(t, err, "download with retries should fail") require.Nil(t, resp, "response body should be nil for failed download with retries") require.Equal(t, d403.timesCalled, 1) @@ -110,6 +114,7 @@ func TestRetriesWith_SwitchDownloaderThenFailWithCorrectErrorMessage(t *testing. d404 := mockDownloader{0, svr.URL + "/status/404"} msiDownloader404 := download.NewBlobWithMsiDownload(svr.URL+"/status/404", mockMsiProvider) resp, err = download.WithRetries(nopLog(), []download.Downloader{&d404, msiDownloader404}, func(d time.Duration) { return }) + requireDownloadCode(t, errorutil.Msi_notFound, err) require.NotNil(t, err, "download with retries should fail") require.Nil(t, resp, "response body should be nil for failed download with retries") require.Equal(t, d404.timesCalled, 1) diff --git a/pkg/download/save.go b/pkg/download/save.go index c04284d..d7aa58e 100644 --- a/pkg/download/save.go +++ b/pkg/download/save.go @@ -4,6 +4,7 @@ import ( "io" "os" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -12,23 +13,27 @@ const ( writeBufSize = 1024 * 8 ) +var openDownloadFile = func(path string, flag int, mode os.FileMode) (io.WriteCloser, error) { + return os.OpenFile(path, flag, mode) +} + // SaveTo uses given downloader to fetch the resource with retries and saves the // given file. Directory of dst is not created by this function. If a file at // dst exists, it will be truncated. If a new file is created, mode is used to // set the permission bits. Written number of bytes are returned on success. func SaveTo(ctx *log.Context, downloaders []Downloader, dst string, mode os.FileMode) (int64, error) { - f, err := os.OpenFile(dst, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, mode) + f, err := openDownloadFile(dst, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, mode) if err != nil { - return 0, errors.Wrapf(err, "failed to open file for writing: %s", dst) + return 0, errorutil.EnsureClarification(errorutil.FileDownload_failedToOpenDestination, errors.Wrapf(err, "failed to open file for writing: %s", dst)) } defer f.Close() body, err := WithRetries(ctx, downloaders, ActualSleep) if err != nil { - return 0, errors.Wrapf(err, "failed to download file '%s'", dst) + return 0, errorutil.EnsureClarification(errorutil.Internal_failedToDownloadToFile, errors.Wrapf(err, "failed to download file '%s'", dst)) } defer body.Close() n, err := io.CopyBuffer(f, body, make([]byte, writeBufSize)) - return n, errors.Wrapf(err, "failed to write to file: %s", dst) + return n, errorutil.EnsureClarification(errorutil.FileDownload_failedToWriteDestination, errors.Wrapf(err, "failed to write to file: %s", dst)) } diff --git a/pkg/download/save_test.go b/pkg/download/save_test.go index d93d185..c5a7773 100644 --- a/pkg/download/save_test.go +++ b/pkg/download/save_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/Azure/run-command-handler-linux/pkg/download" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/ahmetalpbalkan/go-httpbin" "github.com/stretchr/testify/require" ) @@ -20,6 +21,7 @@ func TestSaveTo_invalidDir(t *testing.T) { d := download.NewURLDownload(srv.URL + "/bytes/65536") _, err := download.SaveTo(nopLog(), []download.Downloader{d}, "/nonexistent-dir/dst", 0600) + requireDownloadCode(t, errorutil.FileDownload_failedToOpenDestination, err) require.Contains(t, err.Error(), "failed to open file for writing") } diff --git a/pkg/download/url.go b/pkg/download/url.go index 3dec7ea..fb143b9 100644 --- a/pkg/download/url.go +++ b/pkg/download/url.go @@ -4,6 +4,7 @@ import ( "net/http" "net/url" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/google/uuid" ) @@ -28,7 +29,7 @@ func (u urlDownload) GetRequest() (*http.Request, error) { if req != nil { req.Header.Add(xMsClientRequestIdHeaderName, uuid.New().String()) } - return req, err + return req, errorutil.EnsureClarification(errorutil.FileDownload_invalidRequestUrl, err) } // Scrub query. Used to remove the query parts like SAS token. diff --git a/pkg/download/url_test.go b/pkg/download/url_test.go index caf6670..e79467e 100644 --- a/pkg/download/url_test.go +++ b/pkg/download/url_test.go @@ -3,6 +3,7 @@ package download import ( "testing" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/stretchr/testify/require" ) @@ -15,6 +16,7 @@ func Test_urlDownload_GetRequest_badURL(t *testing.T) { d := NewURLDownload(u) r, err := d.GetRequest() require.NotNil(t, err, u) + requireDownloadCode(t, errorutil.FileDownload_invalidRequestUrl, err) require.Contains(t, err.Error(), "invalid URL", u) require.Nil(t, r, u) } diff --git a/pkg/errorutil/errorclarificationcodes.go b/pkg/errorutil/errorclarificationcodes.go new file mode 100644 index 0000000..91de236 --- /dev/null +++ b/pkg/errorutil/errorclarificationcodes.go @@ -0,0 +1,316 @@ +package errorutil + +// ErrorClarificationCode is the numeric clarification code carried by vmextension. +type ErrorClarificationCode = int + +const ( + // System errors. + Storage_internalServerError ErrorClarificationCode = -1 + Internal_badConfig ErrorClarificationCode = -21 + Imds_internalMsiError ErrorClarificationCode = -30 + FileDownload_badRequest ErrorClarificationCode = -41 + Os_FailedToDeleteDataDir ErrorClarificationCode = -50 + Os_FailedToOpenStdOut ErrorClarificationCode = -51 + Os_FailedToOpenStdErr ErrorClarificationCode = -52 + + // RunCommand-specific system errors. + Internal_failedToCreateDownloadRequest ErrorClarificationCode = -1002 + Internal_failedToAggregateDownloadAttempts ErrorClarificationCode = -1005 + Internal_noDownloaders ErrorClarificationCode = -1006 + Internal_failedToDownloadToFile ErrorClarificationCode = -1008 + FileDownload_internalServerError ErrorClarificationCode = -1010 + Storage_failedToGenerateSas ErrorClarificationCode = -1013 + Storage_sasDownloadInternalServerError ErrorClarificationCode = -1015 + Imds_mockSystemIdentityError ErrorClarificationCode = -1033 + Imds_mockClientIdentityError ErrorClarificationCode = -1034 + Imds_mockObjectIdentityError ErrorClarificationCode = -1035 + Imds_downloadInternalServerError ErrorClarificationCode = -1036 + + Internal_failedToMarshalHandlerSettings ErrorClarificationCode = -2410 + Internal_invalidArtifactSpecification ErrorClarificationCode = -2421 + Internal_inconsistentArtifactSpecification ErrorClarificationCode = -2422 + Internal_emptyPolicyPath ErrorClarificationCode = -2510 + Internal_failedToCreatePolicyManager ErrorClarificationCode = -2511 + Internal_failedToGetPolicySettings ErrorClarificationCode = -2513 + Internal_nilPolicySettings ErrorClarificationCode = -2514 + + Internal_nilGoalStateNotifier ErrorClarificationCode = -3201 + Internal_unknownGoalStateCommand ErrorClarificationCode = -3202 + Internal_failedToMarshalGoalStateInstanceView ErrorClarificationCode = -3203 + Internal_failedToCreateGoalStateStatus ErrorClarificationCode = -3204 + Internal_nilImmediateCommandNotifier ErrorClarificationCode = -3205 + Internal_unknownImmediateCommand ErrorClarificationCode = -3206 + Internal_failedToCreateImmediateCommandStatus ErrorClarificationCode = -3207 + Internal_incompleteImmediateCommandGoalState ErrorClarificationCode = -3208 + Internal_nilGoalStateInstanceView ErrorClarificationCode = -3209 + Internal_nilGoalStateCommunicator ErrorClarificationCode = -3210 + HostGA_failedToRetrieveVmSettings ErrorClarificationCode = -3211 + Internal_failedToProcessImmediateCommand ErrorClarificationCode = -3212 + Internal_failedToParseImmediateInstanceView ErrorClarificationCode = -3213 + HostGA_failedToCreateRequestManager ErrorClarificationCode = -3300 + HostGA_failedToReadVmSettingsResponse ErrorClarificationCode = -3301 + HostGA_failedToParseVmSettingsResponse ErrorClarificationCode = -3302 + HostGA_missingVmSettingsETag ErrorClarificationCode = -3303 + Internal_invalidWireServerFallbackAddress ErrorClarificationCode = -3304 + HostGA_failedToCreateVmSettings ErrorClarificationCode = -3310 + HostGA_failedToParseHandlerEnvironment ErrorClarificationCode = -3311 + HostGA_missingSettingsCertificateThumbprint ErrorClarificationCode = -3312 + HostGA_failedToDecryptProtectedSettings ErrorClarificationCode = -3313 + Internal_incompleteFetchedGoalState ErrorClarificationCode = -3500 + Internal_failedToNotifyObserver ErrorClarificationCode = -3600 + HostGA_failedToCreateRequest ErrorClarificationCode = -3700 + HostGA_failedToSendRequest ErrorClarificationCode = -3701 + HostGA_requestReturnedErrorStatus ErrorClarificationCode = -3702 + Internal_failedToMarshalPublicSettings ErrorClarificationCode = -3903 + Internal_failedToMarshalSettings ErrorClarificationCode = -3904 + Internal_failedToMarshalTelemetryEvent ErrorClarificationCode = -4006 + Internal_emptyCleanupFilePattern ErrorClarificationCode = -4103 + Internal_invalidCleanupFilePattern ErrorClarificationCode = -4105 + Internal_failedToReadSequenceConfigurationDirectory ErrorClarificationCode = -4200 + Internal_noSequenceConfigurationFiles ErrorClarificationCode = -4201 + Internal_failedToReadServiceVersion ErrorClarificationCode = -4311 + HostGA_failedToCreateStatusRequest ErrorClarificationCode = -4400 + HostGA_failedToSendStatusRequest ErrorClarificationCode = -4401 + HostGA_statusUploadReturnedErrorStatus ErrorClarificationCode = -4402 + Internal_emptyStatusUpload ErrorClarificationCode = -4410 + Internal_invalidServiceDefinition ErrorClarificationCode = -4600 + + Internal_incorrectRunAsScriptPath ErrorClarificationCode = -5000 + Internal_failedToMarshalInstanceView ErrorClarificationCode = -5100 + Internal_failedToMarshalTerminalGoalStates ErrorClarificationCode = -5204 + Internal_invalidGoalStateRemovalState ErrorClarificationCode = -5210 + Internal_unexpectedStatusItemCount ErrorClarificationCode = -5211 + Internal_failedToMarshalStatusReport ErrorClarificationCode = -5212 + Internal_failedToMarshalImmediateStatus ErrorClarificationCode = -5213 + HostGA_failedToReportImmediateStatus ErrorClarificationCode = -5214 + HostGA_immediateStatusReturnedErrorStatus ErrorClarificationCode = -5215 + Internal_unclassifiedCommandFailure ErrorClarificationCode = -5300 + Internal_invalidSequenceEnvironmentVariable ErrorClarificationCode = -5301 + Internal_failedToMarshalLocalSettings ErrorClarificationCode = -5302 + Internal_unclassifiedPrecheckFailure ErrorClarificationCode = -5305 + Internal_invalidStatusFileSequenceNumber ErrorClarificationCode = -5408 + Internal_invalidMrseqFileSequenceNumber ErrorClarificationCode = -5412 + Internal_emptyUpgradeDirectories ErrorClarificationCode = -5415 + Internal_invalidDummyStatusSequenceNumber ErrorClarificationCode = -5422 + Internal_emptyDummyStatusSequenceFile ErrorClarificationCode = -5423 + Internal_invalidDummyStatusSequenceFileName ErrorClarificationCode = -5424 + Imds_failedToCreateManagedIdentityCredential ErrorClarificationCode = -5432 + + // User errors. + CommandExecution_failedUnknownError ErrorClarificationCode = 1 + CommandExecution_failureExitCode ErrorClarificationCode = 2 + CommandExecution_interruptedByVmShutdown ErrorClarificationCode = 3 + + CustomerInput_commandToExecuteAndScriptNotSpecified ErrorClarificationCode = 23 + CustomerInput_clientIdObjectIdBothSpecified ErrorClarificationCode = 27 + + FileDownload_unableToCreateDownloadDirectory ErrorClarificationCode = 50 + FileDownload_accessDenied ErrorClarificationCode = 52 + FileDownload_doesNotExist ErrorClarificationCode = 53 + FileDownload_networkingError ErrorClarificationCode = 54 + FileDownload_genericError ErrorClarificationCode = 55 + FileDownload_exceededTimeout ErrorClarificationCode = 56 + + Msi_notFound ErrorClarificationCode = 70 + Msi_doesNotHaveRightPermissions ErrorClarificationCode = 71 + Msi_GenericRetrievalError ErrorClarificationCode = 72 + + ExtensionPolicySettings_invalidPolicyFileFormat ErrorClarificationCode = 80 + ExtensionPolicySettings_policyLoadFailed ErrorClarificationCode = 81 + ExtensionPolicySettings_downloadedScriptNotAllowed ErrorClarificationCode = 82 + ExtensionPolicySettings_commandToExecuteNotAllowed ErrorClarificationCode = 83 + ExtensionPolicySettings_scriptNotAllowed ErrorClarificationCode = 84 + + // RunCommand-specific user errors. + FileDownload_invalidRequestUrl ErrorClarificationCode = 1000 + FileDownload_httpRequestFailed ErrorClarificationCode = 1003 + FileDownload_failedToOpenDestination ErrorClarificationCode = 1007 + FileDownload_failedToWriteDestination ErrorClarificationCode = 1009 + FileDownload_failedToCreateBlobRequest ErrorClarificationCode = 1011 + Storage_failedToInitializeClient ErrorClarificationCode = 1012 + FileDownload_sasRequestFailed ErrorClarificationCode = 1014 + FileDownload_sasDownloadErrorStatus ErrorClarificationCode = 1016 + CustomerInput_invalidSasBlobUrl ErrorClarificationCode = 1017 + CustomerInput_missingSasBlobFileName ErrorClarificationCode = 1018 + FileDownload_failedToOpenSasDestination ErrorClarificationCode = 1019 + FileDownload_failedToWriteSasDestination ErrorClarificationCode = 1020 + CustomerInput_invalidAppendBlobSasUrl ErrorClarificationCode = 1021 + Storage_failedToGetSasContainer ErrorClarificationCode = 1022 + CustomerInput_emptyAppendBlobPath ErrorClarificationCode = 1023 + Storage_failedToCreateAppendBlob ErrorClarificationCode = 1024 + CustomerInput_invalidBlobPathUrl ErrorClarificationCode = 1025 + CustomerInput_missingBlobContainerPath ErrorClarificationCode = 1026 + Msi_failedToGetDownloadToken ErrorClarificationCode = 1027 + Msi_emptyToken ErrorClarificationCode = 1028 + CustomerInput_invalidMsiDownloadUrl ErrorClarificationCode = 1029 + Msi_failedToGetSystemIdentity ErrorClarificationCode = 1030 + Msi_failedToGetClientIdentity ErrorClarificationCode = 1031 + Msi_failedToGetObjectIdentity ErrorClarificationCode = 1032 + + CustomerInput_invalidBlobUrl ErrorClarificationCode = 1100 + CustomerInput_unsupportedBlobUrlScheme ErrorClarificationCode = 1101 + CustomerInput_missingBlobUrlHost ErrorClarificationCode = 1102 + CustomerInput_invalidBlobHost ErrorClarificationCode = 1103 + CustomerInput_invalidStorageEndpoint ErrorClarificationCode = 1104 + CustomerInput_missingBlobContainerOrName ErrorClarificationCode = 1105 + FileDownload_redactedUrlError ErrorClarificationCode = 1200 + Os_FailedToOpenTextFile ErrorClarificationCode = 1300 + Os_FailedToReadTextFile ErrorClarificationCode = 1301 + + Os_FailedToGetWorkingDirectory ErrorClarificationCode = 2400 + Os_FailedToExamineHandlerEnvironment ErrorClarificationCode = 2401 + Os_CouldNotFindHandlerEnvironment ErrorClarificationCode = 2402 + Os_InvalidHandlerEnvironmentJson ErrorClarificationCode = 2403 + Os_UnexpectedHandlerEnvironmentCount ErrorClarificationCode = 2404 + CustomerInput_invalidHandlerSettingsJson ErrorClarificationCode = 2411 + Os_FailedToReadHandlerSettings ErrorClarificationCode = 2412 + CustomerInput_unexpectedRuntimeSettingsCount ErrorClarificationCode = 2414 + CustomerInput_missingCertificateThumbprint ErrorClarificationCode = 2415 + CustomerInput_invalidProtectedSettingsBase64 ErrorClarificationCode = 2416 + Os_FailedToDecryptSettingsWithCms ErrorClarificationCode = 2417 + Os_FailedToDecryptSettingsWithSmime ErrorClarificationCode = 2418 + CustomerInput_invalidDecryptedSettingsJson ErrorClarificationCode = 2419 + ExtensionPolicySettings_inconsistentDownloadedScriptAllowlist ErrorClarificationCode = 2501 + ExtensionPolicySettings_inconsistentCommandIdAllowlist ErrorClarificationCode = 2502 + ExtensionPolicySettings_downloadedScriptTypeNotAllowed ErrorClarificationCode = 2504 + ExtensionPolicySettings_galleryScriptNotAllowed ErrorClarificationCode = 2505 + ExtensionPolicySettings_diagnosticScriptNotAllowed ErrorClarificationCode = 2506 + ExtensionPolicySettings_commandIdScriptNotAllowed ErrorClarificationCode = 2507 + ExtensionPolicySettings_unknownScriptType ErrorClarificationCode = 2508 + ExtensionPolicySettings_runAsUserNotAllowed ErrorClarificationCode = 2516 + ExtensionPolicySettings_outputBlobsNotAllowed ErrorClarificationCode = 2517 + + Os_FailedToCleanOldScriptsAndSettings ErrorClarificationCode = 3000 + Os_FailedToOpenDownloadDirectory ErrorClarificationCode = 3001 + Os_FailedToCleanOldRunAsDirectories ErrorClarificationCode = 3002 + CustomerInput_invalidScriptUrl ErrorClarificationCode = 3100 + FileDownload_scriptSasDownloadFailed ErrorClarificationCode = 3101 + Os_FailedToMakeDownloadedFileExecutable ErrorClarificationCode = 3102 + CustomerInput_emptyDownloadUrl ErrorClarificationCode = 3103 + CustomerInput_invalidDownloadFileUrl ErrorClarificationCode = 3105 + CustomerInput_missingDownloadFileName ErrorClarificationCode = 3106 + Os_FailedToDetermineScriptFileType ErrorClarificationCode = 3107 + Os_FailedToReadScriptForPostprocessing ErrorClarificationCode = 3108 + Os_FailedToWritePostprocessedScript ErrorClarificationCode = 3109 + Os_FailedToOpenEmbeddedScript ErrorClarificationCode = 3110 + Os_FailedToWriteEmbeddedScript ErrorClarificationCode = 3111 + Os_FailedToCloseEmbeddedScript ErrorClarificationCode = 3112 + Os_FailedToMakeDownloadedFileWritable ErrorClarificationCode = 3113 + Os_FailedToRestoreDownloadedFilePermissions ErrorClarificationCode = 3114 + Os_FailedToOpenTailFile ErrorClarificationCode = 3120 + Os_FailedToStatTailFile ErrorClarificationCode = 3121 + Os_FailedToSeekTailFile ErrorClarificationCode = 3122 + Os_FailedToReadTailFile ErrorClarificationCode = 3123 + Os_FailedToOpenOutputFile ErrorClarificationCode = 3124 + Os_FailedToStatOutputFile ErrorClarificationCode = 3125 + Os_FailedToSeekOutputFile ErrorClarificationCode = 3126 + Os_FailedToReadOutputFile ErrorClarificationCode = 3127 + CommandExecution_immediateGoalStateTimedOut ErrorClarificationCode = 3200 + CommandExecution_concurrentGoalStateLimitExceeded ErrorClarificationCode = 3501 + Os_FailedToGetProcessStartTime ErrorClarificationCode = 3650 + Os_FailedToWriteProcessRecord ErrorClarificationCode = 3651 + Os_FailedToDeleteProcessRecord ErrorClarificationCode = 3652 + Os_FailedToReadProcessRecord ErrorClarificationCode = 3653 + Os_InvalidProcessRecordFormat ErrorClarificationCode = 3654 + Os_InvalidProcessRecordId ErrorClarificationCode = 3655 + Os_FailedToKillPreviousProcessGroup ErrorClarificationCode = 3656 + Os_SystemdNotSupported ErrorClarificationCode = 3800 + Os_FailedToMakeServiceBinaryExecutable ErrorClarificationCode = 3801 + CustomerInput_invalidSettingsJson ErrorClarificationCode = 3900 + CustomerInput_invalidPublicSettingsJson ErrorClarificationCode = 3901 + CustomerInput_publicSettingsNotObject ErrorClarificationCode = 3902 + Os_FailedToOpenTelemetryFile ErrorClarificationCode = 4000 + Os_FailedToWriteTelemetryFile ErrorClarificationCode = 4001 + Os_ShortTelemetryFileWrite ErrorClarificationCode = 4002 + Os_FailedToPublishTelemetryFile ErrorClarificationCode = 4003 + Os_FailedToCloseTelemetryFile ErrorClarificationCode = 4004 + Os_FailedToCloseTelemetryWriter ErrorClarificationCode = 4005 + Os_FailedToWriteTelemetryEvent ErrorClarificationCode = 4007 + Os_ShortTelemetryEventWrite ErrorClarificationCode = 4008 + Os_FailedToOpenCleanupParentDirectory ErrorClarificationCode = 4100 + Os_FailedToReadCleanupParentDirectory ErrorClarificationCode = 4101 + Os_FailedToDeleteScriptDirectory ErrorClarificationCode = 4102 + Os_FailedToOpenSettingsCleanupDirectory ErrorClarificationCode = 4104 + Os_FailedToReadSettingsCleanupDirectory ErrorClarificationCode = 4106 + Os_FailedToDeleteSettingsFile ErrorClarificationCode = 4107 + Os_FailedToTruncateSettingsFile ErrorClarificationCode = 4108 + Os_FailedToWriteSequenceNumber ErrorClarificationCode = 4202 + Os_FailedToReadSequenceNumber ErrorClarificationCode = 4203 + Os_InvalidStoredSequenceNumber ErrorClarificationCode = 4204 + Service_failedToStart ErrorClarificationCode = 4300 + Service_failedToStop ErrorClarificationCode = 4301 + Service_failedToEnable ErrorClarificationCode = 4302 + Service_failedToDisable ErrorClarificationCode = 4303 + Service_failedToReloadDaemon ErrorClarificationCode = 4304 + Service_failedToCheckActivity ErrorClarificationCode = 4305 + Service_failedToCheckEnabled ErrorClarificationCode = 4306 + Service_failedToCheckInstalled ErrorClarificationCode = 4307 + Service_failedToRemoveOldConfiguration ErrorClarificationCode = 4308 + Service_failedToCreateConfiguration ErrorClarificationCode = 4309 + Service_failedToUnregister ErrorClarificationCode = 4310 + Systemd_failedToStartUnit ErrorClarificationCode = 4500 + Systemd_failedToStopUnit ErrorClarificationCode = 4501 + Systemd_failedToEnableUnit ErrorClarificationCode = 4502 + Systemd_failedToDisableUnit ErrorClarificationCode = 4503 + Systemd_failedToReloadDaemon ErrorClarificationCode = 4504 + Systemd_failedToCheckUnitActivity ErrorClarificationCode = 4505 + Systemd_failedToCheckUnitEnabled ErrorClarificationCode = 4506 + Systemd_failedToCheckUnitFile ErrorClarificationCode = 4507 + Systemd_failedToWriteUnitConfiguration ErrorClarificationCode = 4508 + Systemd_failedToRemoveUnitConfiguration ErrorClarificationCode = 4509 + Systemd_failedToReadUnitConfiguration ErrorClarificationCode = 4510 + Systemd_failedToInspectUnitDirectory ErrorClarificationCode = 4520 + Systemd_noUnitConfigurationDirectory ErrorClarificationCode = 4521 + CustomerInput_invalidCommandLineArguments ErrorClarificationCode = 4700 + CustomerInput_unknownCommandLineOperation ErrorClarificationCode = 4701 + + Os_FailedToCreateRunAsDirectory ErrorClarificationCode = 5001 + Os_FailedToOpenRunAsSourceScript ErrorClarificationCode = 5002 + Os_FailedToCopyRunAsScript ErrorClarificationCode = 5003 + CommandExecution_runAsUserNotFound ErrorClarificationCode = 5004 + Os_InvalidRunAsUserId ErrorClarificationCode = 5005 + Os_FailedToChangeRunAsScriptOwner ErrorClarificationCode = 5006 + Os_FailedToChangeRunAsScriptPermissions ErrorClarificationCode = 5007 + Os_FailedToRemoveRunAsScript ErrorClarificationCode = 5008 + Os_FailedToCreateRunAsScript ErrorClarificationCode = 5009 + CustomerInput_invalidParameterEnvironmentVariable ErrorClarificationCode = 5010 + Os_FailedToCloseStdOut ErrorClarificationCode = 5013 + Os_FailedToCloseStdErr ErrorClarificationCode = 5014 + Os_FailedToCloseRunAsSourceScript ErrorClarificationCode = 5015 + Os_FailedToCloseRunAsScript ErrorClarificationCode = 5016 + Os_FailedToCreateTemporaryStatusFile ErrorClarificationCode = 5200 + Os_FailedToCloseTemporaryStatusFile ErrorClarificationCode = 5201 + Os_FailedToWriteTemporaryStatusFile ErrorClarificationCode = 5202 + Os_FailedToPublishStatusFile ErrorClarificationCode = 5203 + Os_FailedToWriteTerminalGoalStates ErrorClarificationCode = 5205 + Os_FailedToPublishTerminalGoalStates ErrorClarificationCode = 5206 + Os_FailedToReadTerminalGoalStates ErrorClarificationCode = 5207 + Os_InvalidTerminalGoalStatesJson ErrorClarificationCode = 5208 + Os_FailedToStatTerminalGoalStates ErrorClarificationCode = 5209 + Os_FailedToWriteLocalSettings ErrorClarificationCode = 5303 + Os_FailedToDeleteRuntimePolicy ErrorClarificationCode = 5304 + Os_FailedToCreateDataDir ErrorClarificationCode = 5400 + Os_FailedToStatRuntimePolicy ErrorClarificationCode = 5402 + Storage_failedToAppendOutputWithSas ErrorClarificationCode = 5403 + Storage_failedToAppendOutputWithManagedIdentity ErrorClarificationCode = 5404 + Os_FailedToResetSequenceNumber ErrorClarificationCode = 5405 + Os_FailedToOpenRehydrationStatusDirectory ErrorClarificationCode = 5406 + Os_FailedToReadRehydrationStatusDirectory ErrorClarificationCode = 5407 + Os_FailedToRehydrateSequenceNumber ErrorClarificationCode = 5409 + Os_FailedToStatRehydrationSequenceFile ErrorClarificationCode = 5410 + Os_FailedToReadRehydrationSequenceFile ErrorClarificationCode = 5411 + Os_FailedToUpdateRehydratedSequenceFile ErrorClarificationCode = 5413 + Os_FailedToCreateUpgradeStatusDirectory ErrorClarificationCode = 5414 + Os_FailedToOpenUpgradeSourceDirectory ErrorClarificationCode = 5416 + Os_FailedToReadUpgradeSourceDirectory ErrorClarificationCode = 5417 + Os_FailedToOpenUpgradeSourceFile ErrorClarificationCode = 5418 + Os_FailedToCreateUpgradeDestinationFile ErrorClarificationCode = 5419 + Os_FailedToCopyUpgradeStateFile ErrorClarificationCode = 5420 + Os_FailedToReadDummyStatusSequenceFile ErrorClarificationCode = 5421 + Os_FailedToDeleteBlockedDownloadedScript ErrorClarificationCode = 5425 + CustomerInput_invalidEmbeddedScriptBase64 ErrorClarificationCode = 5427 + CustomerInput_invalidCompressedScript ErrorClarificationCode = 5428 + CustomerInput_outputIdentityObjectIdNotSupported ErrorClarificationCode = 5429 + Storage_failedToCreateManagedAppendBlobClient ErrorClarificationCode = 5430 + Storage_failedToCreateManagedAppendBlob ErrorClarificationCode = 5431 +) diff --git a/pkg/errorutil/errors.go b/pkg/errorutil/errors.go new file mode 100644 index 0000000..ab0cefb --- /dev/null +++ b/pkg/errorutil/errors.go @@ -0,0 +1,38 @@ +// Package errorutil attaches site-specific RunCommand error clarification codes. +package errorutil + +import "github.com/Azure/azure-extension-platform/vmextension" + +// EnsureClarification classifies an error only if a lower layer has not already done so. +// Positive codes identify customer input or VM problems; negative codes identify +// handler or Azure infrastructure failures. Each code belongs to one call site. +func EnsureClarification(code ErrorClarificationCode, err error) error { + if err == nil { + return nil + } + if _, ok := FindClarification(err); ok { + return err + } + return vmextension.NewErrorWithClarificationPtr(code, err) +} + +// FindClarification supports both pointer and value platform errors through wrapping. +func FindClarification(err error) (int, bool) { + switch err := err.(type) { + case *vmextension.ErrorWithClarification: + if err != nil { + return err.ErrorCode, true + } + case vmextension.ErrorWithClarification: + return err.ErrorCode, true + case interface{ Unwrap() []error }: + for _, cause := range err.Unwrap() { + if code, ok := FindClarification(cause); ok { + return code, true + } + } + case interface{ Unwrap() error }: + return FindClarification(err.Unwrap()) + } + return 0, false +} diff --git a/pkg/errorutil/errors_test.go b/pkg/errorutil/errors_test.go new file mode 100644 index 0000000..ee2ffd1 --- /dev/null +++ b/pkg/errorutil/errors_test.go @@ -0,0 +1,214 @@ +package errorutil + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/Azure/azure-extension-platform/vmextension" + "github.com/stretchr/testify/require" +) + +func TestEnsureClarification(t *testing.T) { + cause := errors.New("cause") + require.NoError(t, EnsureClarification(1, nil)) + for _, original := range []error{ + vmextension.NewErrorWithClarificationPtr(25, cause), + vmextension.NewErrorWithClarification(-25, cause), + fmt.Errorf("context: %w", error(vmextension.NewErrorWithClarificationPtr(26, cause))), + fmt.Errorf("context: %w", vmextension.NewErrorWithClarification(-26, cause)), + } { + expected, _ := FindClarification(original) + err := EnsureClarification(999, original) + require.Equal(t, original, err) + code, ok := FindClarification(err) + require.True(t, ok) + require.Equal(t, expected, code) + require.ErrorIs(t, err, cause) + } + for _, code := range []int{1, -1} { + err := EnsureClarification(code, cause) + actual, ok := FindClarification(err) + require.True(t, ok) + require.Equal(t, code, actual) + require.ErrorIs(t, err, cause) + } + _, ok := FindClarification(cause) + require.False(t, ok) + _, ok = FindClarification(nil) + require.False(t, ok) + joined := errors.Join(vmextension.NewErrorWithClarification(27, cause), vmextension.NewErrorWithClarificationPtr(28, cause)) + code, ok := FindClarification(joined) + require.True(t, ok) + require.Equal(t, 27, code, "pointer errors must not take precedence over earlier value errors") + code, ok = FindClarification(errors.Join(cause, vmextension.NewErrorWithClarificationPtr(29, cause))) + require.True(t, ok) + require.Equal(t, 29, code) + _, ok = FindClarification(errors.Join(cause, errors.New("another unclassified cause"))) + require.False(t, ok) + var nilClarification *vmextension.ErrorWithClarification + _, ok = FindClarification(nilClarification) + require.False(t, ok) +} + +func TestCustomScriptClarificationCodeCompatibility(t *testing.T) { + for name, tc := range map[string]struct { + code ErrorClarificationCode + want int + }{ + "Storage_internalServerError": {Storage_internalServerError, -1}, + "Internal_badConfig": {Internal_badConfig, -21}, + "Imds_internalMsiError": {Imds_internalMsiError, -30}, + "FileDownload_badRequest": {FileDownload_badRequest, -41}, + "Os_FailedToDeleteDataDir": {Os_FailedToDeleteDataDir, -50}, + "Os_FailedToOpenStdOut": {Os_FailedToOpenStdOut, -51}, + "Os_FailedToOpenStdErr": {Os_FailedToOpenStdErr, -52}, + "CommandExecution_failedUnknownError": {CommandExecution_failedUnknownError, 1}, + "CommandExecution_failureExitCode": {CommandExecution_failureExitCode, 2}, + "CommandExecution_interruptedByVmShutdown": {CommandExecution_interruptedByVmShutdown, 3}, + "CustomerInput_commandToExecuteAndScriptNotSpecified": {CustomerInput_commandToExecuteAndScriptNotSpecified, 23}, + "CustomerInput_clientIdObjectIdBothSpecified": {CustomerInput_clientIdObjectIdBothSpecified, 27}, + "FileDownload_unableToCreateDownloadDirectory": {FileDownload_unableToCreateDownloadDirectory, 50}, + "FileDownload_accessDenied": {FileDownload_accessDenied, 52}, + "FileDownload_doesNotExist": {FileDownload_doesNotExist, 53}, + "FileDownload_networkingError": {FileDownload_networkingError, 54}, + "FileDownload_genericError": {FileDownload_genericError, 55}, + "FileDownload_exceededTimeout": {FileDownload_exceededTimeout, 56}, + "Msi_notFound": {Msi_notFound, 70}, + "Msi_doesNotHaveRightPermissions": {Msi_doesNotHaveRightPermissions, 71}, + "Msi_GenericRetrievalError": {Msi_GenericRetrievalError, 72}, + "ExtensionPolicySettings_invalidPolicyFileFormat": {ExtensionPolicySettings_invalidPolicyFileFormat, 80}, + "ExtensionPolicySettings_policyLoadFailed": {ExtensionPolicySettings_policyLoadFailed, 81}, + "ExtensionPolicySettings_downloadedScriptNotAllowed": {ExtensionPolicySettings_downloadedScriptNotAllowed, 82}, + "ExtensionPolicySettings_commandToExecuteNotAllowed": {ExtensionPolicySettings_commandToExecuteNotAllowed, 83}, + "ExtensionPolicySettings_scriptNotAllowed": {ExtensionPolicySettings_scriptNotAllowed, 84}, + } { + t.Run(name, func(t *testing.T) { require.Equal(t, tc.want, tc.code) }) + } +} + +// Each named code must identify exactly one production classification site. +func TestClarificationCodesHaveUniqueSites(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..")) + require.NoError(t, err) + files := token.NewFileSet() + registry, err := parser.ParseFile(files, "errorclarificationcodes.go", nil, 0) + require.NoError(t, err) + codes := map[string]int{} + values := map[int]string{} + for _, declaration := range registry.Decls { + group, ok := declaration.(*ast.GenDecl) + if !ok || group.Tok != token.CONST { + continue + } + for _, spec := range group.Specs { + value := spec.(*ast.ValueSpec) + require.Len(t, value.Names, 1) + require.Len(t, value.Values, 1) + codeType, ok := value.Type.(*ast.Ident) + require.True(t, ok) + require.Equal(t, "ErrorClarificationCode", codeType.Name) + expression := value.Values[0] + sign := 1 + if unary, ok := expression.(*ast.UnaryExpr); ok && unary.Op == token.SUB { + sign, expression = -1, unary.X + } + literal, ok := expression.(*ast.BasicLit) + require.True(t, ok, "code values must be explicit integers") + code, err := strconv.Atoi(literal.Value) + require.NoError(t, err) + code *= sign + require.NotZero(t, code) + name := value.Names[0].Name + if previous, exists := values[code]; exists { + t.Errorf("%s and %s both declare code %d", previous, name, code) + } + codes[name], values[code] = code, name + } + } + require.NotEmpty(t, codes) + + sites := map[string]string{} + tested := map[string]bool{} + identifierPattern := regexp.MustCompile(`[A-Za-z_][A-Za-z_0-9]*`) + err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if entry.Name() == ".git" || entry.Name() == "vendor" { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + file, err := parser.ParseFile(files, path, nil, parser.ParseComments) + if err != nil { + return err + } + if strings.HasSuffix(path, "_test.go") { + ast.Inspect(file, func(node ast.Node) bool { + if identifier, ok := node.(*ast.Ident); ok { + if _, exists := codes[identifier.Name]; exists { + tested[identifier.Name] = true + } + } + return true + }) + for _, comment := range file.Comments { + for _, name := range identifierPattern.FindAllString(comment.Text(), -1) { + if _, exists := codes[name]; exists { + tested[name] = true + } + } + } + return nil + } + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok || len(call.Args) != 2 { + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "EnsureClarification" { + return true + } + argument, ok := call.Args[0].(*ast.SelectorExpr) + location := files.Position(call.Pos()).String() + if !ok { + t.Errorf("use a central ErrorClarificationCode constant at %s", location) + return true + } + name := argument.Sel.Name + if _, exists := codes[name]; !exists { + t.Errorf("unregistered clarification constant %s at %s", name, location) + } + if previous, exists := sites[name]; exists { + t.Errorf("%s reused at %s and %s", name, previous, location) + } + sites[name] = location + return true + }) + return nil + }) + require.NoError(t, err) + for name := range codes { + if sites[name] == "" { + t.Errorf("%s needs exactly one production classification site", name) + } + if !tested[name] { + t.Errorf("%s needs an error-path test or a documented untestable-path reason", name) + } + } + t.Logf("verified %d unique named clarification sites", len(sites)) +} diff --git a/pkg/linuxutils/clarification_test.go b/pkg/linuxutils/clarification_test.go new file mode 100644 index 0000000..2fa4ada --- /dev/null +++ b/pkg/linuxutils/clarification_test.go @@ -0,0 +1,274 @@ +package linuxutils + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type fakeDirectory struct { + entries []os.DirEntry + err error +} + +func (d fakeDirectory) ReadDir(int) ([]os.DirEntry, error) { return d.entries, d.err } +func (d fakeDirectory) Close() error { return nil } + +type fakeEntry struct { + directory bool + name string +} + +func (e fakeEntry) Name() string { + if e.name != "" { + return e.name + } + return "test" +} + +func (e fakeEntry) IsDir() bool { return e.directory } +func (e fakeEntry) Type() os.FileMode { return 0 } +func (e fakeEntry) Info() (os.FileInfo, error) { return nil, nil } + +type filesystemErrorLogger struct{ err error } + +func (l *filesystemErrorLogger) Log(values ...interface{}) error { + for _, value := range values { + if err, ok := value.(error); ok { + l.err = err + } + } + return nil +} + +func TestLinuxFilesystemClarifications(t *testing.T) { + oldOpen, oldRemoveDir, oldRemove, oldTruncate := openDirectory, removeDirectory, removeFile, truncateFile + t.Cleanup(func() { + openDirectory, removeDirectory, removeFile, truncateFile = oldOpen, oldRemoveDir, oldRemove, oldTruncate + }) + cause := errors.New("filesystem failure") + for _, want := range []int{errorutil.Os_FailedToOpenCleanupParentDirectory, errorutil.Os_FailedToReadCleanupParentDirectory, errorutil.Os_FailedToDeleteScriptDirectory, errorutil.Internal_emptyCleanupFilePattern, errorutil.Os_FailedToOpenSettingsCleanupDirectory, errorutil.Internal_invalidCleanupFilePattern, errorutil.Os_FailedToReadSettingsCleanupDirectory, errorutil.Os_FailedToDeleteSettingsFile, errorutil.Os_FailedToTruncateSettingsFile} { + t.Run(string(rune(want+5000)), func(t *testing.T) { + openDirectory = func(string) (directoryReader, error) { + if want == errorutil.Os_FailedToOpenCleanupParentDirectory || want == errorutil.Os_FailedToOpenSettingsCleanupDirectory { + return nil, cause + } + d := fakeDirectory{entries: []os.DirEntry{fakeEntry{directory: want == errorutil.Os_FailedToDeleteScriptDirectory}}} + if want == errorutil.Os_FailedToReadCleanupParentDirectory || want == errorutil.Os_FailedToReadSettingsCleanupDirectory { + d.err = cause + } + return d, nil + } + removeDirectory = func(string) error { return cause } + removeFile = func(string) error { return cause } + truncateFile = func(string, int64) error { return cause } + logger := &filesystemErrorLogger{} + ctx := log.NewContext(logger) + var err error + if want >= errorutil.Os_FailedToOpenCleanupParentDirectory && want <= errorutil.Os_FailedToDeleteScriptDirectory { + err = TryDeleteDirectories(ctx, "test") + } else { + pattern := "test" + if want == errorutil.Internal_emptyCleanupFilePattern { + pattern = "" + } else if want == errorutil.Internal_invalidCleanupFilePattern { + pattern = "[" + } + err = tryClearRegexMatchingFiles(ctx, "test", pattern, want != errorutil.Os_FailedToTruncateSettingsFile) + } + if want == errorutil.Os_FailedToDeleteScriptDirectory || want == errorutil.Os_FailedToDeleteSettingsFile || want == errorutil.Os_FailedToTruncateSettingsFile { + require.NoError(t, err, "cleanup remains best effort") + err = logger.err + } + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + if want > 0 { + require.ErrorIs(t, err, cause) + } + }) + } +} + +func TestCleanupExceptMostRecent(t *testing.T) { + oldOpen, oldRemoveDir, oldRemove, oldTruncate := openDirectory, removeDirectory, removeFile, truncateFile + t.Cleanup(func() { + openDirectory, removeDirectory, removeFile, truncateFile = oldOpen, oldRemoveDir, oldRemove, oldTruncate + }) + scripts := filepath.Join("downloads", "extension") + openDirectory = func(name string) (directoryReader, error) { + require.Contains(t, []string{scripts, "config"}, name) + if name == scripts { + return fakeDirectory{entries: []os.DirEntry{ + fakeEntry{name: "1", directory: true}, + fakeEntry{name: "5", directory: true}, + fakeEntry{name: "notes.txt"}, + }}, nil + } + return fakeDirectory{entries: []os.DirEntry{ + fakeEntry{name: "extension.1.settings"}, + fakeEntry{name: "extension.5.settings"}, + fakeEntry{name: "other.1.settings"}, + }}, nil + } + var deletedDirectories, deletedFiles, truncatedFiles []string + removeDirectory = func(name string) error { + deletedDirectories = append(deletedDirectories, name) + return nil + } + removeFile = func(name string) error { + deletedFiles = append(deletedFiles, name) + return nil + } + truncateFile = func(name string, size int64) error { + require.Zero(t, size) + truncatedFiles = append(truncatedFiles, name) + return nil + } + err := TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent( + log.NewContext(log.NewNopLogger()), "downloads", "config", "extension", 5, + `^extension\.\d+\.settings$`, "extension.%d.settings") + require.NoError(t, err) + require.Equal(t, []string{filepath.Join(scripts, "1")}, deletedDirectories) + require.Equal(t, []string{filepath.Join("config", "extension.1.settings")}, truncatedFiles) + require.Empty(t, deletedFiles) +} + +func TestMostRecentCleanupClarifications(t *testing.T) { + oldOpen := openDirectory + t.Cleanup(func() { openDirectory = oldOpen }) + cause := errors.New("filesystem failure") + for _, test := range []struct { + name string + directory string + readError bool + pattern string + want errorutil.ErrorClarificationCode + }{ + {"open scripts", "downloads", false, "settings", errorutil.Os_FailedToOpenCleanupParentDirectory}, + {"read scripts", "downloads", true, "settings", errorutil.Os_FailedToReadCleanupParentDirectory}, + {"open settings", "config", false, "settings", errorutil.Os_FailedToOpenSettingsCleanupDirectory}, + {"read settings", "config", true, "settings", errorutil.Os_FailedToReadSettingsCleanupDirectory}, + {"empty pattern", "", false, "", errorutil.Internal_emptyCleanupFilePattern}, + {"invalid pattern", "", false, "[", errorutil.Internal_invalidCleanupFilePattern}, + } { + t.Run(test.name, func(t *testing.T) { + openDirectory = func(name string) (directoryReader, error) { + if name == test.directory { + if test.readError { + return fakeDirectory{err: cause}, nil + } + return nil, cause + } + return fakeDirectory{}, nil + } + err := TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent( + log.NewContext(log.NewNopLogger()), "downloads", "config", "", 5, test.pattern, "%d.settings") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, test.want, code) + if test.directory != "" { + require.ErrorIs(t, err, cause) + } + }) + } +} + +func TestMostRecentCleanupTruncatesSettingsAfterScriptsFailure(t *testing.T) { + for _, readError := range []bool{false, true} { + name := "open scripts" + code := errorutil.Os_FailedToOpenCleanupParentDirectory + if readError { + name = "read scripts" + code = errorutil.Os_FailedToReadCleanupParentDirectory + } + t.Run(name, func(t *testing.T) { + oldOpen := openDirectory + t.Cleanup(func() { openDirectory = oldOpen }) + config := t.TempDir() + for _, name := range []string{"extension.1.settings", "extension.5.settings", "other.1.settings"} { + require.NoError(t, os.WriteFile(filepath.Join(config, name), []byte("settings"), 0600)) + } + cause := errors.New("script directory failure") + openDirectory = func(name string) (directoryReader, error) { + if name == filepath.Join("downloads", "extension") { + if readError { + return fakeDirectory{err: cause}, nil + } + return nil, cause + } + return os.Open(name) + } + err := TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent( + log.NewContext(log.NewNopLogger()), "downloads", config, "extension", 5, + `^extension\.\d+\.settings$`, "extension.%d.settings") + require.ErrorIs(t, err, cause) + actual, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, code, actual) + for _, name := range []string{"extension.1.settings", "extension.5.settings", "other.1.settings"} { + content, err := os.ReadFile(filepath.Join(config, name)) + require.NoError(t, err) + if name == "extension.1.settings" { + require.Empty(t, content, "older settings must be truncated even when scripts cleanup fails") + } else { + require.Equal(t, "settings", string(content)) + } + } + }) + } +} + +func TestMostRecentCleanupPreservesBothDirectoryErrors(t *testing.T) { + failures := []struct { + name string + read bool + dirCode errorutil.ErrorClarificationCode + fileCode errorutil.ErrorClarificationCode + }{ + {"open", false, errorutil.Os_FailedToOpenCleanupParentDirectory, errorutil.Os_FailedToOpenSettingsCleanupDirectory}, + {"read", true, errorutil.Os_FailedToReadCleanupParentDirectory, errorutil.Os_FailedToReadSettingsCleanupDirectory}, + } + for _, scriptsFailure := range failures { + for _, settingsFailure := range failures { + t.Run(scriptsFailure.name+" scripts/"+settingsFailure.name+" settings", func(t *testing.T) { + oldOpen := openDirectory + t.Cleanup(func() { openDirectory = oldOpen }) + scriptsCause := errors.New("script directory failure") + settingsCause := errors.New("settings directory failure") + var opened []string + openDirectory = func(name string) (directoryReader, error) { + opened = append(opened, name) + read, cause := settingsFailure.read, settingsCause + if name == filepath.Join("downloads", "extension") { + read, cause = scriptsFailure.read, scriptsCause + } + if read { + return fakeDirectory{err: cause}, nil + } + return nil, cause + } + err := TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent( + log.NewContext(log.NewNopLogger()), "downloads", "config", "extension", 5, + `^extension\.\d+\.settings$`, "extension.%d.settings") + require.Equal(t, []string{filepath.Join("downloads", "extension"), "config"}, opened) + require.ErrorIs(t, err, scriptsCause) + require.ErrorIs(t, err, settingsCause) + var combined interface{ Unwrap() []error } + require.ErrorAs(t, err, &combined) + causes := combined.Unwrap() + require.Len(t, causes, 2) + for i, want := range []errorutil.ErrorClarificationCode{scriptsFailure.dirCode, settingsFailure.fileCode} { + code, ok := errorutil.FindClarification(causes[i]) + require.True(t, ok) + require.Equal(t, want, code, "each cause must retain its own clarification code") + } + }) + } + } +} diff --git a/pkg/linuxutils/linuxutils.go b/pkg/linuxutils/linuxutils.go index 8151bc3..0ae6340 100644 --- a/pkg/linuxutils/linuxutils.go +++ b/pkg/linuxutils/linuxutils.go @@ -7,11 +7,23 @@ import ( "os" "path/filepath" "regexp" + "strconv" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) +type directoryReader interface { + ReadDir(int) ([]os.DirEntry, error) + Close() error +} + +var openDirectory = func(name string) (directoryReader, error) { return os.Open(name) } +var removeDirectory = os.RemoveAll +var removeFile = os.Remove +var truncateFile = os.Truncate + func TryClearExtensionScriptsDirectoriesAndSettingsFiles(ctx *log.Context, scriptsDirectory string, runtimeSettingsDirectory string, extensionName string, runtimeSettingsRegexFormatWithAnyExtName string) error { // It is important to call both methods first and then handle the errors or unexpected files could remain in the VM. // E.g., If nothing was dowloaded to the VM, the directory won't exist and thus the `TryDeleteDirectories` will return an error. @@ -29,24 +41,46 @@ func TryClearExtensionScriptsDirectoriesAndSettingsFiles(ctx *log.Context, scrip return nil } +func TryClearExtensionScriptsDirectoriesAndSettingsFilesExceptMostRecent(ctx *log.Context, scriptsDirectory string, runtimeSettingsDirectory string, extensionName string, mostRecentSequenceNumberFinished uint64, runtimeSettingsRegexFormatWithAnyExtName string, runtimeSettingsLastSeqNumFormatWithAnyExtName string) error { + recentSequence := strconv.FormatUint(mostRecentSequenceNumberFinished, 10) + dirErr := TryDeleteDirectoriesExcept(ctx, filepath.Join(scriptsDirectory, extensionName), recentSequence) + recentSettings := fmt.Sprintf(runtimeSettingsLastSeqNumFormatWithAnyExtName, mostRecentSequenceNumberFinished) + filesErr := tryClearRegexMatchingFilesExcept(ctx, runtimeSettingsDirectory, runtimeSettingsRegexFormatWithAnyExtName, recentSettings, false) + + if dirErr != nil && filesErr != nil { + return fmt.Errorf("failed to delete dirs: %w; failed to clear settings: %w", dirErr, filesErr) + } else if dirErr != nil { + return errors.Wrap(dirErr, "could not delete extension scripts directories") + } else if filesErr != nil { + return errors.Wrap(filesErr, "could not clear runtime settings files") + } + + return nil +} + func TryDeleteDirectories(ctx *log.Context, parentDirectory string) error { + return TryDeleteDirectoriesExcept(ctx, parentDirectory, "") +} + +func TryDeleteDirectoriesExcept(ctx *log.Context, parentDirectory string, exceptDirectoryName string) error { // Check if the directory exists - directoryFDRef, err := os.Open(parentDirectory) + directoryFDRef, err := openDirectory(parentDirectory) if err != nil { - return errors.Wrap(err, "could not open parent directory") + return errorutil.EnsureClarification(errorutil.Os_FailedToOpenCleanupParentDirectory, errors.Wrap(err, "could not open parent directory")) } + defer directoryFDRef.Close() dirEntries, err := directoryFDRef.ReadDir(0) if err != nil { - return errors.Wrap(err, "could not read contents from directory") + return errorutil.EnsureClarification(errorutil.Os_FailedToReadCleanupParentDirectory, errors.Wrap(err, "could not read contents from directory")) } if dirEntries != nil { for _, dirEntry := range dirEntries { - if dirEntry.IsDir() { + if dirEntry.IsDir() && dirEntry.Name() != exceptDirectoryName { fullDirectoryPath := filepath.Join(parentDirectory, dirEntry.Name()) ctx.Log("message", "trying to remove directory: "+fullDirectoryPath) - err = os.RemoveAll(fullDirectoryPath) + err = errorutil.EnsureClarification(errorutil.Os_FailedToDeleteScriptDirectory, removeDirectory(fullDirectoryPath)) if err != nil { ctx.Log("warning", "could not delete directory", "error", err) } @@ -60,40 +94,45 @@ func TryDeleteDirectories(ctx *log.Context, parentDirectory string) error { } func tryClearRegexMatchingFiles(ctx *log.Context, directory string, regexFileNamePattern string, deleteFiles bool) error { + return tryClearRegexMatchingFilesExcept(ctx, directory, regexFileNamePattern, "", deleteFiles) +} + +func tryClearRegexMatchingFilesExcept(ctx *log.Context, directory string, regexFileNamePattern string, exceptFileName string, deleteFiles bool) error { if regexFileNamePattern == "" { - return errors.New("empty regexFileNamePattern argument") + return errorutil.EnsureClarification(errorutil.Internal_emptyCleanupFilePattern, errors.New("empty regexFileNamePattern argument")) } // Check if the directory exists - directoryFDRef, err := os.Open(directory) + directoryFDRef, err := openDirectory(directory) if err != nil { - return errors.Wrap(err, "could not open directory") + return errorutil.EnsureClarification(errorutil.Os_FailedToOpenSettingsCleanupDirectory, errors.Wrap(err, "could not open directory")) } + defer directoryFDRef.Close() regex, err := regexp.Compile(regexFileNamePattern) if err != nil { - return errors.Wrap(err, "could not parse given regular expression") + return errorutil.EnsureClarification(errorutil.Internal_invalidCleanupFilePattern, errors.Wrap(err, "could not parse given regular expression")) } dirEntries, err := directoryFDRef.ReadDir(0) if err != nil { - return errors.Wrap(err, "could not read contents from directory") + return errorutil.EnsureClarification(errorutil.Os_FailedToReadSettingsCleanupDirectory, errors.Wrap(err, "could not read contents from directory")) } for _, dirEntry := range dirEntries { fileName := dirEntry.Name() - if regex.MatchString(fileName) { + if fileName != exceptFileName && regex.MatchString(fileName) { fullFilePath := filepath.Join(directory, fileName) if deleteFiles { ctx.Log("message", "deleting file "+fullFilePath) - err = os.Remove(fullFilePath) + err = errorutil.EnsureClarification(errorutil.Os_FailedToDeleteSettingsFile, removeFile(fullFilePath)) if err != nil { ctx.Log("warning", "could not delete file", "error", err) } } else { ctx.Log("message", "cleaning file "+fullFilePath) - err = os.Truncate(fullFilePath, 0) // Calling create on existing file truncates file + err = errorutil.EnsureClarification(errorutil.Os_FailedToTruncateSettingsFile, truncateFile(fullFilePath, 0)) // Calling create on existing file truncates file if err != nil { ctx.Log("warning", "could not truncate file", "error", err) } diff --git a/pkg/preprocess/clarification_test.go b/pkg/preprocess/clarification_test.go new file mode 100644 index 0000000..488ad5e --- /dev/null +++ b/pkg/preprocess/clarification_test.go @@ -0,0 +1,41 @@ +package preprocess + +import ( + "errors" + "io" + "testing" + + "github.com/Azure/azure-extension-platform/vmextension" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +type failingTextReader struct{ err error } + +func (r failingTextReader) Read([]byte) (int, error) { return 0, r.err } +func (r failingTextReader) Close() error { return nil } + +func TestTextFileClarifications(t *testing.T) { + original := openTextFile + t.Cleanup(func() { openTextFile = original }) + for _, code := range []int{errorutil.Os_FailedToOpenTextFile, errorutil.Os_FailedToReadTextFile} { + pointer := vmextension.NewErrorWithClarificationPtr(1998, errors.New("injected classified file failure")) + for _, cause := range []error{errors.New("injected file failure"), pointer, *pointer} { + expected := code + if _, ok := errorutil.FindClarification(cause); ok { + expected = 1998 + } + openTextFile = func(string) (io.ReadCloser, error) { + if code == errorutil.Os_FailedToOpenTextFile { + return nil, cause + } + return failingTextReader{cause}, nil + } + _, err := IsTextFile("script") + actual, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, expected, actual) + require.True(t, errors.Is(err, cause)) + } + } +} diff --git a/pkg/preprocess/file.go b/pkg/preprocess/file.go index 7fb5e5f..780ead0 100644 --- a/pkg/preprocess/file.go +++ b/pkg/preprocess/file.go @@ -7,11 +7,14 @@ import ( "path/filepath" "strings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) const peekLen = 64 // look at first N bytes to figure out if it has shebang +var openTextFile = func(path string) (io.ReadCloser, error) { return os.Open(path) } + // textExtensions is predefined list of script and text // file extensions. var textExtensions = []string{ @@ -28,15 +31,15 @@ func IsTextFile(path string) (bool, error) { if hasTextExtension(path) { return true, nil } - f, err := os.Open(path) + f, err := openTextFile(path) if err != nil { - return false, errors.Wrap(err, "failed to open file") + return false, errorutil.EnsureClarification(errorutil.Os_FailedToOpenTextFile, errors.Wrap(err, "failed to open file")) } defer f.Close() b := make([]byte, peekLen) _, err = f.Read(b) if err != nil && err != io.EOF { - return false, errors.Wrap(err, "failed to read file") + return false, errorutil.EnsureClarification(errorutil.Os_FailedToReadTextFile, errors.Wrap(err, "failed to read file")) } return hasShebang(b), nil } diff --git a/pkg/preprocess/file_test.go b/pkg/preprocess/file_test.go index 3533de6..2f41fe1 100644 --- a/pkg/preprocess/file_test.go +++ b/pkg/preprocess/file_test.go @@ -4,6 +4,7 @@ import ( "path/filepath" "testing" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/stretchr/testify/require" ) @@ -30,6 +31,9 @@ func TestHasShebang(t *testing.T) { func TestIsTextFile_openError(t *testing.T) { _, err := IsTextFile("/non/existing/path") require.NotNil(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Os_FailedToOpenTextFile, code) require.Contains(t, err.Error(), "failed to open file") } diff --git a/pkg/seqnumutil/clarification_test.go b/pkg/seqnumutil/clarification_test.go new file mode 100644 index 0000000..fa274e6 --- /dev/null +++ b/pkg/seqnumutil/clarification_test.go @@ -0,0 +1,44 @@ +package seqnumutil + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func TestSequenceClarifications(t *testing.T) { + dir, err := os.MkdirTemp(".", ".clarification-") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(dir) }) + assertCode := func(err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + } + _, err = FindSequenceNumberFromConfig(dir, ".settings", "[") + assertCode(err, errorutil.Internal_failedToReadSequenceConfigurationDirectory) + _, err = FindSequenceNumberFromConfig(dir, ".settings", "test") + assertCode(err, errorutil.Internal_noSequenceConfigurationFiles) + assertCode(SaveSeqNum(dir, 1), errorutil.Os_FailedToWriteSequenceNumber) + _, err = IsSmallerThan(dir, 1) + assertCode(err, errorutil.Os_FailedToReadSequenceNumber) + file := filepath.Join(dir, "sequence") + require.NoError(t, os.WriteFile(file, []byte("bad"), 0600)) + _, err = IsSmallerThan(file, 1) + assertCode(err, errorutil.Os_InvalidStoredSequenceNumber) + require.NoError(t, SaveSeqNum(file, 1)) + smaller, err := IsSmallerThan(file, 2) + require.NoError(t, err) + require.True(t, smaller) + require.NoError(t, os.WriteFile(filepath.Join(dir, "test.bad.settings"), nil, 0600)) + _, err = FindSequenceNumberFromConfig(dir, ".settings", "test") + assertCode(err, errorutil.Internal_noSequenceConfigurationFiles) + require.NoError(t, os.WriteFile(filepath.Join(dir, "test.9.settings"), nil, 0600)) + seq, err := FindSequenceNumberFromConfig(dir, ".settings", "test") + require.NoError(t, err) + require.Equal(t, 9, seq) +} diff --git a/pkg/seqnumutil/seqnum.go b/pkg/seqnumutil/seqnum.go index 92ffb3d..d06ae59 100644 --- a/pkg/seqnumutil/seqnum.go +++ b/pkg/seqnumutil/seqnum.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/pkg/errors" ) @@ -21,14 +22,17 @@ const ( func FindSequenceNumberFromConfig(path, fileExtension string, extensionName string) (int, error) { g, err := filepath.Glob(filepath.Join(path, fmt.Sprintf("%s.*%s", extensionName, fileExtension))) if err != nil { - return 0, err + return 0, errorutil.EnsureClarification(errorutil.Internal_failedToReadSequenceConfigurationDirectory, err) } - seqs := make([]int, len(g)) + seqs := make([]int, 0, len(g)) for _, v := range g { f := filepath.Base(v) fileNameWithoutExtension := strings.TrimSuffix(f, filepath.Ext(f)) dotAndSequenceNumberString := filepath.Ext(fileNameWithoutExtension) // returns something like "." - sequenceNumberString := dotAndSequenceNumberString[1:] // Remove '.' in the front + if len(dotAndSequenceNumberString) < 2 { + continue + } + sequenceNumberString := dotAndSequenceNumberString[1:] // Remove '.' in the front i, err := strconv.Atoi(sequenceNumberString) if err != nil { continue // continue to the next filename if Atoi fails @@ -36,7 +40,7 @@ func FindSequenceNumberFromConfig(path, fileExtension string, extensionName stri seqs = append(seqs, i) } if len(seqs) == 0 { - return 0, fmt.Errorf("can't find out seqnum from %s, not enough files", path) + return 0, errorutil.EnsureClarification(errorutil.Internal_noSequenceConfigurationFiles, fmt.Errorf("can't find out seqnum from %s, not enough files", path)) } sort.Sort(sort.Reverse(sort.IntSlice(seqs))) return seqs[0], nil @@ -46,7 +50,7 @@ func FindSequenceNumberFromConfig(path, fileExtension string, extensionName stri // path if it does not exist. func SaveSeqNum(path string, num int) error { b := []byte(fmt.Sprintf("%v", num)) - return errors.Wrap(os.WriteFile(path, b, chmod), "seqnum: failed to write") + return errorutil.EnsureClarification(errorutil.Os_FailedToWriteSequenceNumber, errors.Wrap(os.WriteFile(path, b, chmod), "seqnum: failed to write")) } // IsSmallerThan returns true if the sequence number stored at path is smaller @@ -58,9 +62,9 @@ func IsSmallerThan(path string, num int) (bool, error) { if os.IsNotExist(err) { return true, nil } - return false, errors.Wrap(err, "seqnum: failed to read") + return false, errorutil.EnsureClarification(errorutil.Os_FailedToReadSequenceNumber, errors.Wrap(err, "seqnum: failed to read")) } seqNum := strings.TrimSuffix(string(b), "\n") stored, err := strconv.Atoi(seqNum) - return stored < num, errors.Wrapf(err, "seqnum: cannot parse number %q", b) + return stored < num, errorutil.EnsureClarification(errorutil.Os_InvalidStoredSequenceNumber, errors.Wrapf(err, "seqnum: cannot parse number %q", b)) } diff --git a/pkg/servicehandler/clarification_test.go b/pkg/servicehandler/clarification_test.go new file mode 100644 index 0000000..c4373ff --- /dev/null +++ b/pkg/servicehandler/clarification_test.go @@ -0,0 +1,95 @@ +package servicehandler + +import ( + "errors" + "io" + "os" + "testing" + + "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func requireServiceCode(t *testing.T, want errorutil.ErrorClarificationCode, err error) { + t.Helper() + require.Error(t, err) + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "missing clarification: %v", err) + require.Equal(t, want, code) +} + +func TestServiceHandlerClarifications(t *testing.T) { + t.Setenv(constants.VersionEnvName, installedTargetVersion) + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("manager failure") + for _, want := range []int{errorutil.Service_failedToStart, errorutil.Service_failedToStop, errorutil.Service_failedToEnable, errorutil.Service_failedToDisable, errorutil.Service_failedToReloadDaemon, errorutil.Service_failedToCheckActivity, errorutil.Service_failedToCheckEnabled, errorutil.Service_failedToCheckInstalled, errorutil.Service_failedToRemoveOldConfiguration, errorutil.Service_failedToCreateConfiguration, errorutil.Service_failedToUnregister, errorutil.Internal_failedToReadServiceVersion} { + m := getManagerMock() + fail := func(string, *log.Context) error { return cause } + h := NewHandler(m, NewConfiguration("test.service"), ctx) + var err error + switch want { + case errorutil.Service_failedToStart: + m.start_f = fail + err = h.Start() + case errorutil.Service_failedToStop: + m.stop_f = fail + err = h.Stop() + case errorutil.Service_failedToEnable: + m.enable_f = fail + err = h.Enable() + case errorutil.Service_failedToDisable: + m.disable_f = fail + err = h.Disable() + case errorutil.Service_failedToReloadDaemon: + m.daemondreload_f = fail + err = h.DaemonReload() + case errorutil.Service_failedToCheckActivity: + m.isactive_f = fail + _, err = h.IsActive() + case errorutil.Service_failedToCheckEnabled: + m.isenabled_f = func(string, *log.Context) (bool, error) { return false, cause } + _, err = h.IsEnabled() + case errorutil.Service_failedToCheckInstalled: + m.isinstalled_f = func(string, *log.Context) (bool, error) { return false, cause } + _, err = h.IsInstalled() + case errorutil.Service_failedToRemoveOldConfiguration: + m.removeUnitConfiguration_f = fail + err = h.Register(ctx, "") + case errorutil.Service_failedToCreateConfiguration: + m.createUnitConfiguration_f = func(string, []byte, *log.Context) error { return cause } + err = h.Register(ctx, "") + case errorutil.Service_failedToUnregister: + m.removeUnitConfiguration_f = fail + err = h.DeRegister(ctx) + case errorutil.Internal_failedToReadServiceVersion: + m.getInstalledVersion_f = func(string, *log.Context) (string, error) { return "", cause } + _, err = h.GetInstalledVersion(ctx) + } + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, want, code) + require.ErrorIs(t, err, cause) + } +} + +func TestServiceHandlerPreservesLowerClarifications(t *testing.T) { + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("lower failure") + lower := errorutil.EnsureClarification(-999, cause) + m := getManagerMock() + m.createUnitConfiguration_f = func(string, []byte, *log.Context) error { return lower } + h := NewHandler(m, NewConfiguration("test.service"), ctx) + err := h.Register(ctx, "") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, -999, code) + require.ErrorIs(t, err, cause) + m = getManagerMock() + m.removeUnitConfiguration_f = func(string, *log.Context) error { + return errorutil.EnsureClarification(998, &os.PathError{Op: "remove", Path: "test", Err: os.ErrNotExist}) + } + h = NewHandler(m, NewConfiguration("test.service"), ctx) + require.NoError(t, h.Register(ctx, ""), "wrapped ENOENT remains ignorable") +} diff --git a/pkg/servicehandler/servicehandler.go b/pkg/servicehandler/servicehandler.go index 078f025..ada78f2 100644 --- a/pkg/servicehandler/servicehandler.go +++ b/pkg/servicehandler/servicehandler.go @@ -5,6 +5,7 @@ import ( "os" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -43,64 +44,66 @@ func NewConfiguration(unitName string) Configuration { func (handler *Handler) Start() error { err := handler.manager.StartUnit(handler.config.Name, handler.ctx) - return err + return errorutil.EnsureClarification(errorutil.Service_failedToStart, err) } func (handler *Handler) Stop() error { err := handler.manager.StopUnit(handler.config.Name, handler.ctx) - return err + return errorutil.EnsureClarification(errorutil.Service_failedToStop, err) } func (handler *Handler) Enable() error { err := handler.manager.EnableUnit(handler.config.Name, handler.ctx) - return err + return errorutil.EnsureClarification(errorutil.Service_failedToEnable, err) } func (handler *Handler) Disable() error { err := handler.manager.DisableUnit(handler.config.Name, handler.ctx) - return err + return errorutil.EnsureClarification(errorutil.Service_failedToDisable, err) } func (handler *Handler) DaemonReload() error { err := handler.manager.DaemonReload(handler.config.Name, handler.ctx) - return err + return errorutil.EnsureClarification(errorutil.Service_failedToReloadDaemon, err) } func (handler *Handler) IsActive() (bool, error) { err := handler.manager.IsUnitActive(handler.config.Name, handler.ctx) if err != nil { - return false, err + return false, errorutil.EnsureClarification(errorutil.Service_failedToCheckActivity, err) } return true, nil } func (handler *Handler) IsEnabled() (bool, error) { - return handler.manager.IsUnitEnabled(handler.config.Name, handler.ctx) + enabled, err := handler.manager.IsUnitEnabled(handler.config.Name, handler.ctx) + return enabled, errorutil.EnsureClarification(errorutil.Service_failedToCheckEnabled, err) } func (handler *Handler) IsInstalled() (bool, error) { - return handler.manager.IsUnitInstalled(handler.config.Name, handler.ctx) + installed, err := handler.manager.IsUnitInstalled(handler.config.Name, handler.ctx) + return installed, errorutil.EnsureClarification(errorutil.Service_failedToCheckInstalled, err) } func (handler *Handler) Register(ctx *log.Context, unitConfigContent string) error { err := handler.manager.RemoveUnitConfigurationFile(handler.config.Name, ctx) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error while removing old unit configuration file: %v", err) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return errorutil.EnsureClarification(errorutil.Service_failedToRemoveOldConfiguration, fmt.Errorf("error while removing old unit configuration file: %w", err)) } err = handler.manager.CreateUnitConfigurationFile(handler.config.Name, []byte(unitConfigContent), ctx) if err != nil { - return fmt.Errorf("error while creating unit configuration file: %v", err) + return errorutil.EnsureClarification(errorutil.Service_failedToCreateConfiguration, fmt.Errorf("error while creating unit configuration file: %w", err)) } err = handler.DaemonReload() if err != nil { - return fmt.Errorf("error while reloading daemon worker: %v", err) + return fmt.Errorf("error while reloading daemon worker: %w", err) } err = handler.Enable() if err != nil { - return fmt.Errorf("error while enabling unit: %v", err) + return fmt.Errorf("error while enabling unit: %w", err) } return nil @@ -120,17 +123,17 @@ func (handler *Handler) DeRegister(ctx *log.Context) error { if targetVersion == installedVersion { err = handler.Stop() if err != nil { - return fmt.Errorf("error while stopping unit: %v", err) + return fmt.Errorf("error while stopping unit: %w", err) } err = handler.Disable() if err != nil { - return fmt.Errorf("error while disabling unit: %v", err) + return fmt.Errorf("error while disabling unit: %w", err) } err = handler.manager.RemoveUnitConfigurationFile(handler.config.Name, ctx) if err != nil { - return fmt.Errorf("error while removing unit configuration: %v", err) + return errorutil.EnsureClarification(errorutil.Service_failedToUnregister, fmt.Errorf("error while removing unit configuration: %w", err)) } } else { ctx.Log("message", "Skipping action. Target version is not current installed") @@ -140,5 +143,6 @@ func (handler *Handler) DeRegister(ctx *log.Context) error { } func (handler *Handler) GetInstalledVersion(ctx *log.Context) (string, error) { - return handler.manager.GetInstalledVersion(handler.config.Name, ctx) + version, err := handler.manager.GetInstalledVersion(handler.config.Name, ctx) + return version, errorutil.EnsureClarification(errorutil.Internal_failedToReadServiceVersion, err) } diff --git a/pkg/servicehandler/servicehandler_test.go b/pkg/servicehandler/servicehandler_test.go index 0bc610b..d36c6f3 100644 --- a/pkg/servicehandler/servicehandler_test.go +++ b/pkg/servicehandler/servicehandler_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/systemd" "github.com/go-kit/kit/log" ) @@ -229,6 +230,7 @@ func TestHandlerRegisterFailsOnUnitConfigurationFileDeletion(t *testing.T) { if err == nil { t.Errorf("unexpected successful registration call") } + requireServiceCode(t, errorutil.Service_failedToRemoveOldConfiguration, err) } func TestHandlerRegisterFailsOnUnitConfigurationFileCreation(t *testing.T) { @@ -250,6 +252,7 @@ func TestHandlerRegisterFailsOnUnitConfigurationFileCreation(t *testing.T) { if err == nil { t.Errorf("unexpected successful registration call") } + requireServiceCode(t, errorutil.Service_failedToCreateConfiguration, err) } func TestHandlerRegisterFailsOnDaemonReload(t *testing.T) { @@ -271,6 +274,7 @@ func TestHandlerRegisterFailsOnDaemonReload(t *testing.T) { if err == nil { t.Errorf("unexpected successful registration call") } + requireServiceCode(t, errorutil.Service_failedToReloadDaemon, err) } func TestHandlerRegisterFailsOnEnable(t *testing.T) { @@ -292,6 +296,7 @@ func TestHandlerRegisterFailsOnEnable(t *testing.T) { if err == nil { t.Errorf("unexpected successful registration call") } + requireServiceCode(t, errorutil.Service_failedToEnable, err) } func TestHandlerSuccessfulDeRegister(t *testing.T) { @@ -380,6 +385,7 @@ func TestHandlerFailsOnGettingInstalledVersionDeRegister(t *testing.T) { if err == nil { t.Errorf("an error should be thrown due to not being able to get the installed version") } + requireServiceCode(t, errorutil.Internal_failedToReadServiceVersion, err) if !m.functionCalled.getinstalledversion_f { t.Errorf("missing call to verify target version") @@ -441,6 +447,7 @@ func TestHandlerStartFailsOnStartUnit(t *testing.T) { if err == nil { t.Errorf("unexpected successful start unit command") } + requireServiceCode(t, errorutil.Service_failedToStart, err) } func TestHandlerSuccessfulStop(t *testing.T) { @@ -482,6 +489,7 @@ func TestHandlerStopFailsOnStopUnit(t *testing.T) { if err == nil { t.Errorf("unexpected successful stop unit command") } + requireServiceCode(t, errorutil.Service_failedToStop, err) } func TestHandlerSuccessfulEnable(t *testing.T) { @@ -523,6 +531,7 @@ func TestHandlerEnableFailsOnEnableUnit(t *testing.T) { if err == nil { t.Errorf("unexpected successful enable unit command") } + requireServiceCode(t, errorutil.Service_failedToEnable, err) } func TestHandlerSuccessfulDisable(t *testing.T) { @@ -564,6 +573,7 @@ func TestHandlerDisableFailsOnDisableUnit(t *testing.T) { if err == nil { t.Errorf("unexpected successful disable unit command") } + requireServiceCode(t, errorutil.Service_failedToDisable, err) } func TestHandlerSuccessfulDaemonReload(t *testing.T) { @@ -605,6 +615,7 @@ func TestHandlerDaemonReloadFailsOnDaemonReload(t *testing.T) { if err == nil { t.Errorf("unexpected successful daemon reload command") } + requireServiceCode(t, errorutil.Service_failedToReloadDaemon, err) } func TestHandlerIsActiveTrue(t *testing.T) { @@ -646,7 +657,8 @@ func TestHandlerIsActiveFalse(t *testing.T) { return fmt.Errorf("Failed to check if unit is active") } - isActive, _ := handler.IsActive() + isActive, err := handler.IsActive() + requireServiceCode(t, errorutil.Service_failedToCheckActivity, err) if !m.functionCalled.isunitactive_f { t.Errorf("missing IsUnitActive method call") } diff --git a/pkg/statusreporter/clarification_test.go b/pkg/statusreporter/clarification_test.go new file mode 100644 index 0000000..624cada --- /dev/null +++ b/pkg/statusreporter/clarification_test.go @@ -0,0 +1,66 @@ +package statusreporter + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type statusTransport func(*http.Request) (*http.Response, error) + +func (f statusTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestStatusReporterClarifications(t *testing.T) { + oldClient := statusHTTPClient + t.Cleanup(func() { statusHTTPClient = oldClient }) + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("transport failure") + for _, want := range []int{errorutil.HostGA_failedToCreateStatusRequest, errorutil.HostGA_failedToSendStatusRequest, errorutil.HostGA_statusUploadReturnedErrorStatus, errorutil.Internal_emptyStatusUpload} { + statusHTTPClient = &http.Client{Transport: statusTransport(func(*http.Request) (*http.Response, error) { + if want == errorutil.HostGA_failedToSendStatusRequest { + return nil, cause + } + return &http.Response{StatusCode: http.StatusForbidden, Status: "403 Forbidden", Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + endpoint, content := "http://example.invalid", "status" + if want == errorutil.HostGA_failedToCreateStatusRequest { + endpoint = "://" + } + var resp *http.Response + var err error + if want == errorutil.Internal_emptyStatusUpload { + resp, err = NewGuestInformationServiceClient(endpoint).ReportStatus(ctx, "") + } else { + resp, err = ReportStatus(ctx, endpoint, content) + } + if resp != nil { + resp.Body.Close() + } + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + if want == errorutil.HostGA_failedToSendStatusRequest { + require.ErrorIs(t, err, cause) + } else if want == errorutil.HostGA_statusUploadReturnedErrorStatus { + require.Contains(t, err.Error(), "403 Forbidden") + } + } + statusHTTPClient = &http.Client{Transport: statusTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + resp, err := ReportStatus(ctx, "http://example.invalid", "status") + require.ErrorContains(t, err, "404 Not Found") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.HostGA_statusUploadReturnedErrorStatus, code) + resp.Body.Close() + resp, err = ReportStatus(ctx, "://", "") + require.NoError(t, err) + require.Nil(t, resp) +} diff --git a/pkg/statusreporter/guestinformationserviceclient.go b/pkg/statusreporter/guestinformationserviceclient.go index 470c443..3f042ff 100644 --- a/pkg/statusreporter/guestinformationserviceclient.go +++ b/pkg/statusreporter/guestinformationserviceclient.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -36,7 +37,7 @@ func (c GuestInformationServiceClient) GetPutStatusUri() string { func (c GuestInformationServiceClient) ReportStatus(ctx *log.Context, statusToUpload string) (*http.Response, error) { if statusToUpload == "" { - return nil, errors.New("provided status to upload is empty") + return nil, errorutil.EnsureClarification(errorutil.Internal_emptyStatusUpload, errors.New("provided status to upload is empty")) } return ReportStatus(ctx, c.GetPutStatusUri(), statusToUpload) diff --git a/pkg/statusreporter/statusreporter.go b/pkg/statusreporter/statusreporter.go index 7d0c482..13898ee 100644 --- a/pkg/statusreporter/statusreporter.go +++ b/pkg/statusreporter/statusreporter.go @@ -2,12 +2,16 @@ package statusreporter import ( "bytes" + "fmt" "net/http" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) +var statusHTTPClient = &http.Client{} + type PutStatusRequest struct { Content string } @@ -25,15 +29,21 @@ func ReportStatus(ctx *log.Context, putStatusEndpoint string, statusToUpload str func uploadData(putStatusEndpoint string, byteContent []byte) (*http.Response, error) { req, err := http.NewRequest(http.MethodPut, putStatusEndpoint, bytes.NewBuffer(byteContent)) if err != nil { - return nil, errors.Wrap(err, "could not create new http request to send provided content") + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToCreateStatusRequest, errors.Wrap(err, "could not create new http request to send provided content")) } req.Header.Set("Content-Type", "application/json") - client := &http.Client{} - resp, err := client.Do(req) + resp, err := statusHTTPClient.Do(req) if err != nil { - return nil, errors.Wrap(err, "failed to send http request") + return nil, errorutil.EnsureClarification(errorutil.HostGA_failedToSendStatusRequest, errors.Wrap(err, "failed to send http request")) } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + status := resp.Status + if status == "" { + status = fmt.Sprintf("%d %s", resp.StatusCode, http.StatusText(resp.StatusCode)) + } + return resp, errorutil.EnsureClarification(errorutil.HostGA_statusUploadReturnedErrorStatus, fmt.Errorf("status upload returned HTTP %s", status)) + } return resp, nil } diff --git a/pkg/systemd/clarification_test.go b/pkg/systemd/clarification_test.go new file mode 100644 index 0000000..41de071 --- /dev/null +++ b/pkg/systemd/clarification_test.go @@ -0,0 +1,80 @@ +package systemd + +import ( + "errors" + "io" + "os" + "testing" + "time" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +type systemdFileInfo struct{ directory bool } + +func (i systemdFileInfo) Name() string { return "unit" } +func (i systemdFileInfo) Size() int64 { return 0 } +func (i systemdFileInfo) Mode() os.FileMode { return 0 } +func (i systemdFileInfo) ModTime() time.Time { return time.Time{} } +func (i systemdFileInfo) IsDir() bool { return i.directory } +func (i systemdFileInfo) Sys() interface{} { return nil } + +func TestSystemdClarifications(t *testing.T) { + oldRun, oldPath := runSystemctl, GetUnitConfigurationFilePath + oldStat, oldWrite, oldRemove, oldRead, oldBaseStat := statUnitFile, writeUnitFile, removeUnitFile, readUnitFile, statSystemdPath + t.Cleanup(func() { + runSystemctl, GetUnitConfigurationFilePath = oldRun, oldPath + statUnitFile, writeUnitFile, removeUnitFile, readUnitFile, statSystemdPath = oldStat, oldWrite, oldRemove, oldRead, oldBaseStat + }) + ctx := log.NewContext(log.NewLogfmtLogger(io.Discard)) + cause := errors.New("systemd failure") + assertCode := func(err error, want int) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "%v", err) + require.Equal(t, want, code) + } + runSystemctl = func(...string) ([]byte, error) { return nil, cause } + mgr := NewUnitManager() + for _, tc := range []struct { + code int + run func(string, *log.Context) error + }{ + {errorutil.Systemd_failedToStartUnit, mgr.StartUnit}, {errorutil.Systemd_failedToStopUnit, mgr.StopUnit}, + {errorutil.Systemd_failedToEnableUnit, mgr.EnableUnit}, {errorutil.Systemd_failedToDisableUnit, mgr.DisableUnit}, + {errorutil.Systemd_failedToReloadDaemon, mgr.DaemonReload}, {errorutil.Systemd_failedToCheckUnitActivity, mgr.IsUnitActive}, + } { + err := tc.run("test.service", ctx) + assertCode(err, tc.code) + require.ErrorIs(t, err, cause) + } + _, err := mgr.IsUnitEnabled("test", ctx) + assertCode(err, errorutil.Systemd_failedToCheckUnitEnabled) + for _, state := range []string{"enabled\n", "disabled\n"} { + runSystemctl = func(...string) ([]byte, error) { return []byte(state), cause } + enabled, err := mgr.IsUnitEnabled("test", ctx) + require.NoError(t, err) + require.Equal(t, state == "enabled\n", enabled) + } + GetUnitConfigurationFilePath = func(string, *log.Context) (string, error) { return "test", nil } + statUnitFile = func(string) (os.FileInfo, error) { return nil, cause } + _, err = mgr.IsUnitInstalled("test", ctx) + assertCode(err, errorutil.Systemd_failedToCheckUnitFile) + writeUnitFile = func(string, []byte, os.FileMode) error { return cause } + assertCode(mgr.CreateUnitConfigurationFile("test", nil, ctx), errorutil.Systemd_failedToWriteUnitConfiguration) + removeUnitFile = func(string) error { return cause } + assertCode(mgr.RemoveUnitConfigurationFile("test", ctx), errorutil.Systemd_failedToRemoveUnitConfiguration) + readUnitFile = func(string) ([]byte, error) { return nil, cause } + _, err = mgr.GetInstalledVersion("test", ctx) + assertCode(err, errorutil.Systemd_failedToReadUnitConfiguration) + require.ErrorIs(t, err, cause) + statSystemdPath = func(string) (os.FileInfo, error) { return nil, cause } + _, err = GetSystemDConfigurationBasePath(ctx) + assertCode(err, errorutil.Systemd_failedToInspectUnitDirectory) + require.ErrorIs(t, err, cause) + statSystemdPath = func(string) (os.FileInfo, error) { return systemdFileInfo{}, nil } + _, err = GetSystemDConfigurationBasePath(ctx) + assertCode(err, errorutil.Systemd_noUnitConfigurationDirectory) +} diff --git a/pkg/systemd/helper.go b/pkg/systemd/helper.go index 70bb732..7832c9d 100644 --- a/pkg/systemd/helper.go +++ b/pkg/systemd/helper.go @@ -4,10 +4,13 @@ import ( "fmt" "os" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) +var statSystemdPath = os.Stat + func IsSystemDPresent() bool { _, err := os.Stat("/run/systemd/system") return err == nil @@ -15,15 +18,18 @@ func IsSystemDPresent() bool { func GetSystemDConfigurationBasePath(ctx *log.Context) (string, error) { ctx.Log("message", "Getting systemd configuration path available in the system") - info, err := os.Stat(unitConfigurationBasePath_preferred) + info, err := statSystemdPath(unitConfigurationBasePath_preferred) if err != nil || info == nil || !info.IsDir() { ctx.Log("message", fmt.Sprintf("INFO: %s path was not found on the system", unitConfigurationBasePath_preferred)) - info, err = os.Stat(unitConfigurationBasePath_alternative) + info, err = statSystemdPath(unitConfigurationBasePath_alternative) if err != nil || info == nil || !info.IsDir() { errorstring := fmt.Sprintf("ERROR: neither %s nor %s path was not found on the system", unitConfigurationBasePath_preferred, unitConfigurationBasePath_alternative) ctx.Log("message", errorstring) - return "", errors.New(errorstring) + if err != nil { + return "", errorutil.EnsureClarification(errorutil.Systemd_failedToInspectUnitDirectory, errors.Wrap(err, errorstring)) + } + return "", errorutil.EnsureClarification(errorutil.Systemd_noUnitConfigurationDirectory, errors.New(errorstring)) } ctx.Log("message", fmt.Sprintf("Alternative path was found on the system: %s", unitConfigurationBasePath_alternative)) diff --git a/pkg/systemd/manager.go b/pkg/systemd/manager.go index d44725f..41f93f8 100644 --- a/pkg/systemd/manager.go +++ b/pkg/systemd/manager.go @@ -7,6 +7,7 @@ import ( "path" "strings" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/Azure/run-command-handler-linux/pkg/versionutil" "github.com/go-kit/kit/log" "github.com/pkg/errors" @@ -31,49 +32,60 @@ const ( type Manager struct { } +var runSystemctl = func(args ...string) ([]byte, error) { + return exec.Command(systemctl, args...).Output() +} + +var ( + statUnitFile = os.Stat + writeUnitFile = os.WriteFile + removeUnitFile = os.Remove + readUnitFile = os.ReadFile +) + func NewUnitManager() *Manager { return &Manager{} } func (mgr *Manager) StartUnit(unitName string, ctx *log.Context) error { ctx.Log("message", "running command to start unit") - err := exec.Command(systemctl, systemctl_start, unitName).Run() - return err + _, err := runSystemctl(systemctl_start, unitName) + return errorutil.EnsureClarification(errorutil.Systemd_failedToStartUnit, errors.Wrap(err, "failed to start systemd unit")) } func (mgr *Manager) StopUnit(unitName string, ctx *log.Context) error { ctx.Log("message", "running command to stop unit") - err := exec.Command(systemctl, systemctl_stop, unitName).Run() - return err + _, err := runSystemctl(systemctl_stop, unitName) + return errorutil.EnsureClarification(errorutil.Systemd_failedToStopUnit, errors.Wrap(err, "failed to stop systemd unit")) } func (mgr *Manager) EnableUnit(unitName string, ctx *log.Context) error { ctx.Log("message", "running command to enable unit") - err := exec.Command(systemctl, systemctl_enable, unitName).Run() - return err + _, err := runSystemctl(systemctl_enable, unitName) + return errorutil.EnsureClarification(errorutil.Systemd_failedToEnableUnit, errors.Wrap(err, "failed to enable systemd unit")) } func (mgr *Manager) DisableUnit(unitName string, ctx *log.Context) error { ctx.Log("message", "running command to disable unit") - err := exec.Command(systemctl, systemctl_disable, unitName).Run() - return err + _, err := runSystemctl(systemctl_disable, unitName) + return errorutil.EnsureClarification(errorutil.Systemd_failedToDisableUnit, errors.Wrap(err, "failed to disable systemd unit")) } func (mgr *Manager) DaemonReload(unitName string, ctx *log.Context) error { ctx.Log("message", "running command to reload daemon") - err := exec.Command(systemctl, systemctl_daemonreload).Run() - return err + _, err := runSystemctl(systemctl_daemonreload) + return errorutil.EnsureClarification(errorutil.Systemd_failedToReloadDaemon, errors.Wrap(err, "failed to reload systemd daemon")) } func (mgr *Manager) IsUnitActive(unitName string, ctx *log.Context) error { ctx.Log("message", "running command to check if unit is active") - err := exec.Command(systemctl, systemctl_isactive, unitName).Run() - return err + _, err := runSystemctl(systemctl_isactive, unitName) + return errorutil.EnsureClarification(errorutil.Systemd_failedToCheckUnitActivity, errors.Wrap(err, "failed to check systemd unit activity")) } func (mgr *Manager) IsUnitEnabled(unitName string, ctx *log.Context) (bool, error) { ctx.Log("message", "running command to check if unit is already enabled") - output, err := exec.Command(systemctl, systemctl_isenabled, unitName).Output() + output, err := runSystemctl(systemctl_isenabled, unitName) sanitizedOutput := strings.Replace(string(output), "\n", "", -1) ctx.Log("message", fmt.Sprintf("%v %v output: %v", systemctl, systemctl_isenabled, sanitizedOutput)) if sanitizedOutput == "enabled" { @@ -82,7 +94,7 @@ func (mgr *Manager) IsUnitEnabled(unitName string, ctx *log.Context) (bool, erro return false, nil } - return false, err + return false, errorutil.EnsureClarification(errorutil.Systemd_failedToCheckUnitEnabled, errors.Wrap(err, "failed to check whether systemd unit is enabled")) } func (mgr *Manager) IsUnitInstalled(unitName string, ctx *log.Context) (bool, error) { @@ -92,12 +104,12 @@ func (mgr *Manager) IsUnitInstalled(unitName string, ctx *log.Context) (bool, er } ctx.Log("message", fmt.Sprintf("Checking if unit file %v exists to verify presence of installed service", filePath)) - _, err = os.Stat(filePath) + _, err = statUnitFile(filePath) if os.IsNotExist(err) { return false, nil } else if err != nil { - return false, errors.Wrap(err, "Error occurred while checking file existence") + return false, errorutil.EnsureClarification(errorutil.Systemd_failedToCheckUnitFile, errors.Wrap(err, "Error occurred while checking file existence")) } return true, nil @@ -110,7 +122,7 @@ func (*Manager) CreateUnitConfigurationFile(unitName string, content []byte, ctx } ctx.Log("message", "creating unit configuration file in "+unitConfigPath) - return os.WriteFile(unitConfigPath, content, unitConfigurationFilePermission) + return errorutil.EnsureClarification(errorutil.Systemd_failedToWriteUnitConfiguration, errors.Wrap(writeUnitFile(unitConfigPath, content, unitConfigurationFilePermission), "failed to write systemd unit configuration")) } func (*Manager) RemoveUnitConfigurationFile(unitName string, ctx *log.Context) error { @@ -120,7 +132,7 @@ func (*Manager) RemoveUnitConfigurationFile(unitName string, ctx *log.Context) e } ctx.Log("message", "removing unit configuration file from "+unitConfigPath) - return os.Remove(unitConfigPath) + return errorutil.EnsureClarification(errorutil.Systemd_failedToRemoveUnitConfiguration, errors.Wrap(removeUnitFile(unitConfigPath), "failed to remove systemd unit configuration")) } // Gets the installed version of the extension. The version will look at the service definition and extract it from the @@ -133,9 +145,9 @@ func (*Manager) GetInstalledVersion(unitName string, ctx *log.Context) (string, } ctx.Log("message", "getting content from "+unitConfigPath) - content, err := os.ReadFile(unitConfigPath) + content, err := readUnitFile(unitConfigPath) if err != nil { - return "", err + return "", errorutil.EnsureClarification(errorutil.Systemd_failedToReadUnitConfiguration, errors.Wrap(err, "failed to read systemd unit configuration")) } return versionutil.ExtractFromServiceDefinition(string(content), ctx) } diff --git a/pkg/urlutil/clarification_test.go b/pkg/urlutil/clarification_test.go new file mode 100644 index 0000000..cc73b75 --- /dev/null +++ b/pkg/urlutil/clarification_test.go @@ -0,0 +1,34 @@ +package urlutil + +import ( + "errors" + "fmt" + "testing" + + "github.com/Azure/azure-extension-platform/vmextension" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" +) + +func TestRemoveURLClarification(t *testing.T) { + require.Nil(t, RemoveUrlFromErr(nil)) + cause := errors.New(`Get "https://example.invalid/script?sig=secret": failed`) + err := RemoveUrlFromErr(cause) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.FileDownload_redactedUrlError, code) + require.NotContains(t, err.Error(), "secret") + require.True(t, errors.Is(err, cause)) +} + +func TestRemoveURLPreservesClassifications(t *testing.T) { + pointer := vmextension.NewErrorWithClarificationPtr(1998, errors.New("https://example.invalid/?sig=secret failed")) + for _, classified := range []error{pointer, *pointer} { + err := RemoveUrlFromErr(fmt.Errorf("wrapped: %w", classified)) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, 1998, code) + require.NotContains(t, err.Error(), "secret") + require.True(t, errors.Is(err, classified)) + } +} diff --git a/pkg/urlutil/urlerrorhelper.go b/pkg/urlutil/urlerrorhelper.go index 8be50f8..8fd7d98 100644 --- a/pkg/urlutil/urlerrorhelper.go +++ b/pkg/urlutil/urlerrorhelper.go @@ -1,12 +1,24 @@ package urlutil import ( - "errors" "net/url" "strings" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" ) +type redactedError struct { + message string + cause error +} + +func (e redactedError) Error() string { return e.message } +func (e redactedError) Unwrap() error { return e.cause } + func RemoveUrlFromErr(err error) error { + if err == nil { + return nil + } strSegments := strings.Split(err.Error(), " ") for i, v := range strSegments { if IsValidUrl(v) { @@ -14,11 +26,11 @@ func RemoveUrlFromErr(err error) error { strSegments[i] = "[REDACTED]" } } - return errors.New(strings.Join(strSegments, " ")) + return errorutil.EnsureClarification(errorutil.FileDownload_redactedUrlError, redactedError{strings.Join(strSegments, " "), err}) } func IsValidUrl(urlstring string) bool { - u, parseError := url.Parse(urlstring) + u, parseError := url.Parse(strings.Trim(urlstring, `"'`)) if parseError == nil && u.Scheme != "" && u.Host != "" { return true } diff --git a/pkg/urlutil/urlerrorhelper_test.go b/pkg/urlutil/urlerrorhelper_test.go index f2c32e1..c099bf4 100644 --- a/pkg/urlutil/urlerrorhelper_test.go +++ b/pkg/urlutil/urlerrorhelper_test.go @@ -4,12 +4,23 @@ import ( "fmt" "strings" "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/stretchr/testify/require" ) +func requireRedactedErrorCode(t *testing.T, err error) { + t.Helper() + code, ok := errorutil.FindClarification(err) + require.True(t, ok, "missing clarification: %v", err) + require.Equal(t, errorutil.FileDownload_redactedUrlError, code) +} + func Test_expectedErrorString(t *testing.T) { errorString := `Get https://aaaaaaaa.blob.core.windows.net/nodeagentpackage-version9-0-0-381/Ubuntu-16.04/batch_config-ubuntu-16.04-1.5.9.tar.gz?sv=2018-03-28&sr=b&sig=a%secret%2Bsecret&st=2019-05-17T01%3A25%3A42Z&se=2021-05-24T01%3A25%3A42Z&sp=r: dial tcp 13.68.165.64:443: i/o timeout` inputErr := fmt.Errorf("%s", errorString) outputErr := RemoveUrlFromErr(inputErr) + requireRedactedErrorCode(t, outputErr) if strings.Contains(outputErr.Error(), "https://") || strings.Contains(outputErr.Error(), "secret") || !strings.Contains(outputErr.Error(), "[REDACTED]") { t.Error("Url removal failed") } else if !strings.Contains(outputErr.Error(), "dial tcp 13.68.165.64:443: i/o timeout") { @@ -23,6 +34,7 @@ func Test_nonHttpsSchema(t *testing.T) { errorString := `Gethttps://aaaaaaaa.blob.core.windows.net/nodeagentpackage-version9-0-0-381/Ubuntu-16.04/batch_config-ubuntu-16.04-1.5.9.tar.gz?sv=2018-03-28&sr=b&sig=a%secret%2Bsecret&st=2019-05-17T01%3A25%3A42Z&se=2021-05-24T01%3A25%3A42Z&sp=r: dial tcp 13.68.165.64:443: i/o timeout` inputErr := fmt.Errorf("%s", errorString) outputErr := RemoveUrlFromErr(inputErr) + requireRedactedErrorCode(t, outputErr) if strings.Contains(outputErr.Error(), "https://") || strings.Contains(outputErr.Error(), "secret") || !strings.Contains(outputErr.Error(), "[REDACTED]") { t.Error("Url removal failed") } else if !strings.Contains(outputErr.Error(), "dial tcp 13.68.165.64:443: i/o timeout") { @@ -36,6 +48,7 @@ func Test_errorWithMissingSpaces(t *testing.T) { errorString := `Gethttps://aaaaaaaa.blob.core.windows.net/nodeagentpackage-version9-0-0-381/Ubuntu-16.04/batch_config-ubuntu-16.04-1.5.9.tar.gz?sv=2018-03-28&sr=b&sig=a%secret%2Bsecret&st=2019-05-17T01%3A25%3A42Z&se=2021-05-24T01%3A25%3A42Z&sp=r:dial tcp 13.68.165.64:443:i/o timeout` inputErr := fmt.Errorf("%s", errorString) outputErr := RemoveUrlFromErr(inputErr) + requireRedactedErrorCode(t, outputErr) if strings.Contains(outputErr.Error(), "https://") || strings.Contains(outputErr.Error(), "secret") || !strings.Contains(outputErr.Error(), "[REDACTED]") { t.Error("Url removal failed") } else if !strings.Contains(outputErr.Error(), "tcp 13.68.165.64:443:i/o timeout") { @@ -50,6 +63,7 @@ func Test_errorStringIsUrlOnly(t *testing.T) { errorString := `https://aaaaaaaa.blob.core.windows.net/nodeagentpackage-version9-0-0-381/Ubuntu-16.04/batch_config-ubuntu-16.04-1.5.9.tar.gz?sv=2018-03-28&sr=b&sig=a%secret%2Bsecret&st=2019-05-17T01%3A25%3A42Z&se=2021-05-24T01%3A25%3A42Z&sp=r` inputErr := fmt.Errorf("%s", errorString) outputErr := RemoveUrlFromErr(inputErr) + requireRedactedErrorCode(t, outputErr) if strings.Contains(outputErr.Error(), "https://") || strings.Contains(outputErr.Error(), "secret") || !strings.Contains(outputErr.Error(), "[REDACTED]") { t.Error("Url removal failed") } else { @@ -61,6 +75,7 @@ func Test_errorStringIsSpaces(t *testing.T) { errorString := ` ` inputErr := fmt.Errorf("%s", errorString) outputErr := RemoveUrlFromErr(inputErr) + requireRedactedErrorCode(t, outputErr) if outputErr.Error() != errorString { t.Error("Url detected where there was no URL") } else { @@ -72,6 +87,7 @@ func Test_errorStringHasNoUrl(t *testing.T) { errorString := `This: error\\ message \ has \ no? {urls:}` inputErr := fmt.Errorf("%s", errorString) outputErr := RemoveUrlFromErr(inputErr) + requireRedactedErrorCode(t, outputErr) if outputErr.Error() != errorString { t.Error("Url detected where there was no URL") } else { diff --git a/pkg/versionutil/clarification_test.go b/pkg/versionutil/clarification_test.go new file mode 100644 index 0000000..bca4742 --- /dev/null +++ b/pkg/versionutil/clarification_test.go @@ -0,0 +1,17 @@ +package versionutil + +import ( + "io" + "testing" + + "github.com/Azure/run-command-handler-linux/pkg/errorutil" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func TestVersionClarification(t *testing.T) { + _, err := ExtractFromServiceDefinition("invalid", log.NewContext(log.NewLogfmtLogger(io.Discard))) + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Internal_invalidServiceDefinition, code) +} diff --git a/pkg/versionutil/version.go b/pkg/versionutil/version.go index 6fec0d7..71063e0 100644 --- a/pkg/versionutil/version.go +++ b/pkg/versionutil/version.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" ) @@ -50,7 +51,7 @@ func ExtractFromServiceDefinition(content string, ctx *log.Context) (string, err // If parsing still didn't succeed, return an error. if len(firstSplit) < 2 { - return "", errors.New("wrong service definition found. Missing field " + fmt.Sprintf("ExecStart=%s/%s- and ExecStart=%s/%s-", constants.WaAgentDirectory, constants.RunCommandExtensionName, constants.WaAgentDirectory, constants.RunCommandTestExtensionName)) + return "", errorutil.EnsureClarification(errorutil.Internal_invalidServiceDefinition, errors.New("wrong service definition found. Missing field "+fmt.Sprintf("ExecStart=%s/%s- and ExecStart=%s/%s-", constants.WaAgentDirectory, constants.RunCommandExtensionName, constants.WaAgentDirectory, constants.RunCommandTestExtensionName))) } } diff --git a/pkg/versionutil/version_test.go b/pkg/versionutil/version_test.go index 1204a5f..2762304 100644 --- a/pkg/versionutil/version_test.go +++ b/pkg/versionutil/version_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/Azure/run-command-handler-linux/internal/constants" + "github.com/Azure/run-command-handler-linux/pkg/errorutil" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -59,6 +60,9 @@ func TestFailToExtractVersion(t *testing.T) { for _, content := range versionsToTest { extractedVersion, err := ExtractFromServiceDefinition(content, ctx) require.NotNil(t, err, "extracting the version from invalid service definition should return an error") + code, ok := errorutil.FindClarification(err) + require.True(t, ok) + require.Equal(t, errorutil.Internal_invalidServiceDefinition, code) require.Equal(t, "", extractedVersion) } }