From e6c884d66a8f57c8d178f445839a575d8cf08cdf Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Fri, 28 Aug 2026 19:40:07 +0400 Subject: [PATCH 01/15] Update debug archive v1 Signed-off-by: Valery Losev --- .../collect-debug-info/debugtar/debugTar.go | 102 +++++++++++------- 1 file changed, 64 insertions(+), 38 deletions(-) diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index c99c36634..50959c0a0 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -45,12 +45,12 @@ type moduleList struct { // debugCommands - a complete list of commands for collecting debug information. var debugCommands = []Command{ { - File: "queue.txt", + File: "deckhouse-queue.txt", Cmd: "deckhouse-controller", Args: []string{"queue", "list"}, }, { - File: "global-values.json", + File: "cluster-global-values.json", Cmd: "bash", Args: []string{"-c", `deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, }, @@ -75,7 +75,7 @@ var debugCommands = []Command{ Args: []string{"-c", `kubectl get moduleconfig -ojson | jq -r '.items[] | select(.spec.maintenance == "NoResourceReconciliation") | .metadata.name'`}, }, { - File: "events.json", + File: "cluster-events.json", Cmd: "kubectl", Args: []string{"get", "events", "--sort-by=.metadata.creationTimestamp", "-A", "-o", "json"}, }, @@ -85,48 +85,48 @@ var debugCommands = []Command{ Args: []string{"-c", `for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, }, { - File: "node-groups.json", + File: "cluster-node-groups.json", Cmd: "kubectl", Args: []string{"get", "nodegroups", "-A", "-o", "json"}, }, { - File: "node-group-configuration.json", + File: "cluster-node-group-configuration.json", Cmd: "kubectl", Args: []string{"get", "nodegroupconfiguration", "-A", "-o", "json"}, }, { - File: "nodes.json", + File: "cluster-nodes.json", Cmd: "kubectl", Args: []string{"get", "nodes", "-A", "-o", "json"}, }, { - File: "namespace.json", + File: "cluster-namespace.json", Cmd: "kubectl", Args: []string{"get", "namespaces", "-o", "json"}, }, { - File: "machines.json", + File: "instance-manager-machines.json", Cmd: "bash", - Args: []string{"-c", `kubectl get machines.machine.sapcloud.io -A -o json | jq '.items[]'`}, + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.cluster.x-k8s.io -o json | jq '.items[]'`}, }, { - File: "instances.json", + File: "instance-manager-instances.json", Cmd: "bash", Args: []string{"-c", `kubectl get instances.deckhouse.io -o json | jq '.items[]'`}, }, { - File: "staticinstances.json", + File: "instance-manager-staticinstances.json", Cmd: "bash", Args: []string{"-c", `kubectl get staticinstances.deckhouse.io -o json | jq '.items[]'`}, }, { - File: "cloud-machine-deployment.txt", + File: "instance-manager-cloud-machine-deployment.txt", Cmd: "bash", Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machinedeployments.machine.sapcloud.io -o json | jq '.items[]'`}, RequiredModule: "cloud-provider", }, { - File: "static-machine-deployment.txt", + File: "instance-manager-static-machine-deployment.txt", Cmd: "bash", Args: []string{"-c", "kubectl -n d8-cloud-instance-manager get machinedeployments.cluster.x-k8s.io -o json --ignore-not-found | jq '.items[]'"}, }, @@ -146,101 +146,101 @@ var debugCommands = []Command{ Args: []string{"-n", "d8-system", "logs", "-l", "app=deckhouse", "--tail", "3000"}, }, { - File: "capi-controller-manager-logs.txt", + File: "instance-manager-capi-controller-manager-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=capi-controller-manager", "--tail", "3000", "--ignore-errors=true"}, }, { - File: "caps-controller-manager-logs.txt", + File: "instance-manager-caps-controller-manager-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=caps-controller-manager", "--tail", "3000", "--ignore-errors=true"}, }, { - File: "machine-controller-manager.json", + File: "instance-manager-machine-controller-manager.json", Cmd: "bash", Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, RequiredModule: "cloud-provider", }, { - File: "mcm-logs.txt", + File: "instance-manager-mcm-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, }, { - File: "ccm-logs-{module-name}.txt", + File: "{module-name}-ccm-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, RequiredModule: "cloud-provider", ExpandPerModule: true, }, { - File: "csi-controller-logs-{module-name}.txt", + File: "{module-name}-csi-controller-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, RequiredModule: "cloud-provider", ExpandPerModule: true, }, { - File: "cluster-autoscaler-logs.txt", + File: "instance-manager-autoscaler-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail=5000", "-c", "cluster-autoscaler", "--ignore-errors=true"}, }, { - File: "cert-manager-logs.txt", + File: "core-cert-manager-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-cert-manager", "logs", "-l", "app=cert-manager", "--tail=3000", "--ignore-errors=true"}, RequiredModule: "cert-manager", ExpandPerModule: false, }, { - File: "certificate-cert-manager.json", + File: "core-certificate-cert-manager.json", Cmd: "kubectl", Args: []string{"get", "certificate", "-A", "-o", "json", "--ignore-not-found=true"}, RequiredModule: "cert-manager", ExpandPerModule: false, }, { - File: "vpa-admission-controller-logs.txt", + File: "kube-system-vpa-admission-controller-logs.txt", Cmd: "kubectl", Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-admission-controller", "--tail=3000", "-c", "admission-controller", "--ignore-errors=true"}, }, { - File: "vpa-recommender-logs.txt", + File: "kube-system-vpa-recommender-logs.txt", Cmd: "kubectl", Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-recommender", "--tail=3000", "-c", "recommender", "--ignore-errors=true"}, }, { - File: "vpa-updater-logs.txt", + File: "kube-system-vpa-updater-logs.txt", Cmd: "kubectl", Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-updater", "--tail=3000", "-c", "updater", "--ignore-errors=true"}, }, { - File: "prometheus-logs.txt", + File: "monitoring-prometheus-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-monitoring", "logs", "-l", "prometheus=main", "--tail=3000", "-c", "prometheus", "--ignore-errors=true"}, }, { - File: "alerts.json", + File: "cluster-alerts.json", Cmd: "bash", Args: []string{"-c", `kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, }, { - File: "bad-pods.txt", + File: "cluster-bad-pods.txt", Cmd: "bash", Args: []string{"-c", `kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)" || true`}, }, { - File: "cluster-authorization-rules.json", + File: "security-cluster-authorization-rules.json", Cmd: "bash", Args: []string{"-c", `kubectl get clusterauthorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, }, { - File: "authorization-rules.json", + File: "security-authorization-rules.json", Cmd: "bash", Args: []string{"-c", `kubectl get authorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, }, { - File: "module-configs.json", + File: "deckhouse-module-configs.json", Cmd: "kubectl", Args: []string{"get", "moduleconfig", "-o", "json"}, }, @@ -287,14 +287,14 @@ var debugCommands = []Command{ ExpandPerModule: false, }, { - File: "cni-cilium-health-status.txt", + File: "network-cni-cilium-health-status.txt", Cmd: "bash", Args: []string{"-c", `kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, RequiredModule: "cni-cilium", ExpandPerModule: false, }, { - File: "audit-policy.json", + File: "kube-system-audit-policy.json", Cmd: "kubectl", Args: []string{"-n", "kube-system", "get", "secrets", "audit-policy", "-o", "json", "--ignore-not-found=true"}, }, @@ -329,30 +329,56 @@ var debugCommands = []Command{ Args: []string{"-n", "kube-system", "logs", "-l", "k8s-app=kube-dns", "--tail=3000", "--ignore-errors=true"}, }, { - File: "prometheusremotewrites.json", + File: "monitoring-prometheusremotewrites.json", Cmd: "kubectl", Args: []string{"get", "prometheusremotewrites", "-A", "-o", "json", "--ignore-not-found=true"}, }, { - File: "mutatingwebhookconfigurations.json", + File: "other-mutatingwebhookconfigurations.json", Cmd: "kubectl", Args: []string{"get", "mutatingwebhookconfigurations.admissionregistration.k8s.io", "-o", "json"}, }, { - File: "validatingwebhookconfigurations.json", + File: "other-validatingwebhookconfigurations.json", Cmd: "kubectl", Args: []string{"get", "validatingwebhookconfigurations.admissionregistration.k8s.io", "-o", "json"}, }, { - File: "storage-deckhouse-io-terminating.txt", + File: "other-storage-deckhouse-io-terminating.txt", Cmd: "bash", Args: []string{"-c", `kubectl get $(kubectl api-resources --api-group=storage.deckhouse.io --verbs=list -o name | paste -sd, -) --ignore-not-found -A --chunk-size=200 -o json | jq -r '.items[] | select(.apiVersion == "storage.deckhouse.io/v1alpha1") | select(.metadata.deletionTimestamp != null) | "[\(.kind)] \(.metadata.namespace // "-")/\(.metadata.name)"'`}, }, { - File: "ingressnginxcontrollers.json", + File: "network-ingressnginxcontrollers.json", Cmd: "kubectl", Args: []string{"get", "ingressnginxcontrollers.deckhouse.io", "-o", "json", "--ignore-not-found=true"}, }, + { + File: "cluster-crd.json", + Cmd: "kubectl", + Args: []string{"get", "customresourcedefinitions", "-o", "json", "--ignore-not-found=true"}, + }, + { + File: "d8-virtualization-dvcr-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=dvcr", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", + ExpandPerModule: false, + }, + { + File: "d8-virtualization-virt-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "kubevirt.internal.virtualization.deckhouse.io=virt-controller", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", + ExpandPerModule: false, + }, + { + File: "d8-virtualization-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=virtualization-controller", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", + ExpandPerModule: false, + }, } func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) error { From f4924db246265a0731bc64fe3f0328ac3ed7336e Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 1 Sep 2026 17:57:32 +0400 Subject: [PATCH 02/15] Update debug archive v2 Signed-off-by: Valery Losev --- .../collect-debug-info/collect-debug-info.go | 3 + .../collect-debug-info/debugtar/debugTar.go | 43 +++-- .../debugtar/virtualizationTarball.go | 167 ++++++++++++++++++ .../virtualizationtar/virtualizationTar.go | 106 +++++++++++ 4 files changed, 307 insertions(+), 12 deletions(-) create mode 100644 internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go create mode 100644 internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go diff --git a/internal/system/cmd/collect-debug-info/collect-debug-info.go b/internal/system/cmd/collect-debug-info/collect-debug-info.go index 746bf74fe..8354b4201 100644 --- a/internal/system/cmd/collect-debug-info/collect-debug-info.go +++ b/internal/system/cmd/collect-debug-info/collect-debug-info.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/deckhouse/deckhouse-cli/internal/system/cmd/collect-debug-info/debugtar" + "github.com/deckhouse/deckhouse-cli/internal/system/cmd/collect-debug-info/virtualizationtar" "github.com/deckhouse/deckhouse-cli/internal/utilk8s" ) @@ -79,6 +80,8 @@ func NewCommand() *cobra.Command { collectDebugInfoCmd.Flags().DurationVar(&commandTimeout, "command-timeout", 2*time.Minute, "Timeout for each individual debug command execution") collectDebugInfoCmd.Flags().DurationVar(&requestInterval, "request-interval", 0, "Minimum interval between debug command executions to avoid overloading the cluster (e.g. 200ms, 500ms, 1s). Zero disables rate limiting (default 0s)") + collectDebugInfoCmd.AddCommand(virtualizationtar.NewCommand()) + return collectDebugInfoCmd } diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index 50959c0a0..6ac732536 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -186,14 +186,14 @@ var debugCommands = []Command{ Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail=5000", "-c", "cluster-autoscaler", "--ignore-errors=true"}, }, { - File: "core-cert-manager-logs.txt", + File: "d8-cert-manager-logs.txt", Cmd: "kubectl", Args: []string{"-n", "d8-cert-manager", "logs", "-l", "app=cert-manager", "--tail=3000", "--ignore-errors=true"}, RequiredModule: "cert-manager", ExpandPerModule: false, }, { - File: "core-certificate-cert-manager.json", + File: "d8-cert-manager-all-certificate.json", Cmd: "kubectl", Args: []string{"get", "certificate", "-A", "-o", "json", "--ignore-not-found=true"}, RequiredModule: "cert-manager", @@ -404,6 +404,35 @@ func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []st excludeMap[file] = true } + gzipWriter := gzip.NewWriter(os.Stdout) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + fmt.Fprintf(os.Stderr, "Collecting debug info from Deckhouse...\n") + + if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, excludeMap, commandTimeout, requestInterval); err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "Debug archive collection completed.\n") + + return nil +} + +// runCommands executes each command inside the Deckhouse pod and streams its +// output into the tar archive, honoring the exclude list and the optional +// rate limit between command executions. +func runCommands( + tarWriter *tar.Writer, + config *rest.Config, + kubeCl kubernetes.Interface, + podName, namespace, containerName string, + commands []Command, + excludeMap map[string]bool, + commandTimeout, requestInterval time.Duration, +) error { var tickCh <-chan time.Time if requestInterval > 0 { @@ -415,14 +444,6 @@ func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []st var stdout, stderr bytes.Buffer - gzipWriter := gzip.NewWriter(os.Stdout) - defer gzipWriter.Close() - - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() - - fmt.Fprintf(os.Stderr, "Collecting debug info from Deckhouse...\n") - for _, cmd := range commands { if isFileExcluded(cmd.File, excludeMap) { continue @@ -464,8 +485,6 @@ func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []st stderr.Reset() } - fmt.Fprintf(os.Stderr, "Debug archive collection completed.\n") - return nil } diff --git a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go new file mode 100644 index 000000000..f3337252a --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go @@ -0,0 +1,167 @@ +package debugtar + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "os" + "sort" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" +) + +const virtualizationNamespace = "d8-virtualization" + +// virtualizationCommands - additional resource-intensive commands collected only in the virtualization archive +var virtualizationCommands = []Command{ + { + File: "d8-virtualization-pods-wide.txt", + Cmd: "kubectl", + Args: []string{"-n", virtualizationNamespace, "get", "pod", "-o", "wide"}, + }, +} + +type virtualizationPod struct { + Name string + DaemonSetOwned bool +} + +type podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + OwnerReferences []struct { + Kind string `json:"kind"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` +} + +// VirtualizationTarball collects a separate, virtualization-focused debug +// archive: the list of pods in the d8-virtualization namespace +// plus per-pod logs, optionally skipping pods owned by a DaemonSet +// (virt-handler, virtualization-dra, vm-route-forge, ...) since their log +// volume scales with the number of nodes. +func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, commandTimeout, requestInterval time.Duration, skipDsLogs bool) error { + const ( + namespace = "d8-system" + containerName = "deckhouse" + ) + + podName, err := utilk8s.GetDeckhousePod(kubeCl) + if err != nil { + return fmt.Errorf("failed to get Deckhouse pod: %w", err) + } + + pods, err := fetchVirtualizationPods(config, kubeCl, podName, namespace, containerName, commandTimeout) + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: could not list pods in %s: %v\n", virtualizationNamespace, err) + } else if len(pods) == 0 { + fmt.Fprintf(os.Stderr, "WARNING: no pods found in namespace %s, is the virtualization module enabled?\n", virtualizationNamespace) + } + + commands := buildVirtualizationCommands(pods, skipDsLogs) + + gzipWriter := gzip.NewWriter(os.Stdout) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + fmt.Fprintf(os.Stderr, "Collecting virtualization debug info from Deckhouse...\n") + + if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, nil, commandTimeout, requestInterval); err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "Virtualization debug archive collection completed.\n") + + return nil +} + +// fetchVirtualizationPods lists the pods currently running in the +// virtualization namespace and reports which ones are owned by a DaemonSet, +// so the DaemonSet-managed pods can be identified without hardcoding their names. +func fetchVirtualizationPods( + config *rest.Config, + kubeCl kubernetes.Interface, + podName, namespace, containerName string, + timeout time.Duration, +) ([]virtualizationPod, error) { + cmdLine := []string{"kubectl", "-n", virtualizationNamespace, "get", "pods", "-o", "json", "--ignore-not-found=true"} + + executor, err := utilk8s.ExecInPod(config, kubeCl, cmdLine, podName, namespace, containerName) + if err != nil { + return nil, fmt.Errorf("create executor: %w", err) + } + + var stdout, stderr bytes.Buffer + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + if err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + return nil, fmt.Errorf("stream kubectl get pods: %w (stderr: %s)", err, stderr.String()) + } + + if stdout.Len() == 0 { + return nil, nil + } + + var list podList + if err = json.Unmarshal(stdout.Bytes(), &list); err != nil { + return nil, fmt.Errorf("parse pod list: %w", err) + } + + pods := make([]virtualizationPod, 0, len(list.Items)) + for _, item := range list.Items { + daemonSetOwned := false + for _, owner := range item.Metadata.OwnerReferences { + if owner.Kind == "DaemonSet" { + daemonSetOwned = true + break + } + } + + pods = append(pods, virtualizationPod{ + Name: item.Metadata.Name, + DaemonSetOwned: daemonSetOwned, + }) + } + + sort.Slice(pods, func(i, j int) bool { return pods[i].Name < pods[j].Name }) + + return pods, nil +} + +// buildVirtualizationCommands turns the discovered pod list into the final +// the static commands first, then one log-collection command per pod (skipping DaemonSet-owned pods when skipDsLogs is set). +func buildVirtualizationCommands(pods []virtualizationPod, skipDsLogs bool) []Command { + commands := make([]Command, 0, len(virtualizationCommands)+len(pods)) + commands = append(commands, virtualizationCommands...) + + for _, pod := range pods { + if skipDsLogs && pod.DaemonSetOwned { + continue + } + + commands = append(commands, Command{ + File: fmt.Sprintf("d8-virtualization-%s-logs.txt", pod.Name), + Cmd: "kubectl", + Args: []string{"-n", virtualizationNamespace, "logs", pod.Name, "--tail=-1", "--ignore-errors=true"}, + }) + } + + return commands +} diff --git a/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go b/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go new file mode 100644 index 000000000..e59de27d2 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go @@ -0,0 +1,106 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualizationtar + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "golang.org/x/term" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/deckhouse/deckhouse-cli/internal/system/cmd/collect-debug-info/debugtar" + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" +) + +var ( + virtualizationCmdLong = templates.LongDesc(` + Collect a separate debug archive with detailed data from the d8-virtualization namespace. + + Pods are discovered dynamically, so the collected set always matches + the current cluster state. Logs are collected from the default + container of each pod. + + © Flant JSC 2025`) + + virtualizationCmdExample = templates.Examples(` + # Collect the virtualization debug archive: + d8 system collect-debug-info virtualization > deckhouse-debug-virtualization-$(date +"%Y_%m_%d").tar.gz + + # The --skip-ds-logs flag can be used to skip logs from DaemonSet-managed pods + # (virt-handler, virtualization-dra, vm-route-forge, ...) to reduce archive size: + d8 system collect-debug-info virtualization --skip-ds-logs > deckhouse-debug-virtualization-$(date +"%Y_%m_%d").tar.gz + `) +) + +func NewCommand() *cobra.Command { + var ( + commandTimeout time.Duration + requestInterval time.Duration + skipDsLogs bool + ) + + virtualizationCmd := &cobra.Command{ + Use: `virtualization [flags] > deckhouse-debug-virtualization-$(date +"%Y_%m_%d").tar.gz`, + Short: "Collect a separate virtualization debug archive.", + Long: virtualizationCmdLong, + Example: virtualizationCmdExample, + SilenceErrors: true, + SilenceUsage: true, + PreRunE: func(_ *cobra.Command, _ []string) error { + if term.IsTerminal(int(os.Stdout.Fd())) { + return fmt.Errorf("output must be redirected to a file, e.g., \"> dump-logs.tar.gz\"") + } + + return nil + }, + RunE: func(cmd *cobra.Command, _ []string) error { + return collectVirtualizationDebugInfo(cmd, commandTimeout, requestInterval, skipDsLogs) + }, + } + + virtualizationCmd.Flags().DurationVar(&commandTimeout, "command-timeout", 2*time.Minute, "Timeout for each individual debug command execution") + virtualizationCmd.Flags().DurationVar(&requestInterval, "request-interval", 0, "Minimum interval between debug command executions to avoid overloading the cluster (e.g. 200ms, 500ms, 1s). Zero disables rate limiting (default 0s)") + virtualizationCmd.Flags().BoolVar(&skipDsLogs, "skip-ds-logs", false, "Skip collecting logs from pods managed by a DaemonSet (virt-handler, virtualization-dra, vm-route-forge, ...) to reduce archive size on clusters with many nodes") + + return virtualizationCmd +} + +func collectVirtualizationDebugInfo(cmd *cobra.Command, commandTimeout, requestInterval time.Duration, skipDsLogs bool) error { + kubeconfigPath, err := cmd.Flags().GetString("kubeconfig") + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + contextName, err := cmd.Flags().GetString("context") + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + config, kubeCl, err := utilk8s.SetupK8sClientSet(kubeconfigPath, contextName) + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + if err = debugtar.VirtualizationTarball(config, kubeCl, commandTimeout, requestInterval, skipDsLogs); err != nil { + return fmt.Errorf("Error collecting virtualization debug info: %w", err) + } + + return nil +} From e92fd7175b7477ecc444033fba9c999c837e3ad8 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 1 Sep 2026 18:27:12 +0400 Subject: [PATCH 03/15] Update debug archive v3 Signed-off-by: Valery Losev --- .../virtualizationtar/virtualizationTar.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go b/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go index e59de27d2..e494048f1 100644 --- a/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go +++ b/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go @@ -32,12 +32,7 @@ import ( var ( virtualizationCmdLong = templates.LongDesc(` Collect a separate debug archive with detailed data from the d8-virtualization namespace. - - Pods are discovered dynamically, so the collected set always matches - the current cluster state. Logs are collected from the default - container of each pod. - - © Flant JSC 2025`) + `) virtualizationCmdExample = templates.Examples(` # Collect the virtualization debug archive: From 1071427ca8344e59b840e37ad1dbf87bb6154cc6 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 1 Sep 2026 18:38:55 +0400 Subject: [PATCH 04/15] Update debug archive v4 Signed-off-by: Valery Losev --- .../cmd/collect-debug-info/debugtar/virtualizationTarball.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go index f3337252a..7ab217b8c 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go +++ b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go @@ -127,6 +127,7 @@ func fetchVirtualizationPods( pods := make([]virtualizationPod, 0, len(list.Items)) for _, item := range list.Items { daemonSetOwned := false + for _, owner := range item.Metadata.OwnerReferences { if owner.Kind == "DaemonSet" { daemonSetOwned = true From 214e8c65190d8f063e2d18dc7fd9a1cb11ec8c7d Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Mon, 7 Sep 2026 20:45:42 +0400 Subject: [PATCH 05/15] Update debug archive v5 Signed-off-by: Valery Losev --- .../collect-debug-info/debugtar/debugTar.go | 50 +++++++++++++------ .../debugtar/virtualizationTarball.go | 19 ++++--- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index 6ac732536..df7a139d6 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -25,6 +25,10 @@ type Command struct { Args []string File string + // ExcludeKey overrides the identifier used in the --exclude/--list-exclude flags. + // Leave blank to get it from the "File" field. This is set explicitly if {module-name} is not the final "File" segment, as the default output only processes this pattern. + ExcludeKey string + // RequiredModule is the module prefix (status.phase == "Ready"). If the module is enabled, data from it will be collected. An empty string means always run. RequiredModule string // ExpandPerModule — If true, the command is duplicated for each active module matching RequiredModule. The {module-name} placeholder is accepted in File and Args, and is replaced with the actual module name. @@ -168,6 +172,7 @@ var debugCommands = []Command{ }, { File: "{module-name}-ccm-logs.txt", + ExcludeKey: "ccm-logs", Cmd: "kubectl", Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, RequiredModule: "cloud-provider", @@ -175,6 +180,7 @@ var debugCommands = []Command{ }, { File: "{module-name}-csi-controller-logs.txt", + ExcludeKey: "csi-controller-logs", Cmd: "kubectl", Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, RequiredModule: "cloud-provider", @@ -381,7 +387,7 @@ var debugCommands = []Command{ }, } -func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) error { +func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) (err error) { const ( namespace = "d8-system" containerName = "deckhouse" @@ -405,10 +411,17 @@ func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []st } gzipWriter := gzip.NewWriter(os.Stdout) - defer gzipWriter.Close() - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() + + defer func() { + if closeErr := tarWriter.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to finalize tar archive: %w", closeErr) + } + + if closeErr := gzipWriter.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to finalize gzip stream: %w", closeErr) + } + }() fmt.Fprintf(os.Stderr, "Collecting debug info from Deckhouse...\n") @@ -445,7 +458,7 @@ func runCommands( var stdout, stderr bytes.Buffer for _, cmd := range commands { - if isFileExcluded(cmd.File, excludeMap) { + if isFileExcluded(cmd, excludeMap) { continue } @@ -545,9 +558,10 @@ func filterAndExpandCommands(commands []Command, activeModules map[string]bool) matchedModules := matchingModules(activeModules, cmd.RequiredModule) for _, moduleName := range matchedModules { result = append(result, Command{ - Cmd: cmd.Cmd, - File: strings.ReplaceAll(cmd.File, "{module-name}", moduleName), - Args: replaceModuleName(cmd.Args, moduleName), + Cmd: cmd.Cmd, + File: strings.ReplaceAll(cmd.File, "{module-name}", moduleName), + Args: replaceModuleName(cmd.Args, moduleName), + ExcludeKey: cmd.ExcludeKey, }) } } else { @@ -608,20 +622,28 @@ func (c *Command) writeToTar(tarWriter *tar.Writer, fileContent []byte) error { return nil } -func excludeBaseName(file string) string { - name := strings.TrimSuffix(file, ".json") +func excludeBaseName(cmd Command) string { + if cmd.ExcludeKey != "" { + return cmd.ExcludeKey + } + + name := strings.TrimSuffix(cmd.File, ".json") name = strings.TrimSuffix(name, ".txt") name = strings.TrimSuffix(name, "-{module-name}") return name } -func isFileExcluded(fileName string, excludeMap map[string]bool) bool { - if excludeMap[fileName] { +func isFileExcluded(cmd Command, excludeMap map[string]bool) bool { + if excludeMap[cmd.File] { return true } - base := strings.TrimSuffix(fileName, ".json") + if cmd.ExcludeKey != "" { + return excludeMap[cmd.ExcludeKey] + } + + base := strings.TrimSuffix(cmd.File, ".json") base = strings.TrimSuffix(base, ".txt") if excludeMap[base] { @@ -642,7 +664,7 @@ func GetExcludableFiles() []string { files := make([]string, 0, len(debugCommands)) for _, cmd := range debugCommands { - name := excludeBaseName(cmd.File) + name := excludeBaseName(cmd) if !seen[name] { seen[name] = true files = append(files, name) diff --git a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go index 7ab217b8c..29b274bef 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go +++ b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go @@ -50,7 +50,7 @@ type podList struct { // plus per-pod logs, optionally skipping pods owned by a DaemonSet // (virt-handler, virtualization-dra, vm-route-forge, ...) since their log // volume scales with the number of nodes. -func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, commandTimeout, requestInterval time.Duration, skipDsLogs bool) error { +func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, commandTimeout, requestInterval time.Duration, skipDsLogs bool) (err error) { const ( namespace = "d8-system" containerName = "deckhouse" @@ -71,10 +71,17 @@ func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, com commands := buildVirtualizationCommands(pods, skipDsLogs) gzipWriter := gzip.NewWriter(os.Stdout) - defer gzipWriter.Close() - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() + + defer func() { + if closeErr := tarWriter.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to finalize tar archive: %w", closeErr) + } + + if closeErr := gzipWriter.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to finalize gzip stream: %w", closeErr) + } + }() fmt.Fprintf(os.Stderr, "Collecting virtualization debug info from Deckhouse...\n") @@ -146,8 +153,8 @@ func fetchVirtualizationPods( return pods, nil } -// buildVirtualizationCommands turns the discovered pod list into the final -// the static commands first, then one log-collection command per pod (skipping DaemonSet-owned pods when skipDsLogs is set). +// buildVirtualizationCommands transforms the discovered list of pods into a final list. +// first the static commands, then one log collection command for each pod (skipping pods belonging to DaemonSet if skipDsLogs is set). func buildVirtualizationCommands(pods []virtualizationPod, skipDsLogs bool) []Command { commands := make([]Command, 0, len(virtualizationCommands)+len(pods)) commands = append(commands, virtualizationCommands...) From 03e640d9cb920ebd9a9c2645ef6b3ef000903787 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 8 Sep 2026 20:43:05 +0400 Subject: [PATCH 06/15] Update debug archive v6 Signed-off-by: Valery Losev --- .../collect-debug-info/debugtar/debugTar.go | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index df7a139d6..5c9b862d0 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -166,25 +166,30 @@ var debugCommands = []Command{ RequiredModule: "cloud-provider", }, { - File: "instance-manager-mcm-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, + File: "instance-manager-mcm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, + RequiredModule: "cloud-provider", }, { - File: "{module-name}-ccm-logs.txt", - ExcludeKey: "ccm-logs", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, - RequiredModule: "cloud-provider", - ExpandPerModule: true, + File: "instance-manager-mcm-cloud-machines.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, + RequiredModule: "cloud-provider", }, { - File: "{module-name}-csi-controller-logs.txt", - ExcludeKey: "csi-controller-logs", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, - RequiredModule: "cloud-provider", - ExpandPerModule: true, + File: "d8-{module-name}-ccm-logs.txt", + ExcludeKey: "ccm-logs", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, + RequiredModule: "cloud-provider", + }, + { + File: "d8-{module-name}-csi-controller-logs.txt", + ExcludeKey: "csi-controller-logs", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, + RequiredModule: "cloud-provider", }, { File: "instance-manager-autoscaler-logs.txt", From 554fee2417b360d30c053850acf9391b6861b42c Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 8 Sep 2026 22:03:14 +0400 Subject: [PATCH 07/15] Update debug archive v7 Signed-off-by: Valery Losev --- .../collect-debug-info/debugtar/debugTar.go | 163 ++++++++++-------- 1 file changed, 87 insertions(+), 76 deletions(-) diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index 5c9b862d0..e6d5902b0 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "slices" "sort" "strings" "time" @@ -30,9 +31,14 @@ type Command struct { ExcludeKey string // RequiredModule is the module prefix (status.phase == "Ready"). If the module is enabled, data from it will be collected. An empty string means always run. + // + // If File or any Args element contains the {module-name} placeholder, it + // is only substituted (and the command duplicated once per matching + // module, see needsModuleExpansion) when RequiredModule is set — the + // module name to substitute comes from resolving RequiredModule against + // the active modules. Leaving RequiredModule empty while the placeholder + // is present means it is never resolved and stays literal in the output. RequiredModule string - // ExpandPerModule — If true, the command is duplicated for each active module matching RequiredModule. The {module-name} placeholder is accepted in File and Args, and is replaced with the actual module name. - ExpandPerModule bool } type moduleList struct { @@ -73,6 +79,11 @@ var debugCommands = []Command{ Cmd: "bash", Args: []string{"-c", "kubectl get modulepulloverrides -o json | jq '.items[]'"}, }, + { + File: "deckhouse-module-update-policies.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get moduleupdatepolicies -o json | jq '.items[]'"}, + }, { File: "deckhouse-maintenance-modules.txt", Cmd: "bash", @@ -109,7 +120,7 @@ var debugCommands = []Command{ Args: []string{"get", "namespaces", "-o", "json"}, }, { - File: "instance-manager-machines.json", + File: "instance-manager-capi-machines.json", Cmd: "bash", Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.cluster.x-k8s.io -o json | jq '.items[]'`}, }, @@ -160,22 +171,19 @@ var debugCommands = []Command{ Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=caps-controller-manager", "--tail", "3000", "--ignore-errors=true"}, }, { - File: "instance-manager-machine-controller-manager.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, - RequiredModule: "cloud-provider", + File: "instance-manager-machine-controller-manager.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, }, { - File: "instance-manager-mcm-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, - RequiredModule: "cloud-provider", + File: "instance-manager-mcm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, }, { - File: "instance-manager-mcm-cloud-machines.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, - RequiredModule: "cloud-provider", + File: "instance-manager-mcm-cloud-machines.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, }, { File: "d8-{module-name}-ccm-logs.txt", @@ -197,18 +205,16 @@ var debugCommands = []Command{ Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail=5000", "-c", "cluster-autoscaler", "--ignore-errors=true"}, }, { - File: "d8-cert-manager-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cert-manager", "logs", "-l", "app=cert-manager", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "cert-manager", - ExpandPerModule: false, + File: "d8-cert-manager-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cert-manager", "logs", "-l", "app=cert-manager", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "cert-manager", }, { - File: "d8-cert-manager-all-certificate.json", - Cmd: "kubectl", - Args: []string{"get", "certificate", "-A", "-o", "json", "--ignore-not-found=true"}, - RequiredModule: "cert-manager", - ExpandPerModule: false, + File: "d8-cert-manager-all-certificate.json", + Cmd: "kubectl", + Args: []string{"get", "certificate", "-A", "-o", "json", "--ignore-not-found=true"}, + RequiredModule: "cert-manager", }, { File: "kube-system-vpa-admission-controller-logs.txt", @@ -256,53 +262,46 @@ var debugCommands = []Command{ Args: []string{"get", "moduleconfig", "-o", "json"}, }, { - File: "d8-istio-resources.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio get all -o json | jq '.items[]'`}, - RequiredModule: "istio", - ExpandPerModule: false, + File: "d8-istio-resources.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-istio get all -o json | jq '.items[]'`}, + RequiredModule: "istio", }, { - File: "d8-istio-custom-resources.json", - Cmd: "bash", - Args: []string{"-c", `for crd in $(kubectl get crds | grep -E 'istio.io|gateway.networking.k8s.io' | awk '{print $1}'); do echo "Listing resources for CRD: $crd" && kubectl get $crd -A -o json; done`}, - RequiredModule: "istio", - ExpandPerModule: false, + File: "d8-istio-custom-resources.json", + Cmd: "bash", + Args: []string{"-c", `for crd in $(kubectl get crds | grep -E 'istio.io|gateway.networking.k8s.io' | awk '{print $1}'); do echo "Listing resources for CRD: $crd" && kubectl get $crd -A -o json; done`}, + RequiredModule: "istio", }, { - File: "d8-istio-envoy-config.json", - Cmd: "bash", - Args: []string{"-c", `kubectl port-forward daemonset/ingressgateway -n d8-istio 15000:15000 & sleep 5; (curl http://localhost:15000/config_dump?include_eds=true | jq 'del(.configs[6].dynamic_active_secrets)' && kill $!) || { kill $!; exit 0; }`}, - RequiredModule: "istio", - ExpandPerModule: false, + File: "d8-istio-envoy-config.json", + Cmd: "bash", + Args: []string{"-c", `kubectl port-forward daemonset/ingressgateway -n d8-istio 15000:15000 & sleep 5; (curl http://localhost:15000/config_dump?include_eds=true | jq 'del(.configs[6].dynamic_active_secrets)' && kill $!) || { kill $!; exit 0; }`}, + RequiredModule: "istio", }, { - File: "d8-istio-system-logs.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio logs -l app=istiod || true`}, - RequiredModule: "istio", - ExpandPerModule: false, + File: "d8-istio-system-logs.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-istio logs -l app=istiod || true`}, + RequiredModule: "istio", }, { - File: "d8-istio-ingress-logs.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio logs daemonset/ingressgateway || true`}, - RequiredModule: "istio", - ExpandPerModule: false, + File: "d8-istio-ingress-logs.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-istio logs daemonset/ingressgateway || true`}, + RequiredModule: "istio", }, { - File: "d8-istio-users-logs.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl get pods --all-namespaces -o jsonpath='{range .items[?(@.metadata.annotations.istio\.io/rev)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' | awk '/istio-proxy/ {print $0}' | shuf -n 1 | while read namespace pod_name containers; do echo "Collecting logs from istio-proxy in Pod $pod_name (Namespace: $namespace)"; kubectl logs "$pod_name" -n "$namespace" -c istio-proxy; done`}, - RequiredModule: "istio", - ExpandPerModule: false, + File: "d8-istio-users-logs.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get pods --all-namespaces -o jsonpath='{range .items[?(@.metadata.annotations.istio\.io/rev)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' | awk '/istio-proxy/ {print $0}' | shuf -n 1 | while read namespace pod_name containers; do echo "Collecting logs from istio-proxy in Pod $pod_name (Namespace: $namespace)"; kubectl logs "$pod_name" -n "$namespace" -c istio-proxy; done`}, + RequiredModule: "istio", }, { - File: "network-cni-cilium-health-status.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, - RequiredModule: "cni-cilium", - ExpandPerModule: false, + File: "network-cni-cilium-health-status.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, + RequiredModule: "cni-cilium", }, { File: "kube-system-audit-policy.json", @@ -370,25 +369,22 @@ var debugCommands = []Command{ Args: []string{"get", "customresourcedefinitions", "-o", "json", "--ignore-not-found=true"}, }, { - File: "d8-virtualization-dvcr-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=dvcr", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "virtualization", - ExpandPerModule: false, + File: "d8-virtualization-dvcr-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=dvcr", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", }, { - File: "d8-virtualization-virt-controller-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-virtualization", "logs", "-l", "kubevirt.internal.virtualization.deckhouse.io=virt-controller", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "virtualization", - ExpandPerModule: false, + File: "d8-virtualization-virt-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "kubevirt.internal.virtualization.deckhouse.io=virt-controller", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", }, { - File: "d8-virtualization-controller-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=virtualization-controller", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "virtualization", - ExpandPerModule: false, + File: "d8-virtualization-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=virtualization-controller", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", }, } @@ -559,7 +555,7 @@ func filterAndExpandCommands(commands []Command, activeModules map[string]bool) continue } - if cmd.ExpandPerModule { + if needsModuleExpansion(cmd) { matchedModules := matchingModules(activeModules, cmd.RequiredModule) for _, moduleName := range matchedModules { result = append(result, Command{ @@ -600,6 +596,21 @@ func isModuleMatch(moduleName, required string) bool { return moduleName == required || strings.HasPrefix(moduleName, required) } +// needsModuleExpansion reports whether cmd must be duplicated once per active +// module matching RequiredModule (with {module-name} substituted into File +// and Args), rather than run once as-is. This is derived from the template +// itself instead of a separate flag, so File/Args and the expansion behavior +// can never drift apart. +func needsModuleExpansion(cmd Command) bool { + if strings.Contains(cmd.File, "{module-name}") { + return true + } + + return slices.ContainsFunc(cmd.Args, func(arg string) bool { + return strings.Contains(arg, "{module-name}") + }) +} + func replaceModuleName(args []string, moduleName string) []string { expanded := make([]string, len(args)) for i, arg := range args { From 41cbf321b07a05684339b843b350fcbb690ef471 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 8 Sep 2026 22:16:45 +0400 Subject: [PATCH 08/15] Update debug archive v8 Signed-off-by: Valery Losev --- .../debugtar/debugTar_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go new file mode 100644 index 000000000..f3600ec4e --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go @@ -0,0 +1,24 @@ +package debugtar + +import "testing" + +// TestDebugCommandsModuleExpansionInvariant guards the contract documented on +// Command.RequiredModule: a command whose File or Args contains the +// {module-name} placeholder must also set RequiredModule, since that is what +// filterAndExpandCommands uses to resolve the placeholder into a real module +// name. Without RequiredModule, needsModuleExpansion is never even checked, +// so the placeholder would leak into the collected archive as a literal +// string instead of a resolved module name. +func TestDebugCommandsModuleExpansionInvariant(t *testing.T) { + for _, cmd := range debugCommands { + if cmd.RequiredModule != "" { + continue + } + + if !needsModuleExpansion(cmd) { + continue + } + + t.Errorf("command %q uses the {module-name} placeholder but has no RequiredModule set, so it will never be resolved", cmd.File) + } +} From 6b88484b9ba191d66704bf9e2a87ec6c78749f28 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Wed, 9 Sep 2026 19:32:29 +0400 Subject: [PATCH 09/15] Update debug archive v9 Signed-off-by: Valery Losev --- internal/system/cmd/collect-debug-info/debugtar/debugTar.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index e6d5902b0..875c187a1 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -598,9 +598,7 @@ func isModuleMatch(moduleName, required string) bool { // needsModuleExpansion reports whether cmd must be duplicated once per active // module matching RequiredModule (with {module-name} substituted into File -// and Args), rather than run once as-is. This is derived from the template -// itself instead of a separate flag, so File/Args and the expansion behavior -// can never drift apart. +// and Args), rather than run once as-is. func needsModuleExpansion(cmd Command) bool { if strings.Contains(cmd.File, "{module-name}") { return true From e03421d02c7ef386399b84ff0e738fcb910cc347 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Mon, 14 Sep 2026 21:31:02 +0400 Subject: [PATCH 10/15] Update debug archive v10 Signed-off-by: Valery Losev --- internal/system/README.md | 34 +- .../collect-debug-info/collect-debug-info.go | 31 +- .../collect-debug-info_test.go | 33 ++ .../collect-debug-info/debugtar/debugTar.go | 368 +++++++++++++----- .../debugtar/debugTar_test.go | 278 ++++++++++++- .../debugtar/virtualizationTarball.go | 110 ++---- .../virtualizationtar/virtualizationTar.go | 15 +- internal/utilk8s/clientset.go | 15 + internal/utilk8s/operatepod.go | 30 ++ internal/utilk8s/syncbuffer.go | 47 +++ internal/utilk8s/syncbuffer_test.go | 70 ++++ 11 files changed, 815 insertions(+), 216 deletions(-) create mode 100644 internal/system/cmd/collect-debug-info/collect-debug-info_test.go create mode 100644 internal/utilk8s/syncbuffer.go create mode 100644 internal/utilk8s/syncbuffer_test.go diff --git a/internal/system/README.md b/internal/system/README.md index e972efc4b..c7bd292dc 100644 --- a/internal/system/README.md +++ b/internal/system/README.md @@ -52,6 +52,7 @@ d8 system (aliases: s, p, platform) │ └── main Dump the main queue ├── logs Stream deckhouse-controller logs └── collect-debug-info Stream a gzipped debug tarball to stdout + └── virtualization Stream a d8-virtualization-only debug tarball ``` The `s` alias is the recommended short form (`d8 s module list`). `p` and `platform` are legacy aliases kept for backward compatibility with older documentation. @@ -227,16 +228,34 @@ Collects a wide cluster snapshot into a **gzipped tar streamed to stdout**, so y d8 system collect-debug-info > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz ``` -It refuses to run when stdout is a terminal (to avoid dumping binary to your screen) unless you pass `--list-exclude`. The collection runs **inside** the leader pod: it executes on the order of ~60 diagnostic commands there (`deckhouse-controller queue list`, redacted global values, module/source/release inventories, cluster-wide `kubectl get` snapshots, and controller/etcd/apiserver/VPA/Prometheus logs, plus cloud-provider/cert-manager/istio/cni-cilium extras when those modules are Ready), writing each result as a file in the archive. +It refuses to run when stdout is a terminal (to avoid dumping binary to your screen) unless you pass `--list-exclude`. The collection runs **inside** the leader pod: it executes 63 diagnostic commands there (`deckhouse-controller queue list`, redacted global values, module/source/release inventories, cluster-wide `kubectl get` snapshots, and controller/etcd/apiserver/VPA/Prometheus logs, plus cloud-provider/cert-manager/istio/cni-cilium/virtualization extras when those modules are Ready), writing each result as a file in the archive. + +Before collecting, the command reads the list of `Ready` modules to decide which module-gated commands apply. If that read fails, it prints an error and keeps going: module-gated commands run anyway (and may produce empty files), except the per-module log collections whose file name contains the module name - those are skipped, since their archive entry name cannot be resolved. | Flag | Short | Type | Default | Description | |---|---|---|---|---| -| `--exclude` | | string list | (none) | Comma-separated list of elements to leave out of the archive. Matches by base name, so e.g. `ccm-logs` also drops the per-cloud `ccm-logs-.txt`. | -| `--list-exclude` | `-l` | bool | `false` | Print the names of everything that can be excluded, then exit. This path makes no cluster calls. | +| `--exclude` | | string list | (none) | Comma-separated list of entries to leave out of the archive. Accepts exactly the names printed by `--list-exclude`, with or without the file extension; a name matches that entry only, never a group of files sharing a prefix. A name that matches nothing is an error listing close matches, so a typo cannot pass as "collect everything". | +| `--list-exclude` | `-l` | bool | `false` | Print the names accepted by `--exclude`, then exit. This path makes no cluster calls. The names are the archive file names as declared in the command table, except the per-module cloud logs, which are printed as the module-independent keys `ccm-logs` and `csi-controller-logs` (their real entry is `d8--ccm-logs.txt`, and both spellings are accepted). | | `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command. | | `--request-interval` | | duration | `0` | Minimum gap between commands to avoid overloading the cluster (e.g. `200ms`, `1s`). `0` disables rate limiting. | -**Handle the archive as sensitive.** Only `global-values.json` is redacted (its `kubeRBACProxyCA` and registry `dockercfg`); container logs and the raw `audit-policy` Secret are included unredacted. Also note that a file is written even when its source command fails or times out, so an entry may be empty rather than absent. +**Handle the archive as sensitive.** Only `cluster-global-values.json` is redacted (its `kubeRBACProxyCA` and registry `dockercfg`); container logs and the raw audit policy Secret (`kube-system-audit-policy.json`) are included unredacted. Also note that a file is written even when its source command fails or times out, so an entry may be empty rather than absent. + +### `collect-debug-info virtualization` + +Collects a separate, virtualization-focused archive: the pod list of the `d8-virtualization` namespace plus the **full** log of every pod in it (`--tail=-1`, no line cap). Same stdout rules as the parent command. + +```bash +d8 system collect-debug-info virtualization > deckhouse-debug-virtualization-$(date +"%Y_%m_%d").tar.gz +``` + +| Flag | Short | Type | Default | Description | +|---|---|---|---|---| +| `--skip-ds-logs` | | bool | `false` | Skip logs of pods owned by a DaemonSet (`virt-handler`, `virtualization-dra`, `vm-route-forge`, ...), whose volume scales with the number of nodes. | +| `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command. | +| `--request-interval` | | duration | `0` | Minimum gap between commands to avoid overloading the cluster. | + +The pod list is the entire payload of this archive, so the command fails (and writes nothing) when the namespace cannot be listed or holds no pods - instead of producing a valid-looking archive with a single empty file. `--exclude`/`--list-exclude` do not apply here. --- @@ -312,6 +331,10 @@ d8 system collect-debug-info --list-exclude d8 system collect-debug-info --exclude ccm-logs,csi-controller-logs \ > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz +# Collect the virtualization-only archive, without DaemonSet pod logs +d8 system collect-debug-info virtualization --skip-ds-logs \ + > deckhouse-debug-virtualization-$(date +"%Y_%m_%d").tar.gz + # --- Global flags --- @@ -328,5 +351,6 @@ d8 system --kubeconfig ~/.kube/prod.config --context prod module list - **`approve` / `apply-now` are annotation-only and idempotent.** They never error on an already-annotated or non-`Pending` release; they print a notice and exit 0. - **`package scan` does not scan locally and does not wait.** It creates a `PackageRepositoryOperation` and returns; results are reported by the platform, not the CLI. - **In-pod commands need a leader pod.** `module list`/`values`/`snapshots`, `queue`, and `collect-debug-info` exec into the pod labeled `leader=true` in `d8-system`; without it they fail with `no pods deckhouse available in namespace d8-system`. -- **The debug archive is sensitive** (unredacted logs and the raw audit-policy Secret) and must be redirected to a file. +- **The debug archive is sensitive** (unredacted logs and the raw audit policy Secret, `kube-system-audit-policy.json`) and must be redirected to a file. +- **`collect-debug-info` takes no positional arguments.** A misspelled subcommand (`virtualisation`) is rejected with `unknown command` instead of silently running the full cluster-wide collection. - **stdout vs stderr:** `module` state changes print to stdout while notices/warnings/errors print to stderr, which makes it easy to script against applied changes only. diff --git a/internal/system/cmd/collect-debug-info/collect-debug-info.go b/internal/system/cmd/collect-debug-info/collect-debug-info.go index 8354b4201..bcf8ee896 100644 --- a/internal/system/cmd/collect-debug-info/collect-debug-info.go +++ b/internal/system/cmd/collect-debug-info/collect-debug-info.go @@ -54,10 +54,15 @@ func NewCommand() *cobra.Command { ) collectDebugInfoCmd := &cobra.Command{ - Use: `collect-debug-info [flags] > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz`, - Short: "Collect debug info.", - Long: collectDebugInfoCmdLong, - Example: collectDebugInfoCmdExample, + Use: `collect-debug-info [flags] > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz`, + Short: "Collect debug info.", + Long: collectDebugInfoCmdLong, + Example: collectDebugInfoCmdExample, + // Without this, an unknown positional argument (a misspelled + // subcommand, say) is silently accepted by the parent and the full + // cluster-wide collection runs instead: cobra only reports unknown + // commands for the root command, and this one has a parent. + Args: cobra.NoArgs, SilenceErrors: true, SilenceUsage: true, PreRunE: func(_ *cobra.Command, _ []string) error { @@ -75,8 +80,8 @@ func NewCommand() *cobra.Command { return collectDebugInfo(cmd, listExclude, excludeList, commandTimeout, requestInterval) }, } - collectDebugInfoCmd.Flags().StringSliceVar(&excludeList, "exclude", []string{}, "Exclude specific files from the debug archive. Use comma-separated values") - collectDebugInfoCmd.Flags().BoolVarP(&listExclude, "list-exclude", "l", false, "List all files that can be excluded from the debug archive") + collectDebugInfoCmd.Flags().StringSliceVar(&excludeList, "exclude", []string{}, "Exclude specific files from the debug archive. Use comma-separated names as printed by --list-exclude; the file extension is optional. An unknown name is an error") + collectDebugInfoCmd.Flags().BoolVarP(&listExclude, "list-exclude", "l", false, "List all files that can be excluded from the debug archive, then exit") collectDebugInfoCmd.Flags().DurationVar(&commandTimeout, "command-timeout", 2*time.Minute, "Timeout for each individual debug command execution") collectDebugInfoCmd.Flags().DurationVar(&requestInterval, "request-interval", 0, "Minimum interval between debug command executions to avoid overloading the cluster (e.g. 200ms, 500ms, 1s). Zero disables rate limiting (default 0s)") @@ -99,19 +104,9 @@ func collectDebugInfo(cmd *cobra.Command, listExclude bool, excludeList []string return nil } - kubeconfigPath, err := cmd.Flags().GetString("kubeconfig") + config, kubeCl, err := utilk8s.NewClientSet(cmd) if err != nil { - return fmt.Errorf("Failed to setup Kubernetes client: %w", err) - } - - contextName, err := cmd.Flags().GetString("context") - if err != nil { - return fmt.Errorf("Failed to setup Kubernetes client: %w", err) - } - - config, kubeCl, err := utilk8s.SetupK8sClientSet(kubeconfigPath, contextName) - if err != nil { - return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + return err } if err = debugtar.Tarball(config, kubeCl, excludeList, commandTimeout, requestInterval); err != nil { diff --git a/internal/system/cmd/collect-debug-info/collect-debug-info_test.go b/internal/system/cmd/collect-debug-info/collect-debug-info_test.go new file mode 100644 index 000000000..fc9d5e2b2 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/collect-debug-info_test.go @@ -0,0 +1,33 @@ +package collectdebuginfo + +import ( + "io" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// TestSubcommandTypoIsRejected guards the Args: cobra.NoArgs on the parent +// command. Without it cobra accepts an unknown positional argument silently +// (its "unknown command" check only fires for the root command) and the full +// cluster-wide collection runs instead of the requested subcommand. +func TestSubcommandTypoIsRejected(t *testing.T) { + root := &cobra.Command{Use: "d8", SilenceErrors: true, SilenceUsage: true} + system := &cobra.Command{Use: "system"} + root.AddCommand(system) + system.AddCommand(NewCommand()) + + root.SetOut(io.Discard) + root.SetErr(io.Discard) + root.SetArgs([]string{"system", "collect-debug-info", "virtualisation"}) + + err := root.Execute() + if err == nil { + t.Fatal("a misspelled subcommand was accepted, the full collection would have run") + } + + if !strings.Contains(err.Error(), "unknown command") { + t.Errorf("error = %v, want it to mention an unknown command", err) + } +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go index 875c187a1..c231311a9 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go @@ -2,7 +2,6 @@ package debugtar import ( "archive/tar" - "bytes" "compress/gzip" "context" "encoding/json" @@ -16,7 +15,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/remotecommand" "github.com/deckhouse/deckhouse-cli/internal/utilk8s" ) @@ -26,18 +24,26 @@ type Command struct { Args []string File string - // ExcludeKey overrides the identifier used in the --exclude/--list-exclude flags. - // Leave blank to get it from the "File" field. This is set explicitly if {module-name} is not the final "File" segment, as the default output only processes this pattern. - ExcludeKey string - - // RequiredModule is the module prefix (status.phase == "Ready"). If the module is enabled, data from it will be collected. An empty string means always run. + // RequiredModule gates the command on a module being Ready (status.phase == + // "Ready"): the command runs only when the name of some Ready module starts + // with this string, so "cloud-provider" matches cloud-provider-aws. An empty + // string means always run. + // + // Together with the {module-name} placeholder it also means "once per + // matching module": when RequiredModule is set and File or any Args element + // contains the placeholder (see needsModuleExpansion), the command is + // duplicated for every matching Ready module, with the placeholder + // substituted in both File and Args. Put the placeholder in File whenever it + // appears in Args — copies that differ only in Args all end up under the same + // archive entry name, and only the last one survives extraction. + // + // Leaving RequiredModule empty while the placeholder is present means it is + // never resolved and stays literal in the output. // - // If File or any Args element contains the {module-name} placeholder, it - // is only substituted (and the command duplicated once per matching - // module, see needsModuleExpansion) when RequiredModule is set — the - // module name to substitute comes from resolving RequiredModule against - // the active modules. Leaving RequiredModule empty while the placeholder - // is present means it is never resolved and stays literal in the output. + // When the module list cannot be fetched at all, gating is impossible and + // the fallback differs by shape: commands without the placeholder run anyway + // (they either produce data or an empty file), commands with it are skipped, + // since their archive entry name cannot be resolved. RequiredModule string } @@ -187,14 +193,12 @@ var debugCommands = []Command{ }, { File: "d8-{module-name}-ccm-logs.txt", - ExcludeKey: "ccm-logs", Cmd: "kubectl", Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, RequiredModule: "cloud-provider", }, { File: "d8-{module-name}-csi-controller-logs.txt", - ExcludeKey: "csi-controller-logs", Cmd: "kubectl", Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, RequiredModule: "cloud-provider", @@ -365,8 +369,12 @@ var debugCommands = []Command{ }, { File: "cluster-crd.json", - Cmd: "kubectl", - Args: []string{"get", "customresourcedefinitions", "-o", "json", "--ignore-not-found=true"}, + Cmd: "bash", + // The OpenAPI schemas dominate the size of a full CRD dump (tens of MB on + // a cluster with virtualization/istio/cilium/storage) without adding + // diagnostic value, so they are dropped here instead of being buffered, + // transferred and stored. + Args: []string{"-c", `set -o pipefail; kubectl get customresourcedefinitions -o json | jq 'del(.items[].spec.versions[].schema)'`}, }, { File: "d8-virtualization-dvcr-logs.txt", @@ -388,7 +396,7 @@ var debugCommands = []Command{ }, } -func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) (err error) { +func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) error { const ( namespace = "d8-system" containerName = "deckhouse" @@ -399,18 +407,38 @@ func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []st return fmt.Errorf("failed to get Deckhouse pod: %w", err) } - activeModules, err := fetchActiveModules(config, kubeCl, podName, namespace, containerName, commandTimeout) - if err != nil { - fmt.Fprintf(os.Stderr, "WARNING: could not fetch active modules, module-dependent commands will be skipped: %v\n", err) + activeModules, modulesErr := fetchActiveModules(config, kubeCl, podName, namespace, containerName, commandTimeout) + if modulesErr != nil { + fmt.Fprintf(os.Stderr, "ERROR: could not fetch the list of active modules: %v\n", modulesErr) + fmt.Fprintf(os.Stderr, " collection continues without module filtering: module-gated commands run anyway and may produce empty files; per-module commands are skipped because their file names cannot be resolved (%s)\n", + strings.Join(moduleScopedFiles(debugCommands), ", ")) } - commands := filterAndExpandCommands(debugCommands, activeModules) + commands, acceptedNames := filterAndExpandCommands(debugCommands, activeModules, modulesErr == nil, newExcludeSet(excludeFiles)) - excludeMap := make(map[string]bool, len(excludeFiles)) - for _, file := range excludeFiles { - excludeMap[file] = true + if err := validateExcludeNames(excludeFiles, acceptedNames); err != nil { + return err } + return writeArchive( + config, kubeCl, podName, namespace, containerName, + commands, commandTimeout, requestInterval, + "Collecting debug info from Deckhouse...", + "Debug archive collection completed.", + ) +} + +// writeArchive streams a gzipped tar of the given commands' output to stdout. +// It is the shared body of the debug archives: only the command set and the +// progress banners differ between them. +func writeArchive( + config *rest.Config, + kubeCl kubernetes.Interface, + podName, namespace, containerName string, + commands []Command, + commandTimeout, requestInterval time.Duration, + startBanner, doneBanner string, +) (err error) { gzipWriter := gzip.NewWriter(os.Stdout) tarWriter := tar.NewWriter(gzipWriter) @@ -424,27 +452,40 @@ func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []st } }() - fmt.Fprintf(os.Stderr, "Collecting debug info from Deckhouse...\n") + fmt.Fprintf(os.Stderr, "%s\n", startBanner) - if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, excludeMap, commandTimeout, requestInterval); err != nil { + if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, commandTimeout, requestInterval); err != nil { return err } - fmt.Fprintf(os.Stderr, "Debug archive collection completed.\n") + fmt.Fprintf(os.Stderr, "%s\n", doneBanner) return nil } +// moduleScopedFiles lists the File templates that can only be resolved with a +// known module list, for the warning printed when that list is unavailable. +func moduleScopedFiles(commands []Command) []string { + var files []string + + for _, cmd := range commands { + if cmd.RequiredModule != "" && needsModuleExpansion(cmd) { + files = append(files, cmd.File) + } + } + + return files +} + // runCommands executes each command inside the Deckhouse pod and streams its -// output into the tar archive, honoring the exclude list and the optional -// rate limit between command executions. +// output into the tar archive, honoring the optional rate limit between command +// executions. The command list is already filtered by the caller. func runCommands( tarWriter *tar.Writer, config *rest.Config, kubeCl kubernetes.Interface, podName, namespace, containerName string, commands []Command, - excludeMap map[string]bool, commandTimeout, requestInterval time.Duration, ) error { var tickCh <-chan time.Time @@ -456,30 +497,15 @@ func runCommands( tickCh = ticker.C } - var stdout, stderr bytes.Buffer - for _, cmd := range commands { - if isFileExcluded(cmd, excludeMap) { - continue - } - if tickCh != nil { <-tickCh } fullCommand := append([]string{cmd.Cmd}, cmd.Args...) - executor, err := utilk8s.ExecInPod(config, kubeCl, fullCommand, podName, namespace, containerName) - if err != nil { - fmt.Fprintf(os.Stderr, " ERROR: failed to create executor for %s: %v\n", cmd.File, err) - continue - } - cmdCtx, cancel := context.WithTimeout(context.Background(), commandTimeout) - streamErr := executor.StreamWithContext(cmdCtx, remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, - }) + output, stderrOutput, streamErr := utilk8s.ExecCommandInPod(cmdCtx, config, kubeCl, fullCommand, podName, namespace, containerName) cancel() @@ -487,21 +513,40 @@ func runCommands( if errors.Is(streamErr, context.DeadlineExceeded) { fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s\n", cmd.File, commandTimeout) } else { - fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %s\n%s\n", cmd.File, strings.Join(fullCommand, " "), stderr.String()) + fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %s\n%s\n", cmd.File, strings.Join(fullCommand, " "), stderrOutput) } } - if err = cmd.writeToTar(tarWriter, stdout.Bytes()); err != nil { - return fmt.Errorf("failed to write tar file %s: %w", cmd.File, err) + if notice := defaultedContainerNotice(stderrOutput); notice != "" { + output = append([]byte(notice), output...) } - stdout.Reset() - stderr.Reset() + if err := cmd.writeToTar(tarWriter, output); err != nil { + return fmt.Errorf("failed to write tar file %s: %w", cmd.File, err) + } } return nil } +// defaultedContainerNotice extracts kubectl's client-side "Defaulted container +// ... out of: ..." notice(s) from a command's stderr, so they can be prepended +// to the collected log output. Without this, the discarded stderr would take +// with it the only record of which container a `logs` command without +// -c/--all-containers actually collected from a multi-container pod. +func defaultedContainerNotice(stderrOutput string) string { + var notice strings.Builder + + for _, line := range strings.Split(stderrOutput, "\n") { + if strings.Contains(line, "Defaulted container") { + notice.WriteString(line) + notice.WriteString("\n") + } + } + + return notice.String() +} + // fetchActiveModules returns a map with the names of modules that are in the Ready phase. func fetchActiveModules( config *rest.Config, @@ -511,25 +556,20 @@ func fetchActiveModules( ) (map[string]bool, error) { cmdLine := []string{"kubectl", "get", "module", "-o", "json"} - executor, err := utilk8s.ExecInPod(config, kubeCl, cmdLine, podName, namespace, containerName) - if err != nil { - return nil, fmt.Errorf("create executor: %w", err) - } - - var stdout, stderr bytes.Buffer - ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - if err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, - }); err != nil { - return nil, fmt.Errorf("stream kubectl get module: %w (stderr: %s)", err, stderr.String()) + stdout, stderr, err := utilk8s.ExecCommandInPod(ctx, config, kubeCl, cmdLine, podName, namespace, containerName) + if err != nil { + return nil, fmt.Errorf("stream kubectl get module: %w (stderr: %s)", err, stderr) + } + + if len(stdout) == 0 { + return nil, fmt.Errorf("kubectl get module returned no output (stderr: %s)", stderr) } var list moduleList - if err = json.Unmarshal(stdout.Bytes(), &list); err != nil { + if err = json.Unmarshal(stdout, &list); err != nil { return nil, fmt.Errorf("parse module list: %w", err) } @@ -543,39 +583,81 @@ func fetchActiveModules( return active, nil } -func filterAndExpandCommands(commands []Command, activeModules map[string]bool) []Command { - result := make([]Command, 0, len(commands)) +// filterAndExpandCommands selects the commands to run: it resolves the +// {module-name} placeholder against the active modules and drops the entries +// excluded on the command line. It also returns every name --exclude accepts +// for this run, including the names of entries these very excludes dropped, so +// a valid name is never reported as unknown. +// +// Exclusion happens here, and not further down, because this is the only place +// where both spellings of an entry are known at once: the resolved archive name +// (d8-cloud-provider-aws-ccm-logs.txt) and the module-independent token printed +// by --list-exclude (ccm-logs). The resolved name alone does not reveal which +// of its segments is the module. +// +// modulesKnown reports whether activeModules actually describes the cluster. It +// is false when the module list could not be fetched: module-gated commands are +// then collected anyway (an empty file beats a silently missing one), except +// those whose File carries the {module-name} placeholder — their archive entry +// name cannot be resolved, so they are skipped rather than stored under a +// literal placeholder name. +func filterAndExpandCommands(commands []Command, activeModules map[string]bool, modulesKnown bool, excludeSet map[string]bool) (selected []Command, acceptedNames []string) { + selected = make([]Command, 0, len(commands)) + acceptedNames = make([]string, 0, len(commands)) + for _, cmd := range commands { + // The token stays accepted even when the command is gated out below: + // --exclude ccm-logs must not fail on a cluster without a cloud provider. + token := excludeBaseName(cmd) + acceptedNames = append(acceptedNames, token) + + if excludedByName(excludeSet, cmd.File, token) { + continue + } + if cmd.RequiredModule == "" { - result = append(result, cmd) + selected = append(selected, cmd) continue } - if len(activeModules) == 0 { + if !modulesKnown { + if !needsModuleExpansion(cmd) { + selected = append(selected, cmd) + } + continue } + // No explicit guard for an empty activeModules is needed: both branches + // below iterate the matching modules, of which there are none. if needsModuleExpansion(cmd) { matchedModules := matchingModules(activeModules, cmd.RequiredModule) for _, moduleName := range matchedModules { - result = append(result, Command{ - Cmd: cmd.Cmd, - File: strings.ReplaceAll(cmd.File, "{module-name}", moduleName), - Args: replaceModuleName(cmd.Args, moduleName), - ExcludeKey: cmd.ExcludeKey, - }) + // Copy the command and overwrite only what is substituted, so a + // field added to Command later cannot be silently dropped here. + expanded := cmd + expanded.File = strings.ReplaceAll(cmd.File, "{module-name}", moduleName) + expanded.Args = replaceModuleName(cmd.Args, moduleName) + + acceptedNames = append(acceptedNames, expanded.File) + + if excludedByName(excludeSet, expanded.File) { + continue + } + + selected = append(selected, expanded) } } else { for moduleName := range activeModules { if isModuleMatch(moduleName, cmd.RequiredModule) { - result = append(result, cmd) + selected = append(selected, cmd) break } } } } - return result + return selected, acceptedNames } func matchingModules(activeModules map[string]bool, required string) []string { @@ -636,36 +718,58 @@ func (c *Command) writeToTar(tarWriter *tar.Writer, fileContent []byte) error { return nil } +// trimArchiveExt drops the archive entry extension, so --exclude accepts a name +// with or without it. +func trimArchiveExt(name string) string { + return strings.TrimSuffix(strings.TrimSuffix(name, ".json"), ".txt") +} + +// excludeBaseName returns the --exclude token printed by --list-exclude for a +// command template: the archive entry name as written in debugCommands, or — +// when that name is per-module and therefore cluster-specific — the +// module-independent remainder (d8-{module-name}-ccm-logs.txt -> ccm-logs). +// +// The token is always derived from File, so a new per-module command needs no +// extra per-command data and cannot disagree with its own file name. func excludeBaseName(cmd Command) string { - if cmd.ExcludeKey != "" { - return cmd.ExcludeKey + if !strings.Contains(cmd.File, "{module-name}") { + return cmd.File } - name := strings.TrimSuffix(cmd.File, ".json") - name = strings.TrimSuffix(name, ".txt") - name = strings.TrimSuffix(name, "-{module-name}") + name := strings.ReplaceAll(cmd.File, "d8-{module-name}-", "") + name = strings.ReplaceAll(name, "-{module-name}-", "-") + name = strings.ReplaceAll(name, "-{module-name}", "") + name = strings.ReplaceAll(name, "{module-name}-", "") + name = strings.ReplaceAll(name, "{module-name}", "") - return name + return trimArchiveExt(name) } -func isFileExcluded(cmd Command, excludeMap map[string]bool) bool { - if excludeMap[cmd.File] { - return true - } +// newExcludeSet normalizes the raw --exclude values into the form matched +// against command names: surrounding spaces and the extension are irrelevant. +func newExcludeSet(excludeFiles []string) map[string]bool { + set := make(map[string]bool, len(excludeFiles)) - if cmd.ExcludeKey != "" { - return excludeMap[cmd.ExcludeKey] + for _, name := range excludeFiles { + name = trimArchiveExt(strings.TrimSpace(name)) + if name != "" { + set[name] = true + } } - base := strings.TrimSuffix(cmd.File, ".json") + return set +} - base = strings.TrimSuffix(base, ".txt") - if excludeMap[base] { - return true +// excludedByName reports whether any of the spellings of one archive entry was +// excluded on the command line. A name matches only that entry: there is no +// prefix or group matching, so --exclude d8 cannot silently drop every d8-* file. +func excludedByName(excludeSet map[string]bool, names ...string) bool { + if len(excludeSet) == 0 { + return false } - for excluded := range excludeMap { - if strings.HasPrefix(base, excluded+"-") { + for _, name := range names { + if name != "" && excludeSet[trimArchiveExt(name)] { return true } } @@ -673,6 +777,80 @@ func isFileExcluded(cmd Command, excludeMap map[string]bool) bool { return false } +// validateExcludeNames rejects --exclude values that cannot match any archive +// entry, so a typo is reported instead of quietly collecting the full archive. +// acceptedNames comes from filterAndExpandCommands and already covers both the +// resolved entry names of this run and the module-independent tokens. +func validateExcludeNames(excludeFiles, acceptedNames []string) error { + known := make(map[string]bool, len(acceptedNames)) + accepted := make([]string, 0, len(acceptedNames)) + + for _, name := range acceptedNames { + key := trimArchiveExt(name) + if key == "" || known[key] { + continue + } + + known[key] = true + + accepted = append(accepted, name) + } + + var unknown []string + + for _, name := range excludeFiles { + name = trimArchiveExt(strings.TrimSpace(name)) + if name != "" && !known[name] { + unknown = append(unknown, name) + } + } + + if len(unknown) == 0 { + return nil + } + + return fmt.Errorf("unknown --exclude name(s): %s%s\nrun \"d8 system collect-debug-info --list-exclude\" to see the accepted names", + strings.Join(unknown, ", "), suggestExcludeNames(unknown, accepted)) +} + +// suggestExcludeNames offers the accepted names that contain (or are contained +// in) an unknown one, which covers both typos and the group prefixes that used +// to match implicitly. +func suggestExcludeNames(unknown, accepted []string) string { + const maxSuggestions = 5 + + seen := make(map[string]bool, maxSuggestions) + + var matches []string + + for _, name := range unknown { + for _, candidate := range accepted { + key := trimArchiveExt(candidate) + if seen[key] || !strings.Contains(key, name) && !strings.Contains(name, key) { + continue + } + + seen[key] = true + + matches = append(matches, candidate) + } + } + + if len(matches) == 0 { + return "" + } + + sort.Strings(matches) + + if len(matches) > maxSuggestions { + return fmt.Sprintf("; did you mean one of: %s, ... (%d more)", strings.Join(matches[:maxSuggestions], ", "), len(matches)-maxSuggestions) + } + + return fmt.Sprintf("; did you mean: %s", strings.Join(matches, ", ")) +} + +// GetExcludableFiles returns the tokens accepted by --exclude, one per archive +// entry, as printed by --list-exclude. func GetExcludableFiles() []string { seen := make(map[string]bool, len(debugCommands)) diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go index f3600ec4e..d63e39f34 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go @@ -1,6 +1,10 @@ package debugtar -import "testing" +import ( + "slices" + "strings" + "testing" +) // TestDebugCommandsModuleExpansionInvariant guards the contract documented on // Command.RequiredModule: a command whose File or Args contains the @@ -22,3 +26,275 @@ func TestDebugCommandsModuleExpansionInvariant(t *testing.T) { t.Errorf("command %q uses the {module-name} placeholder but has no RequiredModule set, so it will never be resolved", cmd.File) } } + +// TestFilterAndExpandCommandsWithoutModuleList covers the degraded path taken +// when `kubectl get module` fails: the collection must go on, but no command +// may reach the archive with an unresolved {module-name} placeholder in its +// file name. +func TestFilterAndExpandCommandsWithoutModuleList(t *testing.T) { + commands, _ := filterAndExpandCommands(debugCommands, nil, false, nil) + + var expected int + + for _, cmd := range debugCommands { + if !needsModuleExpansion(cmd) { + expected++ + } + } + + if len(commands) != expected { + t.Errorf("got %d commands without a module list, want %d", len(commands), expected) + } + + for _, cmd := range commands { + if needsModuleExpansion(cmd) { + t.Errorf("command %q still carries an unresolved {module-name} placeholder", cmd.File) + } + } +} + +// TestFilterAndExpandCommandsKeepsFields guards the expansion against silently +// dropping a field of Command that a later change adds. +func TestFilterAndExpandCommandsKeepsFields(t *testing.T) { + source := Command{ + File: "d8-{module-name}-ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + } + + expanded, _ := filterAndExpandCommands([]Command{source}, map[string]bool{"cloud-provider-aws": true}, true, nil) + if len(expanded) != 1 { + t.Fatalf("got %d expanded commands, want 1", len(expanded)) + } + + got := expanded[0] + if got.File != "d8-cloud-provider-aws-ccm-logs.txt" { + t.Errorf("File = %q, want %q", got.File, "d8-cloud-provider-aws-ccm-logs.txt") + } + + if got.Cmd != source.Cmd || got.RequiredModule != source.RequiredModule { + t.Errorf("expansion dropped a field: %+v", got) + } +} + +// TestGetExcludableFilesAreUsableTokens guards the contract of --list-exclude: +// every printed name must be usable verbatim with --exclude, so none of them +// may carry an unresolved placeholder. +func TestGetExcludableFilesAreUsableTokens(t *testing.T) { + tokens := GetExcludableFiles() + if len(tokens) != len(debugCommands) { + t.Errorf("got %d tokens for %d commands", len(tokens), len(debugCommands)) + } + + for _, token := range tokens { + if strings.Contains(token, "{module-name}") { + t.Errorf("token %q cannot be typed by a user", token) + } + } + + for _, want := range []string{"cluster-events.json", "ccm-logs", "csi-controller-logs"} { + if !slices.Contains(tokens, want) { + t.Errorf("token %q is missing from --list-exclude", want) + } + } +} + +// TestExcludeAcceptedSpellings pins the accepted --exclude spellings and, just +// as importantly, the rejected ones: a group prefix must not drop a whole family +// of files. The per-module entries are checked through the real selection, since +// that is where a command is matched against the exclude list. +func TestExcludeAcceptedSpellings(t *testing.T) { + templates := []Command{ + {File: "cluster-events.json", Cmd: "kubectl"}, + { + File: "d8-{module-name}-ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + }, + } + activeModules := map[string]bool{"cloud-provider-aws": true, "cloud-provider-yandex": true} + + cases := []struct { + token string + want []string + }{ + {"", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"ccm-logs", []string{"cluster-events.json"}}, + {"ccm-logs.txt", []string{"cluster-events.json"}}, + {"d8-cloud-provider-aws-ccm-logs.txt", []string{"cluster-events.json", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"d8-cloud-provider-aws-ccm-logs", []string{"cluster-events.json", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"cluster-events", []string{"d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {" cluster-events ", []string{"d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"d8", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"cluster", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + } + + for _, tc := range cases { + var exclude []string + if tc.token != "" { + exclude = []string{tc.token} + } + + selected, _ := filterAndExpandCommands(templates, activeModules, true, newExcludeSet(exclude)) + + got := make([]string, 0, len(selected)) + for _, cmd := range selected { + got = append(got, cmd.File) + } + + if !slices.Equal(got, tc.want) { + t.Errorf("--exclude %q left %v, want %v", tc.token, got, tc.want) + } + } +} + +// TestValidateExcludeNames checks that a name which can never match is reported +// instead of silently collecting everything, while a module-independent token +// stays valid on a cluster where that module is not enabled. +func TestValidateExcludeNames(t *testing.T) { + _, accepted := filterAndExpandCommands(debugCommands, nil, false, nil) + + if err := validateExcludeNames([]string{"ccm-logs", "cluster-events", "cluster-events.json"}, accepted); err != nil { + t.Errorf("valid names rejected: %v", err) + } + + err := validateExcludeNames([]string{"d8"}, accepted) + if err == nil { + t.Fatal("group prefix d8 was accepted, it silently excludes nothing now") + } + + if !strings.Contains(err.Error(), "did you mean") { + t.Errorf("error %q offers no suggestion", err) + } + + err = validateExcludeNames([]string{"ccm-log"}, accepted) + if err == nil || !strings.Contains(err.Error(), "did you mean: ccm-logs") { + t.Errorf("a near miss should be pointed at its name, got %v", err) + } + + err = validateExcludeNames([]string{"zzzz"}, accepted) + if err == nil { + t.Fatal("a name matching nothing was accepted") + } + + if strings.Contains(err.Error(), "did you mean") { + t.Errorf("error %q invents a suggestion for a name with no near miss", err) + } + + // A repeated or empty accepted name must not confuse the lookup. + if err := validateExcludeNames([]string{"plain"}, []string{"plain.txt", "plain.txt", ""}); err != nil { + t.Errorf("duplicate accepted names broke the lookup: %v", err) + } +} + +// TestExcludedEntryStaysAValidName guards the interaction between exclusion and +// validation: the name a user just excluded must not then be reported as +// unknown, even though its command is gone from the selection. +func TestExcludedEntryStaysAValidName(t *testing.T) { + for _, name := range []string{"ccm-logs", "cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt"} { + _, accepted := filterAndExpandCommands(debugCommands, map[string]bool{"cloud-provider-aws": true}, true, newExcludeSet([]string{name})) + + if err := validateExcludeNames([]string{name}, accepted); err != nil { + t.Errorf("--exclude %q reported as unknown: %v", name, err) + } + } +} + +// TestSelectionMatrix pins every combination of the two knobs documented on +// Command.RequiredModule: whether the command is gated on a module, whether it +// carries the {module-name} placeholder, and whether the module list could be +// fetched at all. The expected file lists also pin the order, which must stay +// deterministic (templates in declaration order, per-module copies sorted by +// module name) so two runs of the same cluster produce the same archive. +func TestSelectionMatrix(t *testing.T) { + templates := []Command{ + {File: "plain.txt", Cmd: "kubectl"}, + {File: "literal-{module-name}.txt", Cmd: "kubectl"}, + {File: "gated.txt", Cmd: "kubectl", RequiredModule: "istio"}, + { + File: "d8-{module-name}-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + }, + } + + cases := []struct { + name string + activeModules map[string]bool + modulesKnown bool + want []string + }{ + { + name: "gate matched by prefix and by exact name", + activeModules: map[string]bool{"istio": true, "cloud-provider-yandex": true, "cloud-provider-aws": true}, + modulesKnown: true, + want: []string{ + "plain.txt", + "literal-{module-name}.txt", + "gated.txt", + "d8-cloud-provider-aws-logs.txt", + "d8-cloud-provider-yandex-logs.txt", + }, + }, + { + name: "no module matches the gate", + activeModules: map[string]bool{"cert-manager": true}, + modulesKnown: true, + want: []string{"plain.txt", "literal-{module-name}.txt"}, + }, + { + name: "no module is Ready", + activeModules: map[string]bool{}, + modulesKnown: true, + want: []string{"plain.txt", "literal-{module-name}.txt"}, + }, + { + name: "module list unavailable", + modulesKnown: false, + want: []string{"plain.txt", "literal-{module-name}.txt", "gated.txt"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + selected, _ := filterAndExpandCommands(templates, tc.activeModules, tc.modulesKnown, nil) + + got := make([]string, 0, len(selected)) + for _, cmd := range selected { + got = append(got, cmd.File) + } + + if !slices.Equal(got, tc.want) { + t.Errorf("selected %v, want %v", got, tc.want) + } + }) + } +} + +// TestExpansionSubstitutesArgs checks the other half of the substitution: the +// module name must reach Args too, not just the archive entry name. +func TestExpansionSubstitutesArgs(t *testing.T) { + template := Command{ + File: "d8-{module-name}-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + } + + selected, _ := filterAndExpandCommands([]Command{template}, map[string]bool{"cloud-provider-aws": true}, true, nil) + if len(selected) != 1 { + t.Fatalf("got %d commands, want 1", len(selected)) + } + + want := []string{"-n", "d8-cloud-provider-aws", "logs"} + if !slices.Equal(selected[0].Args, want) { + t.Errorf("Args = %v, want %v", selected[0].Args, want) + } + + if !slices.Equal(template.Args, []string{"-n", "d8-{module-name}", "logs"}) { + t.Errorf("expansion mutated the template Args: %v", template.Args) + } +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go index 29b274bef..f64216d7b 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go +++ b/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go @@ -1,19 +1,14 @@ package debugtar import ( - "archive/tar" - "bytes" - "compress/gzip" "context" - "encoding/json" "fmt" - "os" "sort" "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/remotecommand" "github.com/deckhouse/deckhouse-cli/internal/utilk8s" ) @@ -25,7 +20,7 @@ var virtualizationCommands = []Command{ { File: "d8-virtualization-pods-wide.txt", Cmd: "kubectl", - Args: []string{"-n", virtualizationNamespace, "get", "pod", "-o", "wide"}, + Args: []string{"-n", virtualizationNamespace, "get", "pod", "-o", "wide", "--ignore-not-found=true"}, }, } @@ -34,23 +29,15 @@ type virtualizationPod struct { DaemonSetOwned bool } -type podList struct { - Items []struct { - Metadata struct { - Name string `json:"name"` - OwnerReferences []struct { - Kind string `json:"kind"` - } `json:"ownerReferences"` - } `json:"metadata"` - } `json:"items"` -} - // VirtualizationTarball collects a separate, virtualization-focused debug // archive: the list of pods in the d8-virtualization namespace // plus per-pod logs, optionally skipping pods owned by a DaemonSet // (virt-handler, virtualization-dra, vm-route-forge, ...) since their log // volume scales with the number of nodes. -func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, commandTimeout, requestInterval time.Duration, skipDsLogs bool) (err error) { +// +// The pod list is the entire payload of this archive, so a failure to obtain it +// aborts the collection instead of producing an archive that looks complete. +func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, commandTimeout, requestInterval time.Duration, skipDsLogs bool) error { const ( namespace = "d8-system" containerName = "deckhouse" @@ -61,90 +48,43 @@ func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, com return fmt.Errorf("failed to get Deckhouse pod: %w", err) } - pods, err := fetchVirtualizationPods(config, kubeCl, podName, namespace, containerName, commandTimeout) + pods, err := fetchVirtualizationPods(kubeCl, commandTimeout) if err != nil { - fmt.Fprintf(os.Stderr, "WARNING: could not list pods in %s: %v\n", virtualizationNamespace, err) - } else if len(pods) == 0 { - fmt.Fprintf(os.Stderr, "WARNING: no pods found in namespace %s, is the virtualization module enabled?\n", virtualizationNamespace) + return fmt.Errorf("failed to list pods in namespace %s: %w", virtualizationNamespace, err) } - commands := buildVirtualizationCommands(pods, skipDsLogs) - - gzipWriter := gzip.NewWriter(os.Stdout) - tarWriter := tar.NewWriter(gzipWriter) - - defer func() { - if closeErr := tarWriter.Close(); closeErr != nil && err == nil { - err = fmt.Errorf("failed to finalize tar archive: %w", closeErr) - } - - if closeErr := gzipWriter.Close(); closeErr != nil && err == nil { - err = fmt.Errorf("failed to finalize gzip stream: %w", closeErr) - } - }() - - fmt.Fprintf(os.Stderr, "Collecting virtualization debug info from Deckhouse...\n") - - if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, nil, commandTimeout, requestInterval); err != nil { - return err + if len(pods) == 0 { + return fmt.Errorf("no pods found in namespace %s, is the virtualization module enabled?", virtualizationNamespace) } - fmt.Fprintf(os.Stderr, "Virtualization debug archive collection completed.\n") - - return nil + return writeArchive( + config, kubeCl, podName, namespace, containerName, + buildVirtualizationCommands(pods, skipDsLogs), commandTimeout, requestInterval, + "Collecting virtualization debug info from Deckhouse...", + "Virtualization debug archive collection completed.", + ) } // fetchVirtualizationPods lists the pods currently running in the // virtualization namespace and reports which ones are owned by a DaemonSet, // so the DaemonSet-managed pods can be identified without hardcoding their names. -func fetchVirtualizationPods( - config *rest.Config, - kubeCl kubernetes.Interface, - podName, namespace, containerName string, - timeout time.Duration, -) ([]virtualizationPod, error) { - cmdLine := []string{"kubectl", "-n", virtualizationNamespace, "get", "pods", "-o", "json", "--ignore-not-found=true"} - - executor, err := utilk8s.ExecInPod(config, kubeCl, cmdLine, podName, namespace, containerName) - if err != nil { - return nil, fmt.Errorf("create executor: %w", err) - } - - var stdout, stderr bytes.Buffer - +func fetchVirtualizationPods(kubeCl kubernetes.Interface, timeout time.Duration) ([]virtualizationPod, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - if err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, - }); err != nil { - return nil, fmt.Errorf("stream kubectl get pods: %w (stderr: %s)", err, stderr.String()) - } - - if stdout.Len() == 0 { - return nil, nil - } - - var list podList - if err = json.Unmarshal(stdout.Bytes(), &list); err != nil { - return nil, fmt.Errorf("parse pod list: %w", err) + list, err := kubeCl.CoreV1().Pods(virtualizationNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err } pods := make([]virtualizationPod, 0, len(list.Items)) - for _, item := range list.Items { - daemonSetOwned := false - - for _, owner := range item.Metadata.OwnerReferences { - if owner.Kind == "DaemonSet" { - daemonSetOwned = true - break - } - } + + for i := range list.Items { + owner := metav1.GetControllerOf(&list.Items[i]) pods = append(pods, virtualizationPod{ - Name: item.Metadata.Name, - DaemonSetOwned: daemonSetOwned, + Name: list.Items[i].Name, + DaemonSetOwned: owner != nil && owner.Kind == "DaemonSet", }) } diff --git a/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go b/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go index e494048f1..270aa2a35 100644 --- a/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go +++ b/internal/system/cmd/collect-debug-info/virtualizationtar/virtualizationTar.go @@ -56,6 +56,7 @@ func NewCommand() *cobra.Command { Short: "Collect a separate virtualization debug archive.", Long: virtualizationCmdLong, Example: virtualizationCmdExample, + Args: cobra.NoArgs, SilenceErrors: true, SilenceUsage: true, PreRunE: func(_ *cobra.Command, _ []string) error { @@ -78,19 +79,9 @@ func NewCommand() *cobra.Command { } func collectVirtualizationDebugInfo(cmd *cobra.Command, commandTimeout, requestInterval time.Duration, skipDsLogs bool) error { - kubeconfigPath, err := cmd.Flags().GetString("kubeconfig") + config, kubeCl, err := utilk8s.NewClientSet(cmd) if err != nil { - return fmt.Errorf("Failed to setup Kubernetes client: %w", err) - } - - contextName, err := cmd.Flags().GetString("context") - if err != nil { - return fmt.Errorf("Failed to setup Kubernetes client: %w", err) - } - - config, kubeCl, err := utilk8s.SetupK8sClientSet(kubeconfigPath, contextName) - if err != nil { - return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + return err } if err = debugtar.VirtualizationTarball(config, kubeCl, commandTimeout, requestInterval, skipDsLogs); err != nil { diff --git a/internal/utilk8s/clientset.go b/internal/utilk8s/clientset.go index 194ad0d45..372d7958d 100644 --- a/internal/utilk8s/clientset.go +++ b/internal/utilk8s/clientset.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -37,6 +38,20 @@ func WithInsecureSkipTLSVerify(insecure bool) ClientOption { } } +// NewClientSet builds a clientset from the global --kubeconfig/--context flags +// of cmd, the way NewDynamicClient does for the dynamic client. +func NewClientSet(cmd *cobra.Command) (*rest.Config, *kubernetes.Clientset, error) { + kubeconfigPath, _ := cmd.Flags().GetString("kubeconfig") + contextName, _ := cmd.Flags().GetString("context") + + config, kubeCl, err := SetupK8sClientSet(kubeconfigPath, contextName) + if err != nil { + return nil, nil, fmt.Errorf("failed to setup Kubernetes client: %w", err) + } + + return config, kubeCl, nil +} + // SetupK8sClientSet reads kubeconfig file at kubeconfigPath and constructs a kubernetes clientset from it. // If contextName is not empty, context under that name is used instead of default. func SetupK8sClientSet(kubeconfigPath, contextName string, opts ...ClientOption) (*rest.Config, *kubernetes.Clientset, error) { diff --git a/internal/utilk8s/operatepod.go b/internal/utilk8s/operatepod.go index 7e5ab0d8a..e37aa10ce 100644 --- a/internal/utilk8s/operatepod.go +++ b/internal/utilk8s/operatepod.go @@ -60,3 +60,33 @@ func ExecInPod(config *rest.Config, kubeCl kubernetes.Interface, cmdLine []strin return executor, nil } + +// ExecCommandInPod runs cmdLine in the given container and returns whatever the +// command wrote to stdout and stderr. Output collected before an error (a +// timeout in particular) is returned along with that error, so callers can keep +// a partial result. +// +// The buffers are goroutine-safe on purpose: StreamWithContext returns as soon +// as ctx is done without joining the goroutines that copy the remote streams, +// so those goroutines may still write into them after this call returned. +func ExecCommandInPod( + ctx context.Context, + config *rest.Config, + kubeCl kubernetes.Interface, + cmdLine []string, + podName, namespace, containerName string, +) (stdout []byte, stderr string, err error) { + executor, err := ExecInPod(config, kubeCl, cmdLine, podName, namespace, containerName) + if err != nil { + return nil, "", err + } + + var stdoutBuf, stderrBuf syncBuffer + + streamErr := executor.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdoutBuf, + Stderr: &stderrBuf, + }) + + return stdoutBuf.Bytes(), stderrBuf.String(), streamErr +} diff --git a/internal/utilk8s/syncbuffer.go b/internal/utilk8s/syncbuffer.go new file mode 100644 index 000000000..d258f377e --- /dev/null +++ b/internal/utilk8s/syncbuffer.go @@ -0,0 +1,47 @@ +package utilk8s + +import ( + "bytes" + "sync" +) + +// syncBuffer is a goroutine-safe sink for the output of a remote command. +// +// It is needed because remotecommand.Executor.StreamWithContext returns as +// soon as the context is done (client-go tools/remotecommand/spdy.go) without +// joining the goroutines that io.Copy the remote streams into the writers +// passed in StreamOptions. After a command times out those goroutines may keep +// writing, so the writer outlives the call and a plain bytes.Buffer would be +// accessed concurrently: a racing Bytes() can return a slice already grown past +// the bytes actually copied into it, and a late Write can resurrect a buffer the +// caller considers finished. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write implements io.Writer and is safe to call concurrently with the readers +// below. +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.Write(p) +} + +// Bytes returns a copy of everything written so far. The copy keeps the caller +// isolated from writes a late stream goroutine may still perform. +func (b *syncBuffer) Bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + + return bytes.Clone(b.buf.Bytes()) +} + +// String returns everything written so far as a string. +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.String() +} diff --git a/internal/utilk8s/syncbuffer_test.go b/internal/utilk8s/syncbuffer_test.go new file mode 100644 index 000000000..0e8344d4f --- /dev/null +++ b/internal/utilk8s/syncbuffer_test.go @@ -0,0 +1,70 @@ +package utilk8s + +import ( + "strings" + "sync" + "testing" +) + +// TestSyncBufferConcurrentAccess reproduces the situation left behind by a +// timed out StreamWithContext: a stream goroutine keeps writing into the +// buffer while the collecting goroutine reads it. Run with -race. +func TestSyncBufferConcurrentAccess(t *testing.T) { + var buf syncBuffer + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for i := 0; i < 1000; i++ { + if _, err := buf.Write([]byte(strings.Repeat("a", 64))); err != nil { + t.Errorf("write: %v", err) + return + } + } + }() + + go func() { + defer wg.Done() + + for i := 0; i < 1000; i++ { + for _, b := range buf.Bytes() { + if b != 'a' { + t.Errorf("Bytes() exposed a byte that was never written: %q", b) + return + } + } + + _ = buf.String() + } + }() + + wg.Wait() +} + +// TestSyncBufferBytesIsSnapshot guards that a collected command output cannot +// be mutated or extended by a late write from an abandoned stream goroutine. +func TestSyncBufferBytesIsSnapshot(t *testing.T) { + var buf syncBuffer + + if _, err := buf.Write([]byte("first")); err != nil { + t.Fatalf("write: %v", err) + } + + snapshot := buf.Bytes() + + if _, err := buf.Write([]byte("late")); err != nil { + t.Fatalf("write: %v", err) + } + + if got := string(snapshot); got != "first" { + t.Errorf("snapshot changed after a later write: got %q, want %q", got, "first") + } + + if got := string(buf.Bytes()); got != "firstlate" { + t.Errorf("buffer content: got %q, want %q", got, "firstlate") + } +} From d036c3489c781167d5bed3551c5201db699c96ec Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 22 Sep 2026 00:26:35 +0400 Subject: [PATCH 11/15] Update debug archive v11 Signed-off-by: Valery Losev --- internal/system/README.md | 8 +- .../collect-debug-info/debugtar/archive.go | 131 +++ .../debugtar/commandtype.go | 79 ++ .../collect-debug-info/debugtar/debugTar.go | 869 ------------------ .../debugtar/debugTar_test.go | 300 ------ .../debugtar/debugcommands.go | 341 +++++++ .../debugtar/debugcommands_test.go | 26 + .../collect-debug-info/debugtar/exclude.go | 158 ++++ .../debugtar/exclude_test.go | 131 +++ .../collect-debug-info/debugtar/selection.go | 110 +++ .../debugtar/selection_test.go | 154 ++++ .../collect-debug-info/debugtar/tarball.go | 94 ++ ...ualizationTarball.go => virtualization.go} | 10 +- internal/utilk8s/operatepod.go | 43 + ...{syncbuffer_test.go => operatepod_test.go} | 0 internal/utilk8s/syncbuffer.go | 47 - 16 files changed, 1277 insertions(+), 1224 deletions(-) create mode 100644 internal/system/cmd/collect-debug-info/debugtar/archive.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/commandtype.go delete mode 100644 internal/system/cmd/collect-debug-info/debugtar/debugTar.go delete mode 100644 internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/debugcommands.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/exclude.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/exclude_test.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/selection.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/selection_test.go create mode 100644 internal/system/cmd/collect-debug-info/debugtar/tarball.go rename internal/system/cmd/collect-debug-info/debugtar/{virtualizationTarball.go => virtualization.go} (93%) rename internal/utilk8s/{syncbuffer_test.go => operatepod_test.go} (100%) delete mode 100644 internal/utilk8s/syncbuffer.go diff --git a/internal/system/README.md b/internal/system/README.md index c7bd292dc..f7159633e 100644 --- a/internal/system/README.md +++ b/internal/system/README.md @@ -228,13 +228,13 @@ Collects a wide cluster snapshot into a **gzipped tar streamed to stdout**, so y d8 system collect-debug-info > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz ``` -It refuses to run when stdout is a terminal (to avoid dumping binary to your screen) unless you pass `--list-exclude`. The collection runs **inside** the leader pod: it executes 63 diagnostic commands there (`deckhouse-controller queue list`, redacted global values, module/source/release inventories, cluster-wide `kubectl get` snapshots, and controller/etcd/apiserver/VPA/Prometheus logs, plus cloud-provider/cert-manager/istio/cni-cilium/virtualization extras when those modules are Ready), writing each result as a file in the archive. +It refuses to run when stdout is a terminal (to avoid dumping binary to your screen) unless you pass `--list-exclude`. The collection runs **inside** the leader pod: it executes the 63 declared commands there (`deckhouse-controller queue list`, redacted global values, module/source/release inventories, cluster-wide `kubectl get` snapshots, and controller/etcd/apiserver/VPA/Prometheus logs, plus cloud-provider/cert-manager/istio/cni-cilium/virtualization extras when those modules are Ready), writing each result as a file in the archive. The two per-cloud log collections are repeated once per matching provider module, so a cloud cluster ends up with slightly more archive entries than commands. Before collecting, the command reads the list of `Ready` modules to decide which module-gated commands apply. If that read fails, it prints an error and keeps going: module-gated commands run anyway (and may produce empty files), except the per-module log collections whose file name contains the module name - those are skipped, since their archive entry name cannot be resolved. | Flag | Short | Type | Default | Description | |---|---|---|---|---| -| `--exclude` | | string list | (none) | Comma-separated list of entries to leave out of the archive. Accepts exactly the names printed by `--list-exclude`, with or without the file extension; a name matches that entry only, never a group of files sharing a prefix. A name that matches nothing is an error listing close matches, so a typo cannot pass as "collect everything". | +| `--exclude` | | string list | (none) | Comma-separated list of entries to leave out of the archive. Accepts exactly the names printed by `--list-exclude`, with or without the file extension and ignoring surrounding spaces; a name matches that entry only, never a group of files sharing a prefix. A name that matches nothing is an error listing close matches, so a typo cannot pass as "collect everything". | | `--list-exclude` | `-l` | bool | `false` | Print the names accepted by `--exclude`, then exit. This path makes no cluster calls. The names are the archive file names as declared in the command table, except the per-module cloud logs, which are printed as the module-independent keys `ccm-logs` and `csi-controller-logs` (their real entry is `d8--ccm-logs.txt`, and both spellings are accepted). | | `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command. | | `--request-interval` | | duration | `0` | Minimum gap between commands to avoid overloading the cluster (e.g. `200ms`, `1s`). `0` disables rate limiting. | @@ -245,6 +245,8 @@ Before collecting, the command reads the list of `Ready` modules to decide which Collects a separate, virtualization-focused archive: the pod list of the `d8-virtualization` namespace plus the **full** log of every pod in it (`--tail=-1`, no line cap). Same stdout rules as the parent command. +The pod list is read through the Kubernetes API with **your own** kubeconfig, so the account you run `d8` with needs `list pods` in `d8-virtualization`; the logs themselves are still collected by `kubectl` running inside the leader pod, like every other archive entry. + ```bash d8 system collect-debug-info virtualization > deckhouse-debug-virtualization-$(date +"%Y_%m_%d").tar.gz ``` @@ -252,7 +254,7 @@ d8 system collect-debug-info virtualization > deckhouse-debug-virtualization-$(d | Flag | Short | Type | Default | Description | |---|---|---|---|---| | `--skip-ds-logs` | | bool | `false` | Skip logs of pods owned by a DaemonSet (`virt-handler`, `virtualization-dra`, `vm-route-forge`, ...), whose volume scales with the number of nodes. | -| `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command. | +| `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command, and to the pod list request. | | `--request-interval` | | duration | `0` | Minimum gap between commands to avoid overloading the cluster. | The pod list is the entire payload of this archive, so the command fails (and writes nothing) when the namespace cannot be listed or holds no pods - instead of producing a valid-looking archive with a single empty file. `--exclude`/`--list-exclude` do not apply here. diff --git a/internal/system/cmd/collect-debug-info/debugtar/archive.go b/internal/system/cmd/collect-debug-info/debugtar/archive.go new file mode 100644 index 000000000..95039fa83 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/archive.go @@ -0,0 +1,131 @@ +package debugtar + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" +) + +// writeArchive streams a gzipped tar of the given commands' output to stdout. +// It is the shared body of the debug archives: only the command set and the +// progress banners differ between them. +func writeArchive( + config *rest.Config, + kubeCl kubernetes.Interface, + podName, namespace, containerName string, + commands []command, + commandTimeout, requestInterval time.Duration, + startBanner, doneBanner string, +) (err error) { + gzipWriter := gzip.NewWriter(os.Stdout) + tarWriter := tar.NewWriter(gzipWriter) + + defer func() { + if closeErr := tarWriter.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to finalize tar archive: %w", closeErr) + } + + if closeErr := gzipWriter.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to finalize gzip stream: %w", closeErr) + } + }() + + fmt.Fprintf(os.Stderr, "%s\n", startBanner) + + if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, commandTimeout, requestInterval); err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "%s\n", doneBanner) + + return nil +} + +// runCommands executes each command inside the Deckhouse pod and streams its +// output into the tar archive, honoring the optional rate limit between command +// executions. The command list is already filtered by the caller. +func runCommands( + tarWriter *tar.Writer, + config *rest.Config, + kubeCl kubernetes.Interface, + podName, namespace, containerName string, + commands []command, + commandTimeout, requestInterval time.Duration, +) error { + var tickCh <-chan time.Time + + if requestInterval > 0 { + ticker := time.NewTicker(requestInterval) + defer ticker.Stop() + + tickCh = ticker.C + } + + for _, cmd := range commands { + if tickCh != nil { + <-tickCh + } + + fullCommand := append([]string{cmd.Cmd}, cmd.Args...) + + cmdCtx, cancel := context.WithTimeout(context.Background(), commandTimeout) + output, stderrOutput, streamErr := utilk8s.ExecCommandInPod(cmdCtx, config, kubeCl, fullCommand, podName, namespace, containerName) + + cancel() + + if streamErr != nil { + // Report the error itself, the command that produced it and how much + // output survived: the entry is written either way, so without the + // byte count an operator cannot tell an empty file from a truncated + // one, and without the error a non-zero exit code looks the same as a + // broken stream. + if errors.Is(streamErr, context.DeadlineExceeded) { + fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s, keeping %d bytes collected so far\n", + cmd.File, commandTimeout, len(output)) + } else { + fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %v, keeping %d bytes\n command: %s\n", + cmd.File, streamErr, len(output), strings.Join(fullCommand, " ")) + } + + if trimmed := strings.TrimSpace(stderrOutput); trimmed != "" { + fmt.Fprintf(os.Stderr, " stderr: %s\n", trimmed) + } + } + + if notice := defaultedContainerNotice(stderrOutput); notice != "" { + output = append([]byte(notice), output...) + } + + if err := cmd.writeToTar(tarWriter, output); err != nil { + return fmt.Errorf("failed to write tar file %s: %w", cmd.File, err) + } + } + + return nil +} + +// defaultedContainerNotice extracts kubectl's client-side "Defaulted container +// ... out of: ..." notice(s) from a command's stderr, so they can be prepended +// to the collected log output. +func defaultedContainerNotice(stderrOutput string) string { + var notice strings.Builder + + for _, line := range strings.Split(stderrOutput, "\n") { + if strings.Contains(line, "Defaulted container") { + notice.WriteString(line) + notice.WriteString("\n") + } + } + + return notice.String() +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/commandtype.go b/internal/system/cmd/collect-debug-info/debugtar/commandtype.go new file mode 100644 index 000000000..636d21176 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/commandtype.go @@ -0,0 +1,79 @@ +package debugtar + +import ( + "archive/tar" + "fmt" + "slices" + "strings" +) + +// command is one entry of a debug archive: the shell command executed inside +// the Deckhouse pod and the archive file its output lands in. The tables of +// such entries live in debugcommands.go and virtualization.go. +type command struct { + Cmd string + Args []string + File string + + // RequiredModule gates the command on a module being Ready (status.phase == + // "Ready"): the command runs only when the name of some Ready module starts + // with this string, so "cloud-provider" matches cloud-provider-aws. An empty + // string means always run. + // + // Together with the {module-name} placeholder it also means "once per + // matching module": when RequiredModule is set and File or any Args element + // contains the placeholder (see needsModuleExpansion), the command is + // duplicated for every matching Ready module, with the placeholder + // substituted in both File and Args. Put the placeholder in File whenever it + // appears in Args — copies that differ only in Args all end up under the same + // archive entry name, and only the last one survives extraction. + // + // Leaving RequiredModule empty while the placeholder is present means it is + // never resolved and stays literal in the output. + // + // When the module list cannot be fetched at all, gating is impossible and + // the fallback differs by shape: commands without the placeholder run anyway + // (they either produce data or an empty file), commands with it are skipped, + // since their archive entry name cannot be resolved. + RequiredModule string +} + +// needsModuleExpansion reports whether cmd must be duplicated once per active +// module matching RequiredModule (with {module-name} substituted into File +// and Args), rather than run once as-is. +func needsModuleExpansion(cmd command) bool { + if strings.Contains(cmd.File, "{module-name}") { + return true + } + + return slices.ContainsFunc(cmd.Args, func(arg string) bool { + return strings.Contains(arg, "{module-name}") + }) +} + +func replaceModuleName(args []string, moduleName string) []string { + expanded := make([]string, len(args)) + for i, arg := range args { + expanded[i] = strings.ReplaceAll(arg, "{module-name}", moduleName) + } + + return expanded +} + +func (c *command) writeToTar(tarWriter *tar.Writer, fileContent []byte) error { + header := &tar.Header{ + Name: c.File, + Mode: 0o600, + Size: int64(len(fileContent)), + } + + if err := tarWriter.WriteHeader(header); err != nil { + return fmt.Errorf("write tar header: %v", err) + } + + if _, err := tarWriter.Write(fileContent); err != nil { + return fmt.Errorf("copy content: %v", err) + } + + return nil +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar.go deleted file mode 100644 index c231311a9..000000000 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar.go +++ /dev/null @@ -1,869 +0,0 @@ -package debugtar - -import ( - "archive/tar" - "compress/gzip" - "context" - "encoding/json" - "errors" - "fmt" - "os" - "slices" - "sort" - "strings" - "time" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - - "github.com/deckhouse/deckhouse-cli/internal/utilk8s" -) - -type Command struct { - Cmd string - Args []string - File string - - // RequiredModule gates the command on a module being Ready (status.phase == - // "Ready"): the command runs only when the name of some Ready module starts - // with this string, so "cloud-provider" matches cloud-provider-aws. An empty - // string means always run. - // - // Together with the {module-name} placeholder it also means "once per - // matching module": when RequiredModule is set and File or any Args element - // contains the placeholder (see needsModuleExpansion), the command is - // duplicated for every matching Ready module, with the placeholder - // substituted in both File and Args. Put the placeholder in File whenever it - // appears in Args — copies that differ only in Args all end up under the same - // archive entry name, and only the last one survives extraction. - // - // Leaving RequiredModule empty while the placeholder is present means it is - // never resolved and stays literal in the output. - // - // When the module list cannot be fetched at all, gating is impossible and - // the fallback differs by shape: commands without the placeholder run anyway - // (they either produce data or an empty file), commands with it are skipped, - // since their archive entry name cannot be resolved. - RequiredModule string -} - -type moduleList struct { - Items []struct { - Metadata struct { - Name string `json:"name"` - } `json:"metadata"` - Status struct { - Phase string `json:"phase"` - } `json:"status"` - } `json:"items"` -} - -// debugCommands - a complete list of commands for collecting debug information. -var debugCommands = []Command{ - { - File: "deckhouse-queue.txt", - Cmd: "deckhouse-controller", - Args: []string{"queue", "list"}, - }, - { - File: "cluster-global-values.json", - Cmd: "bash", - Args: []string{"-c", `deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, - }, - { - File: "deckhouse-enabled-modules.json", - Cmd: "bash", - Args: []string{"-c", "kubectl get modules -o json | jq '.items[]'"}, - }, - { - File: "deckhouse-module-sources.json", - Cmd: "bash", - Args: []string{"-c", "kubectl get modulesources -o json | jq '.items[]'"}, - }, - { - File: "deckhouse-module-pull-overrides.json", - Cmd: "bash", - Args: []string{"-c", "kubectl get modulepulloverrides -o json | jq '.items[]'"}, - }, - { - File: "deckhouse-module-update-policies.json", - Cmd: "bash", - Args: []string{"-c", "kubectl get moduleupdatepolicies -o json | jq '.items[]'"}, - }, - { - File: "deckhouse-maintenance-modules.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl get moduleconfig -ojson | jq -r '.items[] | select(.spec.maintenance == "NoResourceReconciliation") | .metadata.name'`}, - }, - { - File: "cluster-events.json", - Cmd: "kubectl", - Args: []string{"get", "events", "--sort-by=.metadata.creationTimestamp", "-A", "-o", "json"}, - }, - { - File: "d8-all.json", - Cmd: "bash", - Args: []string{"-c", `for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, - }, - { - File: "cluster-node-groups.json", - Cmd: "kubectl", - Args: []string{"get", "nodegroups", "-A", "-o", "json"}, - }, - { - File: "cluster-node-group-configuration.json", - Cmd: "kubectl", - Args: []string{"get", "nodegroupconfiguration", "-A", "-o", "json"}, - }, - { - File: "cluster-nodes.json", - Cmd: "kubectl", - Args: []string{"get", "nodes", "-A", "-o", "json"}, - }, - { - File: "cluster-namespace.json", - Cmd: "kubectl", - Args: []string{"get", "namespaces", "-o", "json"}, - }, - { - File: "instance-manager-capi-machines.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.cluster.x-k8s.io -o json | jq '.items[]'`}, - }, - { - File: "instance-manager-instances.json", - Cmd: "bash", - Args: []string{"-c", `kubectl get instances.deckhouse.io -o json | jq '.items[]'`}, - }, - { - File: "instance-manager-staticinstances.json", - Cmd: "bash", - Args: []string{"-c", `kubectl get staticinstances.deckhouse.io -o json | jq '.items[]'`}, - }, - { - File: "instance-manager-cloud-machine-deployment.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machinedeployments.machine.sapcloud.io -o json | jq '.items[]'`}, - RequiredModule: "cloud-provider", - }, - { - File: "instance-manager-static-machine-deployment.txt", - Cmd: "bash", - Args: []string{"-c", "kubectl -n d8-cloud-instance-manager get machinedeployments.cluster.x-k8s.io -o json --ignore-not-found | jq '.items[]'"}, - }, - { - File: "deckhouse-version.json", - Cmd: "bash", - Args: []string{"-c", "jq -s add <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.metadata.annotations | {\"core.deckhouse.io/edition\",\"core.deckhouse.io/version\"}') <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.spec.template.spec.containers[] | select(.name == \"deckhouse\") | {image}')"}, - }, - { - File: "deckhouse-releases.json", - Cmd: "kubectl", - Args: []string{"get", "deckhousereleases", "-o", "json"}, - }, - { - File: "deckhouse-logs.json", - Cmd: "kubectl", - Args: []string{"-n", "d8-system", "logs", "-l", "app=deckhouse", "--tail", "3000"}, - }, - { - File: "instance-manager-capi-controller-manager-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=capi-controller-manager", "--tail", "3000", "--ignore-errors=true"}, - }, - { - File: "instance-manager-caps-controller-manager-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=caps-controller-manager", "--tail", "3000", "--ignore-errors=true"}, - }, - { - File: "instance-manager-machine-controller-manager.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, - }, - { - File: "instance-manager-mcm-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, - }, - { - File: "instance-manager-mcm-cloud-machines.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, - }, - { - File: "d8-{module-name}-ccm-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, - RequiredModule: "cloud-provider", - }, - { - File: "d8-{module-name}-csi-controller-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, - RequiredModule: "cloud-provider", - }, - { - File: "instance-manager-autoscaler-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail=5000", "-c", "cluster-autoscaler", "--ignore-errors=true"}, - }, - { - File: "d8-cert-manager-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-cert-manager", "logs", "-l", "app=cert-manager", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "cert-manager", - }, - { - File: "d8-cert-manager-all-certificate.json", - Cmd: "kubectl", - Args: []string{"get", "certificate", "-A", "-o", "json", "--ignore-not-found=true"}, - RequiredModule: "cert-manager", - }, - { - File: "kube-system-vpa-admission-controller-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-admission-controller", "--tail=3000", "-c", "admission-controller", "--ignore-errors=true"}, - }, - { - File: "kube-system-vpa-recommender-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-recommender", "--tail=3000", "-c", "recommender", "--ignore-errors=true"}, - }, - { - File: "kube-system-vpa-updater-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-updater", "--tail=3000", "-c", "updater", "--ignore-errors=true"}, - }, - { - File: "monitoring-prometheus-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-monitoring", "logs", "-l", "prometheus=main", "--tail=3000", "-c", "prometheus", "--ignore-errors=true"}, - }, - { - File: "cluster-alerts.json", - Cmd: "bash", - Args: []string{"-c", `kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, - }, - { - File: "cluster-bad-pods.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)" || true`}, - }, - { - File: "security-cluster-authorization-rules.json", - Cmd: "bash", - Args: []string{"-c", `kubectl get clusterauthorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, - }, - { - File: "security-authorization-rules.json", - Cmd: "bash", - Args: []string{"-c", `kubectl get authorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, - }, - { - File: "deckhouse-module-configs.json", - Cmd: "kubectl", - Args: []string{"get", "moduleconfig", "-o", "json"}, - }, - { - File: "d8-istio-resources.json", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio get all -o json | jq '.items[]'`}, - RequiredModule: "istio", - }, - { - File: "d8-istio-custom-resources.json", - Cmd: "bash", - Args: []string{"-c", `for crd in $(kubectl get crds | grep -E 'istio.io|gateway.networking.k8s.io' | awk '{print $1}'); do echo "Listing resources for CRD: $crd" && kubectl get $crd -A -o json; done`}, - RequiredModule: "istio", - }, - { - File: "d8-istio-envoy-config.json", - Cmd: "bash", - Args: []string{"-c", `kubectl port-forward daemonset/ingressgateway -n d8-istio 15000:15000 & sleep 5; (curl http://localhost:15000/config_dump?include_eds=true | jq 'del(.configs[6].dynamic_active_secrets)' && kill $!) || { kill $!; exit 0; }`}, - RequiredModule: "istio", - }, - { - File: "d8-istio-system-logs.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio logs -l app=istiod || true`}, - RequiredModule: "istio", - }, - { - File: "d8-istio-ingress-logs.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio logs daemonset/ingressgateway || true`}, - RequiredModule: "istio", - }, - { - File: "d8-istio-users-logs.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl get pods --all-namespaces -o jsonpath='{range .items[?(@.metadata.annotations.istio\.io/rev)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' | awk '/istio-proxy/ {print $0}' | shuf -n 1 | while read namespace pod_name containers; do echo "Collecting logs from istio-proxy in Pod $pod_name (Namespace: $namespace)"; kubectl logs "$pod_name" -n "$namespace" -c istio-proxy; done`}, - RequiredModule: "istio", - }, - { - File: "network-cni-cilium-health-status.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, - RequiredModule: "cni-cilium", - }, - { - File: "kube-system-audit-policy.json", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "get", "secrets", "audit-policy", "-o", "json", "--ignore-not-found=true"}, - }, - { - File: "kube-system-control-plane-manager-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "app=d8-control-plane-manager", "--tail=3000", "--ignore-errors=true"}, - }, - { - File: "kube-system-etcd-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "component=etcd", "--tail=3000", "--ignore-errors=true"}, - }, - { - File: "kube-system-kube-apiserver-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "component=kube-apiserver", "--tail=3000", "--ignore-errors=true"}, - }, - { - File: "kube-system-kube-controller-manager-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "component=kube-controller-manager", "--tail=3000", "--ignore-errors=true"}, - }, - { - File: "kube-system-kube-scheduler-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "component=kube-scheduler", "--tail=3000", "--ignore-errors=true"}, - }, - { - File: "kube-system-kube-dns-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "kube-system", "logs", "-l", "k8s-app=kube-dns", "--tail=3000", "--ignore-errors=true"}, - }, - { - File: "monitoring-prometheusremotewrites.json", - Cmd: "kubectl", - Args: []string{"get", "prometheusremotewrites", "-A", "-o", "json", "--ignore-not-found=true"}, - }, - { - File: "other-mutatingwebhookconfigurations.json", - Cmd: "kubectl", - Args: []string{"get", "mutatingwebhookconfigurations.admissionregistration.k8s.io", "-o", "json"}, - }, - { - File: "other-validatingwebhookconfigurations.json", - Cmd: "kubectl", - Args: []string{"get", "validatingwebhookconfigurations.admissionregistration.k8s.io", "-o", "json"}, - }, - { - File: "other-storage-deckhouse-io-terminating.txt", - Cmd: "bash", - Args: []string{"-c", `kubectl get $(kubectl api-resources --api-group=storage.deckhouse.io --verbs=list -o name | paste -sd, -) --ignore-not-found -A --chunk-size=200 -o json | jq -r '.items[] | select(.apiVersion == "storage.deckhouse.io/v1alpha1") | select(.metadata.deletionTimestamp != null) | "[\(.kind)] \(.metadata.namespace // "-")/\(.metadata.name)"'`}, - }, - { - File: "network-ingressnginxcontrollers.json", - Cmd: "kubectl", - Args: []string{"get", "ingressnginxcontrollers.deckhouse.io", "-o", "json", "--ignore-not-found=true"}, - }, - { - File: "cluster-crd.json", - Cmd: "bash", - // The OpenAPI schemas dominate the size of a full CRD dump (tens of MB on - // a cluster with virtualization/istio/cilium/storage) without adding - // diagnostic value, so they are dropped here instead of being buffered, - // transferred and stored. - Args: []string{"-c", `set -o pipefail; kubectl get customresourcedefinitions -o json | jq 'del(.items[].spec.versions[].schema)'`}, - }, - { - File: "d8-virtualization-dvcr-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=dvcr", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "virtualization", - }, - { - File: "d8-virtualization-virt-controller-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-virtualization", "logs", "-l", "kubevirt.internal.virtualization.deckhouse.io=virt-controller", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "virtualization", - }, - { - File: "d8-virtualization-controller-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=virtualization-controller", "--tail=3000", "--ignore-errors=true"}, - RequiredModule: "virtualization", - }, -} - -func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) error { - const ( - namespace = "d8-system" - containerName = "deckhouse" - ) - - podName, err := utilk8s.GetDeckhousePod(kubeCl) - if err != nil { - return fmt.Errorf("failed to get Deckhouse pod: %w", err) - } - - activeModules, modulesErr := fetchActiveModules(config, kubeCl, podName, namespace, containerName, commandTimeout) - if modulesErr != nil { - fmt.Fprintf(os.Stderr, "ERROR: could not fetch the list of active modules: %v\n", modulesErr) - fmt.Fprintf(os.Stderr, " collection continues without module filtering: module-gated commands run anyway and may produce empty files; per-module commands are skipped because their file names cannot be resolved (%s)\n", - strings.Join(moduleScopedFiles(debugCommands), ", ")) - } - - commands, acceptedNames := filterAndExpandCommands(debugCommands, activeModules, modulesErr == nil, newExcludeSet(excludeFiles)) - - if err := validateExcludeNames(excludeFiles, acceptedNames); err != nil { - return err - } - - return writeArchive( - config, kubeCl, podName, namespace, containerName, - commands, commandTimeout, requestInterval, - "Collecting debug info from Deckhouse...", - "Debug archive collection completed.", - ) -} - -// writeArchive streams a gzipped tar of the given commands' output to stdout. -// It is the shared body of the debug archives: only the command set and the -// progress banners differ between them. -func writeArchive( - config *rest.Config, - kubeCl kubernetes.Interface, - podName, namespace, containerName string, - commands []Command, - commandTimeout, requestInterval time.Duration, - startBanner, doneBanner string, -) (err error) { - gzipWriter := gzip.NewWriter(os.Stdout) - tarWriter := tar.NewWriter(gzipWriter) - - defer func() { - if closeErr := tarWriter.Close(); closeErr != nil && err == nil { - err = fmt.Errorf("failed to finalize tar archive: %w", closeErr) - } - - if closeErr := gzipWriter.Close(); closeErr != nil && err == nil { - err = fmt.Errorf("failed to finalize gzip stream: %w", closeErr) - } - }() - - fmt.Fprintf(os.Stderr, "%s\n", startBanner) - - if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, commandTimeout, requestInterval); err != nil { - return err - } - - fmt.Fprintf(os.Stderr, "%s\n", doneBanner) - - return nil -} - -// moduleScopedFiles lists the File templates that can only be resolved with a -// known module list, for the warning printed when that list is unavailable. -func moduleScopedFiles(commands []Command) []string { - var files []string - - for _, cmd := range commands { - if cmd.RequiredModule != "" && needsModuleExpansion(cmd) { - files = append(files, cmd.File) - } - } - - return files -} - -// runCommands executes each command inside the Deckhouse pod and streams its -// output into the tar archive, honoring the optional rate limit between command -// executions. The command list is already filtered by the caller. -func runCommands( - tarWriter *tar.Writer, - config *rest.Config, - kubeCl kubernetes.Interface, - podName, namespace, containerName string, - commands []Command, - commandTimeout, requestInterval time.Duration, -) error { - var tickCh <-chan time.Time - - if requestInterval > 0 { - ticker := time.NewTicker(requestInterval) - defer ticker.Stop() - - tickCh = ticker.C - } - - for _, cmd := range commands { - if tickCh != nil { - <-tickCh - } - - fullCommand := append([]string{cmd.Cmd}, cmd.Args...) - - cmdCtx, cancel := context.WithTimeout(context.Background(), commandTimeout) - output, stderrOutput, streamErr := utilk8s.ExecCommandInPod(cmdCtx, config, kubeCl, fullCommand, podName, namespace, containerName) - - cancel() - - if streamErr != nil { - if errors.Is(streamErr, context.DeadlineExceeded) { - fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s\n", cmd.File, commandTimeout) - } else { - fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %s\n%s\n", cmd.File, strings.Join(fullCommand, " "), stderrOutput) - } - } - - if notice := defaultedContainerNotice(stderrOutput); notice != "" { - output = append([]byte(notice), output...) - } - - if err := cmd.writeToTar(tarWriter, output); err != nil { - return fmt.Errorf("failed to write tar file %s: %w", cmd.File, err) - } - } - - return nil -} - -// defaultedContainerNotice extracts kubectl's client-side "Defaulted container -// ... out of: ..." notice(s) from a command's stderr, so they can be prepended -// to the collected log output. Without this, the discarded stderr would take -// with it the only record of which container a `logs` command without -// -c/--all-containers actually collected from a multi-container pod. -func defaultedContainerNotice(stderrOutput string) string { - var notice strings.Builder - - for _, line := range strings.Split(stderrOutput, "\n") { - if strings.Contains(line, "Defaulted container") { - notice.WriteString(line) - notice.WriteString("\n") - } - } - - return notice.String() -} - -// fetchActiveModules returns a map with the names of modules that are in the Ready phase. -func fetchActiveModules( - config *rest.Config, - kubeCl kubernetes.Interface, - podName, namespace, containerName string, - timeout time.Duration, -) (map[string]bool, error) { - cmdLine := []string{"kubectl", "get", "module", "-o", "json"} - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - stdout, stderr, err := utilk8s.ExecCommandInPod(ctx, config, kubeCl, cmdLine, podName, namespace, containerName) - if err != nil { - return nil, fmt.Errorf("stream kubectl get module: %w (stderr: %s)", err, stderr) - } - - if len(stdout) == 0 { - return nil, fmt.Errorf("kubectl get module returned no output (stderr: %s)", stderr) - } - - var list moduleList - if err = json.Unmarshal(stdout, &list); err != nil { - return nil, fmt.Errorf("parse module list: %w", err) - } - - active := make(map[string]bool, len(list.Items)) - for _, item := range list.Items { - if item.Status.Phase == "Ready" { - active[item.Metadata.Name] = true - } - } - - return active, nil -} - -// filterAndExpandCommands selects the commands to run: it resolves the -// {module-name} placeholder against the active modules and drops the entries -// excluded on the command line. It also returns every name --exclude accepts -// for this run, including the names of entries these very excludes dropped, so -// a valid name is never reported as unknown. -// -// Exclusion happens here, and not further down, because this is the only place -// where both spellings of an entry are known at once: the resolved archive name -// (d8-cloud-provider-aws-ccm-logs.txt) and the module-independent token printed -// by --list-exclude (ccm-logs). The resolved name alone does not reveal which -// of its segments is the module. -// -// modulesKnown reports whether activeModules actually describes the cluster. It -// is false when the module list could not be fetched: module-gated commands are -// then collected anyway (an empty file beats a silently missing one), except -// those whose File carries the {module-name} placeholder — their archive entry -// name cannot be resolved, so they are skipped rather than stored under a -// literal placeholder name. -func filterAndExpandCommands(commands []Command, activeModules map[string]bool, modulesKnown bool, excludeSet map[string]bool) (selected []Command, acceptedNames []string) { - selected = make([]Command, 0, len(commands)) - acceptedNames = make([]string, 0, len(commands)) - - for _, cmd := range commands { - // The token stays accepted even when the command is gated out below: - // --exclude ccm-logs must not fail on a cluster without a cloud provider. - token := excludeBaseName(cmd) - acceptedNames = append(acceptedNames, token) - - if excludedByName(excludeSet, cmd.File, token) { - continue - } - - if cmd.RequiredModule == "" { - selected = append(selected, cmd) - continue - } - - if !modulesKnown { - if !needsModuleExpansion(cmd) { - selected = append(selected, cmd) - } - - continue - } - - // No explicit guard for an empty activeModules is needed: both branches - // below iterate the matching modules, of which there are none. - if needsModuleExpansion(cmd) { - matchedModules := matchingModules(activeModules, cmd.RequiredModule) - for _, moduleName := range matchedModules { - // Copy the command and overwrite only what is substituted, so a - // field added to Command later cannot be silently dropped here. - expanded := cmd - expanded.File = strings.ReplaceAll(cmd.File, "{module-name}", moduleName) - expanded.Args = replaceModuleName(cmd.Args, moduleName) - - acceptedNames = append(acceptedNames, expanded.File) - - if excludedByName(excludeSet, expanded.File) { - continue - } - - selected = append(selected, expanded) - } - } else { - for moduleName := range activeModules { - if isModuleMatch(moduleName, cmd.RequiredModule) { - selected = append(selected, cmd) - break - } - } - } - } - - return selected, acceptedNames -} - -func matchingModules(activeModules map[string]bool, required string) []string { - var matched []string - - for name := range activeModules { - if isModuleMatch(name, required) { - matched = append(matched, name) - } - } - - sort.Strings(matched) - - return matched -} - -func isModuleMatch(moduleName, required string) bool { - return moduleName == required || strings.HasPrefix(moduleName, required) -} - -// needsModuleExpansion reports whether cmd must be duplicated once per active -// module matching RequiredModule (with {module-name} substituted into File -// and Args), rather than run once as-is. -func needsModuleExpansion(cmd Command) bool { - if strings.Contains(cmd.File, "{module-name}") { - return true - } - - return slices.ContainsFunc(cmd.Args, func(arg string) bool { - return strings.Contains(arg, "{module-name}") - }) -} - -func replaceModuleName(args []string, moduleName string) []string { - expanded := make([]string, len(args)) - for i, arg := range args { - expanded[i] = strings.ReplaceAll(arg, "{module-name}", moduleName) - } - - return expanded -} - -func (c *Command) writeToTar(tarWriter *tar.Writer, fileContent []byte) error { - header := &tar.Header{ - Name: c.File, - Mode: 0o600, - Size: int64(len(fileContent)), - } - - if err := tarWriter.WriteHeader(header); err != nil { - return fmt.Errorf("write tar header: %v", err) - } - - if _, err := tarWriter.Write(fileContent); err != nil { - return fmt.Errorf("copy content: %v", err) - } - - return nil -} - -// trimArchiveExt drops the archive entry extension, so --exclude accepts a name -// with or without it. -func trimArchiveExt(name string) string { - return strings.TrimSuffix(strings.TrimSuffix(name, ".json"), ".txt") -} - -// excludeBaseName returns the --exclude token printed by --list-exclude for a -// command template: the archive entry name as written in debugCommands, or — -// when that name is per-module and therefore cluster-specific — the -// module-independent remainder (d8-{module-name}-ccm-logs.txt -> ccm-logs). -// -// The token is always derived from File, so a new per-module command needs no -// extra per-command data and cannot disagree with its own file name. -func excludeBaseName(cmd Command) string { - if !strings.Contains(cmd.File, "{module-name}") { - return cmd.File - } - - name := strings.ReplaceAll(cmd.File, "d8-{module-name}-", "") - name = strings.ReplaceAll(name, "-{module-name}-", "-") - name = strings.ReplaceAll(name, "-{module-name}", "") - name = strings.ReplaceAll(name, "{module-name}-", "") - name = strings.ReplaceAll(name, "{module-name}", "") - - return trimArchiveExt(name) -} - -// newExcludeSet normalizes the raw --exclude values into the form matched -// against command names: surrounding spaces and the extension are irrelevant. -func newExcludeSet(excludeFiles []string) map[string]bool { - set := make(map[string]bool, len(excludeFiles)) - - for _, name := range excludeFiles { - name = trimArchiveExt(strings.TrimSpace(name)) - if name != "" { - set[name] = true - } - } - - return set -} - -// excludedByName reports whether any of the spellings of one archive entry was -// excluded on the command line. A name matches only that entry: there is no -// prefix or group matching, so --exclude d8 cannot silently drop every d8-* file. -func excludedByName(excludeSet map[string]bool, names ...string) bool { - if len(excludeSet) == 0 { - return false - } - - for _, name := range names { - if name != "" && excludeSet[trimArchiveExt(name)] { - return true - } - } - - return false -} - -// validateExcludeNames rejects --exclude values that cannot match any archive -// entry, so a typo is reported instead of quietly collecting the full archive. -// acceptedNames comes from filterAndExpandCommands and already covers both the -// resolved entry names of this run and the module-independent tokens. -func validateExcludeNames(excludeFiles, acceptedNames []string) error { - known := make(map[string]bool, len(acceptedNames)) - accepted := make([]string, 0, len(acceptedNames)) - - for _, name := range acceptedNames { - key := trimArchiveExt(name) - if key == "" || known[key] { - continue - } - - known[key] = true - - accepted = append(accepted, name) - } - - var unknown []string - - for _, name := range excludeFiles { - name = trimArchiveExt(strings.TrimSpace(name)) - if name != "" && !known[name] { - unknown = append(unknown, name) - } - } - - if len(unknown) == 0 { - return nil - } - - return fmt.Errorf("unknown --exclude name(s): %s%s\nrun \"d8 system collect-debug-info --list-exclude\" to see the accepted names", - strings.Join(unknown, ", "), suggestExcludeNames(unknown, accepted)) -} - -// suggestExcludeNames offers the accepted names that contain (or are contained -// in) an unknown one, which covers both typos and the group prefixes that used -// to match implicitly. -func suggestExcludeNames(unknown, accepted []string) string { - const maxSuggestions = 5 - - seen := make(map[string]bool, maxSuggestions) - - var matches []string - - for _, name := range unknown { - for _, candidate := range accepted { - key := trimArchiveExt(candidate) - if seen[key] || !strings.Contains(key, name) && !strings.Contains(name, key) { - continue - } - - seen[key] = true - - matches = append(matches, candidate) - } - } - - if len(matches) == 0 { - return "" - } - - sort.Strings(matches) - - if len(matches) > maxSuggestions { - return fmt.Sprintf("; did you mean one of: %s, ... (%d more)", strings.Join(matches[:maxSuggestions], ", "), len(matches)-maxSuggestions) - } - - return fmt.Sprintf("; did you mean: %s", strings.Join(matches, ", ")) -} - -// GetExcludableFiles returns the tokens accepted by --exclude, one per archive -// entry, as printed by --list-exclude. -func GetExcludableFiles() []string { - seen := make(map[string]bool, len(debugCommands)) - - files := make([]string, 0, len(debugCommands)) - for _, cmd := range debugCommands { - name := excludeBaseName(cmd) - if !seen[name] { - seen[name] = true - files = append(files, name) - } - } - - sort.Strings(files) - - return files -} diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go b/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go deleted file mode 100644 index d63e39f34..000000000 --- a/internal/system/cmd/collect-debug-info/debugtar/debugTar_test.go +++ /dev/null @@ -1,300 +0,0 @@ -package debugtar - -import ( - "slices" - "strings" - "testing" -) - -// TestDebugCommandsModuleExpansionInvariant guards the contract documented on -// Command.RequiredModule: a command whose File or Args contains the -// {module-name} placeholder must also set RequiredModule, since that is what -// filterAndExpandCommands uses to resolve the placeholder into a real module -// name. Without RequiredModule, needsModuleExpansion is never even checked, -// so the placeholder would leak into the collected archive as a literal -// string instead of a resolved module name. -func TestDebugCommandsModuleExpansionInvariant(t *testing.T) { - for _, cmd := range debugCommands { - if cmd.RequiredModule != "" { - continue - } - - if !needsModuleExpansion(cmd) { - continue - } - - t.Errorf("command %q uses the {module-name} placeholder but has no RequiredModule set, so it will never be resolved", cmd.File) - } -} - -// TestFilterAndExpandCommandsWithoutModuleList covers the degraded path taken -// when `kubectl get module` fails: the collection must go on, but no command -// may reach the archive with an unresolved {module-name} placeholder in its -// file name. -func TestFilterAndExpandCommandsWithoutModuleList(t *testing.T) { - commands, _ := filterAndExpandCommands(debugCommands, nil, false, nil) - - var expected int - - for _, cmd := range debugCommands { - if !needsModuleExpansion(cmd) { - expected++ - } - } - - if len(commands) != expected { - t.Errorf("got %d commands without a module list, want %d", len(commands), expected) - } - - for _, cmd := range commands { - if needsModuleExpansion(cmd) { - t.Errorf("command %q still carries an unresolved {module-name} placeholder", cmd.File) - } - } -} - -// TestFilterAndExpandCommandsKeepsFields guards the expansion against silently -// dropping a field of Command that a later change adds. -func TestFilterAndExpandCommandsKeepsFields(t *testing.T) { - source := Command{ - File: "d8-{module-name}-ccm-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs"}, - RequiredModule: "cloud-provider", - } - - expanded, _ := filterAndExpandCommands([]Command{source}, map[string]bool{"cloud-provider-aws": true}, true, nil) - if len(expanded) != 1 { - t.Fatalf("got %d expanded commands, want 1", len(expanded)) - } - - got := expanded[0] - if got.File != "d8-cloud-provider-aws-ccm-logs.txt" { - t.Errorf("File = %q, want %q", got.File, "d8-cloud-provider-aws-ccm-logs.txt") - } - - if got.Cmd != source.Cmd || got.RequiredModule != source.RequiredModule { - t.Errorf("expansion dropped a field: %+v", got) - } -} - -// TestGetExcludableFilesAreUsableTokens guards the contract of --list-exclude: -// every printed name must be usable verbatim with --exclude, so none of them -// may carry an unresolved placeholder. -func TestGetExcludableFilesAreUsableTokens(t *testing.T) { - tokens := GetExcludableFiles() - if len(tokens) != len(debugCommands) { - t.Errorf("got %d tokens for %d commands", len(tokens), len(debugCommands)) - } - - for _, token := range tokens { - if strings.Contains(token, "{module-name}") { - t.Errorf("token %q cannot be typed by a user", token) - } - } - - for _, want := range []string{"cluster-events.json", "ccm-logs", "csi-controller-logs"} { - if !slices.Contains(tokens, want) { - t.Errorf("token %q is missing from --list-exclude", want) - } - } -} - -// TestExcludeAcceptedSpellings pins the accepted --exclude spellings and, just -// as importantly, the rejected ones: a group prefix must not drop a whole family -// of files. The per-module entries are checked through the real selection, since -// that is where a command is matched against the exclude list. -func TestExcludeAcceptedSpellings(t *testing.T) { - templates := []Command{ - {File: "cluster-events.json", Cmd: "kubectl"}, - { - File: "d8-{module-name}-ccm-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs"}, - RequiredModule: "cloud-provider", - }, - } - activeModules := map[string]bool{"cloud-provider-aws": true, "cloud-provider-yandex": true} - - cases := []struct { - token string - want []string - }{ - {"", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - {"ccm-logs", []string{"cluster-events.json"}}, - {"ccm-logs.txt", []string{"cluster-events.json"}}, - {"d8-cloud-provider-aws-ccm-logs.txt", []string{"cluster-events.json", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - {"d8-cloud-provider-aws-ccm-logs", []string{"cluster-events.json", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - {"cluster-events", []string{"d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - {" cluster-events ", []string{"d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - {"d8", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - {"cluster", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, - } - - for _, tc := range cases { - var exclude []string - if tc.token != "" { - exclude = []string{tc.token} - } - - selected, _ := filterAndExpandCommands(templates, activeModules, true, newExcludeSet(exclude)) - - got := make([]string, 0, len(selected)) - for _, cmd := range selected { - got = append(got, cmd.File) - } - - if !slices.Equal(got, tc.want) { - t.Errorf("--exclude %q left %v, want %v", tc.token, got, tc.want) - } - } -} - -// TestValidateExcludeNames checks that a name which can never match is reported -// instead of silently collecting everything, while a module-independent token -// stays valid on a cluster where that module is not enabled. -func TestValidateExcludeNames(t *testing.T) { - _, accepted := filterAndExpandCommands(debugCommands, nil, false, nil) - - if err := validateExcludeNames([]string{"ccm-logs", "cluster-events", "cluster-events.json"}, accepted); err != nil { - t.Errorf("valid names rejected: %v", err) - } - - err := validateExcludeNames([]string{"d8"}, accepted) - if err == nil { - t.Fatal("group prefix d8 was accepted, it silently excludes nothing now") - } - - if !strings.Contains(err.Error(), "did you mean") { - t.Errorf("error %q offers no suggestion", err) - } - - err = validateExcludeNames([]string{"ccm-log"}, accepted) - if err == nil || !strings.Contains(err.Error(), "did you mean: ccm-logs") { - t.Errorf("a near miss should be pointed at its name, got %v", err) - } - - err = validateExcludeNames([]string{"zzzz"}, accepted) - if err == nil { - t.Fatal("a name matching nothing was accepted") - } - - if strings.Contains(err.Error(), "did you mean") { - t.Errorf("error %q invents a suggestion for a name with no near miss", err) - } - - // A repeated or empty accepted name must not confuse the lookup. - if err := validateExcludeNames([]string{"plain"}, []string{"plain.txt", "plain.txt", ""}); err != nil { - t.Errorf("duplicate accepted names broke the lookup: %v", err) - } -} - -// TestExcludedEntryStaysAValidName guards the interaction between exclusion and -// validation: the name a user just excluded must not then be reported as -// unknown, even though its command is gone from the selection. -func TestExcludedEntryStaysAValidName(t *testing.T) { - for _, name := range []string{"ccm-logs", "cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt"} { - _, accepted := filterAndExpandCommands(debugCommands, map[string]bool{"cloud-provider-aws": true}, true, newExcludeSet([]string{name})) - - if err := validateExcludeNames([]string{name}, accepted); err != nil { - t.Errorf("--exclude %q reported as unknown: %v", name, err) - } - } -} - -// TestSelectionMatrix pins every combination of the two knobs documented on -// Command.RequiredModule: whether the command is gated on a module, whether it -// carries the {module-name} placeholder, and whether the module list could be -// fetched at all. The expected file lists also pin the order, which must stay -// deterministic (templates in declaration order, per-module copies sorted by -// module name) so two runs of the same cluster produce the same archive. -func TestSelectionMatrix(t *testing.T) { - templates := []Command{ - {File: "plain.txt", Cmd: "kubectl"}, - {File: "literal-{module-name}.txt", Cmd: "kubectl"}, - {File: "gated.txt", Cmd: "kubectl", RequiredModule: "istio"}, - { - File: "d8-{module-name}-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs"}, - RequiredModule: "cloud-provider", - }, - } - - cases := []struct { - name string - activeModules map[string]bool - modulesKnown bool - want []string - }{ - { - name: "gate matched by prefix and by exact name", - activeModules: map[string]bool{"istio": true, "cloud-provider-yandex": true, "cloud-provider-aws": true}, - modulesKnown: true, - want: []string{ - "plain.txt", - "literal-{module-name}.txt", - "gated.txt", - "d8-cloud-provider-aws-logs.txt", - "d8-cloud-provider-yandex-logs.txt", - }, - }, - { - name: "no module matches the gate", - activeModules: map[string]bool{"cert-manager": true}, - modulesKnown: true, - want: []string{"plain.txt", "literal-{module-name}.txt"}, - }, - { - name: "no module is Ready", - activeModules: map[string]bool{}, - modulesKnown: true, - want: []string{"plain.txt", "literal-{module-name}.txt"}, - }, - { - name: "module list unavailable", - modulesKnown: false, - want: []string{"plain.txt", "literal-{module-name}.txt", "gated.txt"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - selected, _ := filterAndExpandCommands(templates, tc.activeModules, tc.modulesKnown, nil) - - got := make([]string, 0, len(selected)) - for _, cmd := range selected { - got = append(got, cmd.File) - } - - if !slices.Equal(got, tc.want) { - t.Errorf("selected %v, want %v", got, tc.want) - } - }) - } -} - -// TestExpansionSubstitutesArgs checks the other half of the substitution: the -// module name must reach Args too, not just the archive entry name. -func TestExpansionSubstitutesArgs(t *testing.T) { - template := Command{ - File: "d8-{module-name}-logs.txt", - Cmd: "kubectl", - Args: []string{"-n", "d8-{module-name}", "logs"}, - RequiredModule: "cloud-provider", - } - - selected, _ := filterAndExpandCommands([]Command{template}, map[string]bool{"cloud-provider-aws": true}, true, nil) - if len(selected) != 1 { - t.Fatalf("got %d commands, want 1", len(selected)) - } - - want := []string{"-n", "d8-cloud-provider-aws", "logs"} - if !slices.Equal(selected[0].Args, want) { - t.Errorf("Args = %v, want %v", selected[0].Args, want) - } - - if !slices.Equal(template.Args, []string{"-n", "d8-{module-name}", "logs"}) { - t.Errorf("expansion mutated the template Args: %v", template.Args) - } -} diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go b/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go new file mode 100644 index 000000000..c7b6694b6 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go @@ -0,0 +1,341 @@ +package debugtar + +// This file holds the command table of the cluster-wide debug archive and +// nothing else: which command produces which archive entry. The table of the +// separate virtualization archive lives in virtualization.go, the shape of a +// single entry in commandtype.go, and the mechanics of selecting, excluding and +// running these commands in selection.go, exclude.go and archive.go. + +// debugCommands - a complete list of commands for collecting debug information. +var debugCommands = []command{ + { + File: "deckhouse-queue.txt", + Cmd: "deckhouse-controller", + Args: []string{"queue", "list"}, + }, + { + File: "cluster-global-values.json", + Cmd: "bash", + Args: []string{"-c", `deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, + }, + { + File: "deckhouse-enabled-modules.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get modules -o json | jq '.items[]'"}, + }, + { + File: "deckhouse-module-sources.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get modulesources -o json | jq '.items[]'"}, + }, + { + File: "deckhouse-module-pull-overrides.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get modulepulloverrides -o json | jq '.items[]'"}, + }, + { + File: "deckhouse-module-update-policies.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get moduleupdatepolicies -o json | jq '.items[]'"}, + }, + { + File: "deckhouse-maintenance-modules.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get moduleconfig -ojson | jq -r '.items[] | select(.spec.maintenance == "NoResourceReconciliation") | .metadata.name'`}, + }, + { + File: "cluster-events.json", + Cmd: "kubectl", + Args: []string{"get", "events", "--sort-by=.metadata.creationTimestamp", "-A", "-o", "json"}, + }, + { + File: "d8-all.json", + Cmd: "bash", + Args: []string{"-c", `for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, + }, + { + File: "cluster-node-groups.json", + Cmd: "kubectl", + Args: []string{"get", "nodegroups", "-A", "-o", "json"}, + }, + { + File: "cluster-node-group-configuration.json", + Cmd: "kubectl", + Args: []string{"get", "nodegroupconfiguration", "-A", "-o", "json"}, + }, + { + File: "cluster-nodes.json", + Cmd: "kubectl", + Args: []string{"get", "nodes", "-A", "-o", "json"}, + }, + { + File: "cluster-namespace.json", + Cmd: "kubectl", + Args: []string{"get", "namespaces", "-o", "json"}, + }, + { + File: "instance-manager-capi-machines.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.cluster.x-k8s.io -o json | jq '.items[]'`}, + }, + { + File: "instance-manager-instances.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get instances.deckhouse.io -o json | jq '.items[]'`}, + }, + { + File: "instance-manager-staticinstances.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get staticinstances.deckhouse.io -o json | jq '.items[]'`}, + }, + { + File: "instance-manager-cloud-machine-deployment.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machinedeployments.machine.sapcloud.io -o json | jq '.items[]'`}, + RequiredModule: "cloud-provider", + }, + { + File: "instance-manager-static-machine-deployment.txt", + Cmd: "bash", + Args: []string{"-c", "kubectl -n d8-cloud-instance-manager get machinedeployments.cluster.x-k8s.io -o json --ignore-not-found | jq '.items[]'"}, + }, + { + File: "deckhouse-version.json", + Cmd: "bash", + Args: []string{"-c", "jq -s add <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.metadata.annotations | {\"core.deckhouse.io/edition\",\"core.deckhouse.io/version\"}') <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.spec.template.spec.containers[] | select(.name == \"deckhouse\") | {image}')"}, + }, + { + File: "deckhouse-releases.json", + Cmd: "kubectl", + Args: []string{"get", "deckhousereleases", "-o", "json"}, + }, + { + File: "deckhouse-logs.json", + Cmd: "kubectl", + Args: []string{"-n", "d8-system", "logs", "-l", "app=deckhouse", "--tail", "3000"}, + }, + { + File: "instance-manager-capi-controller-manager-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=capi-controller-manager", "--tail", "3000", "--ignore-errors=true"}, + }, + { + File: "instance-manager-caps-controller-manager-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=caps-controller-manager", "--tail", "3000", "--ignore-errors=true"}, + }, + { + File: "instance-manager-machine-controller-manager.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, + }, + { + File: "instance-manager-mcm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail=3000", "-c", "controller", "--ignore-errors=true"}, + }, + { + File: "instance-manager-mcm-cloud-machines.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, + }, + { + File: "d8-{module-name}-ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=cloud-controller-manager", "--tail=3000"}, + RequiredModule: "cloud-provider", + }, + { + File: "d8-{module-name}-csi-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs", "-l", "app=csi-controller", "--tail=3000"}, + RequiredModule: "cloud-provider", + }, + { + File: "instance-manager-autoscaler-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail=5000", "-c", "cluster-autoscaler", "--ignore-errors=true"}, + }, + { + File: "d8-cert-manager-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cert-manager", "logs", "-l", "app=cert-manager", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "cert-manager", + }, + { + File: "d8-cert-manager-all-certificate.json", + Cmd: "kubectl", + Args: []string{"get", "certificate", "-A", "-o", "json", "--ignore-not-found=true"}, + RequiredModule: "cert-manager", + }, + { + File: "kube-system-vpa-admission-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-admission-controller", "--tail=3000", "-c", "admission-controller", "--ignore-errors=true"}, + }, + { + File: "kube-system-vpa-recommender-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-recommender", "--tail=3000", "-c", "recommender", "--ignore-errors=true"}, + }, + { + File: "kube-system-vpa-updater-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-updater", "--tail=3000", "-c", "updater", "--ignore-errors=true"}, + }, + { + File: "monitoring-prometheus-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-monitoring", "logs", "-l", "prometheus=main", "--tail=3000", "-c", "prometheus", "--ignore-errors=true"}, + }, + { + File: "cluster-alerts.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, + }, + { + File: "cluster-bad-pods.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)" || true`}, + }, + { + File: "security-cluster-authorization-rules.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get clusterauthorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, + }, + { + File: "security-authorization-rules.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get authorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, + }, + { + File: "deckhouse-module-configs.json", + Cmd: "kubectl", + Args: []string{"get", "moduleconfig", "-o", "json"}, + }, + { + File: "d8-istio-resources.json", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-istio get all -o json | jq '.items[]'`}, + RequiredModule: "istio", + }, + { + File: "d8-istio-custom-resources.json", + Cmd: "bash", + Args: []string{"-c", `for crd in $(kubectl get crds | grep -E 'istio.io|gateway.networking.k8s.io' | awk '{print $1}'); do echo "Listing resources for CRD: $crd" && kubectl get $crd -A -o json; done`}, + RequiredModule: "istio", + }, + { + File: "d8-istio-envoy-config.json", + Cmd: "bash", + Args: []string{"-c", `kubectl port-forward daemonset/ingressgateway -n d8-istio 15000:15000 & sleep 5; (curl http://localhost:15000/config_dump?include_eds=true | jq 'del(.configs[6].dynamic_active_secrets)' && kill $!) || { kill $!; exit 0; }`}, + RequiredModule: "istio", + }, + { + File: "d8-istio-system-logs.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-istio logs -l app=istiod || true`}, + RequiredModule: "istio", + }, + { + File: "d8-istio-ingress-logs.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-istio logs daemonset/ingressgateway || true`}, + RequiredModule: "istio", + }, + { + File: "d8-istio-users-logs.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get pods --all-namespaces -o jsonpath='{range .items[?(@.metadata.annotations.istio\.io/rev)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' | awk '/istio-proxy/ {print $0}' | shuf -n 1 | while read namespace pod_name containers; do echo "Collecting logs from istio-proxy in Pod $pod_name (Namespace: $namespace)"; kubectl logs "$pod_name" -n "$namespace" -c istio-proxy; done`}, + RequiredModule: "istio", + }, + { + File: "network-cni-cilium-health-status.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, + RequiredModule: "cni-cilium", + }, + { + File: "kube-system-audit-policy.json", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "get", "secrets", "audit-policy", "-o", "json", "--ignore-not-found=true"}, + }, + { + File: "kube-system-control-plane-manager-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=d8-control-plane-manager", "--tail=3000", "--ignore-errors=true"}, + }, + { + File: "kube-system-etcd-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "component=etcd", "--tail=3000", "--ignore-errors=true"}, + }, + { + File: "kube-system-kube-apiserver-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "component=kube-apiserver", "--tail=3000", "--ignore-errors=true"}, + }, + { + File: "kube-system-kube-controller-manager-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "component=kube-controller-manager", "--tail=3000", "--ignore-errors=true"}, + }, + { + File: "kube-system-kube-scheduler-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "component=kube-scheduler", "--tail=3000", "--ignore-errors=true"}, + }, + { + File: "kube-system-kube-dns-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "k8s-app=kube-dns", "--tail=3000", "--ignore-errors=true"}, + }, + { + File: "monitoring-prometheusremotewrites.json", + Cmd: "kubectl", + Args: []string{"get", "prometheusremotewrites", "-A", "-o", "json", "--ignore-not-found=true"}, + }, + { + File: "other-mutatingwebhookconfigurations.json", + Cmd: "kubectl", + Args: []string{"get", "mutatingwebhookconfigurations.admissionregistration.k8s.io", "-o", "json"}, + }, + { + File: "other-validatingwebhookconfigurations.json", + Cmd: "kubectl", + Args: []string{"get", "validatingwebhookconfigurations.admissionregistration.k8s.io", "-o", "json"}, + }, + { + File: "other-storage-deckhouse-io-terminating.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get $(kubectl api-resources --api-group=storage.deckhouse.io --verbs=list -o name | paste -sd, -) --ignore-not-found -A --chunk-size=200 -o json | jq -r '.items[] | select(.apiVersion == "storage.deckhouse.io/v1alpha1") | select(.metadata.deletionTimestamp != null) | "[\(.kind)] \(.metadata.namespace // "-")/\(.metadata.name)"'`}, + }, + { + File: "network-ingressnginxcontrollers.json", + Cmd: "kubectl", + Args: []string{"get", "ingressnginxcontrollers.deckhouse.io", "-o", "json", "--ignore-not-found=true"}, + }, + { + File: "cluster-crd.json", + Cmd: "bash", + Args: []string{"-c", `set -o pipefail; kubectl get customresourcedefinitions -o json | jq 'del(.items[].spec.versions[].schema)'`}, + }, + { + File: "d8-virtualization-dvcr-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=dvcr", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", + }, + { + File: "d8-virtualization-virt-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "kubevirt.internal.virtualization.deckhouse.io=virt-controller", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", + }, + { + File: "d8-virtualization-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-virtualization", "logs", "-l", "app=virtualization-controller", "--tail=3000", "--ignore-errors=true"}, + RequiredModule: "virtualization", + }, +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go b/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go new file mode 100644 index 000000000..4b04008a6 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go @@ -0,0 +1,26 @@ +package debugtar + +import ( + "testing" +) + +// TestDebugCommandsModuleExpansionInvariant guards the contract documented on +// command.RequiredModule: a command whose File or Args contains the +// {module-name} placeholder must also set RequiredModule, since that is what +// filterAndExpandCommands uses to resolve the placeholder into a real module +// name. Without RequiredModule, needsModuleExpansion is never even checked, +// so the placeholder would leak into the collected archive as a literal +// string instead of a resolved module name. +func TestDebugCommandsModuleExpansionInvariant(t *testing.T) { + for _, cmd := range debugCommands { + if cmd.RequiredModule != "" { + continue + } + + if !needsModuleExpansion(cmd) { + continue + } + + t.Errorf("command %q uses the {module-name} placeholder but has no RequiredModule set, so it will never be resolved", cmd.File) + } +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/exclude.go b/internal/system/cmd/collect-debug-info/debugtar/exclude.go new file mode 100644 index 000000000..52dd4fa80 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/exclude.go @@ -0,0 +1,158 @@ +package debugtar + +import ( + "fmt" + "sort" + "strings" +) + +// trimArchiveExt drops the archive entry extension, so --exclude accepts a name +// with or without it. +func trimArchiveExt(name string) string { + return strings.TrimSuffix(strings.TrimSuffix(name, ".json"), ".txt") +} + +// excludeBaseName returns the --exclude token printed by --list-exclude for a +// command template: the archive entry name as written in debugCommands, or — +// when that name is per-module and therefore cluster-specific — the +// module-independent remainder (d8-{module-name}-ccm-logs.txt -> ccm-logs). +// +// The token is always derived from File, so a new per-module command needs no +// extra per-command data and cannot disagree with its own file name. +func excludeBaseName(cmd command) string { + if !strings.Contains(cmd.File, "{module-name}") { + return cmd.File + } + + name := strings.ReplaceAll(cmd.File, "d8-{module-name}-", "") + name = strings.ReplaceAll(name, "-{module-name}-", "-") + name = strings.ReplaceAll(name, "-{module-name}", "") + name = strings.ReplaceAll(name, "{module-name}-", "") + name = strings.ReplaceAll(name, "{module-name}", "") + + return trimArchiveExt(name) +} + +// newExcludeSet normalizes the raw --exclude values into the form matched +// against command names: surrounding spaces and the extension are irrelevant. +func newExcludeSet(excludeFiles []string) map[string]bool { + set := make(map[string]bool, len(excludeFiles)) + + for _, name := range excludeFiles { + name = trimArchiveExt(strings.TrimSpace(name)) + if name != "" { + set[name] = true + } + } + + return set +} + +// excludedByName reports whether any of the spellings of one archive entry was +// excluded on the command line. A name matches only that entry: there is no +// prefix or group matching. +// P.S. --exclude d8 cannot silently drop every d8-* file. +func excludedByName(excludeSet map[string]bool, names ...string) bool { + if len(excludeSet) == 0 { + return false + } + + for _, name := range names { + if name != "" && excludeSet[trimArchiveExt(name)] { + return true + } + } + + return false +} + +// validateExcludeNames rejects --exclude values that cannot match any archive +// entry, so a typo is reported instead of quietly collecting the full archive. +// acceptedNames comes from filterAndExpandCommands and already covers both the +// resolved entry names of this run and the module-independent tokens. +func validateExcludeNames(excludeFiles, acceptedNames []string) error { + known := make(map[string]bool, len(acceptedNames)) + accepted := make([]string, 0, len(acceptedNames)) + + for _, name := range acceptedNames { + key := trimArchiveExt(name) + if key == "" || known[key] { + continue + } + + known[key] = true + + accepted = append(accepted, name) + } + + var unknown []string + + for _, name := range excludeFiles { + name = trimArchiveExt(strings.TrimSpace(name)) + if name != "" && !known[name] { + unknown = append(unknown, name) + } + } + + if len(unknown) == 0 { + return nil + } + + return fmt.Errorf("unknown --exclude name(s): %s%s\nrun \"d8 system collect-debug-info --list-exclude\" to see the accepted names", + strings.Join(unknown, ", "), suggestExcludeNames(unknown, accepted)) +} + +// suggestExcludeNames offers the accepted names that contain (or are contained +// in) an unknown one, which covers both typos and the group prefixes that used +// to match implicitly. +func suggestExcludeNames(unknown, accepted []string) string { + const maxSuggestions = 5 + + seen := make(map[string]bool, maxSuggestions) + + var matches []string + + for _, name := range unknown { + for _, candidate := range accepted { + key := trimArchiveExt(candidate) + if seen[key] || !strings.Contains(key, name) && !strings.Contains(name, key) { + continue + } + + seen[key] = true + + matches = append(matches, candidate) + } + } + + if len(matches) == 0 { + return "" + } + + sort.Strings(matches) + + if len(matches) > maxSuggestions { + return fmt.Sprintf("; did you mean one of: %s, ... (%d more)", strings.Join(matches[:maxSuggestions], ", "), len(matches)-maxSuggestions) + } + + return fmt.Sprintf("; did you mean: %s", strings.Join(matches, ", ")) +} + +// GetExcludableFiles returns the tokens accepted by --exclude, one per archive +// entry, as printed by --list-exclude. +func GetExcludableFiles() []string { + seen := make(map[string]bool, len(debugCommands)) + + files := make([]string, 0, len(debugCommands)) + for _, cmd := range debugCommands { + name := excludeBaseName(cmd) + if !seen[name] { + seen[name] = true + files = append(files, name) + } + } + + sort.Strings(files) + + return files +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/exclude_test.go b/internal/system/cmd/collect-debug-info/debugtar/exclude_test.go new file mode 100644 index 000000000..60bae917e --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/exclude_test.go @@ -0,0 +1,131 @@ +package debugtar + +import ( + "slices" + "strings" + "testing" +) + +// TestGetExcludableFilesAreUsableTokens guards the contract of --list-exclude: +// every printed name must be usable verbatim with --exclude, so none of them +// may carry an unresolved placeholder. +func TestGetExcludableFilesAreUsableTokens(t *testing.T) { + tokens := GetExcludableFiles() + if len(tokens) != len(debugCommands) { + t.Errorf("got %d tokens for %d commands", len(tokens), len(debugCommands)) + } + + for _, token := range tokens { + if strings.Contains(token, "{module-name}") { + t.Errorf("token %q cannot be typed by a user", token) + } + } + + for _, want := range []string{"cluster-events.json", "ccm-logs", "csi-controller-logs"} { + if !slices.Contains(tokens, want) { + t.Errorf("token %q is missing from --list-exclude", want) + } + } +} + +// TestExcludeAcceptedSpellings pins the accepted --exclude spellings and, just +// as importantly, the rejected ones: a group prefix must not drop a whole family +// of files. The per-module entries are checked through the real selection, since +// that is where a command is matched against the exclude list. +func TestExcludeAcceptedSpellings(t *testing.T) { + templates := []command{ + {File: "cluster-events.json", Cmd: "kubectl"}, + { + File: "d8-{module-name}-ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + }, + } + activeModules := map[string]bool{"cloud-provider-aws": true, "cloud-provider-yandex": true} + + cases := []struct { + token string + want []string + }{ + {"", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"ccm-logs", []string{"cluster-events.json"}}, + {"ccm-logs.txt", []string{"cluster-events.json"}}, + {"d8-cloud-provider-aws-ccm-logs.txt", []string{"cluster-events.json", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"d8-cloud-provider-aws-ccm-logs", []string{"cluster-events.json", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"cluster-events", []string{"d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {" cluster-events ", []string{"d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"d8", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + {"cluster", []string{"cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt", "d8-cloud-provider-yandex-ccm-logs.txt"}}, + } + + for _, tc := range cases { + var exclude []string + if tc.token != "" { + exclude = []string{tc.token} + } + + selected, _ := filterAndExpandCommands(templates, activeModules, true, newExcludeSet(exclude)) + + got := make([]string, 0, len(selected)) + for _, cmd := range selected { + got = append(got, cmd.File) + } + + if !slices.Equal(got, tc.want) { + t.Errorf("--exclude %q left %v, want %v", tc.token, got, tc.want) + } + } +} + +// TestValidateExcludeNames checks that a name which can never match is reported +// instead of silently collecting everything, while a module-independent token +// stays valid on a cluster where that module is not enabled. +func TestValidateExcludeNames(t *testing.T) { + _, accepted := filterAndExpandCommands(debugCommands, nil, false, nil) + + if err := validateExcludeNames([]string{"ccm-logs", "cluster-events", "cluster-events.json"}, accepted); err != nil { + t.Errorf("valid names rejected: %v", err) + } + + err := validateExcludeNames([]string{"d8"}, accepted) + if err == nil { + t.Fatal("group prefix d8 was accepted, it silently excludes nothing now") + } + + if !strings.Contains(err.Error(), "did you mean") { + t.Errorf("error %q offers no suggestion", err) + } + + err = validateExcludeNames([]string{"ccm-log"}, accepted) + if err == nil || !strings.Contains(err.Error(), "did you mean: ccm-logs") { + t.Errorf("a near miss should be pointed at its name, got %v", err) + } + + err = validateExcludeNames([]string{"zzzz"}, accepted) + if err == nil { + t.Fatal("a name matching nothing was accepted") + } + + if strings.Contains(err.Error(), "did you mean") { + t.Errorf("error %q invents a suggestion for a name with no near miss", err) + } + + // A repeated or empty accepted name must not confuse the lookup. + if err := validateExcludeNames([]string{"plain"}, []string{"plain.txt", "plain.txt", ""}); err != nil { + t.Errorf("duplicate accepted names broke the lookup: %v", err) + } +} + +// TestExcludedEntryStaysAValidName guards the interaction between exclusion and +// validation: the name a user just excluded must not then be reported as +// unknown, even though its command is gone from the selection. +func TestExcludedEntryStaysAValidName(t *testing.T) { + for _, name := range []string{"ccm-logs", "cluster-events.json", "d8-cloud-provider-aws-ccm-logs.txt"} { + _, accepted := filterAndExpandCommands(debugCommands, map[string]bool{"cloud-provider-aws": true}, true, newExcludeSet([]string{name})) + + if err := validateExcludeNames([]string{name}, accepted); err != nil { + t.Errorf("--exclude %q reported as unknown: %v", name, err) + } + } +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/selection.go b/internal/system/cmd/collect-debug-info/debugtar/selection.go new file mode 100644 index 000000000..0ac89d535 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/selection.go @@ -0,0 +1,110 @@ +package debugtar + +import ( + "sort" + "strings" +) + +// filterAndExpandCommands selects the commands to run: it resolves the +// {module-name} placeholder against the active modules and drops the entries +// excluded on the command line. It also returns every name --exclude accepts +// for this run, including the names of entries these very excludes dropped, so +// a valid name is never reported as unknown. +// +// Exclusion happens here, and not further down, because this is the only place +// where both spellings of an entry are known at once: the resolved archive name +// (d8-cloud-provider-aws-ccm-logs.txt) and the module-independent token printed +// by --list-exclude (ccm-logs). +// +// modulesKnown reports whether activeModules actually describes the cluster. It +// is false when the module list could not be fetched: module-gated commands are +// then collected anyway (an empty file beats a silently missing one), except +// those whose File carries the {module-name} placeholder — their archive entry +// name cannot be resolved, so they are skipped rather than stored under a +// literal placeholder name. +func filterAndExpandCommands(commands []command, activeModules map[string]bool, modulesKnown bool, excludeSet map[string]bool) (selected []command, acceptedNames []string) { + selected = make([]command, 0, len(commands)) + acceptedNames = make([]string, 0, len(commands)) + + for _, cmd := range commands { + // The token stays accepted even when the command is gated out below: + // --exclude ccm-logs must not fail on a cluster without a cloud provider. + token := excludeBaseName(cmd) + acceptedNames = append(acceptedNames, token) + + if excludedByName(excludeSet, cmd.File, token) { + continue + } + + if cmd.RequiredModule == "" { + selected = append(selected, cmd) + continue + } + + if !modulesKnown { + if !needsModuleExpansion(cmd) { + selected = append(selected, cmd) + } + + continue + } + + if needsModuleExpansion(cmd) { + matchedModules := matchingModules(activeModules, cmd.RequiredModule) + for _, moduleName := range matchedModules { + expanded := cmd + expanded.File = strings.ReplaceAll(cmd.File, "{module-name}", moduleName) + expanded.Args = replaceModuleName(cmd.Args, moduleName) + + acceptedNames = append(acceptedNames, expanded.File) + + if excludedByName(excludeSet, expanded.File) { + continue + } + + selected = append(selected, expanded) + } + } else { + for moduleName := range activeModules { + if isModuleMatch(moduleName, cmd.RequiredModule) { + selected = append(selected, cmd) + break + } + } + } + } + + return selected, acceptedNames +} + +func matchingModules(activeModules map[string]bool, required string) []string { + var matched []string + + for name := range activeModules { + if isModuleMatch(name, required) { + matched = append(matched, name) + } + } + + sort.Strings(matched) + + return matched +} + +func isModuleMatch(moduleName, required string) bool { + return moduleName == required || strings.HasPrefix(moduleName, required) +} + +// moduleScopedFiles lists the File templates that can only be resolved with a +// known module list, for the warning printed when that list is unavailable. +func moduleScopedFiles(commands []command) []string { + var files []string + + for _, cmd := range commands { + if cmd.RequiredModule != "" && needsModuleExpansion(cmd) { + files = append(files, cmd.File) + } + } + + return files +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/selection_test.go b/internal/system/cmd/collect-debug-info/debugtar/selection_test.go new file mode 100644 index 000000000..f5caae94f --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/selection_test.go @@ -0,0 +1,154 @@ +package debugtar + +import ( + "slices" + "testing" +) + +// TestFilterAndExpandCommandsWithoutModuleList covers the degraded path taken +// when `kubectl get module` fails: the collection must go on, but no command +// may reach the archive with an unresolved {module-name} placeholder in its +// file name. +func TestFilterAndExpandCommandsWithoutModuleList(t *testing.T) { + commands, _ := filterAndExpandCommands(debugCommands, nil, false, nil) + + var expected int + + for _, cmd := range debugCommands { + if !needsModuleExpansion(cmd) { + expected++ + } + } + + if len(commands) != expected { + t.Errorf("got %d commands without a module list, want %d", len(commands), expected) + } + + for _, cmd := range commands { + if needsModuleExpansion(cmd) { + t.Errorf("command %q still carries an unresolved {module-name} placeholder", cmd.File) + } + } +} + +// TestFilterAndExpandCommandsKeepsFields guards the expansion against silently +// dropping a field of command that a later change adds. +func TestFilterAndExpandCommandsKeepsFields(t *testing.T) { + source := command{ + File: "d8-{module-name}-ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + } + + expanded, _ := filterAndExpandCommands([]command{source}, map[string]bool{"cloud-provider-aws": true}, true, nil) + if len(expanded) != 1 { + t.Fatalf("got %d expanded commands, want 1", len(expanded)) + } + + got := expanded[0] + if got.File != "d8-cloud-provider-aws-ccm-logs.txt" { + t.Errorf("File = %q, want %q", got.File, "d8-cloud-provider-aws-ccm-logs.txt") + } + + if got.Cmd != source.Cmd || got.RequiredModule != source.RequiredModule { + t.Errorf("expansion dropped a field: %+v", got) + } +} + +// TestSelectionMatrix pins every combination of the two knobs documented on +// command.RequiredModule: whether the command is gated on a module, whether it +// carries the {module-name} placeholder, and whether the module list could be +// fetched at all. The expected file lists also pin the order, which must stay +// deterministic (templates in declaration order, per-module copies sorted by +// module name) so two runs of the same cluster produce the same archive. +func TestSelectionMatrix(t *testing.T) { + templates := []command{ + {File: "plain.txt", Cmd: "kubectl"}, + {File: "literal-{module-name}.txt", Cmd: "kubectl"}, + {File: "gated.txt", Cmd: "kubectl", RequiredModule: "istio"}, + { + File: "d8-{module-name}-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + }, + } + + cases := []struct { + name string + activeModules map[string]bool + modulesKnown bool + want []string + }{ + { + name: "gate matched by prefix and by exact name", + activeModules: map[string]bool{"istio": true, "cloud-provider-yandex": true, "cloud-provider-aws": true}, + modulesKnown: true, + want: []string{ + "plain.txt", + "literal-{module-name}.txt", + "gated.txt", + "d8-cloud-provider-aws-logs.txt", + "d8-cloud-provider-yandex-logs.txt", + }, + }, + { + name: "no module matches the gate", + activeModules: map[string]bool{"cert-manager": true}, + modulesKnown: true, + want: []string{"plain.txt", "literal-{module-name}.txt"}, + }, + { + name: "no module is Ready", + activeModules: map[string]bool{}, + modulesKnown: true, + want: []string{"plain.txt", "literal-{module-name}.txt"}, + }, + { + name: "module list unavailable", + modulesKnown: false, + want: []string{"plain.txt", "literal-{module-name}.txt", "gated.txt"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + selected, _ := filterAndExpandCommands(templates, tc.activeModules, tc.modulesKnown, nil) + + got := make([]string, 0, len(selected)) + for _, cmd := range selected { + got = append(got, cmd.File) + } + + if !slices.Equal(got, tc.want) { + t.Errorf("selected %v, want %v", got, tc.want) + } + }) + } +} + +// TestExpansionSubstitutesArgs checks the other half of the substitution: the +// module name must reach Args too, not just the archive entry name. +func TestExpansionSubstitutesArgs(t *testing.T) { + template := command{ + File: "d8-{module-name}-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-{module-name}", "logs"}, + RequiredModule: "cloud-provider", + } + + selected, _ := filterAndExpandCommands([]command{template}, map[string]bool{"cloud-provider-aws": true}, true, nil) + if len(selected) != 1 { + t.Fatalf("got %d commands, want 1", len(selected)) + } + + want := []string{"-n", "d8-cloud-provider-aws", "logs"} + if !slices.Equal(selected[0].Args, want) { + t.Errorf("Args = %v, want %v", selected[0].Args, want) + } + + if !slices.Equal(template.Args, []string{"-n", "d8-{module-name}", "logs"}) { + t.Errorf("expansion mutated the template Args: %v", template.Args) + } +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/tarball.go b/internal/system/cmd/collect-debug-info/debugtar/tarball.go new file mode 100644 index 000000000..192805bd3 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/tarball.go @@ -0,0 +1,94 @@ +package debugtar + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" +) + +func Tarball(config *rest.Config, kubeCl kubernetes.Interface, excludeFiles []string, commandTimeout time.Duration, requestInterval time.Duration) error { + const ( + namespace = "d8-system" + containerName = "deckhouse" + ) + + podName, err := utilk8s.GetDeckhousePod(kubeCl) + if err != nil { + return fmt.Errorf("failed to get Deckhouse pod: %w", err) + } + + activeModules, modulesErr := fetchActiveModules(config, kubeCl, podName, namespace, containerName, commandTimeout) + if modulesErr != nil { + fmt.Fprintf(os.Stderr, "ERROR: could not fetch the list of active modules: %v\n", modulesErr) + fmt.Fprintf(os.Stderr, " collection continues without module filtering: module-gated commands run anyway and may produce empty files; per-module commands are skipped because their file names cannot be resolved (%s)\n", + strings.Join(moduleScopedFiles(debugCommands), ", ")) + } + + commands, acceptedNames := filterAndExpandCommands(debugCommands, activeModules, modulesErr == nil, newExcludeSet(excludeFiles)) + + if err := validateExcludeNames(excludeFiles, acceptedNames); err != nil { + return err + } + + return writeArchive( + config, kubeCl, podName, namespace, containerName, + commands, commandTimeout, requestInterval, + "Collecting debug info from Deckhouse...", + "Debug archive collection completed.", + ) +} + +type moduleList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Phase string `json:"phase"` + } `json:"status"` + } `json:"items"` +} + +// fetchActiveModules returns a map with the names of modules that are in the Ready phase. +func fetchActiveModules( + config *rest.Config, + kubeCl kubernetes.Interface, + podName, namespace, containerName string, + timeout time.Duration, +) (map[string]bool, error) { + cmdLine := []string{"kubectl", "get", "module", "-o", "json"} + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + stdout, stderr, err := utilk8s.ExecCommandInPod(ctx, config, kubeCl, cmdLine, podName, namespace, containerName) + if err != nil { + return nil, fmt.Errorf("stream kubectl get module: %w (stderr: %s)", err, stderr) + } + + if len(stdout) == 0 { + return nil, fmt.Errorf("kubectl get module returned no output (stderr: %s)", stderr) + } + + var list moduleList + if err = json.Unmarshal(stdout, &list); err != nil { + return nil, fmt.Errorf("parse module list: %w", err) + } + + active := make(map[string]bool, len(list.Items)) + for _, item := range list.Items { + if item.Status.Phase == "Ready" { + active[item.Metadata.Name] = true + } + } + + return active, nil +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go b/internal/system/cmd/collect-debug-info/debugtar/virtualization.go similarity index 93% rename from internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go rename to internal/system/cmd/collect-debug-info/debugtar/virtualization.go index f64216d7b..ef199102f 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/virtualizationTarball.go +++ b/internal/system/cmd/collect-debug-info/debugtar/virtualization.go @@ -16,7 +16,7 @@ import ( const virtualizationNamespace = "d8-virtualization" // virtualizationCommands - additional resource-intensive commands collected only in the virtualization archive -var virtualizationCommands = []Command{ +var virtualizationCommands = []command{ { File: "d8-virtualization-pods-wide.txt", Cmd: "kubectl", @@ -36,7 +36,7 @@ type virtualizationPod struct { // volume scales with the number of nodes. // // The pod list is the entire payload of this archive, so a failure to obtain it -// aborts the collection instead of producing an archive that looks complete. +// aborts the collection. func VirtualizationTarball(config *rest.Config, kubeCl kubernetes.Interface, commandTimeout, requestInterval time.Duration, skipDsLogs bool) error { const ( namespace = "d8-system" @@ -95,8 +95,8 @@ func fetchVirtualizationPods(kubeCl kubernetes.Interface, timeout time.Duration) // buildVirtualizationCommands transforms the discovered list of pods into a final list. // first the static commands, then one log collection command for each pod (skipping pods belonging to DaemonSet if skipDsLogs is set). -func buildVirtualizationCommands(pods []virtualizationPod, skipDsLogs bool) []Command { - commands := make([]Command, 0, len(virtualizationCommands)+len(pods)) +func buildVirtualizationCommands(pods []virtualizationPod, skipDsLogs bool) []command { + commands := make([]command, 0, len(virtualizationCommands)+len(pods)) commands = append(commands, virtualizationCommands...) for _, pod := range pods { @@ -104,7 +104,7 @@ func buildVirtualizationCommands(pods []virtualizationPod, skipDsLogs bool) []Co continue } - commands = append(commands, Command{ + commands = append(commands, command{ File: fmt.Sprintf("d8-virtualization-%s-logs.txt", pod.Name), Cmd: "kubectl", Args: []string{"-n", virtualizationNamespace, "logs", pod.Name, "--tail=-1", "--ignore-errors=true"}, diff --git a/internal/utilk8s/operatepod.go b/internal/utilk8s/operatepod.go index e37aa10ce..076099970 100644 --- a/internal/utilk8s/operatepod.go +++ b/internal/utilk8s/operatepod.go @@ -1,8 +1,10 @@ package utilk8s import ( + "bytes" "context" "fmt" + "sync" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -90,3 +92,44 @@ func ExecCommandInPod( return stdoutBuf.Bytes(), stderrBuf.String(), streamErr } + +// syncBuffer is a goroutine-safe sink for the output of a remote command. +// +// It is needed because remotecommand.Executor.StreamWithContext returns as +// soon as the context is done (client-go tools/remotecommand/spdy.go) without +// joining the goroutines that io.Copy the remote streams into the writers +// passed in StreamOptions. After a command times out those goroutines may keep +// writing, so the writer outlives the call and a plain bytes.Buffer would be +// accessed concurrently: a racing Bytes() can return a slice already grown past +// the bytes actually copied into it, and a late Write can resurrect a buffer the +// caller considers finished. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write implements io.Writer and is safe to call concurrently with the readers +// below. +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.Write(p) +} + +// Bytes returns a copy of everything written so far. The copy keeps the caller +// isolated from writes a late stream goroutine may still perform. +func (b *syncBuffer) Bytes() []byte { + b.mu.Lock() + defer b.mu.Unlock() + + return bytes.Clone(b.buf.Bytes()) +} + +// String returns everything written so far as a string. +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.String() +} diff --git a/internal/utilk8s/syncbuffer_test.go b/internal/utilk8s/operatepod_test.go similarity index 100% rename from internal/utilk8s/syncbuffer_test.go rename to internal/utilk8s/operatepod_test.go diff --git a/internal/utilk8s/syncbuffer.go b/internal/utilk8s/syncbuffer.go deleted file mode 100644 index d258f377e..000000000 --- a/internal/utilk8s/syncbuffer.go +++ /dev/null @@ -1,47 +0,0 @@ -package utilk8s - -import ( - "bytes" - "sync" -) - -// syncBuffer is a goroutine-safe sink for the output of a remote command. -// -// It is needed because remotecommand.Executor.StreamWithContext returns as -// soon as the context is done (client-go tools/remotecommand/spdy.go) without -// joining the goroutines that io.Copy the remote streams into the writers -// passed in StreamOptions. After a command times out those goroutines may keep -// writing, so the writer outlives the call and a plain bytes.Buffer would be -// accessed concurrently: a racing Bytes() can return a slice already grown past -// the bytes actually copied into it, and a late Write can resurrect a buffer the -// caller considers finished. -type syncBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -// Write implements io.Writer and is safe to call concurrently with the readers -// below. -func (b *syncBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - - return b.buf.Write(p) -} - -// Bytes returns a copy of everything written so far. The copy keeps the caller -// isolated from writes a late stream goroutine may still perform. -func (b *syncBuffer) Bytes() []byte { - b.mu.Lock() - defer b.mu.Unlock() - - return bytes.Clone(b.buf.Bytes()) -} - -// String returns everything written so far as a string. -func (b *syncBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - - return b.buf.String() -} From bccdb0a62c6420673ee99f16af15c14263de6d15 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Tue, 22 Sep 2026 16:53:44 +0400 Subject: [PATCH 12/15] Update debug archive v12 + optimization Signed-off-by: Valery Losev --- internal/system/README.md | 6 +- .../collect-debug-info/debugtar/archive.go | 193 ++++++++++++++++-- .../debugtar/commandtype.go | 20 +- .../debugtar/debugcommands.go | 50 ++--- .../debugtar/debugcommands_test.go | 53 +++++ internal/utilk8s/operatepod.go | 28 ++- 6 files changed, 303 insertions(+), 47 deletions(-) diff --git a/internal/system/README.md b/internal/system/README.md index f7159633e..255eaec3e 100644 --- a/internal/system/README.md +++ b/internal/system/README.md @@ -239,7 +239,11 @@ Before collecting, the command reads the list of `Ready` modules to decide which | `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command. | | `--request-interval` | | duration | `0` | Minimum gap between commands to avoid overloading the cluster (e.g. `200ms`, `1s`). `0` disables rate limiting. | -**Handle the archive as sensitive.** Only `cluster-global-values.json` is redacted (its `kubeRBACProxyCA` and registry `dockercfg`); container logs and the raw audit policy Secret (`kube-system-audit-policy.json`) are included unredacted. Also note that a file is written even when its source command fails or times out, so an entry may be empty rather than absent. +While collecting, the command prints one progress line per entry to stderr (position, entry name, duration, bytes), so a slow command can be told from a stuck one. + +A file is written even when its source command fails or times out, so an entry may be empty or truncated rather than absent. Every such command is listed in a **`collection-errors.txt`** entry added to the archive, naming the entry, the command, the error (or the timeout) and how many bytes were kept. The archive has no `collection-errors.txt` when everything succeeded. Check for it before treating an empty entry as "the resource holds nothing" - the warnings printed during collection go to stderr, which is not part of the archive. + +**Handle the archive as sensitive.** Only `cluster-global-values.json` is redacted (its `kubeRBACProxyCA` and registry `dockercfg`); container logs and the raw audit policy Secret (`kube-system-audit-policy.json`) are included unredacted. ### `collect-debug-info virtualization` diff --git a/internal/system/cmd/collect-debug-info/debugtar/archive.go b/internal/system/cmd/collect-debug-info/debugtar/archive.go index 95039fa83..35867820a 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/archive.go +++ b/internal/system/cmd/collect-debug-info/debugtar/archive.go @@ -2,6 +2,7 @@ package debugtar import ( "archive/tar" + "bufio" "compress/gzip" "context" "errors" @@ -16,6 +17,36 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/utilk8s" ) +const ( + // stdoutBufferSize keeps the gzip stream from reaching os.Stdout one small + // block at a time: without it every flush of the compressor is a syscall. + stdoutBufferSize = 256 << 10 + + // collectionErrorsFile is the archive entry that lists the commands which + // failed or timed out. The warnings printed during collection go to stderr, + // which is not part of the archive and is gone by the time anyone opens it, + // so without this entry a truncated file is indistinguishable from a + // complete one. + collectionErrorsFile = "collection-errors.txt" + + // reportStderrLimit caps how much of a command's stderr is quoted in + // collectionErrorsFile: a failing log collection can produce a lot of it, + // and the report is meant to be read, not to hold the output again. + reportStderrLimit = 2 << 10 +) + +// commandFailure records one command that did not complete cleanly, so the +// archive can describe its own gaps. +type commandFailure struct { + file string + command string + err error + timedOut bool + timeout time.Duration + kept int + stderr string +} + // writeArchive streams a gzipped tar of the given commands' output to stdout. // It is the shared body of the debug archives: only the command set and the // progress banners differ between them. @@ -27,7 +58,12 @@ func writeArchive( commandTimeout, requestInterval time.Duration, startBanner, doneBanner string, ) (err error) { - gzipWriter := gzip.NewWriter(os.Stdout) + if err = validateCommands(commands); err != nil { + return err + } + + bufferedStdout := bufio.NewWriterSize(os.Stdout, stdoutBufferSize) + gzipWriter := gzip.NewWriter(bufferedStdout) tarWriter := tar.NewWriter(gzipWriter) defer func() { @@ -38,22 +74,70 @@ func writeArchive( if closeErr := gzipWriter.Close(); closeErr != nil && err == nil { err = fmt.Errorf("failed to finalize gzip stream: %w", closeErr) } + + if flushErr := bufferedStdout.Flush(); flushErr != nil && err == nil { + err = fmt.Errorf("failed to flush the archive to stdout: %w", flushErr) + } }() fmt.Fprintf(os.Stderr, "%s\n", startBanner) - if err = runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, commandTimeout, requestInterval); err != nil { + failures, err := runCommands(tarWriter, config, kubeCl, podName, namespace, containerName, commands, commandTimeout, requestInterval) + if err != nil { + return err + } + + if err = writeCollectionErrors(tarWriter, failures); err != nil { return err } fmt.Fprintf(os.Stderr, "%s\n", doneBanner) + if len(failures) > 0 { + fmt.Fprintf(os.Stderr, "%d of %d commands failed or timed out, see %s inside the archive\n", + len(failures), len(commands), collectionErrorsFile) + } + + return nil +} + +// validateCommands rejects a command list that cannot produce a well-formed +// archive, before a single command runs: a duplicate entry name means only the +// last copy survives extraction, and an unresolved {module-name} placeholder +// means the entry is stored under a template name nobody can use. Both are +// mistakes in the command tables rather than cluster conditions, and both are +// far cheaper to report up front than after several minutes of collection. +// +// It sits here because runCommands is the only writer of archive entries, so +// this covers the cluster-wide table and the virtualization one alike -- the +// latter never passes through filterAndExpandCommands. +func validateCommands(commands []command) error { + seen := make(map[string]bool, len(commands)) + + for _, cmd := range commands { + if strings.Contains(cmd.File, "{module-name}") { + return fmt.Errorf("unresolved {module-name} placeholder in archive entry %q", cmd.File) + } + + if cmd.File == collectionErrorsFile { + return fmt.Errorf("archive entry %q is reserved for the collection error report", cmd.File) + } + + if seen[cmd.File] { + return fmt.Errorf("duplicate archive entry %q: only the last copy would survive extraction", cmd.File) + } + + seen[cmd.File] = true + } + return nil } // runCommands executes each command inside the Deckhouse pod and streams its // output into the tar archive, honoring the optional rate limit between command -// executions. The command list is already filtered by the caller. +// executions. The command list is already filtered by the caller. It returns +// the commands that did not complete cleanly; a returned error means the +// archive itself could not be written and collection cannot continue. func runCommands( tarWriter *tar.Writer, config *rest.Config, @@ -61,8 +145,11 @@ func runCommands( podName, namespace, containerName string, commands []command, commandTimeout, requestInterval time.Duration, -) error { - var tickCh <-chan time.Time +) ([]commandFailure, error) { + var ( + tickCh <-chan time.Time + failures []commandFailure + ) if requestInterval > 0 { ticker := time.NewTicker(requestInterval) @@ -71,49 +158,123 @@ func runCommands( tickCh = ticker.C } - for _, cmd := range commands { - if tickCh != nil { + for i, cmd := range commands { + // The interval separates executions from one another, so the first one + // does not wait for it: the ticker starts before the loop, and waiting + // for its first tick is idle time that protects nothing. + if tickCh != nil && i > 0 { <-tickCh } fullCommand := append([]string{cmd.Cmd}, cmd.Args...) + started := time.Now() + cmdCtx, cancel := context.WithTimeout(context.Background(), commandTimeout) output, stderrOutput, streamErr := utilk8s.ExecCommandInPod(cmdCtx, config, kubeCl, fullCommand, podName, namespace, containerName) cancel() + // One line per command: the collection otherwise prints a banner and then + // goes silent for minutes, with no way to tell a slow command from a stuck + // one, and no evidence afterwards about where the time went. + fmt.Fprintf(os.Stderr, " [%d/%d] %s (%s, %d bytes)\n", + i+1, len(commands), cmd.File, time.Since(started).Round(time.Millisecond), len(output)) + if streamErr != nil { + timedOut := errors.Is(streamErr, context.DeadlineExceeded) + // Report the error itself, the command that produced it and how much // output survived: the entry is written either way, so without the // byte count an operator cannot tell an empty file from a truncated // one, and without the error a non-zero exit code looks the same as a // broken stream. - if errors.Is(streamErr, context.DeadlineExceeded) { - fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s, keeping %d bytes collected so far\n", + if timedOut { + fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s, keeping %d bytes collected so far\n", cmd.File, commandTimeout, len(output)) } else { - fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %v, keeping %d bytes\n command: %s\n", + fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %v, keeping %d bytes\n command: %s\n", cmd.File, streamErr, len(output), strings.Join(fullCommand, " ")) } if trimmed := strings.TrimSpace(stderrOutput); trimmed != "" { - fmt.Fprintf(os.Stderr, " stderr: %s\n", trimmed) + fmt.Fprintf(os.Stderr, " stderr: %s\n", trimmed) } - } - if notice := defaultedContainerNotice(stderrOutput); notice != "" { - output = append([]byte(notice), output...) + failures = append(failures, commandFailure{ + file: cmd.File, + command: strings.Join(fullCommand, " "), + err: streamErr, + timedOut: timedOut, + timeout: commandTimeout, + kept: len(output), + stderr: stderrOutput, + }) } - if err := cmd.writeToTar(tarWriter, output); err != nil { - return fmt.Errorf("failed to write tar file %s: %w", cmd.File, err) + // The notice is passed as a separate chunk rather than prepended to + // output: prepending would copy the whole collected output to add a + // couple of lines in front of it. + if err := cmd.writeToTar(tarWriter, []byte(defaultedContainerNotice(stderrOutput)), output); err != nil { + return nil, fmt.Errorf("failed to write tar file %s: %w", cmd.File, err) } } + return failures, nil +} + +// writeCollectionErrors stores the list of failed commands as the last archive +// entry, so the archive states its own gaps to whoever opens it. +func writeCollectionErrors(tarWriter *tar.Writer, failures []commandFailure) error { + if len(failures) == 0 { + return nil + } + + entry := command{File: collectionErrorsFile} + + if err := entry.writeToTar(tarWriter, formatCollectionErrors(failures)); err != nil { + return fmt.Errorf("failed to write tar file %s: %w", collectionErrorsFile, err) + } + return nil } +// formatCollectionErrors renders the failure list as the body of +// collectionErrorsFile. +func formatCollectionErrors(failures []commandFailure) []byte { + var report strings.Builder + + report.WriteString("Commands that did not complete during this collection.\n") + report.WriteString("The archive entries they produced are empty or truncated.\n\n") + + for _, failure := range failures { + if failure.timedOut { + fmt.Fprintf(&report, "%s: TIMED OUT after %s, %d bytes kept\n", failure.file, failure.timeout, failure.kept) + } else { + fmt.Fprintf(&report, "%s: %v, %d bytes kept\n", failure.file, failure.err, failure.kept) + } + + fmt.Fprintf(&report, " command: %s\n", failure.command) + + if trimmed := strings.TrimSpace(failure.stderr); trimmed != "" { + fmt.Fprintf(&report, " stderr: %s\n", truncateForReport(trimmed)) + } + + report.WriteString("\n") + } + + return []byte(report.String()) +} + +// truncateForReport keeps one quoted stderr from taking over the report. +func truncateForReport(s string) string { + if len(s) <= reportStderrLimit { + return s + } + + return s[:reportStderrLimit] + "... (truncated)" +} + // defaultedContainerNotice extracts kubectl's client-side "Defaulted container // ... out of: ..." notice(s) from a command's stderr, so they can be prepended // to the collected log output. diff --git a/internal/system/cmd/collect-debug-info/debugtar/commandtype.go b/internal/system/cmd/collect-debug-info/debugtar/commandtype.go index 636d21176..d992b1b00 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/commandtype.go +++ b/internal/system/cmd/collect-debug-info/debugtar/commandtype.go @@ -60,19 +60,31 @@ func replaceModuleName(args []string, moduleName string) []string { return expanded } -func (c *command) writeToTar(tarWriter *tar.Writer, fileContent []byte) error { +// writeToTar stores the concatenation of chunks as one archive entry. Taking +// the content in pieces is what keeps a prefix (the "Defaulted container" +// notice) from being prepended by copying: a tar entry needs its size up front, +// but not its bytes in one slice, and the collected output can be hundreds of +// megabytes. +func (c *command) writeToTar(tarWriter *tar.Writer, chunks ...[]byte) error { + var size int64 + for _, chunk := range chunks { + size += int64(len(chunk)) + } + header := &tar.Header{ Name: c.File, Mode: 0o600, - Size: int64(len(fileContent)), + Size: size, } if err := tarWriter.WriteHeader(header); err != nil { return fmt.Errorf("write tar header: %v", err) } - if _, err := tarWriter.Write(fileContent); err != nil { - return fmt.Errorf("copy content: %v", err) + for _, chunk := range chunks { + if _, err := tarWriter.Write(chunk); err != nil { + return fmt.Errorf("copy content: %v", err) + } } return nil diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go b/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go index c7b6694b6..6445383e8 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugcommands.go @@ -16,32 +16,32 @@ var debugCommands = []command{ { File: "cluster-global-values.json", Cmd: "bash", - Args: []string{"-c", `deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, + Args: []string{"-c", `set -o pipefail; deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, }, { File: "deckhouse-enabled-modules.json", Cmd: "bash", - Args: []string{"-c", "kubectl get modules -o json | jq '.items[]'"}, + Args: []string{"-c", "set -o pipefail; kubectl get modules -o json | jq '.items[]'"}, }, { File: "deckhouse-module-sources.json", Cmd: "bash", - Args: []string{"-c", "kubectl get modulesources -o json | jq '.items[]'"}, + Args: []string{"-c", "set -o pipefail; kubectl get modulesources -o json | jq '.items[]'"}, }, { File: "deckhouse-module-pull-overrides.json", Cmd: "bash", - Args: []string{"-c", "kubectl get modulepulloverrides -o json | jq '.items[]'"}, + Args: []string{"-c", "set -o pipefail; kubectl get modulepulloverrides -o json | jq '.items[]'"}, }, { File: "deckhouse-module-update-policies.json", Cmd: "bash", - Args: []string{"-c", "kubectl get moduleupdatepolicies -o json | jq '.items[]'"}, + Args: []string{"-c", "set -o pipefail; kubectl get moduleupdatepolicies -o json | jq '.items[]'"}, }, { File: "deckhouse-maintenance-modules.txt", Cmd: "bash", - Args: []string{"-c", `kubectl get moduleconfig -ojson | jq -r '.items[] | select(.spec.maintenance == "NoResourceReconciliation") | .metadata.name'`}, + Args: []string{"-c", `set -o pipefail; kubectl get moduleconfig -ojson | jq -r '.items[] | select(.spec.maintenance == "NoResourceReconciliation") | .metadata.name'`}, }, { File: "cluster-events.json", @@ -51,7 +51,7 @@ var debugCommands = []command{ { File: "d8-all.json", Cmd: "bash", - Args: []string{"-c", `for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, + Args: []string{"-c", `set -o pipefail; for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, }, { File: "cluster-node-groups.json", @@ -76,33 +76,33 @@ var debugCommands = []command{ { File: "instance-manager-capi-machines.json", Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.cluster.x-k8s.io -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl -n d8-cloud-instance-manager get machines.cluster.x-k8s.io -o json | jq '.items[]'`}, }, { File: "instance-manager-instances.json", Cmd: "bash", - Args: []string{"-c", `kubectl get instances.deckhouse.io -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl get instances.deckhouse.io -o json | jq '.items[]'`}, }, { File: "instance-manager-staticinstances.json", Cmd: "bash", - Args: []string{"-c", `kubectl get staticinstances.deckhouse.io -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl get staticinstances.deckhouse.io -o json | jq '.items[]'`}, }, { File: "instance-manager-cloud-machine-deployment.txt", Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machinedeployments.machine.sapcloud.io -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl -n d8-cloud-instance-manager get machinedeployments.machine.sapcloud.io -o json | jq '.items[]'`}, RequiredModule: "cloud-provider", }, { File: "instance-manager-static-machine-deployment.txt", Cmd: "bash", - Args: []string{"-c", "kubectl -n d8-cloud-instance-manager get machinedeployments.cluster.x-k8s.io -o json --ignore-not-found | jq '.items[]'"}, + Args: []string{"-c", "set -o pipefail; kubectl -n d8-cloud-instance-manager get machinedeployments.cluster.x-k8s.io -o json --ignore-not-found | jq '.items[]'"}, }, { File: "deckhouse-version.json", Cmd: "bash", - Args: []string{"-c", "jq -s add <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.metadata.annotations | {\"core.deckhouse.io/edition\",\"core.deckhouse.io/version\"}') <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.spec.template.spec.containers[] | select(.name == \"deckhouse\") | {image}')"}, + Args: []string{"-c", "set -o pipefail; jq -s add <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.metadata.annotations | {\"core.deckhouse.io/edition\",\"core.deckhouse.io/version\"}') <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.spec.template.spec.containers[] | select(.name == \"deckhouse\") | {image}')"}, }, { File: "deckhouse-releases.json", @@ -127,7 +127,7 @@ var debugCommands = []command{ { File: "instance-manager-machine-controller-manager.json", Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl -n d8-cloud-instance-manager get pods -l app=machine-controller-manager -o json | jq '.items[]'`}, }, { File: "instance-manager-mcm-logs.txt", @@ -137,7 +137,7 @@ var debugCommands = []command{ { File: "instance-manager-mcm-cloud-machines.json", Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl -n d8-cloud-instance-manager get machines.machine.sapcloud.io -o json | jq '.items[]'`}, }, { File: "d8-{module-name}-ccm-logs.txt", @@ -191,22 +191,22 @@ var debugCommands = []command{ { File: "cluster-alerts.json", Cmd: "bash", - Args: []string{"-c", `kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, }, { File: "cluster-bad-pods.txt", Cmd: "bash", - Args: []string{"-c", `kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)" || true`}, + Args: []string{"-c", `set -o pipefail; kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)" || true`}, }, { File: "security-cluster-authorization-rules.json", Cmd: "bash", - Args: []string{"-c", `kubectl get clusterauthorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl get clusterauthorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, }, { File: "security-authorization-rules.json", Cmd: "bash", - Args: []string{"-c", `kubectl get authorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl get authorizationrules.deckhouse.io -A -o json | jq '.items[]'`}, }, { File: "deckhouse-module-configs.json", @@ -216,19 +216,19 @@ var debugCommands = []command{ { File: "d8-istio-resources.json", Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-istio get all -o json | jq '.items[]'`}, + Args: []string{"-c", `set -o pipefail; kubectl -n d8-istio get all -o json | jq '.items[]'`}, RequiredModule: "istio", }, { File: "d8-istio-custom-resources.json", Cmd: "bash", - Args: []string{"-c", `for crd in $(kubectl get crds | grep -E 'istio.io|gateway.networking.k8s.io' | awk '{print $1}'); do echo "Listing resources for CRD: $crd" && kubectl get $crd -A -o json; done`}, + Args: []string{"-c", `set -o pipefail; for crd in $(kubectl get crds | grep -E 'istio.io|gateway.networking.k8s.io' | awk '{print $1}'); do echo "Listing resources for CRD: $crd" && kubectl get $crd -A -o json; done`}, RequiredModule: "istio", }, { File: "d8-istio-envoy-config.json", Cmd: "bash", - Args: []string{"-c", `kubectl port-forward daemonset/ingressgateway -n d8-istio 15000:15000 & sleep 5; (curl http://localhost:15000/config_dump?include_eds=true | jq 'del(.configs[6].dynamic_active_secrets)' && kill $!) || { kill $!; exit 0; }`}, + Args: []string{"-c", `set -o pipefail; kubectl port-forward daemonset/ingressgateway -n d8-istio 15000:15000 & sleep 5; (curl http://localhost:15000/config_dump?include_eds=true | jq 'del(.configs[6].dynamic_active_secrets)' && kill $!) || { kill $!; exit 0; }`}, RequiredModule: "istio", }, { @@ -246,13 +246,13 @@ var debugCommands = []command{ { File: "d8-istio-users-logs.txt", Cmd: "bash", - Args: []string{"-c", `kubectl get pods --all-namespaces -o jsonpath='{range .items[?(@.metadata.annotations.istio\.io/rev)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' | awk '/istio-proxy/ {print $0}' | shuf -n 1 | while read namespace pod_name containers; do echo "Collecting logs from istio-proxy in Pod $pod_name (Namespace: $namespace)"; kubectl logs "$pod_name" -n "$namespace" -c istio-proxy; done`}, + Args: []string{"-c", `set -o pipefail; kubectl get pods --all-namespaces -o jsonpath='{range .items[?(@.metadata.annotations.istio\.io/rev)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' | awk '/istio-proxy/ {print $0}' | shuf -n 1 | while read namespace pod_name containers; do echo "Collecting logs from istio-proxy in Pod $pod_name (Namespace: $namespace)"; kubectl logs "$pod_name" -n "$namespace" -c istio-proxy; done`}, RequiredModule: "istio", }, { File: "network-cni-cilium-health-status.txt", Cmd: "bash", - Args: []string{"-c", `kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, + Args: []string{"-c", `set -o pipefail; kubectl -n d8-cni-cilium exec -it $(kubectl -n d8-cni-cilium get pod -o name | grep agent | head -n 1) -c cilium-agent -- cilium-health status`}, RequiredModule: "cni-cilium", }, { @@ -308,7 +308,7 @@ var debugCommands = []command{ { File: "other-storage-deckhouse-io-terminating.txt", Cmd: "bash", - Args: []string{"-c", `kubectl get $(kubectl api-resources --api-group=storage.deckhouse.io --verbs=list -o name | paste -sd, -) --ignore-not-found -A --chunk-size=200 -o json | jq -r '.items[] | select(.apiVersion == "storage.deckhouse.io/v1alpha1") | select(.metadata.deletionTimestamp != null) | "[\(.kind)] \(.metadata.namespace // "-")/\(.metadata.name)"'`}, + Args: []string{"-c", `set -o pipefail; kubectl get $(kubectl api-resources --api-group=storage.deckhouse.io --verbs=list -o name | paste -sd, -) --ignore-not-found -A --chunk-size=200 -o json | jq -r '.items[] | select(.apiVersion == "storage.deckhouse.io/v1alpha1") | select(.metadata.deletionTimestamp != null) | "[\(.kind)] \(.metadata.namespace // "-")/\(.metadata.name)"'`}, }, { File: "network-ingressnginxcontrollers.json", diff --git a/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go b/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go index 4b04008a6..b1cd43450 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go +++ b/internal/system/cmd/collect-debug-info/debugtar/debugcommands_test.go @@ -1,6 +1,8 @@ package debugtar import ( + "slices" + "strings" "testing" ) @@ -24,3 +26,54 @@ func TestDebugCommandsModuleExpansionInvariant(t *testing.T) { t.Errorf("command %q uses the {module-name} placeholder but has no RequiredModule set, so it will never be resolved", cmd.File) } } + +// TestBashPipelinesSetPipefail guards the exit status the collection sees. In a +// pipeline bash reports only the status of the last stage, so +// `kubectl get ... | jq ...` exits 0 when kubectl fails and jq happily consumes +// the empty input: ExecCommandInPod returns no error, the ERROR branch in +// runCommands never fires, the failure is absent from collection-errors.txt, +// and the archive gets an empty file indistinguishable from "the resource +// exists but holds nothing". `set -o pipefail` is what makes that failure +// observable. +// +// The check deliberately over-approximates: a "|" inside a quoted regexp or +// jsonpath counts as a pipeline too. Deciding otherwise would mean parsing +// shell here, and an extra `set -o pipefail` costs nothing. +func TestBashPipelinesSetPipefail(t *testing.T) { + for _, cmd := range slices.Concat(debugCommands, virtualizationCommands) { + if cmd.Cmd != "bash" { + continue + } + + for _, arg := range cmd.Args { + if !hasShellPipeline(arg) || strings.Contains(arg, "set -o pipefail") { + continue + } + + t.Errorf("command %q runs a pipeline without `set -o pipefail`, a failure of its left-hand side would be collected as an empty file: %s", cmd.File, arg) + } + } +} + +// hasShellPipeline reports whether script contains a "|" that is not part of +// the "||" operator. +func hasShellPipeline(script string) bool { + for i := 0; i < len(script); i++ { + if script[i] != '|' { + continue + } + + if i+1 < len(script) && script[i+1] == '|' { + i++ + continue + } + + if i > 0 && script[i-1] == '|' { + continue + } + + return true + } + + return false +} diff --git a/internal/utilk8s/operatepod.go b/internal/utilk8s/operatepod.go index 076099970..4b1d1110b 100644 --- a/internal/utilk8s/operatepod.go +++ b/internal/utilk8s/operatepod.go @@ -71,6 +71,14 @@ func ExecInPod(config *rest.Config, kubeCl kubernetes.Interface, cmdLine []strin // The buffers are goroutine-safe on purpose: StreamWithContext returns as soon // as ctx is done without joining the goroutines that copy the remote streams, // so those goroutines may still write into them after this call returned. +// +// That only happens on one branch, though. The stream protocol handlers join +// their copy goroutines before returning (wg.Wait() in client-go +// tools/remotecommand/v2.go and v4.go), so once StreamWithContext returns +// anything other than the context error, nothing can write into the buffers any +// more and the defensive copy of Bytes() is pure overhead. It is not a cheap +// overhead: a single archive entry can hold hundreds of megabytes of logs, and +// the copy costs that much memory again on top of the buffer itself. func ExecCommandInPod( ctx context.Context, config *rest.Config, @@ -90,7 +98,14 @@ func ExecCommandInPod( Stderr: &stderrBuf, }) - return stdoutBuf.Bytes(), stderrBuf.String(), streamErr + // Only the ctx.Done() branch of StreamWithContext leaves the copy goroutines + // running, and it is the only branch that can report anything but nil here, + // so the snapshot is taken exactly when a writer may still be alive. + if streamErr != nil { + return stdoutBuf.Bytes(), stderrBuf.String(), streamErr + } + + return stdoutBuf.detach(), stderrBuf.String(), nil } // syncBuffer is a goroutine-safe sink for the output of a remote command. @@ -126,6 +141,17 @@ func (b *syncBuffer) Bytes() []byte { return bytes.Clone(b.buf.Bytes()) } +// detach returns everything written so far without copying it. The returned +// slice aliases the buffer's storage, so it may only be used once no writer is +// left: see the branch in ExecCommandInPod that calls it. Callers that cannot +// prove that must use Bytes instead. +func (b *syncBuffer) detach() []byte { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.Bytes() +} + // String returns everything written so far as a string. func (b *syncBuffer) String() string { b.mu.Lock() From c3d45ee40a468fc67edd8786f04b679011675fc6 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Thu, 24 Sep 2026 14:21:20 +0400 Subject: [PATCH 13/15] Update debug archive v13 Signed-off-by: Valery Losev --- internal/system/README.md | 2 +- .../collect-debug-info/debugtar/archive.go | 26 ++++--------------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/internal/system/README.md b/internal/system/README.md index 255eaec3e..b0db82356 100644 --- a/internal/system/README.md +++ b/internal/system/README.md @@ -239,7 +239,7 @@ Before collecting, the command reads the list of `Ready` modules to decide which | `--command-timeout` | | duration | `2m` | Timeout applied to each individual in-pod command. | | `--request-interval` | | duration | `0` | Minimum gap between commands to avoid overloading the cluster (e.g. `200ms`, `1s`). `0` disables rate limiting. | -While collecting, the command prints one progress line per entry to stderr (position, entry name, duration, bytes), so a slow command can be told from a stuck one. +While collecting, the command prints only its start and completion banners to stderr; individual entries are not announced. Failures are the exception - they are reported as they happen. A file is written even when its source command fails or times out, so an entry may be empty or truncated rather than absent. Every such command is listed in a **`collection-errors.txt`** entry added to the archive, naming the entry, the command, the error (or the timeout) and how many bytes were kept. The archive has no `collection-errors.txt` when everything succeeded. Check for it before treating an empty entry as "the resource holds nothing" - the warnings printed during collection go to stderr, which is not part of the archive. diff --git a/internal/system/cmd/collect-debug-info/debugtar/archive.go b/internal/system/cmd/collect-debug-info/debugtar/archive.go index 35867820a..043530cdb 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/archive.go +++ b/internal/system/cmd/collect-debug-info/debugtar/archive.go @@ -94,8 +94,8 @@ func writeArchive( fmt.Fprintf(os.Stderr, "%s\n", doneBanner) if len(failures) > 0 { - fmt.Fprintf(os.Stderr, "%d of %d commands failed or timed out, see %s inside the archive\n", - len(failures), len(commands), collectionErrorsFile) + fmt.Fprintf(os.Stderr, "%d command(s) failed or timed out, see %s inside the archive\n", + len(failures), collectionErrorsFile) } return nil @@ -159,46 +159,30 @@ func runCommands( } for i, cmd := range commands { - // The interval separates executions from one another, so the first one - // does not wait for it: the ticker starts before the loop, and waiting - // for its first tick is idle time that protects nothing. if tickCh != nil && i > 0 { <-tickCh } fullCommand := append([]string{cmd.Cmd}, cmd.Args...) - started := time.Now() - cmdCtx, cancel := context.WithTimeout(context.Background(), commandTimeout) output, stderrOutput, streamErr := utilk8s.ExecCommandInPod(cmdCtx, config, kubeCl, fullCommand, podName, namespace, containerName) cancel() - // One line per command: the collection otherwise prints a banner and then - // goes silent for minutes, with no way to tell a slow command from a stuck - // one, and no evidence afterwards about where the time went. - fmt.Fprintf(os.Stderr, " [%d/%d] %s (%s, %d bytes)\n", - i+1, len(commands), cmd.File, time.Since(started).Round(time.Millisecond), len(output)) - if streamErr != nil { timedOut := errors.Is(streamErr, context.DeadlineExceeded) - // Report the error itself, the command that produced it and how much - // output survived: the entry is written either way, so without the - // byte count an operator cannot tell an empty file from a truncated - // one, and without the error a non-zero exit code looks the same as a - // broken stream. if timedOut { - fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s, keeping %d bytes collected so far\n", + fmt.Fprintf(os.Stderr, " WARNING: timed out collecting %s after %s, keeping %d bytes collected so far\n", cmd.File, commandTimeout, len(output)) } else { - fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %v, keeping %d bytes\n command: %s\n", + fmt.Fprintf(os.Stderr, " ERROR: collecting %s: %v, keeping %d bytes\n command: %s\n", cmd.File, streamErr, len(output), strings.Join(fullCommand, " ")) } if trimmed := strings.TrimSpace(stderrOutput); trimmed != "" { - fmt.Fprintf(os.Stderr, " stderr: %s\n", trimmed) + fmt.Fprintf(os.Stderr, " stderr: %s\n", trimmed) } failures = append(failures, commandFailure{ From bbc8c526ac224be9e7a9a9c80d56a00094edbff9 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Thu, 24 Sep 2026 16:19:44 +0400 Subject: [PATCH 14/15] Update debug archive v13.1 Signed-off-by: Valery Losev --- internal/utilk8s/operatepod.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/utilk8s/operatepod.go b/internal/utilk8s/operatepod.go index 4b1d1110b..9ac8d4872 100644 --- a/internal/utilk8s/operatepod.go +++ b/internal/utilk8s/operatepod.go @@ -71,14 +71,6 @@ func ExecInPod(config *rest.Config, kubeCl kubernetes.Interface, cmdLine []strin // The buffers are goroutine-safe on purpose: StreamWithContext returns as soon // as ctx is done without joining the goroutines that copy the remote streams, // so those goroutines may still write into them after this call returned. -// -// That only happens on one branch, though. The stream protocol handlers join -// their copy goroutines before returning (wg.Wait() in client-go -// tools/remotecommand/v2.go and v4.go), so once StreamWithContext returns -// anything other than the context error, nothing can write into the buffers any -// more and the defensive copy of Bytes() is pure overhead. It is not a cheap -// overhead: a single archive entry can hold hundreds of megabytes of logs, and -// the copy costs that much memory again on top of the buffer itself. func ExecCommandInPod( ctx context.Context, config *rest.Config, @@ -98,9 +90,6 @@ func ExecCommandInPod( Stderr: &stderrBuf, }) - // Only the ctx.Done() branch of StreamWithContext leaves the copy goroutines - // running, and it is the only branch that can report anything but nil here, - // so the snapshot is taken exactly when a writer may still be alive. if streamErr != nil { return stdoutBuf.Bytes(), stderrBuf.String(), streamErr } From 148ef55221aea15594d305a25a173cf55e213861 Mon Sep 17 00:00:00 2001 From: Valery Losev Date: Thu, 24 Sep 2026 16:47:43 +0400 Subject: [PATCH 15/15] Update debug archive v14 Signed-off-by: Valery Losev --- .../collect-debug-info/collect-debug-info.go | 12 +- .../debugtar/archive_test.go | 214 ++++++++++++++++++ .../collect-debug-info/debugtar/selection.go | 18 +- internal/utilk8s/operatepod.go | 3 +- 4 files changed, 222 insertions(+), 25 deletions(-) create mode 100644 internal/system/cmd/collect-debug-info/debugtar/archive_test.go diff --git a/internal/system/cmd/collect-debug-info/collect-debug-info.go b/internal/system/cmd/collect-debug-info/collect-debug-info.go index bcf8ee896..427809a1a 100644 --- a/internal/system/cmd/collect-debug-info/collect-debug-info.go +++ b/internal/system/cmd/collect-debug-info/collect-debug-info.go @@ -54,14 +54,10 @@ func NewCommand() *cobra.Command { ) collectDebugInfoCmd := &cobra.Command{ - Use: `collect-debug-info [flags] > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz`, - Short: "Collect debug info.", - Long: collectDebugInfoCmdLong, - Example: collectDebugInfoCmdExample, - // Without this, an unknown positional argument (a misspelled - // subcommand, say) is silently accepted by the parent and the full - // cluster-wide collection runs instead: cobra only reports unknown - // commands for the root command, and this one has a parent. + Use: `collect-debug-info [flags] > deckhouse-debug-$(date +"%Y_%m_%d").tar.gz`, + Short: "Collect debug info.", + Long: collectDebugInfoCmdLong, + Example: collectDebugInfoCmdExample, Args: cobra.NoArgs, SilenceErrors: true, SilenceUsage: true, diff --git a/internal/system/cmd/collect-debug-info/debugtar/archive_test.go b/internal/system/cmd/collect-debug-info/debugtar/archive_test.go new file mode 100644 index 000000000..82e1ee0d0 --- /dev/null +++ b/internal/system/cmd/collect-debug-info/debugtar/archive_test.go @@ -0,0 +1,214 @@ +package debugtar + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" +) + +// TestWriteToTarConcatenatesChunks guards the contract that lets the +// "Defaulted container" notice be stored without copying the collected output: +// the entry header must declare the summed size of all chunks. A size that +// disagrees with the bytes written does not produce a wrong file, it produces a +// broken archive -- tar.Writer refuses the extra bytes or reports the missing +// ones on Close, and every entry after this one is lost. +func TestWriteToTarConcatenatesChunks(t *testing.T) { + tests := []struct { + name string + chunks [][]byte + want string + }{ + { + name: "notice in front of the output", + chunks: [][]byte{[]byte("Defaulted container \"virt-handler\" out of: virt-handler, kube-rbac-proxy\n"), []byte("log line\n")}, + want: "Defaulted container \"virt-handler\" out of: virt-handler, kube-rbac-proxy\nlog line\n", + }, + { + name: "no notice, the common case", + chunks: [][]byte{[]byte(""), []byte("log line\n")}, + want: "log line\n", + }, + { + name: "command produced nothing", + chunks: [][]byte{[]byte(""), nil}, + want: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var archive bytes.Buffer + + tarWriter := tar.NewWriter(&archive) + entry := command{File: "d8-virtualization-virt-handler-logs.txt"} + + if err := entry.writeToTar(tarWriter, test.chunks...); err != nil { + t.Fatalf("writeToTar: %v", err) + } + + // Close reports a header size that disagrees with the bytes written. + if err := tarWriter.Close(); err != nil { + t.Fatalf("finalize tar: %v", err) + } + + reader := tar.NewReader(&archive) + + header, err := reader.Next() + if err != nil { + t.Fatalf("read header: %v", err) + } + + if header.Name != entry.File { + t.Errorf("entry name = %q, want %q", header.Name, entry.File) + } + + if header.Size != int64(len(test.want)) { + t.Errorf("header size = %d, want %d", header.Size, len(test.want)) + } + + content, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("read content: %v", err) + } + + if string(content) != test.want { + t.Errorf("content = %q, want %q", content, test.want) + } + }) + } +} + +// TestValidateCommandsRejectsBrokenTables covers the mistakes that a command +// table can carry into the archive: two entries under one name (tar stores +// both, extraction keeps only the last) and a file name still holding the +// {module-name} template. Neither is visible in the produced archive, which is +// why they are rejected before any command runs. +func TestValidateCommandsRejectsBrokenTables(t *testing.T) { + tests := []struct { + name string + commands []command + wantErr string + }{ + { + name: "duplicate entry name", + commands: []command{{File: "cluster-nodes.json"}, {File: "cluster-nodes.json"}}, + wantErr: "duplicate archive entry", + }, + { + name: "placeholder left in the file name", + commands: []command{{File: "d8-{module-name}-ccm-logs.txt"}}, + wantErr: "unresolved {module-name} placeholder", + }, + { + name: "entry name reserved for the error report", + commands: []command{{File: collectionErrorsFile}}, + wantErr: "reserved", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateCommands(test.commands) + if err == nil { + t.Fatalf("validateCommands accepted %v", test.commands) + } + + if !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("error = %v, want it to mention %q", err, test.wantErr) + } + }) + } +} + +// TestBuiltinCommandTablesProduceValidArchives runs the real tables through the +// same validation the collection does, in the shapes they actually reach it: +// the cluster-wide table after module expansion, and the virtualization table +// after the per-pod log commands are generated. +func TestBuiltinCommandTablesProduceValidArchives(t *testing.T) { + activeModules := map[string]bool{ + "cloud-provider-aws": true, + "cloud-provider-yandex": true, + "cert-manager": true, + "istio": true, + "cni-cilium": true, + "virtualization": true, + } + + expanded, _ := filterAndExpandCommands(debugCommands, activeModules, true, nil) + if err := validateCommands(expanded); err != nil { + t.Errorf("expanded cluster-wide commands: %v", err) + } + + pods := []virtualizationPod{ + {Name: "virt-handler-abcde"}, + {Name: "virt-handler-fghij"}, + {Name: "virtualization-controller-0"}, + {Name: "dvcr-0"}, + } + + if err := validateCommands(buildVirtualizationCommands(pods, false)); err != nil { + t.Errorf("virtualization commands: %v", err) + } +} + +// TestFormatCollectionErrorsNamesIncompleteEntries guards the only reason +// collection-errors.txt exists: the archive has to name the entries it could +// not fill. The warnings printed while collecting go to stderr, which is not +// part of the archive and is gone by the time anyone opens it, so a truncated +// entry would otherwise look exactly like a complete one. +func TestFormatCollectionErrorsNamesIncompleteEntries(t *testing.T) { + report := string(formatCollectionErrors([]commandFailure{ + { + file: "cluster-crd.json", + command: "bash -c set -o pipefail; kubectl get customresourcedefinitions -o json | jq ...", + err: context.DeadlineExceeded, + timedOut: true, + timeout: 2 * time.Minute, + kept: 4096, + }, + { + file: "d8-istio-resources.json", + command: "bash -c set -o pipefail; kubectl -n d8-istio get all -o json | jq '.items[]'", + err: errors.New("command terminated with exit code 1"), + stderr: "Error from server (NotFound): namespaces \"d8-istio\" not found", + }, + })) + + for _, want := range []string{ + "cluster-crd.json", + "TIMED OUT after 2m0s", + "4096 bytes kept", + "d8-istio-resources.json", + "exit code 1", + "namespaces \"d8-istio\" not found", + } { + if !strings.Contains(report, want) { + t.Errorf("report does not mention %q:\n%s", want, report) + } + } +} + +// TestFormatCollectionErrorsCapsQuotedStderr keeps one noisy command from +// turning the report into a second copy of its output. +func TestFormatCollectionErrorsCapsQuotedStderr(t *testing.T) { + report := string(formatCollectionErrors([]commandFailure{ + { + file: "kube-system-etcd-logs.txt", + err: errors.New("command terminated with exit code 1"), + stderr: strings.Repeat("unable to retrieve container logs\n", 10000), + }, + })) + + if len(report) > 4*reportStderrLimit { + t.Errorf("report length = %d bytes, want it capped near the %d byte stderr limit", len(report), reportStderrLimit) + } + + if !strings.Contains(report, "truncated") { + t.Errorf("report does not say the stderr was truncated:\n%s", report[:200]) + } +} diff --git a/internal/system/cmd/collect-debug-info/debugtar/selection.go b/internal/system/cmd/collect-debug-info/debugtar/selection.go index 0ac89d535..94e0c3b80 100644 --- a/internal/system/cmd/collect-debug-info/debugtar/selection.go +++ b/internal/system/cmd/collect-debug-info/debugtar/selection.go @@ -10,21 +10,9 @@ import ( // excluded on the command line. It also returns every name --exclude accepts // for this run, including the names of entries these very excludes dropped, so // a valid name is never reported as unknown. -// -// Exclusion happens here, and not further down, because this is the only place -// where both spellings of an entry are known at once: the resolved archive name -// (d8-cloud-provider-aws-ccm-logs.txt) and the module-independent token printed -// by --list-exclude (ccm-logs). -// -// modulesKnown reports whether activeModules actually describes the cluster. It -// is false when the module list could not be fetched: module-gated commands are -// then collected anyway (an empty file beats a silently missing one), except -// those whose File carries the {module-name} placeholder — their archive entry -// name cannot be resolved, so they are skipped rather than stored under a -// literal placeholder name. -func filterAndExpandCommands(commands []command, activeModules map[string]bool, modulesKnown bool, excludeSet map[string]bool) (selected []command, acceptedNames []string) { - selected = make([]command, 0, len(commands)) - acceptedNames = make([]string, 0, len(commands)) +func filterAndExpandCommands(commands []command, activeModules map[string]bool, modulesKnown bool, excludeSet map[string]bool) ([]command, []string) { + selected := make([]command, 0, len(commands)) + acceptedNames := make([]string, 0, len(commands)) for _, cmd := range commands { // The token stays accepted even when the command is gated out below: diff --git a/internal/utilk8s/operatepod.go b/internal/utilk8s/operatepod.go index 9ac8d4872..7279ff6fc 100644 --- a/internal/utilk8s/operatepod.go +++ b/internal/utilk8s/operatepod.go @@ -77,7 +77,7 @@ func ExecCommandInPod( kubeCl kubernetes.Interface, cmdLine []string, podName, namespace, containerName string, -) (stdout []byte, stderr string, err error) { +) ([]byte, string, error) { executor, err := ExecInPod(config, kubeCl, cmdLine, podName, namespace, containerName) if err != nil { return nil, "", err @@ -89,7 +89,6 @@ func ExecCommandInPod( Stdout: &stdoutBuf, Stderr: &stderrBuf, }) - if streamErr != nil { return stdoutBuf.Bytes(), stderrBuf.String(), streamErr }