package main import ( "context" "encoding/json" "fmt" "io" "os" "os/signal" "path/filepath" "sync" "syscall" "time" "git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/config" "git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/runner" "git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/storage" "github.com/spf13/cobra" ) type batchRetry struct { Command string Hosts []string Extra []string } type batchResult struct { Host string Code int } var batchExecutable = os.Executable func batchPath() string { return filepath.Join(filepath.Dir(config.Path()), "netssh", "last-failed.json") } func batchCmd() *cobra.Command { var parallel int var retry, yes, dry bool cmd := &cobra.Command{Use: "batch COMMAND HOST... [-- EXTRA_ARGS...]", Short: "Run a configured command on hosts with bounded parallelism", RunE: func(cmd *cobra.Command, args []string) error { if parallel < 1 || parallel > 64 { return fmt.Errorf("parallel must be between 1 and 64") } cfg, err := config.Load() if err != nil { return err } work := batchRetry{} if retry { if len(args) > 0 { return fmt.Errorf("--retry-failed takes no positional arguments") } data, err := os.ReadFile(batchPath()) if err != nil { return err } if err = json.Unmarshal(data, &work); err != nil { return err } } else { split := cmd.ArgsLenAtDash() if split >= 0 { work.Extra = args[split:] args = args[:split] } if len(args) < 2 { return fmt.Errorf("specify COMMAND and at least one HOST") } work.Command, work.Hosts = args[0], args[1:] } command, ok := cfg.Commands[work.Command] if !ok { return fmt.Errorf("unknown command %q", work.Command) } if len(work.Hosts) == 0 { fmt.Fprintln(cmd.OutOrStdout(), "No failed hosts to retry.") return nil } // Resolve and validate the entire list before launching any action. var targets []string seen := map[string]bool{} for _, target := range work.Hosts { h, err := resolveTargetE(cfg, target, false) if err != nil { return err } if !command.Allowed(commandHost(h)) { return fmt.Errorf("%s is not allowed on %s", work.Command, h.Name) } if !seen[h.Name] { targets = append(targets, h.Name) seen[h.Name] = true } } if command.Confirm && !dry { if err := confirmAction(fmt.Sprintf("Run %s on %d hosts", work.Command, len(targets)), yes); err != nil { return err } } executable, err := batchExecutable() if err != nil { return err } // Temporary output files bound memory even for very verbose commands. dir, err := os.MkdirTemp("", "netssh-batch-") if err != nil { return err } defer os.RemoveAll(dir) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() jobs := make(chan int) results := make(chan batchResult, len(targets)) var wg sync.WaitGroup var output sync.Mutex for n := 0; n < parallel; n++ { wg.Add(1) go func() { defer wg.Done() for i := range jobs { host := targets[i] if ctx.Err() != nil { results <- batchResult{host, 130} continue } output.Lock() fmt.Fprintf(cmd.OutOrStdout(), "[%s] starting\n", host) output.Unlock() f, err := os.CreateTemp(dir, "output-") if err != nil { results <- batchResult{host, 1} continue } argv := []string{executable, "run", "--yes"} if dry { argv = append(argv, "--dry-run") } argv = append(argv, work.Command, "--", host) argv = append(argv, work.Extra...) r := runner.Run(ctx, argv, runner.Options{Stdout: f, Stderr: f}) f.Close() output.Lock() fmt.Fprintf(cmd.OutOrStdout(), "[%s] exit=%d duration=%s\n", host, r.Code, r.Duration.Round(time.Millisecond)) read, err := os.Open(f.Name()) if err == nil { _, _ = io.Copy(cmd.OutOrStdout(), read) read.Close() } output.Unlock() results <- batchResult{host, r.Code} } }() } for i := range targets { jobs <- i } close(jobs) wg.Wait() close(results) failed := map[string]bool{} for r := range results { if r.Code != 0 { failed[r.Host] = true } } work.Hosts = nil for _, host := range targets { if failed[host] { work.Hosts = append(work.Hosts, host) } } if !dry { data, _ := json.Marshal(work) if err := os.MkdirAll(filepath.Dir(batchPath()), 0700); err != nil { return err } if err := storage.WriteFile(batchPath(), data, 0600); err != nil { return err } } fmt.Fprintf(cmd.OutOrStdout(), "%d succeeded, %d failed\n", len(targets)-len(failed), len(failed)) if len(failed) > 0 { return fmt.Errorf("batch failed; retry with netssh batch --retry-failed") } return nil }} cmd.Flags().IntVar(¶llel, "parallel", 4, "Maximum simultaneous hosts (1-64)") cmd.Flags().BoolVar(&retry, "retry-failed", false, "Repeat failed targets from the last batch") cmd.Flags().BoolVar(&yes, "yes", false, "Confirm the batch action") cmd.Flags().BoolVar(&dry, "dry-run", false, "Resolve and print commands without running") return cmd }