192 lines
5.0 KiB
Go
192 lines
5.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/config"
|
|
"git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/runner"
|
|
"git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/storage"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
type batchRetry struct {
|
|
Command string
|
|
Hosts []string
|
|
Extra []string
|
|
}
|
|
type batchResult struct {
|
|
Host string
|
|
Code int
|
|
}
|
|
|
|
var batchExecutable = os.Executable
|
|
|
|
func batchPath() string {
|
|
return filepath.Join(filepath.Dir(config.Path()), "netssh", "last-failed.json")
|
|
}
|
|
func batchCmd() *cobra.Command {
|
|
var parallel int
|
|
var retry, yes, dry bool
|
|
cmd := &cobra.Command{Use: "batch COMMAND HOST... [-- EXTRA_ARGS...]", Short: "Run a configured command on hosts with bounded parallelism", RunE: func(cmd *cobra.Command, args []string) error {
|
|
if parallel < 1 || parallel > 64 {
|
|
return fmt.Errorf("parallel must be between 1 and 64")
|
|
}
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
work := batchRetry{}
|
|
if retry {
|
|
if len(args) > 0 {
|
|
return fmt.Errorf("--retry-failed takes no positional arguments")
|
|
}
|
|
data, err := os.ReadFile(batchPath())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = json.Unmarshal(data, &work); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
split := cmd.ArgsLenAtDash()
|
|
if split >= 0 {
|
|
work.Extra = args[split:]
|
|
args = args[:split]
|
|
}
|
|
if len(args) < 2 {
|
|
return fmt.Errorf("specify COMMAND and at least one HOST")
|
|
}
|
|
work.Command, work.Hosts = args[0], args[1:]
|
|
}
|
|
command, ok := cfg.Commands[work.Command]
|
|
if !ok {
|
|
return fmt.Errorf("unknown command %q", work.Command)
|
|
}
|
|
if len(work.Hosts) == 0 {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "No failed hosts to retry.")
|
|
return nil
|
|
}
|
|
// Resolve and validate the entire list before launching any action.
|
|
var targets []string
|
|
seen := map[string]bool{}
|
|
for _, target := range work.Hosts {
|
|
h, err := resolveTargetE(cfg, target, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !command.Allowed(commandHost(h)) {
|
|
return fmt.Errorf("%s is not allowed on %s", work.Command, h.Name)
|
|
}
|
|
if !seen[h.Name] {
|
|
targets = append(targets, h.Name)
|
|
seen[h.Name] = true
|
|
}
|
|
}
|
|
if command.Confirm && !dry {
|
|
if err := confirmAction(fmt.Sprintf("Run %s on %d hosts", work.Command, len(targets)), yes); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
executable, err := batchExecutable()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Temporary output files bound memory even for very verbose commands.
|
|
dir, err := os.MkdirTemp("", "netssh-batch-")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer cancel()
|
|
jobs := make(chan int)
|
|
results := make(chan batchResult, len(targets))
|
|
var wg sync.WaitGroup
|
|
var output sync.Mutex
|
|
for n := 0; n < parallel; n++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := range jobs {
|
|
host := targets[i]
|
|
if ctx.Err() != nil {
|
|
results <- batchResult{host, 130}
|
|
continue
|
|
}
|
|
output.Lock()
|
|
fmt.Fprintf(cmd.OutOrStdout(), "[%s] starting\n", host)
|
|
output.Unlock()
|
|
f, err := os.CreateTemp(dir, "output-")
|
|
if err != nil {
|
|
results <- batchResult{host, 1}
|
|
continue
|
|
}
|
|
argv := []string{executable, "run", "--yes"}
|
|
if dry {
|
|
argv = append(argv, "--dry-run")
|
|
}
|
|
argv = append(argv, work.Command, "--", host)
|
|
argv = append(argv, work.Extra...)
|
|
r := runner.Run(ctx, argv, runner.Options{Stdout: f, Stderr: f})
|
|
f.Close()
|
|
output.Lock()
|
|
fmt.Fprintf(cmd.OutOrStdout(), "[%s] exit=%d duration=%s\n", host, r.Code, r.Duration.Round(time.Millisecond))
|
|
read, err := os.Open(f.Name())
|
|
if err == nil {
|
|
_, _ = io.Copy(cmd.OutOrStdout(), read)
|
|
read.Close()
|
|
}
|
|
output.Unlock()
|
|
results <- batchResult{host, r.Code}
|
|
}
|
|
}()
|
|
}
|
|
for i := range targets {
|
|
jobs <- i
|
|
}
|
|
close(jobs)
|
|
wg.Wait()
|
|
close(results)
|
|
failed := map[string]bool{}
|
|
for r := range results {
|
|
if r.Code != 0 {
|
|
failed[r.Host] = true
|
|
}
|
|
}
|
|
work.Hosts = nil
|
|
for _, host := range targets {
|
|
if failed[host] {
|
|
work.Hosts = append(work.Hosts, host)
|
|
}
|
|
}
|
|
if !dry {
|
|
data, _ := json.Marshal(work)
|
|
if err := os.MkdirAll(filepath.Dir(batchPath()), 0700); err != nil {
|
|
return err
|
|
}
|
|
if err := storage.WriteFile(batchPath(), data, 0600); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%d succeeded, %d failed\n", len(targets)-len(failed), len(failed))
|
|
if len(failed) > 0 {
|
|
return fmt.Errorf("batch failed; retry with netssh batch --retry-failed")
|
|
}
|
|
return nil
|
|
}}
|
|
cmd.Flags().IntVar(¶llel, "parallel", 4, "Maximum simultaneous hosts (1-64)")
|
|
cmd.Flags().BoolVar(&retry, "retry-failed", false, "Repeat failed targets from the last batch")
|
|
cmd.Flags().BoolVar(&yes, "yes", false, "Confirm the batch action")
|
|
cmd.Flags().BoolVar(&dry, "dry-run", false, "Resolve and print commands without running")
|
|
return cmd
|
|
}
|