feat: add host actions, lifecycle hooks and operational tools
This commit is contained in:
1 parent
8d4915543e
commit
e5bde9c61f
32 files changed
+2083
-64
No files matched your search
@@ -0,0 +1,191 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/config"
|
||||
"git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/runner"
|
||||
"git.zb-server.de/Sebi/ssh-netbox-wrapper/internal/storage"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type batchRetry struct {
|
||||
Command string
|
||||
Hosts []string
|
||||
Extra []string
|
||||
}
|
||||
type batchResult struct {
|
||||
Host string
|
||||
Code int
|
||||
}
|
||||
|
||||
var batchExecutable = os.Executable
|
||||
|
||||
func batchPath() string {
|
||||
return filepath.Join(filepath.Dir(config.Path()), "netssh", "last-failed.json")
|
||||
}
|
||||
func batchCmd() *cobra.Command {
|
||||
var parallel int
|
||||
var retry, yes, dry bool
|
||||
cmd := &cobra.Command{Use: "batch COMMAND HOST... [-- EXTRA_ARGS...]", Short: "Run a configured command on hosts with bounded parallelism", RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if parallel < 1 || parallel > 64 {
|
||||
return fmt.Errorf("parallel must be between 1 and 64")
|
||||
}
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
work := batchRetry{}
|
||||
if retry {
|
||||
if len(args) > 0 {
|
||||
return fmt.Errorf("--retry-failed takes no positional arguments")
|
||||
}
|
||||
data, err := os.ReadFile(batchPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = json.Unmarshal(data, &work); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
split := cmd.ArgsLenAtDash()
|
||||
if split >= 0 {
|
||||
work.Extra = args[split:]
|
||||
args = args[:split]
|
||||
}
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("specify COMMAND and at least one HOST")
|
||||
}
|
||||
work.Command, work.Hosts = args[0], args[1:]
|
||||
}
|
||||
command, ok := cfg.Commands[work.Command]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown command %q", work.Command)
|
||||
}
|
||||
if len(work.Hosts) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "No failed hosts to retry.")
|
||||
return nil
|
||||
}
|
||||
// Resolve and validate the entire list before launching any action.
|
||||
var targets []string
|
||||
seen := map[string]bool{}
|
||||
for _, target := range work.Hosts {
|
||||
h, err := resolveTargetE(cfg, target, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !command.Allowed(commandHost(h)) {
|
||||
return fmt.Errorf("%s is not allowed on %s", work.Command, h.Name)
|
||||
}
|
||||
if !seen[h.Name] {
|
||||
targets = append(targets, h.Name)
|
||||
seen[h.Name] = true
|
||||
}
|
||||
}
|
||||
if command.Confirm && !dry {
|
||||
if err := confirmAction(fmt.Sprintf("Run %s on %d hosts", work.Command, len(targets)), yes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
executable, err := batchExecutable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Temporary output files bound memory even for very verbose commands.
|
||||
dir, err := os.MkdirTemp("", "netssh-batch-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
jobs := make(chan int)
|
||||
results := make(chan batchResult, len(targets))
|
||||
var wg sync.WaitGroup
|
||||
var output sync.Mutex
|
||||
for n := 0; n < parallel; n++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := range jobs {
|
||||
host := targets[i]
|
||||
if ctx.Err() != nil {
|
||||
results <- batchResult{host, 130}
|
||||
continue
|
||||
}
|
||||
output.Lock()
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "[%s] starting\n", host)
|
||||
output.Unlock()
|
||||
f, err := os.CreateTemp(dir, "output-")
|
||||
if err != nil {
|
||||
results <- batchResult{host, 1}
|
||||
continue
|
||||
}
|
||||
argv := []string{executable, "run", "--yes"}
|
||||
if dry {
|
||||
argv = append(argv, "--dry-run")
|
||||
}
|
||||
argv = append(argv, work.Command, "--", host)
|
||||
argv = append(argv, work.Extra...)
|
||||
r := runner.Run(ctx, argv, runner.Options{Stdout: f, Stderr: f})
|
||||
f.Close()
|
||||
output.Lock()
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "[%s] exit=%d duration=%s\n", host, r.Code, r.Duration.Round(time.Millisecond))
|
||||
read, err := os.Open(f.Name())
|
||||
if err == nil {
|
||||
_, _ = io.Copy(cmd.OutOrStdout(), read)
|
||||
read.Close()
|
||||
}
|
||||
output.Unlock()
|
||||
results <- batchResult{host, r.Code}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := range targets {
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
failed := map[string]bool{}
|
||||
for r := range results {
|
||||
if r.Code != 0 {
|
||||
failed[r.Host] = true
|
||||
}
|
||||
}
|
||||
work.Hosts = nil
|
||||
for _, host := range targets {
|
||||
if failed[host] {
|
||||
work.Hosts = append(work.Hosts, host)
|
||||
}
|
||||
}
|
||||
if !dry {
|
||||
data, _ := json.Marshal(work)
|
||||
if err := os.MkdirAll(filepath.Dir(batchPath()), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.WriteFile(batchPath(), data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%d succeeded, %d failed\n", len(targets)-len(failed), len(failed))
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("batch failed; retry with netssh batch --retry-failed")
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
cmd.Flags().IntVar(¶llel, "parallel", 4, "Maximum simultaneous hosts (1-64)")
|
||||
cmd.Flags().BoolVar(&retry, "retry-failed", false, "Repeat failed targets from the last batch")
|
||||
cmd.Flags().BoolVar(&yes, "yes", false, "Confirm the batch action")
|
||||
cmd.Flags().BoolVar(&dry, "dry-run", false, "Resolve and print commands without running")
|
||||
return cmd
|
||||
}
|
||||
Reference in new issue
Block a user