From 7ae391451e8245f6b51758b48f924e51b68c15fa Mon Sep 17 00:00:00 2001 From: Nick White Date: Tue, 24 Sep 2019 18:25:56 +0100 Subject: Get ssh logs from all running servers --- bookpipeline/cmd/lspipeline/main.go | 52 +++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 13 deletions(-) (limited to 'bookpipeline') diff --git a/bookpipeline/cmd/lspipeline/main.go b/bookpipeline/cmd/lspipeline/main.go index a491cbc..9fb7a78 100644 --- a/bookpipeline/cmd/lspipeline/main.go +++ b/bookpipeline/cmd/lspipeline/main.go @@ -4,13 +4,13 @@ import ( "flag" "fmt" "log" - "os" + "os/exec" "strings" "rescribe.xyz/go.git/bookpipeline" ) -const usage = `Usage: lspipeline [-v] +const usage = `Usage: lspipeline [-i key] [-n num] Lists useful things related to the pipeline. @@ -18,7 +18,7 @@ Lists useful things related to the pipeline. - Messages in each queue - Books not completed - Books done -- Last 5 lines of bookpipeline logs from each running instance (with -v) +- Last n lines of bookpipeline logs from each running instance ` type LsPipeliner interface { @@ -144,8 +144,25 @@ func getBookStatusChan(conn LsPipeliner, inprogressc chan string, donec chan str close(donec) } +func getRecentSSHLogs(ip string, id string, n int) (string, error) { + addr := fmt.Sprintf("%s@%s", "admin", ip) + logcmd := fmt.Sprintf("journalctl -n %d -u bookpipeline", n) + var cmd *exec.Cmd + if id == "" { + cmd = exec.Command("ssh", addr, logcmd) + } else { + cmd = exec.Command("ssh", "-i", id, addr, logcmd) + } + out, err := cmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} + func main() { - verbose := flag.Bool("v", false, "verbose") + keyfile := flag.String("i", "", "private key file for SSH") + lognum := flag.Int("n", 5, "number of lines to include in SSH logs") flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), usage) flag.PrintDefaults() @@ -153,12 +170,8 @@ func main() { flag.Parse() var verboselog *log.Logger - if *verbose { - verboselog = log.New(os.Stdout, "", 0) - } else { - var n NullWriter - verboselog = log.New(n, "", 0) - } + var n NullWriter + verboselog = log.New(n, "", 0) var conn LsPipeliner conn = &bookpipeline.AwsConn{Region: "eu-west-2", Logger: verboselog} @@ -169,13 +182,15 @@ func main() { instances := make(chan bookpipeline.InstanceDetails, 100) queues := make(chan queueDetails) - inprogress := make(chan string) - done := make(chan string) + inprogress := make(chan string, 100) + done := make(chan string, 100) go getInstances(conn, instances) go getQueueDetails(conn, queues) go getBookStatusChan(conn, inprogress, done) + var ips []string + fmt.Println("# Instances") for i := range instances { fmt.Printf("ID: %s, Type: %s, LaunchTime: %s, State: %s", i.Id, i.Type, i.LaunchTime, i.State) @@ -184,6 +199,7 @@ func main() { } if i.Ip != "" { fmt.Printf(", IP: %s", i.Ip) + ips = append(ips, i.Ip) } if i.Spot != "" { fmt.Printf(", SpotRequest: %s", i.Spot) @@ -206,5 +222,15 @@ func main() { fmt.Println(i) } - // TODO: See remaining items in the usage statement + if len(ips) > 0 { + fmt.Println("\n# Recent logs") + for _, ip := range ips { + logs, err := getRecentSSHLogs(ip, *keyfile, *lognum) + if err != nil { + log.Printf("Error running ssh for %s: %v\n", ip, err) + continue + } + fmt.Printf("\n%s\n%s\n", ip, logs) + } + } } -- cgit v1.2.1-24-ge1ad