diff options
| -rw-r--r-- | bookpipeline/cmd/lspipeline/main.go | 90 |
1 files changed, 88 insertions, 2 deletions
diff --git a/bookpipeline/cmd/lspipeline/main.go b/bookpipeline/cmd/lspipeline/main.go index 3cbc893..a491cbc 100644 --- a/bookpipeline/cmd/lspipeline/main.go +++ b/bookpipeline/cmd/lspipeline/main.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "strings" "rescribe.xyz/go.git/bookpipeline" ) @@ -15,8 +16,8 @@ Lists useful things related to the pipeline. - Instances running - Messages in each queue -- Books not completed (from S3 without a best file) -- Books completed (from S3 with a best file) +- Books not completed +- Books done - Last 5 lines of bookpipeline logs from each running instance (with -v) ` @@ -27,6 +28,8 @@ type LsPipeliner interface { AnalyseQueueId() string GetQueueDetails(url string) (string, string, error) GetInstanceDetails() ([]bookpipeline.InstanceDetails, error) + ListObjects(bucket string, prefix string) ([]string, error) + WIPStorageId() string } // NullWriter is used so non-verbose logging may be discarded @@ -71,6 +74,76 @@ func getQueueDetails(conn LsPipeliner, qdetails chan queueDetails) { close(qdetails) } +// getBookStatus returns a list of in progress and done books. +// It determines this by listing all objects, and splitting the +// prefixes into two lists, those which have a 'graph.png' file, +// which are classed as done, and those which are not. +func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err error) { + allfiles, err := conn.ListObjects(conn.WIPStorageId(), "") + if err != nil { + log.Println("Error getting list of objects:", err) + return inprogress, done, err + } + for _, f := range allfiles { + parts := strings.Split(f, "/") + if parts[1] != "graph.png" { + continue + } + prefix := parts[0] + found := false + for _, i := range done { + if i == prefix { + found = true + continue + } + } + if !found { + done = append(done, prefix) + } + } + + for _, f := range allfiles { + parts := strings.Split(f, "/") + prefix := parts[0] + found := false + for _, i := range done { + if i == prefix { + found = true + continue + } + } + for _, i := range inprogress { + if i == prefix { + found = true + continue + } + } + if !found { + inprogress = append(inprogress, prefix) + } + } + + return inprogress, done, err +} + +func getBookStatusChan(conn LsPipeliner, inprogressc chan string, donec chan string) { + inprogress, done, err := getBookStatus(conn) + if err != nil { + log.Println("Error getting book status:", err) + close(inprogressc) + close(donec) + return + } + for _, i := range inprogress { + inprogressc <- i + } + close(inprogressc) + for _, i := range done { + donec <- i + } + close(donec) +} + func main() { verbose := flag.Bool("v", false, "verbose") flag.Usage = func() { @@ -96,9 +169,12 @@ func main() { instances := make(chan bookpipeline.InstanceDetails, 100) queues := make(chan queueDetails) + inprogress := make(chan string) + done := make(chan string) go getInstances(conn, instances) go getQueueDetails(conn, queues) + go getBookStatusChan(conn, inprogress, done) fmt.Println("# Instances") for i := range instances { @@ -120,5 +196,15 @@ func main() { fmt.Printf("%s: %s available, %s in progress\n", i.name, i.numAvailable, i.numInProgress) } + fmt.Println("\n# Books not completed") + for i := range inprogress { + fmt.Println(i) + } + + fmt.Println("\n# Books done") + for i := range done { + fmt.Println(i) + } + // TODO: See remaining items in the usage statement } |