1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
package main
import (
"flag"
"fmt"
"log"
"os"
"rescribe.xyz/go.git/bookpipeline"
)
const usage = `Usage: lspipeline [-v]
Lists useful things related to the pipeline.
- Instances running
- Messages in each queue
- Books not completed (from S3 without a best file)
- Books completed (from S3 with a best file)
- Last 5 lines of bookpipeline logs from each running instance (with -v)
`
type LsPipeliner interface {
Init() error
PreQueueId() string
OCRQueueId() string
AnalyseQueueId() string
GetQueueDetails(url string) (string, string, error)
GetInstanceDetails() ([]bookpipeline.InstanceDetails, error)
}
// NullWriter is used so non-verbose logging may be discarded
type NullWriter bool
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
type queueDetails struct {
name, numAvailable, numInProgress string
}
func getInstances(conn LsPipeliner, detailsc chan bookpipeline.InstanceDetails) {
details, err := conn.GetInstanceDetails()
if err != nil {
log.Println("Error getting instance details:", err)
}
for _, d := range details {
detailsc <- d
}
close(detailsc)
}
func getQueueDetails(conn LsPipeliner, qdetails chan queueDetails) {
queues := []struct{ name, id string }{
{"preprocess", conn.PreQueueId()},
{"ocr", conn.OCRQueueId()},
{"analyse", conn.AnalyseQueueId()},
}
for _, q := range queues {
avail, inprog, err := conn.GetQueueDetails(q.id)
if err != nil {
log.Println("Error getting queue details:", err)
}
var qd queueDetails
qd.name = q.name
qd.numAvailable = avail
qd.numInProgress = inprog
qdetails <- qd
}
close(qdetails)
}
func main() {
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), usage)
flag.PrintDefaults()
}
flag.Parse()
var verboselog *log.Logger
if *verbose {
verboselog = log.New(os.Stdout, "", 0)
} else {
var n NullWriter
verboselog = log.New(n, "", 0)
}
var conn LsPipeliner
conn = &bookpipeline.AwsConn{Region: "eu-west-2", Logger: verboselog}
err := conn.Init()
if err != nil {
log.Fatalln("Failed to set up cloud connection:", err)
}
instances := make(chan bookpipeline.InstanceDetails, 100)
queues := make(chan queueDetails)
go getInstances(conn, instances)
go getQueueDetails(conn, queues)
fmt.Println("# Instances")
for i := range instances {
fmt.Printf("ID: %s, Type: %s, LaunchTime: %s, State: %s", i.Id, i.Type, i.LaunchTime, i.State)
if i.Name != "" {
fmt.Printf(", Name: %s", i.Name)
}
if i.Ip != "" {
fmt.Printf(", IP: %s", i.Ip)
}
if i.Spot != "" {
fmt.Printf(", SpotRequest: %s", i.Spot)
}
fmt.Printf("\n")
}
fmt.Println("\n# Queues")
for i := range queues {
fmt.Printf("%s: %s available, %s in progress\n", i.name, i.numAvailable, i.numInProgress)
}
// TODO: See remaining items in the usage statement
}
|
