diff options
-rw-r--r-- | bookpipeline/aws.go | 2 | ||||
-rw-r--r-- | bookpipeline/cmd/bookpipeline/main.go | 25 | ||||
-rw-r--r-- | bookpipeline/cmd/booktopipeline/main.go | 7 | ||||
-rw-r--r-- | bookpipeline/cmd/confgraph/main.go | 2 | ||||
-rw-r--r-- | bookpipeline/cmd/getpipelinebook/main.go | 7 | ||||
-rw-r--r-- | bookpipeline/cmd/mkpipeline/main.go | 4 | ||||
-rw-r--r-- | bookpipeline/graph.go | 24 |
7 files changed, 40 insertions, 31 deletions
diff --git a/bookpipeline/aws.go b/bookpipeline/aws.go index a111ebf..a956cf1 100644 --- a/bookpipeline/aws.go +++ b/bookpipeline/aws.go @@ -16,7 +16,7 @@ import ( const PreprocPattern = `_bin[0-9].[0-9].png` type Qmsg struct { - Handle, Body string + Handle, Body string } type AwsConn struct { diff --git a/bookpipeline/cmd/bookpipeline/main.go b/bookpipeline/cmd/bookpipeline/main.go index 3698873..520311b 100644 --- a/bookpipeline/cmd/bookpipeline/main.go +++ b/bookpipeline/cmd/bookpipeline/main.go @@ -72,7 +72,8 @@ func download(dl chan string, process chan string, conn Pipeliner, dir string, e logger.Println("Downloading", key) err := conn.Download(conn.WIPStorageId(), key, fn) if err != nil { - for range dl {} // consume the rest of the receiving channel so it isn't blocked + for range dl { + } // consume the rest of the receiving channel so it isn't blocked close(process) errc <- err return @@ -89,7 +90,8 @@ func up(c chan string, done chan bool, conn Pipeliner, bookname string, errc cha logger.Println("Uploading", key) err := conn.Upload(conn.WIPStorageId(), key, path) if err != nil { - for range c {} // consume the rest of the receiving channel so it isn't blocked + for range c { + } // consume the rest of the receiving channel so it isn't blocked errc <- err return } @@ -103,7 +105,8 @@ func preprocess(pre chan string, up chan string, errc chan error, logger *log.Lo logger.Println("Preprocessing", path) done, err := preproc.PreProcMulti(path, []float64{0.1, 0.2, 0.4, 0.5}, "binary", 0, true, 5, 30) if err != nil { - for range pre {} // consume the rest of the receiving channel so it isn't blocked + for range pre { + } // consume the rest of the receiving channel so it isn't blocked close(up) errc <- err return @@ -116,14 +119,15 @@ func preprocess(pre chan string, up chan string, errc chan error, logger *log.Lo } func ocr(training string) func(chan string, chan string, chan error, *log.Logger) { - return func (toocr chan string, up chan string, errc chan error, logger *log.Logger) { + return func(toocr chan string, up chan string, errc chan error, logger *log.Logger) { for path := range toocr { logger.Println("OCRing", path) name := strings.Replace(path, ".png", "", 1) cmd := exec.Command("tesseract", "-l", training, path, name, "hocr") err := cmd.Run() if err != nil { - for range toocr {} // consume the rest of the receiving channel so it isn't blocked + for range toocr { + } // consume the rest of the receiving channel so it isn't blocked close(up) errc <- errors.New(fmt.Sprintf("Error ocring %s: %s", path, err)) return @@ -146,7 +150,8 @@ func analyse(toanalyse chan string, up chan string, errc chan error, logger *log logger.Println("Calculating confidence for", path) avg, err := hocr.GetAvgConf(path) if err != nil { - for range toanalyse {} // consume the rest of the receiving channel so it isn't blocked + for range toanalyse { + } // consume the rest of the receiving channel so it isn't blocked close(up) errc <- errors.New(fmt.Sprintf("Error retreiving confidence for %s: %s", path, err)) return @@ -226,7 +231,7 @@ func analyse(toanalyse chan string, up chan string, errc chan error, logger *log func heartbeat(conn Pipeliner, t *time.Ticker, msg string, queue string, errc chan error) { for range t.C { - err := conn.QueueHeartbeat(msg, queue, HeartbeatTime * 2) + err := conn.QueueHeartbeat(msg, queue, HeartbeatTime*2) if err != nil { errc <- err t.Stop() @@ -355,7 +360,7 @@ func main() { for { select { case <-checkPreQueue: - msg, err := conn.CheckQueue(conn.PreQueueId(), HeartbeatTime * 2) + msg, err := conn.CheckQueue(conn.PreQueueId(), HeartbeatTime*2) checkPreQueue = time.After(PauseBetweenChecks) if err != nil { log.Println("Error checking preprocess queue", err) @@ -371,7 +376,7 @@ func main() { log.Println("Error during preprocess", err) } case <-checkOCRQueue: - msg, err := conn.CheckQueue(conn.OCRQueueId(), HeartbeatTime * 2) + msg, err := conn.CheckQueue(conn.OCRQueueId(), HeartbeatTime*2) checkOCRQueue = time.After(PauseBetweenChecks) if err != nil { log.Println("Error checking OCR queue", err) @@ -387,7 +392,7 @@ func main() { log.Println("Error during OCR process", err) } case <-checkAnalyseQueue: - msg, err := conn.CheckQueue(conn.AnalyseQueueId(), HeartbeatTime * 2) + msg, err := conn.CheckQueue(conn.AnalyseQueueId(), HeartbeatTime*2) checkAnalyseQueue = time.After(PauseBetweenChecks) if err != nil { log.Println("Error checking analyse queue", err) diff --git a/bookpipeline/cmd/booktopipeline/main.go b/bookpipeline/cmd/booktopipeline/main.go index a0fcb41..06bc648 100644 --- a/bookpipeline/cmd/booktopipeline/main.go +++ b/bookpipeline/cmd/booktopipeline/main.go @@ -1,4 +1,5 @@ package main + // TODO: use bookpipeline package to do aws stuff import ( @@ -15,9 +16,11 @@ import ( // null writer to enable non-verbose logging to be discarded type NullWriter bool + func (w NullWriter) Write(p []byte) (n int, err error) { return len(p), nil } + var verboselog *log.Logger type fileWalk chan string @@ -52,7 +55,7 @@ func main() { verboselog = log.New(os.Stdout, "", log.LstdFlags) } else { var n NullWriter - verboselog = log.New(n, "", log.LstdFlags) + verboselog = log.New(n, "", log.LstdFlags) } verboselog.Println("Setting up AWS session") @@ -108,7 +111,7 @@ func main() { verboselog.Println("Sending message", bookname, "to queue", qurl) _, err = sqssvc.SendMessage(&sqs.SendMessageInput{ MessageBody: aws.String(bookname), - QueueUrl: &qurl, + QueueUrl: &qurl, }) if err != nil { log.Fatalln("Error adding book to queue:", err) diff --git a/bookpipeline/cmd/confgraph/main.go b/bookpipeline/cmd/confgraph/main.go index 20a5838..b60821e 100644 --- a/bookpipeline/cmd/confgraph/main.go +++ b/bookpipeline/cmd/confgraph/main.go @@ -17,7 +17,7 @@ func walker(confs *[]*bookpipeline.Conf) filepath.WalkFunc { if info.IsDir() { return nil } - if ! strings.HasSuffix(path, ".hocr") { + if !strings.HasSuffix(path, ".hocr") { return nil } avg, err := hocr.GetAvgConf(path) diff --git a/bookpipeline/cmd/getpipelinebook/main.go b/bookpipeline/cmd/getpipelinebook/main.go index 55401e6..66e3f70 100644 --- a/bookpipeline/cmd/getpipelinebook/main.go +++ b/bookpipeline/cmd/getpipelinebook/main.go @@ -15,6 +15,7 @@ const usage = "Usage: getpipelinebook [-a] [-v] bookname\n\nDownloads the pipeli // null writer to enable non-verbose logging to be discarded type NullWriter bool + func (w NullWriter) Write(p []byte) (n int, err error) { return len(p), nil } @@ -72,9 +73,9 @@ func main() { if *all { verboselog.Println("Downloading all files for", bookname) objs, err := conn.ListObjects(conn.WIPStorageId(), bookname) - if err != nil { - log.Fatalln("Failed to get list of files for book", bookname, err) - } + if err != nil { + log.Fatalln("Failed to get list of files for book", bookname, err) + } for _, i := range objs { verboselog.Println("Downloading", i) err = conn.Download(conn.WIPStorageId(), i, i) diff --git a/bookpipeline/cmd/mkpipeline/main.go b/bookpipeline/cmd/mkpipeline/main.go index 611a091..970543e 100644 --- a/bookpipeline/cmd/mkpipeline/main.go +++ b/bookpipeline/cmd/mkpipeline/main.go @@ -58,8 +58,8 @@ func main() { _, err = sqssvc.CreateQueue(&sqs.CreateQueueInput{ QueueName: aws.String(qname), Attributes: map[string]*string{ - "VisibilityTimeout": aws.String("120"), // 2 minutes - "MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs + "VisibilityTimeout": aws.String("120"), // 2 minutes + "MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs "ReceiveMessageWaitTimeSeconds": aws.String("20"), }, }) diff --git a/bookpipeline/graph.go b/bookpipeline/graph.go index 67deb33..e17bdd0 100644 --- a/bookpipeline/graph.go +++ b/bookpipeline/graph.go @@ -18,8 +18,8 @@ const mediumCutoff = 65 const badCutoff = 60 type Conf struct { - Path, Code string - Conf float64 + Path, Code string + Conf float64 } type GraphConf struct { @@ -76,7 +76,7 @@ func Graph(confs map[string]*Conf, bookname string, w io.Writer) error { if c.Conf < goodCutoff { annotations = append(annotations, chart.Value2{Label: fmt.Sprintf("%.0f", c.Pgnum), XValue: c.Pgnum, YValue: c.Conf}) } - if tickevery % i == 0 { + if tickevery%i == 0 { ticks = append(ticks, chart.Tick{c.Pgnum, fmt.Sprintf("%.0f", c.Pgnum)}) } } @@ -92,8 +92,8 @@ func Graph(confs map[string]*Conf, bookname string, w io.Writer) error { // Create lines marking top and bottom 10% confidence sort.Slice(graphconf, func(i, j int) bool { return graphconf[i].Conf < graphconf[j].Conf }) - lowconf := graphconf[int(len(graphconf) / 10)].Conf - highconf := graphconf[int((len(graphconf) / 10) * 9)].Conf + lowconf := graphconf[int(len(graphconf)/10)].Conf + highconf := graphconf[int((len(graphconf)/10)*9)].Conf yvalues = []float64{} for _ = range graphconf { yvalues = append(yvalues, lowconf) @@ -122,23 +122,23 @@ func Graph(confs map[string]*Conf, bookname string, w io.Writer) error { } graph := chart.Chart{ - Title: bookname, + Title: bookname, TitleStyle: chart.StyleShow(), - Width: 1920, - Height: 1080, + Width: 1920, + Height: 1080, XAxis: chart.XAxis{ - Name: "Page number", + Name: "Page number", NameStyle: chart.StyleShow(), - Style: chart.StyleShow(), + Style: chart.StyleShow(), Range: &chart.ContinuousRange{ Min: 0.0, }, Ticks: ticks, }, YAxis: chart.YAxis{ - Name: "Confidence", + Name: "Confidence", NameStyle: chart.StyleShow(), - Style: chart.StyleShow(), + Style: chart.StyleShow(), Range: &chart.ContinuousRange{ Min: 0.0, Max: 100.0, |