summaryrefslogtreecommitdiff
path: root/bookpipeline/cmd
diff options
context:
space:
mode:
Diffstat (limited to 'bookpipeline/cmd')
-rw-r--r--bookpipeline/cmd/bookpipeline/main.go25
-rw-r--r--bookpipeline/cmd/booktopipeline/main.go7
-rw-r--r--bookpipeline/cmd/confgraph/main.go2
-rw-r--r--bookpipeline/cmd/getpipelinebook/main.go7
-rw-r--r--bookpipeline/cmd/mkpipeline/main.go4
5 files changed, 27 insertions, 18 deletions
diff --git a/bookpipeline/cmd/bookpipeline/main.go b/bookpipeline/cmd/bookpipeline/main.go
index 3698873..520311b 100644
--- a/bookpipeline/cmd/bookpipeline/main.go
+++ b/bookpipeline/cmd/bookpipeline/main.go
@@ -72,7 +72,8 @@ func download(dl chan string, process chan string, conn Pipeliner, dir string, e
logger.Println("Downloading", key)
err := conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
- for range dl {} // consume the rest of the receiving channel so it isn't blocked
+ for range dl {
+ } // consume the rest of the receiving channel so it isn't blocked
close(process)
errc <- err
return
@@ -89,7 +90,8 @@ func up(c chan string, done chan bool, conn Pipeliner, bookname string, errc cha
logger.Println("Uploading", key)
err := conn.Upload(conn.WIPStorageId(), key, path)
if err != nil {
- for range c {} // consume the rest of the receiving channel so it isn't blocked
+ for range c {
+ } // consume the rest of the receiving channel so it isn't blocked
errc <- err
return
}
@@ -103,7 +105,8 @@ func preprocess(pre chan string, up chan string, errc chan error, logger *log.Lo
logger.Println("Preprocessing", path)
done, err := preproc.PreProcMulti(path, []float64{0.1, 0.2, 0.4, 0.5}, "binary", 0, true, 5, 30)
if err != nil {
- for range pre {} // consume the rest of the receiving channel so it isn't blocked
+ for range pre {
+ } // consume the rest of the receiving channel so it isn't blocked
close(up)
errc <- err
return
@@ -116,14 +119,15 @@ func preprocess(pre chan string, up chan string, errc chan error, logger *log.Lo
}
func ocr(training string) func(chan string, chan string, chan error, *log.Logger) {
- return func (toocr chan string, up chan string, errc chan error, logger *log.Logger) {
+ return func(toocr chan string, up chan string, errc chan error, logger *log.Logger) {
for path := range toocr {
logger.Println("OCRing", path)
name := strings.Replace(path, ".png", "", 1)
cmd := exec.Command("tesseract", "-l", training, path, name, "hocr")
err := cmd.Run()
if err != nil {
- for range toocr {} // consume the rest of the receiving channel so it isn't blocked
+ for range toocr {
+ } // consume the rest of the receiving channel so it isn't blocked
close(up)
errc <- errors.New(fmt.Sprintf("Error ocring %s: %s", path, err))
return
@@ -146,7 +150,8 @@ func analyse(toanalyse chan string, up chan string, errc chan error, logger *log
logger.Println("Calculating confidence for", path)
avg, err := hocr.GetAvgConf(path)
if err != nil {
- for range toanalyse {} // consume the rest of the receiving channel so it isn't blocked
+ for range toanalyse {
+ } // consume the rest of the receiving channel so it isn't blocked
close(up)
errc <- errors.New(fmt.Sprintf("Error retreiving confidence for %s: %s", path, err))
return
@@ -226,7 +231,7 @@ func analyse(toanalyse chan string, up chan string, errc chan error, logger *log
func heartbeat(conn Pipeliner, t *time.Ticker, msg string, queue string, errc chan error) {
for range t.C {
- err := conn.QueueHeartbeat(msg, queue, HeartbeatTime * 2)
+ err := conn.QueueHeartbeat(msg, queue, HeartbeatTime*2)
if err != nil {
errc <- err
t.Stop()
@@ -355,7 +360,7 @@ func main() {
for {
select {
case <-checkPreQueue:
- msg, err := conn.CheckQueue(conn.PreQueueId(), HeartbeatTime * 2)
+ msg, err := conn.CheckQueue(conn.PreQueueId(), HeartbeatTime*2)
checkPreQueue = time.After(PauseBetweenChecks)
if err != nil {
log.Println("Error checking preprocess queue", err)
@@ -371,7 +376,7 @@ func main() {
log.Println("Error during preprocess", err)
}
case <-checkOCRQueue:
- msg, err := conn.CheckQueue(conn.OCRQueueId(), HeartbeatTime * 2)
+ msg, err := conn.CheckQueue(conn.OCRQueueId(), HeartbeatTime*2)
checkOCRQueue = time.After(PauseBetweenChecks)
if err != nil {
log.Println("Error checking OCR queue", err)
@@ -387,7 +392,7 @@ func main() {
log.Println("Error during OCR process", err)
}
case <-checkAnalyseQueue:
- msg, err := conn.CheckQueue(conn.AnalyseQueueId(), HeartbeatTime * 2)
+ msg, err := conn.CheckQueue(conn.AnalyseQueueId(), HeartbeatTime*2)
checkAnalyseQueue = time.After(PauseBetweenChecks)
if err != nil {
log.Println("Error checking analyse queue", err)
diff --git a/bookpipeline/cmd/booktopipeline/main.go b/bookpipeline/cmd/booktopipeline/main.go
index a0fcb41..06bc648 100644
--- a/bookpipeline/cmd/booktopipeline/main.go
+++ b/bookpipeline/cmd/booktopipeline/main.go
@@ -1,4 +1,5 @@
package main
+
// TODO: use bookpipeline package to do aws stuff
import (
@@ -15,9 +16,11 @@ import (
// null writer to enable non-verbose logging to be discarded
type NullWriter bool
+
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
+
var verboselog *log.Logger
type fileWalk chan string
@@ -52,7 +55,7 @@ func main() {
verboselog = log.New(os.Stdout, "", log.LstdFlags)
} else {
var n NullWriter
- verboselog = log.New(n, "", log.LstdFlags)
+ verboselog = log.New(n, "", log.LstdFlags)
}
verboselog.Println("Setting up AWS session")
@@ -108,7 +111,7 @@ func main() {
verboselog.Println("Sending message", bookname, "to queue", qurl)
_, err = sqssvc.SendMessage(&sqs.SendMessageInput{
MessageBody: aws.String(bookname),
- QueueUrl: &qurl,
+ QueueUrl: &qurl,
})
if err != nil {
log.Fatalln("Error adding book to queue:", err)
diff --git a/bookpipeline/cmd/confgraph/main.go b/bookpipeline/cmd/confgraph/main.go
index 20a5838..b60821e 100644
--- a/bookpipeline/cmd/confgraph/main.go
+++ b/bookpipeline/cmd/confgraph/main.go
@@ -17,7 +17,7 @@ func walker(confs *[]*bookpipeline.Conf) filepath.WalkFunc {
if info.IsDir() {
return nil
}
- if ! strings.HasSuffix(path, ".hocr") {
+ if !strings.HasSuffix(path, ".hocr") {
return nil
}
avg, err := hocr.GetAvgConf(path)
diff --git a/bookpipeline/cmd/getpipelinebook/main.go b/bookpipeline/cmd/getpipelinebook/main.go
index 55401e6..66e3f70 100644
--- a/bookpipeline/cmd/getpipelinebook/main.go
+++ b/bookpipeline/cmd/getpipelinebook/main.go
@@ -15,6 +15,7 @@ const usage = "Usage: getpipelinebook [-a] [-v] bookname\n\nDownloads the pipeli
// null writer to enable non-verbose logging to be discarded
type NullWriter bool
+
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
@@ -72,9 +73,9 @@ func main() {
if *all {
verboselog.Println("Downloading all files for", bookname)
objs, err := conn.ListObjects(conn.WIPStorageId(), bookname)
- if err != nil {
- log.Fatalln("Failed to get list of files for book", bookname, err)
- }
+ if err != nil {
+ log.Fatalln("Failed to get list of files for book", bookname, err)
+ }
for _, i := range objs {
verboselog.Println("Downloading", i)
err = conn.Download(conn.WIPStorageId(), i, i)
diff --git a/bookpipeline/cmd/mkpipeline/main.go b/bookpipeline/cmd/mkpipeline/main.go
index 611a091..970543e 100644
--- a/bookpipeline/cmd/mkpipeline/main.go
+++ b/bookpipeline/cmd/mkpipeline/main.go
@@ -58,8 +58,8 @@ func main() {
_, err = sqssvc.CreateQueue(&sqs.CreateQueueInput{
QueueName: aws.String(qname),
Attributes: map[string]*string{
- "VisibilityTimeout": aws.String("120"), // 2 minutes
- "MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs
+ "VisibilityTimeout": aws.String("120"), // 2 minutes
+ "MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs
"ReceiveMessageWaitTimeSeconds": aws.String("20"),
},
})
generated by cgit v1.2.1-24-ge1ad (git 2.21.0) at 2024-11-04 21:10:38 +0000