summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--bookpipeline/aws.go2
-rw-r--r--bookpipeline/cmd/bookpipeline/main.go25
-rw-r--r--bookpipeline/cmd/booktopipeline/main.go7
-rw-r--r--bookpipeline/cmd/confgraph/main.go2
-rw-r--r--bookpipeline/cmd/getpipelinebook/main.go7
-rw-r--r--bookpipeline/cmd/mkpipeline/main.go4
-rw-r--r--bookpipeline/graph.go24
7 files changed, 40 insertions, 31 deletions
diff --git a/bookpipeline/aws.go b/bookpipeline/aws.go
index a111ebf..a956cf1 100644
--- a/bookpipeline/aws.go
+++ b/bookpipeline/aws.go
@@ -16,7 +16,7 @@ import (
const PreprocPattern = `_bin[0-9].[0-9].png`
type Qmsg struct {
- Handle, Body string
+ Handle, Body string
}
type AwsConn struct {
diff --git a/bookpipeline/cmd/bookpipeline/main.go b/bookpipeline/cmd/bookpipeline/main.go
index 3698873..520311b 100644
--- a/bookpipeline/cmd/bookpipeline/main.go
+++ b/bookpipeline/cmd/bookpipeline/main.go
@@ -72,7 +72,8 @@ func download(dl chan string, process chan string, conn Pipeliner, dir string, e
logger.Println("Downloading", key)
err := conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
- for range dl {} // consume the rest of the receiving channel so it isn't blocked
+ for range dl {
+ } // consume the rest of the receiving channel so it isn't blocked
close(process)
errc <- err
return
@@ -89,7 +90,8 @@ func up(c chan string, done chan bool, conn Pipeliner, bookname string, errc cha
logger.Println("Uploading", key)
err := conn.Upload(conn.WIPStorageId(), key, path)
if err != nil {
- for range c {} // consume the rest of the receiving channel so it isn't blocked
+ for range c {
+ } // consume the rest of the receiving channel so it isn't blocked
errc <- err
return
}
@@ -103,7 +105,8 @@ func preprocess(pre chan string, up chan string, errc chan error, logger *log.Lo
logger.Println("Preprocessing", path)
done, err := preproc.PreProcMulti(path, []float64{0.1, 0.2, 0.4, 0.5}, "binary", 0, true, 5, 30)
if err != nil {
- for range pre {} // consume the rest of the receiving channel so it isn't blocked
+ for range pre {
+ } // consume the rest of the receiving channel so it isn't blocked
close(up)
errc <- err
return
@@ -116,14 +119,15 @@ func preprocess(pre chan string, up chan string, errc chan error, logger *log.Lo
}
func ocr(training string) func(chan string, chan string, chan error, *log.Logger) {
- return func (toocr chan string, up chan string, errc chan error, logger *log.Logger) {
+ return func(toocr chan string, up chan string, errc chan error, logger *log.Logger) {
for path := range toocr {
logger.Println("OCRing", path)
name := strings.Replace(path, ".png", "", 1)
cmd := exec.Command("tesseract", "-l", training, path, name, "hocr")
err := cmd.Run()
if err != nil {
- for range toocr {} // consume the rest of the receiving channel so it isn't blocked
+ for range toocr {
+ } // consume the rest of the receiving channel so it isn't blocked
close(up)
errc <- errors.New(fmt.Sprintf("Error ocring %s: %s", path, err))
return
@@ -146,7 +150,8 @@ func analyse(toanalyse chan string, up chan string, errc chan error, logger *log
logger.Println("Calculating confidence for", path)
avg, err := hocr.GetAvgConf(path)
if err != nil {
- for range toanalyse {} // consume the rest of the receiving channel so it isn't blocked
+ for range toanalyse {
+ } // consume the rest of the receiving channel so it isn't blocked
close(up)
errc <- errors.New(fmt.Sprintf("Error retreiving confidence for %s: %s", path, err))
return
@@ -226,7 +231,7 @@ func analyse(toanalyse chan string, up chan string, errc chan error, logger *log
func heartbeat(conn Pipeliner, t *time.Ticker, msg string, queue string, errc chan error) {
for range t.C {
- err := conn.QueueHeartbeat(msg, queue, HeartbeatTime * 2)
+ err := conn.QueueHeartbeat(msg, queue, HeartbeatTime*2)
if err != nil {
errc <- err
t.Stop()
@@ -355,7 +360,7 @@ func main() {
for {
select {
case <-checkPreQueue:
- msg, err := conn.CheckQueue(conn.PreQueueId(), HeartbeatTime * 2)
+ msg, err := conn.CheckQueue(conn.PreQueueId(), HeartbeatTime*2)
checkPreQueue = time.After(PauseBetweenChecks)
if err != nil {
log.Println("Error checking preprocess queue", err)
@@ -371,7 +376,7 @@ func main() {
log.Println("Error during preprocess", err)
}
case <-checkOCRQueue:
- msg, err := conn.CheckQueue(conn.OCRQueueId(), HeartbeatTime * 2)
+ msg, err := conn.CheckQueue(conn.OCRQueueId(), HeartbeatTime*2)
checkOCRQueue = time.After(PauseBetweenChecks)
if err != nil {
log.Println("Error checking OCR queue", err)
@@ -387,7 +392,7 @@ func main() {
log.Println("Error during OCR process", err)
}
case <-checkAnalyseQueue:
- msg, err := conn.CheckQueue(conn.AnalyseQueueId(), HeartbeatTime * 2)
+ msg, err := conn.CheckQueue(conn.AnalyseQueueId(), HeartbeatTime*2)
checkAnalyseQueue = time.After(PauseBetweenChecks)
if err != nil {
log.Println("Error checking analyse queue", err)
diff --git a/bookpipeline/cmd/booktopipeline/main.go b/bookpipeline/cmd/booktopipeline/main.go
index a0fcb41..06bc648 100644
--- a/bookpipeline/cmd/booktopipeline/main.go
+++ b/bookpipeline/cmd/booktopipeline/main.go
@@ -1,4 +1,5 @@
package main
+
// TODO: use bookpipeline package to do aws stuff
import (
@@ -15,9 +16,11 @@ import (
// null writer to enable non-verbose logging to be discarded
type NullWriter bool
+
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
+
var verboselog *log.Logger
type fileWalk chan string
@@ -52,7 +55,7 @@ func main() {
verboselog = log.New(os.Stdout, "", log.LstdFlags)
} else {
var n NullWriter
- verboselog = log.New(n, "", log.LstdFlags)
+ verboselog = log.New(n, "", log.LstdFlags)
}
verboselog.Println("Setting up AWS session")
@@ -108,7 +111,7 @@ func main() {
verboselog.Println("Sending message", bookname, "to queue", qurl)
_, err = sqssvc.SendMessage(&sqs.SendMessageInput{
MessageBody: aws.String(bookname),
- QueueUrl: &qurl,
+ QueueUrl: &qurl,
})
if err != nil {
log.Fatalln("Error adding book to queue:", err)
diff --git a/bookpipeline/cmd/confgraph/main.go b/bookpipeline/cmd/confgraph/main.go
index 20a5838..b60821e 100644
--- a/bookpipeline/cmd/confgraph/main.go
+++ b/bookpipeline/cmd/confgraph/main.go
@@ -17,7 +17,7 @@ func walker(confs *[]*bookpipeline.Conf) filepath.WalkFunc {
if info.IsDir() {
return nil
}
- if ! strings.HasSuffix(path, ".hocr") {
+ if !strings.HasSuffix(path, ".hocr") {
return nil
}
avg, err := hocr.GetAvgConf(path)
diff --git a/bookpipeline/cmd/getpipelinebook/main.go b/bookpipeline/cmd/getpipelinebook/main.go
index 55401e6..66e3f70 100644
--- a/bookpipeline/cmd/getpipelinebook/main.go
+++ b/bookpipeline/cmd/getpipelinebook/main.go
@@ -15,6 +15,7 @@ const usage = "Usage: getpipelinebook [-a] [-v] bookname\n\nDownloads the pipeli
// null writer to enable non-verbose logging to be discarded
type NullWriter bool
+
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
@@ -72,9 +73,9 @@ func main() {
if *all {
verboselog.Println("Downloading all files for", bookname)
objs, err := conn.ListObjects(conn.WIPStorageId(), bookname)
- if err != nil {
- log.Fatalln("Failed to get list of files for book", bookname, err)
- }
+ if err != nil {
+ log.Fatalln("Failed to get list of files for book", bookname, err)
+ }
for _, i := range objs {
verboselog.Println("Downloading", i)
err = conn.Download(conn.WIPStorageId(), i, i)
diff --git a/bookpipeline/cmd/mkpipeline/main.go b/bookpipeline/cmd/mkpipeline/main.go
index 611a091..970543e 100644
--- a/bookpipeline/cmd/mkpipeline/main.go
+++ b/bookpipeline/cmd/mkpipeline/main.go
@@ -58,8 +58,8 @@ func main() {
_, err = sqssvc.CreateQueue(&sqs.CreateQueueInput{
QueueName: aws.String(qname),
Attributes: map[string]*string{
- "VisibilityTimeout": aws.String("120"), // 2 minutes
- "MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs
+ "VisibilityTimeout": aws.String("120"), // 2 minutes
+ "MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs
"ReceiveMessageWaitTimeSeconds": aws.String("20"),
},
})
diff --git a/bookpipeline/graph.go b/bookpipeline/graph.go
index 67deb33..e17bdd0 100644
--- a/bookpipeline/graph.go
+++ b/bookpipeline/graph.go
@@ -18,8 +18,8 @@ const mediumCutoff = 65
const badCutoff = 60
type Conf struct {
- Path, Code string
- Conf float64
+ Path, Code string
+ Conf float64
}
type GraphConf struct {
@@ -76,7 +76,7 @@ func Graph(confs map[string]*Conf, bookname string, w io.Writer) error {
if c.Conf < goodCutoff {
annotations = append(annotations, chart.Value2{Label: fmt.Sprintf("%.0f", c.Pgnum), XValue: c.Pgnum, YValue: c.Conf})
}
- if tickevery % i == 0 {
+ if tickevery%i == 0 {
ticks = append(ticks, chart.Tick{c.Pgnum, fmt.Sprintf("%.0f", c.Pgnum)})
}
}
@@ -92,8 +92,8 @@ func Graph(confs map[string]*Conf, bookname string, w io.Writer) error {
// Create lines marking top and bottom 10% confidence
sort.Slice(graphconf, func(i, j int) bool { return graphconf[i].Conf < graphconf[j].Conf })
- lowconf := graphconf[int(len(graphconf) / 10)].Conf
- highconf := graphconf[int((len(graphconf) / 10) * 9)].Conf
+ lowconf := graphconf[int(len(graphconf)/10)].Conf
+ highconf := graphconf[int((len(graphconf)/10)*9)].Conf
yvalues = []float64{}
for _ = range graphconf {
yvalues = append(yvalues, lowconf)
@@ -122,23 +122,23 @@ func Graph(confs map[string]*Conf, bookname string, w io.Writer) error {
}
graph := chart.Chart{
- Title: bookname,
+ Title: bookname,
TitleStyle: chart.StyleShow(),
- Width: 1920,
- Height: 1080,
+ Width: 1920,
+ Height: 1080,
XAxis: chart.XAxis{
- Name: "Page number",
+ Name: "Page number",
NameStyle: chart.StyleShow(),
- Style: chart.StyleShow(),
+ Style: chart.StyleShow(),
Range: &chart.ContinuousRange{
Min: 0.0,
},
Ticks: ticks,
},
YAxis: chart.YAxis{
- Name: "Confidence",
+ Name: "Confidence",
NameStyle: chart.StyleShow(),
- Style: chart.StyleShow(),
+ Style: chart.StyleShow(),
Range: &chart.ContinuousRange{
Min: 0.0,
Max: 100.0,
generated by cgit v1.2.1-24-ge1ad (git 2.21.0) at 2024-11-22 06:22:41 +0000