summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--aws.go10
-rw-r--r--cmd/getsamplepages/main.go4
-rw-r--r--cmd/getstats/main.go16
-rw-r--r--cmd/rescribe/getembeds.go4
-rw-r--r--cmd/rescribe/main.go7
-rw-r--r--cmd/rmbook/main.go4
-rw-r--r--internal/pipeline/pipeline_test.go36
7 files changed, 41 insertions, 40 deletions
diff --git a/aws.go b/aws.go
index 2411c24..b1681f1 100644
--- a/aws.go
+++ b/aws.go
@@ -383,8 +383,8 @@ func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta,
func (a *AwsConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) {
var obj ObjMeta
err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
- Bucket: aws.String(bucket),
- Prefix: aws.String(prefix),
+ Bucket: aws.String(bucket),
+ Prefix: aws.String(prefix),
MaxKeys: aws.Int64(1),
}, func(page *s3.ListObjectsV2Output, last bool) bool {
for _, r := range page.Contents {
@@ -421,12 +421,12 @@ func (a *AwsConn) DeleteObjects(bucket string, keys []string) error {
// s3.DeleteObjects can only take up to 1000 keys at a time,
// so if necessary delete those collected so far and empty
// the objs queue
- if i % 1000 == 1 {
+ if i%1000 == 1 {
_, err := a.s3svc.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &s3.Delete{
Objects: objs,
- Quiet: aws.Bool(true),
+ Quiet: aws.Bool(true),
},
})
if err != nil {
@@ -439,7 +439,7 @@ func (a *AwsConn) DeleteObjects(bucket string, keys []string) error {
Bucket: aws.String(bucket),
Delete: &s3.Delete{
Objects: objs,
- Quiet: aws.Bool(true),
+ Quiet: aws.Bool(true),
},
})
return err
diff --git a/cmd/getsamplepages/main.go b/cmd/getsamplepages/main.go
index 8e35055..cb32700 100644
--- a/cmd/getsamplepages/main.go
+++ b/cmd/getsamplepages/main.go
@@ -74,7 +74,7 @@ func main() {
err = conn.Download(conn.WIPStorageId(), p+"best", name+"best")
if err != nil {
}
- b, err := ioutil.ReadFile(name+"best")
+ b, err := ioutil.ReadFile(name + "best")
if err != nil {
log.Fatalf("Failed to read file %s\n", name+"best")
}
@@ -85,7 +85,7 @@ func main() {
}
pg := strings.TrimSuffix(lines[0], ".hocr")
- err = os.Remove(name+"best")
+ err = os.Remove(name + "best")
if err != nil {
log.Fatalf("Failed to remove temporary best file for %s", name)
}
diff --git a/cmd/getstats/main.go b/cmd/getstats/main.go
index 5ed1414..b51be2a 100644
--- a/cmd/getstats/main.go
+++ b/cmd/getstats/main.go
@@ -62,16 +62,16 @@ func main() {
log.Println("Downloading all best and conf files found")
for _, i := range objs {
parts := strings.Split(i, "/")
- if parts[len(parts) - 1] == "best" {
- fmt.Printf("Downloading %s to %s\n", i, parts[0] + "-best")
- err = conn.Download(conn.WIPStorageId(), i, parts[0] + "-best")
+ if parts[len(parts)-1] == "best" {
+ fmt.Printf("Downloading %s to %s\n", i, parts[0]+"-best")
+ err = conn.Download(conn.WIPStorageId(), i, parts[0]+"-best")
if err != nil {
log.Fatalln("Failed to download file", i, err)
}
}
- if parts[len(parts) - 1] == "conf" {
- fmt.Printf("Downloading %s to %s\n", i, parts[0] + "-conf")
- err = conn.Download(conn.WIPStorageId(), i, parts[0] + "-conf")
+ if parts[len(parts)-1] == "conf" {
+ fmt.Printf("Downloading %s to %s\n", i, parts[0]+"-conf")
+ err = conn.Download(conn.WIPStorageId(), i, parts[0]+"-conf")
if err != nil {
log.Fatalln("Failed to download file", i, err)
}
@@ -99,8 +99,8 @@ func main() {
}
if strings.HasSuffix(parts[1], ".hocr") {
- fmt.Printf("Downloading %s to %s\n", i, parts[0] + "-hocr")
- err = conn.Download(conn.WIPStorageId(), i, parts[0] + "-hocr")
+ fmt.Printf("Downloading %s to %s\n", i, parts[0]+"-hocr")
+ err = conn.Download(conn.WIPStorageId(), i, parts[0]+"-hocr")
if err != nil {
log.Fatalln("Failed to download file", i, err)
}
diff --git a/cmd/rescribe/getembeds.go b/cmd/rescribe/getembeds.go
index a3edaf2..1747d82 100644
--- a/cmd/rescribe/getembeds.go
+++ b/cmd/rescribe/getembeds.go
@@ -11,9 +11,9 @@ package main
import (
"fmt"
"io"
+ "net/http"
"os"
"path"
- "net/http"
)
func dl(url string) error {
@@ -40,7 +40,7 @@ func dl(url string) error {
}
func main() {
- urls := []string {
+ urls := []string{
"https://rescribe.xyz/rescribe/embeds/tessdata.20210622.zip",
"https://rescribe.xyz/rescribe/embeds/tesseract-linux-v5.0.0-alpha.20210510.zip",
"https://rescribe.xyz/rescribe/embeds/tesseract-osx-v4.1.1.20191227.zip",
diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go
index 4404789..db8eb41 100644
--- a/cmd/rescribe/main.go
+++ b/cmd/rescribe/main.go
@@ -46,6 +46,7 @@ var tessdatazip []byte
const QueueTimeoutSecs = 2 * 60
const PauseBetweenChecks = 1 * time.Second
const LogSaveTime = 1 * time.Minute
+
var thresholds = []float64{0.1, 0.2, 0.3}
// null writer to enable non-verbose logging to be discarded
@@ -322,8 +323,8 @@ These training files are included in rescribe, and are always available:
}
// For simplicity, remove .binarised.pdf and rename .colour.pdf to .pdf
- _ = os.Remove(filepath.Join(savedir, bookname + ".binarised.pdf"))
- _ = os.Rename(filepath.Join(savedir, bookname + ".colour.pdf"), filepath.Join(savedir, bookname + ".pdf"))
+ _ = os.Remove(filepath.Join(savedir, bookname+".binarised.pdf"))
+ _ = os.Rename(filepath.Join(savedir, bookname+".colour.pdf"), filepath.Join(savedir, bookname+".pdf"))
}
func addTxtVersion(hocrfn string) error {
@@ -342,7 +343,7 @@ func addTxtVersion(hocrfn string) error {
for _, v := range thresholds {
basefn = strings.TrimSuffix(basefn, fmt.Sprintf("_bin%.1f.hocr", v))
}
- fn := filepath.Join(dir, "text", basefn + ".txt")
+ fn := filepath.Join(dir, "text", basefn+".txt")
err = ioutil.WriteFile(fn, []byte(t), 0644)
if err != nil {
diff --git a/cmd/rmbook/main.go b/cmd/rmbook/main.go
index fcacc2e..a26c553 100644
--- a/cmd/rmbook/main.go
+++ b/cmd/rmbook/main.go
@@ -26,8 +26,8 @@ func (w NullWriter) Write(p []byte) (n int, err error) {
}
type RmPipeliner interface {
- MinimalInit() error
- WIPStorageId() string
+ MinimalInit() error
+ WIPStorageId() string
DeleteObjects(bucket string, keys []string) error
ListObjects(bucket string, prefix string) ([]string, error)
}
diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go
index 806f9be..4a08d3b 100644
--- a/internal/pipeline/pipeline_test.go
+++ b/internal/pipeline/pipeline_test.go
@@ -36,7 +36,7 @@ type PipelineTester interface {
type connection struct {
name string
- c PipelineTester
+ c PipelineTester
}
// Test_download tests the download() function inside the pipeline
@@ -53,11 +53,11 @@ func Test_download(t *testing.T) {
}
cases := []struct {
- dl string
+ dl string
contents []byte
- process string
- errs []error
- } {
+ process string
+ errs []error
+ }{
{"notpresent", []byte(""), "", []error{errors.New("no such file or directory"), errors.New("NoSuchKey: The specified key does not exist")}},
{"empty", []byte{}, "empty", []error{}},
{"justastring", []byte("I am just a basic string"), "justastring", []error{}},
@@ -73,7 +73,7 @@ func Test_download(t *testing.T) {
slog.log = ""
tempDir := filepath.Join(os.TempDir(), "pipelinetest")
err = os.MkdirAll(tempDir, 0700)
- if err != nil && ! os.IsExist(err) {
+ if err != nil && !os.IsExist(err) {
t.Fatalf("Could not create temporary directory %s: %v\nLog: %s", tempDir, err, slog.log)
}
@@ -174,11 +174,11 @@ func Test_up(t *testing.T) {
}
cases := []struct {
- ul string
+ ul string
contents []byte
- process string
- errs []error
- } {
+ process string
+ errs []error
+ }{
{"notpresent", []byte(""), "", []error{errors.New("no such file or directory"), errors.New("NoSuchKey: The specified key does not exist")}},
{"empty", []byte{}, "empty", []error{}},
{"justastring", []byte("I am just a basic string"), "justastring", []error{}},
@@ -194,7 +194,7 @@ func Test_up(t *testing.T) {
slog.log = ""
tempDir := filepath.Join(os.TempDir(), "pipelinetest")
err = os.MkdirAll(tempDir, 0700)
- if err != nil && ! os.IsExist(err) {
+ if err != nil && !os.IsExist(err) {
t.Fatalf("Could not create temporary directory %s: %v\nLog: %s", tempDir, err, slog.log)
}
@@ -244,7 +244,7 @@ func Test_up(t *testing.T) {
t.Fatalf("Uploaded file not removed as it should have been after uploading %s: %v\nLog: %s", tempFile, err, slog.log)
}
- err = conn.c.Download(conn.c.WIPStorageId(), "pipelinetest/" + c.ul, tempFile)
+ err = conn.c.Download(conn.c.WIPStorageId(), "pipelinetest/"+c.ul, tempFile)
if err != nil {
t.Fatalf("Could not download file %s: %v\nLog: %s", tempFile, err, slog.log)
}
@@ -292,11 +292,11 @@ func Test_upAndQueue(t *testing.T) {
}
cases := []struct {
- ul string
+ ul string
contents []byte
- process string
- errs []error
- } {
+ process string
+ errs []error
+ }{
{"notpresent", []byte(""), "", []error{errors.New("no such file or directory"), errors.New("NoSuchKey: The specified key does not exist")}},
{"empty", []byte{}, "empty", []error{}},
{"justastring", []byte("I am just a basic string"), "justastring", []error{}},
@@ -312,7 +312,7 @@ func Test_upAndQueue(t *testing.T) {
slog.log = ""
tempDir := filepath.Join(os.TempDir(), "pipelinetest")
err = os.MkdirAll(tempDir, 0700)
- if err != nil && ! os.IsExist(err) {
+ if err != nil && !os.IsExist(err) {
t.Fatalf("Could not create temporary directory %s: %v\nLog: %s", tempDir, err, slog.log)
}
@@ -376,7 +376,7 @@ func Test_upAndQueue(t *testing.T) {
t.Fatalf("Uploaded file not removed as it should have been after uploading %s: %v\nLog: %s", tempFile, err, slog.log)
}
- err = conn.c.Download(conn.c.WIPStorageId(), "pipelinetest/" + c.ul, tempFile)
+ err = conn.c.Download(conn.c.WIPStorageId(), "pipelinetest/"+c.ul, tempFile)
if err != nil {
t.Fatalf("Could not download file %s: %v\nLog: %s", tempFile, err, slog.log)
}
generated by cgit v1.2.1-24-ge1ad (git 2.21.0) at 2024-11-15 05:28:45 +0000