summaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
Diffstat (limited to 'internal')
-rw-r--r--internal/pipeline/get.go33
-rw-r--r--internal/pipeline/pipeline.go5
2 files changed, 37 insertions, 1 deletions
diff --git a/internal/pipeline/get.go b/internal/pipeline/get.go
index 8492d99..6949062 100644
--- a/internal/pipeline/get.go
+++ b/internal/pipeline/get.go
@@ -9,9 +9,10 @@ import (
"fmt"
"os"
"path/filepath"
+ "strings"
)
-func DownloadBestPages(name string, conn Pipeliner) error {
+func DownloadBestPages(name string, conn Pipeliner, pluspngs bool) error {
fn := filepath.Join(name, "best")
err := conn.Download(conn.WIPStorageId(), fn, fn)
if err != nil {
@@ -26,12 +27,27 @@ func DownloadBestPages(name string, conn Pipeliner) error {
s := bufio.NewScanner(f)
for s.Scan() {
fn = filepath.Join(name, s.Text())
+ conn.Log("Downloading file", fn)
err = conn.Download(conn.WIPStorageId(), fn, fn)
if err != nil {
return fmt.Errorf("Failed to download file %s: %v", fn, err)
}
}
+ if !pluspngs {
+ return nil
+ }
+
+ s = bufio.NewScanner(f)
+ for s.Scan() {
+ txtfn := filepath.Join(name, s.Text())
+ fn = strings.Replace(txtfn, ".hocr", ".png", 1)
+ conn.Log("Downloading file", fn)
+ err = conn.Download(conn.WIPStorageId(), fn, fn)
+ if err != nil {
+ return fmt.Errorf("Failed to download file", fn, err)
+ }
+ }
return nil
}
@@ -56,3 +72,18 @@ func DownloadAnalyses(name string, conn Pipeliner) error {
}
return nil
}
+
+func DownloadAll(name string, conn Pipeliner) error {
+ objs, err := conn.ListObjects(conn.WIPStorageId(), name)
+ if err != nil {
+ return fmt.Errorf("Failed to get list of files for book", name, err)
+ }
+ for _, i := range objs {
+ conn.Log("Downloading", i)
+ err = conn.Download(conn.WIPStorageId(), i, i)
+ if err != nil {
+ return fmt.Errorf("Failed to download file", i, err)
+ }
+ }
+ return nil
+}
diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go
index cce5c19..c0accdb 100644
--- a/internal/pipeline/pipeline.go
+++ b/internal/pipeline/pipeline.go
@@ -52,6 +52,11 @@ type Pipeliner interface {
Log(v ...interface{})
}
+type MinPipeliner interface {
+ Pipeliner
+ MinimalInit() error
+}
+
type pageimg struct {
hocr, img string
}