summaryrefslogtreecommitdiff
path: root/internal/pipeline
diff options
context:
space:
mode:
Diffstat (limited to 'internal/pipeline')
-rw-r--r--internal/pipeline/get.go53
1 files changed, 30 insertions, 23 deletions
diff --git a/internal/pipeline/get.go b/internal/pipeline/get.go
index 6949062..6c5b92c 100644
--- a/internal/pipeline/get.go
+++ b/internal/pipeline/get.go
@@ -12,9 +12,10 @@ import (
"strings"
)
-func DownloadBestPages(name string, conn Pipeliner, pluspngs bool) error {
- fn := filepath.Join(name, "best")
- err := conn.Download(conn.WIPStorageId(), fn, fn)
+func DownloadBestPages(dir string, name string, conn Pipeliner, pluspngs bool) error {
+ key := filepath.Join(name, "best")
+ fn := filepath.Join(dir, "best")
+ err := conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
return fmt.Errorf("Failed to download 'best' file: %v", err)
}
@@ -26,11 +27,12 @@ func DownloadBestPages(name string, conn Pipeliner, pluspngs bool) error {
s := bufio.NewScanner(f)
for s.Scan() {
- fn = filepath.Join(name, s.Text())
- conn.Log("Downloading file", fn)
- err = conn.Download(conn.WIPStorageId(), fn, fn)
+ key = filepath.Join(name, s.Text())
+ fn = filepath.Join(dir, s.Text())
+ conn.Log("Downloading file", key)
+ err = conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
- return fmt.Errorf("Failed to download file %s: %v", fn, err)
+ return fmt.Errorf("Failed to download file %s: %v", key, err)
}
}
@@ -40,49 +42,54 @@ func DownloadBestPages(name string, conn Pipeliner, pluspngs bool) error {
s = bufio.NewScanner(f)
for s.Scan() {
- txtfn := filepath.Join(name, s.Text())
- fn = strings.Replace(txtfn, ".hocr", ".png", 1)
- conn.Log("Downloading file", fn)
- err = conn.Download(conn.WIPStorageId(), fn, fn)
+ imgname := strings.Replace(s.Text(), ".hocr", ".png", 1)
+ key = filepath.Join(name, imgname)
+ fn = filepath.Join(dir, imgname)
+ conn.Log("Downloading file", key)
+ err = conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
- return fmt.Errorf("Failed to download file", fn, err)
+ return fmt.Errorf("Failed to download file %s: %v", key, err)
}
}
return nil
}
-func DownloadPdfs(name string, conn Pipeliner) error {
+func DownloadPdfs(dir string, name string, conn Pipeliner) error {
for _, suffix := range []string{".colour.pdf", ".binarised.pdf"} {
- fn := filepath.Join(name, name+suffix)
- err := conn.Download(conn.WIPStorageId(), fn, fn)
+ key := filepath.Join(name, name+suffix)
+ fn := filepath.Join(dir, name+suffix)
+ err := conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
- return fmt.Errorf("Failed to download PDF %s: %v", fn, err)
+ return fmt.Errorf("Failed to download PDF %s: %v", key, err)
}
}
return nil
}
-func DownloadAnalyses(name string, conn Pipeliner) error {
+func DownloadAnalyses(dir string, name string, conn Pipeliner) error {
for _, a := range []string{"conf", "graph.png"} {
- fn := filepath.Join(name, a)
- err := conn.Download(conn.WIPStorageId(), fn, fn)
+ key := filepath.Join(name, a)
+ fn := filepath.Join(dir, a)
+ err := conn.Download(conn.WIPStorageId(), key, fn)
if err != nil {
- return fmt.Errorf("Failed to download analysis file %s: %v", fn, err)
+ return fmt.Errorf("Failed to download analysis file %s: %v", key, err)
}
}
return nil
}
-func DownloadAll(name string, conn Pipeliner) error {
+func DownloadAll(dir string, name string, conn Pipeliner) error {
objs, err := conn.ListObjects(conn.WIPStorageId(), name)
if err != nil {
return fmt.Errorf("Failed to get list of files for book", name, err)
}
for _, i := range objs {
+ base := filepath.Base(i)
+ fn := filepath.Join(dir, base)
conn.Log("Downloading", i)
- err = conn.Download(conn.WIPStorageId(), i, i)
+ err = conn.Download(conn.WIPStorageId(), i, fn)
if err != nil {
- return fmt.Errorf("Failed to download file", i, err)
+ return fmt.Errorf("Failed to download file %s: %v", i, err)
}
}
return nil