summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--aws.go8
-rw-r--r--cloudsettings.go1
-rw-r--r--cmd/booktopipeline/main.go2
-rw-r--r--cmd/rescribe/main.go2
-rw-r--r--internal/pipeline/pipeline_test.go1
-rw-r--r--internal/pipeline/put.go7
-rw-r--r--internal/pipeline/put_test.go36
-rw-r--r--local.go11
8 files changed, 63 insertions, 5 deletions
diff --git a/aws.go b/aws.go
index 21a215c..8e9410f 100644
--- a/aws.go
+++ b/aws.go
@@ -52,6 +52,7 @@ type AwsConn struct {
uploader *s3manager.Uploader
wipequrl, prequrl, ocrpgqurl, analysequrl string
testqurl string
+ teststorageid string
wipstorageid string
}
@@ -139,6 +140,7 @@ func (a *AwsConn) TestInit() error {
return errors.New(fmt.Sprintf("Error getting test queue URL: %s\n", err))
}
a.testqurl = *result.QueueUrl
+ a.teststorageid = storageTest
return nil
}
@@ -346,6 +348,10 @@ func (a *AwsConn) AnalyseQueueId() string {
return a.analysequrl
}
+func (a *AwsConn) TestStorageId() string {
+ return a.teststorageid
+}
+
func (a *AwsConn) WIPStorageId() string {
return a.wipstorageid
}
@@ -612,7 +618,7 @@ func (a *AwsConn) Log(v ...interface{}) {
// mkpipeline sets up necessary buckets and queues for the pipeline
// TODO: also set up the necessary security group and iam stuff
func (a *AwsConn) MkPipeline() error {
- buckets := []string{storageWip}
+ buckets := []string{storageTest, storageWip}
queues := []string{queuePreProc, queueWipeOnly, queueAnalyse, queueOcrPage, queueTest}
for _, bucket := range buckets {
diff --git a/cloudsettings.go b/cloudsettings.go
index fa55742..befb4b4 100644
--- a/cloudsettings.go
+++ b/cloudsettings.go
@@ -36,4 +36,5 @@ const (
// Storage bucket names. Can be anything unique in S3.
const (
storageWip = "rescribeinprogress"
+ storageTest = "rescribetest"
)
diff --git a/cmd/booktopipeline/main.go b/cmd/booktopipeline/main.go
index b4f4d99..1b16cee 100644
--- a/cmd/booktopipeline/main.go
+++ b/cmd/booktopipeline/main.go
@@ -112,7 +112,7 @@ func main() {
}
verboselog.Println("Uploading all images are valid in", bookdir)
- err = pipeline.UploadImages(bookdir, bookname, conn)
+ err = pipeline.UploadImages(bookdir, bookname, conn, conn.WIPStorageId())
if err != nil {
log.Fatalln(err)
}
diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go
index 251c333..e249138 100644
--- a/cmd/rescribe/main.go
+++ b/cmd/rescribe/main.go
@@ -362,7 +362,7 @@ func uploadbook(dir string, name string, conn Pipeliner) error {
if err != nil {
return fmt.Errorf("Error with images in %s: %v", dir, err)
}
- err = pipeline.UploadImages(dir, name, conn)
+ err = pipeline.UploadImages(dir, name, conn, conn.WIPStorageId())
if err != nil {
return fmt.Errorf("Error saving images to process from %s: %v", dir, err)
}
diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go
index 73e8223..fc223b3 100644
--- a/internal/pipeline/pipeline_test.go
+++ b/internal/pipeline/pipeline_test.go
@@ -33,6 +33,7 @@ type PipelineTester interface {
TestInit() error
DeleteObjects(bucket string, keys []string) error
TestQueueId() string
+ TestStorageId() string
}
type connection struct {
diff --git a/internal/pipeline/put.go b/internal/pipeline/put.go
index c08df53..26d581c 100644
--- a/internal/pipeline/put.go
+++ b/internal/pipeline/put.go
@@ -77,7 +77,10 @@ func DetectQueueType(dir string, conn Queuer) string {
}
}
-func UploadImages(dir string, bookname string, conn Uploader) error {
+// UploadImages uploads all files (except those which start with a ".")
+// from a directory (recursively) into a given storage id, prefixed with
+// the given bookname and a slash
+func UploadImages(dir string, bookname string, conn Uploader, storageId string) error {
walker := make(fileWalk)
go func() {
_ = filepath.Walk(dir, walker.Walk)
@@ -86,7 +89,7 @@ func UploadImages(dir string, bookname string, conn Uploader) error {
for path := range walker {
name := filepath.Base(path)
- err := conn.Upload(conn.WIPStorageId(), filepath.Join(bookname, name), path)
+ err := conn.Upload(storageId, bookname + "/" + name, path)
if err != nil {
return fmt.Errorf("Failed to upload %s: %v", path, err)
}
diff --git a/internal/pipeline/put_test.go b/internal/pipeline/put_test.go
index af18ab4..912a4ef 100644
--- a/internal/pipeline/put_test.go
+++ b/internal/pipeline/put_test.go
@@ -6,6 +6,7 @@ package pipeline
import (
"errors"
+ "log"
"os"
"rescribe.xyz/bookpipeline"
"testing"
@@ -73,3 +74,38 @@ func Test_DetectQueueType(t *testing.T) {
})
}
}
+
+func Test_UploadImages(t *testing.T) {
+ var slog StrLog
+ vlog := log.New(&slog, "", 0)
+ var conns []connection
+
+ conns = append(conns, connection{name: "local", c: &bookpipeline.LocalConn{Logger: vlog}})
+
+ if !testing.Short() {
+ conns = append(conns, connection{name: "aws", c: &bookpipeline.AwsConn{Logger: vlog}})
+ }
+
+ for _, conn := range conns {
+ t.Run(conn.name, func(t *testing.T) {
+ err := conn.c.Init()
+ if err != nil {
+ t.Fatalf("Could not initialise %s connection: %v\nLog: %s", conn.name, err, slog.log)
+ }
+ err = conn.c.TestInit()
+ if err != nil {
+ t.Fatalf("Failed in test initialisalisation for %s: %v\nLog: %s", conn.name, err, slog.log)
+ }
+ slog.log = ""
+
+ err = UploadImages("testdata/good", "good", conn.c, conn.c.TestStorageId())
+ if err != nil {
+ t.Fatalf("Error in UploadImages for %s: %v\nLog: %s", conn.name, err, slog.log)
+ }
+
+ // TODO: decide on whether to use TestStorageId or just the WIPStorageId as with other tests, and align other tests to this if needed
+ // TODO: download all files and test that they match
+ // TODO: remove test files from conn storage
+ })
+ }
+}
diff --git a/local.go b/local.go
index 615f9a6..58830c7 100644
--- a/local.go
+++ b/local.go
@@ -21,6 +21,7 @@ const qidOCR = "queueOCR"
const qidAnalyse = "queueAnalyse"
const qidTest = "queueTest"
const storageId = "storage"
+const testStorageId = "test"
// LocalConn is a simple implementation of the pipeliner interface
// that doesn't rely on any "cloud" services, instead doing everything
@@ -66,6 +67,11 @@ func (a *LocalConn) Init() error {
// TestInit does nothing for local connections
func (a *LocalConn) TestInit() error {
+ err := os.Mkdir(filepath.Join(a.TempDir, testStorageId), 0700)
+ if err != nil && !os.IsExist(err) {
+ return fmt.Errorf("Error creating test storage directory: %v", err)
+ }
+
return nil
}
@@ -134,6 +140,11 @@ func (a *LocalConn) WIPStorageId() string {
return storageId
}
+func (a *LocalConn) TestStorageId() string {
+ return testStorageId
+}
+
+
func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkFunc {
return func(path string, info os.FileInfo, err error) error {
if err != nil {
generated by cgit v1.2.1-24-ge1ad (git 2.21.0) at 2025-10-30 03:27:13 +0000