internal/pipeline: Add test (incomplete but working) for UploadImages

8 files changed, 63 insertions, 5 deletions

diff --git a/aws.go b/aws.go
index 21a215c..8e9410f 100644
--- a/aws.go
+++ b/aws.go
@@ -52,6 +52,7 @@ type AwsConn struct {
 	uploader                                  *s3manager.Uploader
 	wipequrl, prequrl, ocrpgqurl, analysequrl string
 	testqurl                                  string
+	teststorageid                             string
 	wipstorageid                              string
 }
 
@@ -139,6 +140,7 @@ func (a *AwsConn) TestInit() error {
 		return errors.New(fmt.Sprintf("Error getting test queue URL: %s\n", err))
 	}
 	a.testqurl = *result.QueueUrl
+	a.teststorageid = storageTest
 	return nil
 }
 
@@ -346,6 +348,10 @@ func (a *AwsConn) AnalyseQueueId() string {
 	return a.analysequrl
 }
 
+func (a *AwsConn) TestStorageId() string {
+	return a.teststorageid
+}
+
 func (a *AwsConn) WIPStorageId() string {
 	return a.wipstorageid
 }
@@ -612,7 +618,7 @@ func (a *AwsConn) Log(v ...interface{}) {
 // mkpipeline sets up necessary buckets and queues for the pipeline
 // TODO: also set up the necessary security group and iam stuff
 func (a *AwsConn) MkPipeline() error {
-	buckets := []string{storageWip}
+	buckets := []string{storageTest, storageWip}
 	queues := []string{queuePreProc, queueWipeOnly, queueAnalyse, queueOcrPage, queueTest}
 
 	for _, bucket := range buckets {
diff --git a/cloudsettings.go b/cloudsettings.go
index fa55742..befb4b4 100644
--- a/cloudsettings.go
+++ b/cloudsettings.go
@@ -36,4 +36,5 @@ const (
 // Storage bucket names. Can be anything unique in S3.
 const (
 	storageWip = "rescribeinprogress"
+	storageTest = "rescribetest"
 )
diff --git a/cmd/booktopipeline/main.go b/cmd/booktopipeline/main.go
index b4f4d99..1b16cee 100644
--- a/cmd/booktopipeline/main.go
+++ b/cmd/booktopipeline/main.go
@@ -112,7 +112,7 @@ func main() {
 	}
 
 	verboselog.Println("Uploading all images are valid in", bookdir)
-	err = pipeline.UploadImages(bookdir, bookname, conn)
+	err = pipeline.UploadImages(bookdir, bookname, conn, conn.WIPStorageId())
 	if err != nil {
 		log.Fatalln(err)
 	}
diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go
index 251c333..e249138 100644
--- a/cmd/rescribe/main.go
+++ b/cmd/rescribe/main.go
@@ -362,7 +362,7 @@ func uploadbook(dir string, name string, conn Pipeliner) error {
 	if err != nil {
 		return fmt.Errorf("Error with images in %s: %v", dir, err)
 	}
-	err = pipeline.UploadImages(dir, name, conn)
+	err = pipeline.UploadImages(dir, name, conn, conn.WIPStorageId())
 	if err != nil {
 		return fmt.Errorf("Error saving images to process from %s: %v", dir, err)
 	}
diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go
index 73e8223..fc223b3 100644
--- a/internal/pipeline/pipeline_test.go
+++ b/internal/pipeline/pipeline_test.go
@@ -33,6 +33,7 @@ type PipelineTester interface {
 	TestInit() error
 	DeleteObjects(bucket string, keys []string) error
 	TestQueueId() string
+	TestStorageId() string
 }
 
 type connection struct {
diff --git a/internal/pipeline/put.go b/internal/pipeline/put.go
index c08df53..26d581c 100644
--- a/internal/pipeline/put.go
+++ b/internal/pipeline/put.go
@@ -77,7 +77,10 @@ func DetectQueueType(dir string, conn Queuer) string {
 	}
 }
 
-func UploadImages(dir string, bookname string, conn Uploader) error {
+// UploadImages uploads all files (except those which start with a ".")
+// from a directory (recursively) into a given storage id, prefixed with
+// the given bookname and a slash
+func UploadImages(dir string, bookname string, conn Uploader, storageId string) error {
 	walker := make(fileWalk)
 	go func() {
 		_ = filepath.Walk(dir, walker.Walk)
@@ -86,7 +89,7 @@ func UploadImages(dir string, bookname string, conn Uploader) error {
 
 	for path := range walker {
 		name := filepath.Base(path)
-		err := conn.Upload(conn.WIPStorageId(), filepath.Join(bookname, name), path)
+		err := conn.Upload(storageId, bookname + "/" + name, path)
 		if err != nil {
 			return fmt.Errorf("Failed to upload %s: %v", path, err)
 		}
diff --git a/internal/pipeline/put_test.go b/internal/pipeline/put_test.go
index af18ab4..912a4ef 100644
--- a/internal/pipeline/put_test.go
+++ b/internal/pipeline/put_test.go
@@ -6,6 +6,7 @@ package pipeline
 
 import (
 	"errors"
+	"log"
 	"os"
 	"rescribe.xyz/bookpipeline"
 	"testing"
@@ -73,3 +74,38 @@ func Test_DetectQueueType(t *testing.T) {
 		})
 	}
 }
+
+func Test_UploadImages(t *testing.T) {
+	var slog StrLog
+        vlog := log.New(&slog, "", 0)
+	var conns []connection
+
+	conns = append(conns, connection{name: "local", c: &bookpipeline.LocalConn{Logger: vlog}})
+
+	if !testing.Short() {
+		conns = append(conns, connection{name: "aws", c: &bookpipeline.AwsConn{Logger: vlog}})
+	}
+
+	for _, conn := range conns {
+		t.Run(conn.name, func(t *testing.T) {
+			err := conn.c.Init()
+			if err != nil {
+				t.Fatalf("Could not initialise %s connection: %v\nLog: %s", conn.name, err, slog.log)
+			}
+			err = conn.c.TestInit()
+			if err != nil {
+				t.Fatalf("Failed in test initialisalisation for %s: %v\nLog: %s", conn.name, err, slog.log)
+			}
+			slog.log = ""
+
+			err = UploadImages("testdata/good", "good", conn.c, conn.c.TestStorageId())
+			if err != nil {
+				t.Fatalf("Error in UploadImages for %s: %v\nLog: %s", conn.name, err, slog.log)
+			}
+
+			// TODO: decide on whether to use TestStorageId or just the WIPStorageId as with other tests, and align other tests to this if needed
+			// TODO: download all files and test that they match
+			// TODO: remove test files from conn storage
+		})
+	}
+}
diff --git a/local.go b/local.go
index 615f9a6..58830c7 100644
--- a/local.go
+++ b/local.go
@@ -21,6 +21,7 @@ const qidOCR = "queueOCR"
 const qidAnalyse = "queueAnalyse"
 const qidTest = "queueTest"
 const storageId = "storage"
+const testStorageId = "test"
 
 // LocalConn is a simple implementation of the pipeliner interface
 // that doesn't rely on any "cloud" services, instead doing everything
@@ -66,6 +67,11 @@ func (a *LocalConn) Init() error {
 
 // TestInit does nothing for local connections
 func (a *LocalConn) TestInit() error {
+	err := os.Mkdir(filepath.Join(a.TempDir, testStorageId), 0700)
+	if err != nil && !os.IsExist(err) {
+		return fmt.Errorf("Error creating test storage directory: %v", err)
+	}
+
 	return nil
 }
 
@@ -134,6 +140,11 @@ func (a *LocalConn) WIPStorageId() string {
 	return storageId
 }
 
+func (a *LocalConn) TestStorageId() string {
+	return testStorageId
+}
+
+
 func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkFunc {
 	return func(path string, info os.FileInfo, err error) error {
 		if err != nil {