pipeline: use regular storage for tests, rather than a separate one

7 files changed, 19 insertions, 39 deletions

diff --git a/aws.go b/aws.go
index 8e9410f..0ca657f 100644
--- a/aws.go
+++ b/aws.go
@@ -44,16 +44,18 @@ type AwsConn struct {
 	Region string
 	Logger *log.Logger
 
-	sess                                      *session.Session
-	ec2svc                                    *ec2.EC2
-	s3svc                                     *s3.S3
-	sqssvc                                    *sqs.SQS
-	downloader                                *s3manager.Downloader
-	uploader                                  *s3manager.Uploader
-	wipequrl, prequrl, ocrpgqurl, analysequrl string
-	testqurl                                  string
-	teststorageid                             string
-	wipstorageid                              string
+	sess         *session.Session
+	ec2svc       *ec2.EC2
+	s3svc        *s3.S3
+	sqssvc       *sqs.SQS
+	downloader   *s3manager.Downloader
+	uploader     *s3manager.Uploader
+	wipequrl     string
+	prequrl      string
+	ocrpgqurl    string
+	analysequrl  string
+	testqurl     string
+	wipstorageid string
 }
 
 // MinimalInit does the bare minimum to initialise aws services
@@ -140,7 +142,6 @@ func (a *AwsConn) TestInit() error {
 		return errors.New(fmt.Sprintf("Error getting test queue URL: %s\n", err))
 	}
 	a.testqurl = *result.QueueUrl
-	a.teststorageid = storageTest
 	return nil
 }
 
@@ -348,10 +349,6 @@ func (a *AwsConn) AnalyseQueueId() string {
 	return a.analysequrl
 }
 
-func (a *AwsConn) TestStorageId() string {
-	return a.teststorageid
-}
-
 func (a *AwsConn) WIPStorageId() string {
 	return a.wipstorageid
 }
@@ -618,7 +615,7 @@ func (a *AwsConn) Log(v ...interface{}) {
 // mkpipeline sets up necessary buckets and queues for the pipeline
 // TODO: also set up the necessary security group and iam stuff
 func (a *AwsConn) MkPipeline() error {
-	buckets := []string{storageTest, storageWip}
+	buckets := []string{storageWip}
 	queues := []string{queuePreProc, queueWipeOnly, queueAnalyse, queueOcrPage, queueTest}
 
 	for _, bucket := range buckets {
diff --git a/cloudsettings.go b/cloudsettings.go
index befb4b4..fa55742 100644
--- a/cloudsettings.go
+++ b/cloudsettings.go
@@ -36,5 +36,4 @@ const (
 // Storage bucket names. Can be anything unique in S3.
 const (
 	storageWip = "rescribeinprogress"
-	storageTest = "rescribetest"
 )
diff --git a/cmd/booktopipeline/main.go b/cmd/booktopipeline/main.go
index 1b16cee..b4f4d99 100644
--- a/cmd/booktopipeline/main.go
+++ b/cmd/booktopipeline/main.go
@@ -112,7 +112,7 @@ func main() {
 	}
 
 	verboselog.Println("Uploading all images are valid in", bookdir)
-	err = pipeline.UploadImages(bookdir, bookname, conn, conn.WIPStorageId())
+	err = pipeline.UploadImages(bookdir, bookname, conn)
 	if err != nil {
 		log.Fatalln(err)
 	}
diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go
index e249138..251c333 100644
--- a/cmd/rescribe/main.go
+++ b/cmd/rescribe/main.go
@@ -362,7 +362,7 @@ func uploadbook(dir string, name string, conn Pipeliner) error {
 	if err != nil {
 		return fmt.Errorf("Error with images in %s: %v", dir, err)
 	}
-	err = pipeline.UploadImages(dir, name, conn, conn.WIPStorageId())
+	err = pipeline.UploadImages(dir, name, conn)
 	if err != nil {
 		return fmt.Errorf("Error saving images to process from %s: %v", dir, err)
 	}
diff --git a/internal/pipeline/put.go b/internal/pipeline/put.go
index 26d581c..8cbecac 100644
--- a/internal/pipeline/put.go
+++ b/internal/pipeline/put.go
@@ -78,9 +78,9 @@ func DetectQueueType(dir string, conn Queuer) string {
 }
 
 // UploadImages uploads all files (except those which start with a ".")
-// from a directory (recursively) into a given storage id, prefixed with
+// from a directory (recursively) into conn.WIPStorageId(), prefixed with
 // the given bookname and a slash
-func UploadImages(dir string, bookname string, conn Uploader, storageId string) error {
+func UploadImages(dir string, bookname string, conn Uploader) error {
 	walker := make(fileWalk)
 	go func() {
 		_ = filepath.Walk(dir, walker.Walk)
@@ -89,7 +89,7 @@ func UploadImages(dir string, bookname string, conn Uploader, storageId string)
 
 	for path := range walker {
 		name := filepath.Base(path)
-		err := conn.Upload(storageId, bookname + "/" + name, path)
+		err := conn.Upload(conn.WIPStorageId(), bookname + "/" + name, path)
 		if err != nil {
 			return fmt.Errorf("Failed to upload %s: %v", path, err)
 		}
diff --git a/internal/pipeline/put_test.go b/internal/pipeline/put_test.go
index 912a4ef..5fb5b43 100644
--- a/internal/pipeline/put_test.go
+++ b/internal/pipeline/put_test.go
@@ -92,18 +92,13 @@ func Test_UploadImages(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Could not initialise %s connection: %v\nLog: %s", conn.name, err, slog.log)
 			}
-			err = conn.c.TestInit()
-			if err != nil {
-				t.Fatalf("Failed in test initialisalisation for %s: %v\nLog: %s", conn.name, err, slog.log)
-			}
 			slog.log = ""
 
-			err = UploadImages("testdata/good", "good", conn.c, conn.c.TestStorageId())
+			err = UploadImages("testdata/good", "good", conn.c)
 			if err != nil {
 				t.Fatalf("Error in UploadImages for %s: %v\nLog: %s", conn.name, err, slog.log)
 			}
 
-			// TODO: decide on whether to use TestStorageId or just the WIPStorageId as with other tests, and align other tests to this if needed
 			// TODO: download all files and test that they match
 			// TODO: remove test files from conn storage
 		})
diff --git a/local.go b/local.go
index 58830c7..615f9a6 100644
--- a/local.go
+++ b/local.go
@@ -21,7 +21,6 @@ const qidOCR = "queueOCR"
 const qidAnalyse = "queueAnalyse"
 const qidTest = "queueTest"
 const storageId = "storage"
-const testStorageId = "test"
 
 // LocalConn is a simple implementation of the pipeliner interface
 // that doesn't rely on any "cloud" services, instead doing everything
@@ -67,11 +66,6 @@ func (a *LocalConn) Init() error {
 
 // TestInit does nothing for local connections
 func (a *LocalConn) TestInit() error {
-	err := os.Mkdir(filepath.Join(a.TempDir, testStorageId), 0700)
-	if err != nil && !os.IsExist(err) {
-		return fmt.Errorf("Error creating test storage directory: %v", err)
-	}
-
 	return nil
 }
 
@@ -140,11 +134,6 @@ func (a *LocalConn) WIPStorageId() string {
 	return storageId
 }
 
-func (a *LocalConn) TestStorageId() string {
-	return testStorageId
-}
-
-
 func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkFunc {
 	return func(path string, info os.FileInfo, err error) error {
 		if err != nil {