summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2021-08-17 13:18:04 +0100
committerNick White <git@njw.name>2021-08-17 13:18:04 +0100
commit48c68cfa7f18992b26765c7b67c52c11687ad74a (patch)
tree6194e21f03f381e1886534ca2ffd3830591bb461
parent2bac685f600b97676de14bb850e5b087f0d54b83 (diff)
pipeline: use regular storage for tests, rather than a separate one
-rw-r--r--aws.go29
-rw-r--r--cloudsettings.go1
-rw-r--r--cmd/booktopipeline/main.go2
-rw-r--r--cmd/rescribe/main.go2
-rw-r--r--internal/pipeline/put.go6
-rw-r--r--internal/pipeline/put_test.go7
-rw-r--r--local.go11
7 files changed, 19 insertions, 39 deletions
diff --git a/aws.go b/aws.go
index 8e9410f..0ca657f 100644
--- a/aws.go
+++ b/aws.go
@@ -44,16 +44,18 @@ type AwsConn struct {
Region string
Logger *log.Logger
- sess *session.Session
- ec2svc *ec2.EC2
- s3svc *s3.S3
- sqssvc *sqs.SQS
- downloader *s3manager.Downloader
- uploader *s3manager.Uploader
- wipequrl, prequrl, ocrpgqurl, analysequrl string
- testqurl string
- teststorageid string
- wipstorageid string
+ sess *session.Session
+ ec2svc *ec2.EC2
+ s3svc *s3.S3
+ sqssvc *sqs.SQS
+ downloader *s3manager.Downloader
+ uploader *s3manager.Uploader
+ wipequrl string
+ prequrl string
+ ocrpgqurl string
+ analysequrl string
+ testqurl string
+ wipstorageid string
}
// MinimalInit does the bare minimum to initialise aws services
@@ -140,7 +142,6 @@ func (a *AwsConn) TestInit() error {
return errors.New(fmt.Sprintf("Error getting test queue URL: %s\n", err))
}
a.testqurl = *result.QueueUrl
- a.teststorageid = storageTest
return nil
}
@@ -348,10 +349,6 @@ func (a *AwsConn) AnalyseQueueId() string {
return a.analysequrl
}
-func (a *AwsConn) TestStorageId() string {
- return a.teststorageid
-}
-
func (a *AwsConn) WIPStorageId() string {
return a.wipstorageid
}
@@ -618,7 +615,7 @@ func (a *AwsConn) Log(v ...interface{}) {
// mkpipeline sets up necessary buckets and queues for the pipeline
// TODO: also set up the necessary security group and iam stuff
func (a *AwsConn) MkPipeline() error {
- buckets := []string{storageTest, storageWip}
+ buckets := []string{storageWip}
queues := []string{queuePreProc, queueWipeOnly, queueAnalyse, queueOcrPage, queueTest}
for _, bucket := range buckets {
diff --git a/cloudsettings.go b/cloudsettings.go
index befb4b4..fa55742 100644
--- a/cloudsettings.go
+++ b/cloudsettings.go
@@ -36,5 +36,4 @@ const (
// Storage bucket names. Can be anything unique in S3.
const (
storageWip = "rescribeinprogress"
- storageTest = "rescribetest"
)
diff --git a/cmd/booktopipeline/main.go b/cmd/booktopipeline/main.go
index 1b16cee..b4f4d99 100644
--- a/cmd/booktopipeline/main.go
+++ b/cmd/booktopipeline/main.go
@@ -112,7 +112,7 @@ func main() {
}
verboselog.Println("Uploading all images are valid in", bookdir)
- err = pipeline.UploadImages(bookdir, bookname, conn, conn.WIPStorageId())
+ err = pipeline.UploadImages(bookdir, bookname, conn)
if err != nil {
log.Fatalln(err)
}
diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go
index e249138..251c333 100644
--- a/cmd/rescribe/main.go
+++ b/cmd/rescribe/main.go
@@ -362,7 +362,7 @@ func uploadbook(dir string, name string, conn Pipeliner) error {
if err != nil {
return fmt.Errorf("Error with images in %s: %v", dir, err)
}
- err = pipeline.UploadImages(dir, name, conn, conn.WIPStorageId())
+ err = pipeline.UploadImages(dir, name, conn)
if err != nil {
return fmt.Errorf("Error saving images to process from %s: %v", dir, err)
}
diff --git a/internal/pipeline/put.go b/internal/pipeline/put.go
index 26d581c..8cbecac 100644
--- a/internal/pipeline/put.go
+++ b/internal/pipeline/put.go
@@ -78,9 +78,9 @@ func DetectQueueType(dir string, conn Queuer) string {
}
// UploadImages uploads all files (except those which start with a ".")
-// from a directory (recursively) into a given storage id, prefixed with
+// from a directory (recursively) into conn.WIPStorageId(), prefixed with
// the given bookname and a slash
-func UploadImages(dir string, bookname string, conn Uploader, storageId string) error {
+func UploadImages(dir string, bookname string, conn Uploader) error {
walker := make(fileWalk)
go func() {
_ = filepath.Walk(dir, walker.Walk)
@@ -89,7 +89,7 @@ func UploadImages(dir string, bookname string, conn Uploader, storageId string)
for path := range walker {
name := filepath.Base(path)
- err := conn.Upload(storageId, bookname + "/" + name, path)
+ err := conn.Upload(conn.WIPStorageId(), bookname + "/" + name, path)
if err != nil {
return fmt.Errorf("Failed to upload %s: %v", path, err)
}
diff --git a/internal/pipeline/put_test.go b/internal/pipeline/put_test.go
index 912a4ef..5fb5b43 100644
--- a/internal/pipeline/put_test.go
+++ b/internal/pipeline/put_test.go
@@ -92,18 +92,13 @@ func Test_UploadImages(t *testing.T) {
if err != nil {
t.Fatalf("Could not initialise %s connection: %v\nLog: %s", conn.name, err, slog.log)
}
- err = conn.c.TestInit()
- if err != nil {
- t.Fatalf("Failed in test initialisalisation for %s: %v\nLog: %s", conn.name, err, slog.log)
- }
slog.log = ""
- err = UploadImages("testdata/good", "good", conn.c, conn.c.TestStorageId())
+ err = UploadImages("testdata/good", "good", conn.c)
if err != nil {
t.Fatalf("Error in UploadImages for %s: %v\nLog: %s", conn.name, err, slog.log)
}
- // TODO: decide on whether to use TestStorageId or just the WIPStorageId as with other tests, and align other tests to this if needed
// TODO: download all files and test that they match
// TODO: remove test files from conn storage
})
diff --git a/local.go b/local.go
index 58830c7..615f9a6 100644
--- a/local.go
+++ b/local.go
@@ -21,7 +21,6 @@ const qidOCR = "queueOCR"
const qidAnalyse = "queueAnalyse"
const qidTest = "queueTest"
const storageId = "storage"
-const testStorageId = "test"
// LocalConn is a simple implementation of the pipeliner interface
// that doesn't rely on any "cloud" services, instead doing everything
@@ -67,11 +66,6 @@ func (a *LocalConn) Init() error {
// TestInit does nothing for local connections
func (a *LocalConn) TestInit() error {
- err := os.Mkdir(filepath.Join(a.TempDir, testStorageId), 0700)
- if err != nil && !os.IsExist(err) {
- return fmt.Errorf("Error creating test storage directory: %v", err)
- }
-
return nil
}
@@ -140,11 +134,6 @@ func (a *LocalConn) WIPStorageId() string {
return storageId
}
-func (a *LocalConn) TestStorageId() string {
- return testStorageId
-}
-
-
func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkFunc {
return func(path string, info os.FileInfo, err error) error {
if err != nil {