diff options
| -rw-r--r-- | aws.go | 29 | ||||
| -rw-r--r-- | cloudsettings.go | 1 | ||||
| -rw-r--r-- | cmd/booktopipeline/main.go | 2 | ||||
| -rw-r--r-- | cmd/rescribe/main.go | 2 | ||||
| -rw-r--r-- | internal/pipeline/put.go | 6 | ||||
| -rw-r--r-- | internal/pipeline/put_test.go | 7 | ||||
| -rw-r--r-- | local.go | 11 |
7 files changed, 19 insertions, 39 deletions
@@ -44,16 +44,18 @@ type AwsConn struct { Region string Logger *log.Logger - sess *session.Session - ec2svc *ec2.EC2 - s3svc *s3.S3 - sqssvc *sqs.SQS - downloader *s3manager.Downloader - uploader *s3manager.Uploader - wipequrl, prequrl, ocrpgqurl, analysequrl string - testqurl string - teststorageid string - wipstorageid string + sess *session.Session + ec2svc *ec2.EC2 + s3svc *s3.S3 + sqssvc *sqs.SQS + downloader *s3manager.Downloader + uploader *s3manager.Uploader + wipequrl string + prequrl string + ocrpgqurl string + analysequrl string + testqurl string + wipstorageid string } // MinimalInit does the bare minimum to initialise aws services @@ -140,7 +142,6 @@ func (a *AwsConn) TestInit() error { return errors.New(fmt.Sprintf("Error getting test queue URL: %s\n", err)) } a.testqurl = *result.QueueUrl - a.teststorageid = storageTest return nil } @@ -348,10 +349,6 @@ func (a *AwsConn) AnalyseQueueId() string { return a.analysequrl } -func (a *AwsConn) TestStorageId() string { - return a.teststorageid -} - func (a *AwsConn) WIPStorageId() string { return a.wipstorageid } @@ -618,7 +615,7 @@ func (a *AwsConn) Log(v ...interface{}) { // mkpipeline sets up necessary buckets and queues for the pipeline // TODO: also set up the necessary security group and iam stuff func (a *AwsConn) MkPipeline() error { - buckets := []string{storageTest, storageWip} + buckets := []string{storageWip} queues := []string{queuePreProc, queueWipeOnly, queueAnalyse, queueOcrPage, queueTest} for _, bucket := range buckets { diff --git a/cloudsettings.go b/cloudsettings.go index befb4b4..fa55742 100644 --- a/cloudsettings.go +++ b/cloudsettings.go @@ -36,5 +36,4 @@ const ( // Storage bucket names. Can be anything unique in S3. const ( storageWip = "rescribeinprogress" - storageTest = "rescribetest" ) diff --git a/cmd/booktopipeline/main.go b/cmd/booktopipeline/main.go index 1b16cee..b4f4d99 100644 --- a/cmd/booktopipeline/main.go +++ b/cmd/booktopipeline/main.go @@ -112,7 +112,7 @@ func main() { } verboselog.Println("Uploading all images are valid in", bookdir) - err = pipeline.UploadImages(bookdir, bookname, conn, conn.WIPStorageId()) + err = pipeline.UploadImages(bookdir, bookname, conn) if err != nil { log.Fatalln(err) } diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go index e249138..251c333 100644 --- a/cmd/rescribe/main.go +++ b/cmd/rescribe/main.go @@ -362,7 +362,7 @@ func uploadbook(dir string, name string, conn Pipeliner) error { if err != nil { return fmt.Errorf("Error with images in %s: %v", dir, err) } - err = pipeline.UploadImages(dir, name, conn, conn.WIPStorageId()) + err = pipeline.UploadImages(dir, name, conn) if err != nil { return fmt.Errorf("Error saving images to process from %s: %v", dir, err) } diff --git a/internal/pipeline/put.go b/internal/pipeline/put.go index 26d581c..8cbecac 100644 --- a/internal/pipeline/put.go +++ b/internal/pipeline/put.go @@ -78,9 +78,9 @@ func DetectQueueType(dir string, conn Queuer) string { } // UploadImages uploads all files (except those which start with a ".") -// from a directory (recursively) into a given storage id, prefixed with +// from a directory (recursively) into conn.WIPStorageId(), prefixed with // the given bookname and a slash -func UploadImages(dir string, bookname string, conn Uploader, storageId string) error { +func UploadImages(dir string, bookname string, conn Uploader) error { walker := make(fileWalk) go func() { _ = filepath.Walk(dir, walker.Walk) @@ -89,7 +89,7 @@ func UploadImages(dir string, bookname string, conn Uploader, storageId string) for path := range walker { name := filepath.Base(path) - err := conn.Upload(storageId, bookname + "/" + name, path) + err := conn.Upload(conn.WIPStorageId(), bookname + "/" + name, path) if err != nil { return fmt.Errorf("Failed to upload %s: %v", path, err) } diff --git a/internal/pipeline/put_test.go b/internal/pipeline/put_test.go index 912a4ef..5fb5b43 100644 --- a/internal/pipeline/put_test.go +++ b/internal/pipeline/put_test.go @@ -92,18 +92,13 @@ func Test_UploadImages(t *testing.T) { if err != nil { t.Fatalf("Could not initialise %s connection: %v\nLog: %s", conn.name, err, slog.log) } - err = conn.c.TestInit() - if err != nil { - t.Fatalf("Failed in test initialisalisation for %s: %v\nLog: %s", conn.name, err, slog.log) - } slog.log = "" - err = UploadImages("testdata/good", "good", conn.c, conn.c.TestStorageId()) + err = UploadImages("testdata/good", "good", conn.c) if err != nil { t.Fatalf("Error in UploadImages for %s: %v\nLog: %s", conn.name, err, slog.log) } - // TODO: decide on whether to use TestStorageId or just the WIPStorageId as with other tests, and align other tests to this if needed // TODO: download all files and test that they match // TODO: remove test files from conn storage }) @@ -21,7 +21,6 @@ const qidOCR = "queueOCR" const qidAnalyse = "queueAnalyse" const qidTest = "queueTest" const storageId = "storage" -const testStorageId = "test" // LocalConn is a simple implementation of the pipeliner interface // that doesn't rely on any "cloud" services, instead doing everything @@ -67,11 +66,6 @@ func (a *LocalConn) Init() error { // TestInit does nothing for local connections func (a *LocalConn) TestInit() error { - err := os.Mkdir(filepath.Join(a.TempDir, testStorageId), 0700) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("Error creating test storage directory: %v", err) - } - return nil } @@ -140,11 +134,6 @@ func (a *LocalConn) WIPStorageId() string { return storageId } -func (a *LocalConn) TestStorageId() string { - return testStorageId -} - - func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkFunc { return func(path string, info os.FileInfo, err error) error { if err != nil { |