From 4b7a4bd1b87d1dbba283e577aa03ce7e390d85d8 Mon Sep 17 00:00:00 2001 From: Nick White Date: Mon, 2 Aug 2021 10:45:53 +0100 Subject: internal/pipeline: Add test (incomplete but working) for UploadImages --- aws.go | 8 +++++++- cloudsettings.go | 1 + cmd/booktopipeline/main.go | 2 +- cmd/rescribe/main.go | 2 +- internal/pipeline/pipeline_test.go | 1 + internal/pipeline/put.go | 7 +++++-- internal/pipeline/put_test.go | 36 ++++++++++++++++++++++++++++++++++++ local.go | 11 +++++++++++ 8 files changed, 63 insertions(+), 5 deletions(-) diff --git a/aws.go b/aws.go index 21a215c..8e9410f 100644 --- a/aws.go +++ b/aws.go @@ -52,6 +52,7 @@ type AwsConn struct { uploader *s3manager.Uploader wipequrl, prequrl, ocrpgqurl, analysequrl string testqurl string + teststorageid string wipstorageid string } @@ -139,6 +140,7 @@ func (a *AwsConn) TestInit() error { return errors.New(fmt.Sprintf("Error getting test queue URL: %s\n", err)) } a.testqurl = *result.QueueUrl + a.teststorageid = storageTest return nil } @@ -346,6 +348,10 @@ func (a *AwsConn) AnalyseQueueId() string { return a.analysequrl } +func (a *AwsConn) TestStorageId() string { + return a.teststorageid +} + func (a *AwsConn) WIPStorageId() string { return a.wipstorageid } @@ -612,7 +618,7 @@ func (a *AwsConn) Log(v ...interface{}) { // mkpipeline sets up necessary buckets and queues for the pipeline // TODO: also set up the necessary security group and iam stuff func (a *AwsConn) MkPipeline() error { - buckets := []string{storageWip} + buckets := []string{storageTest, storageWip} queues := []string{queuePreProc, queueWipeOnly, queueAnalyse, queueOcrPage, queueTest} for _, bucket := range buckets { diff --git a/cloudsettings.go b/cloudsettings.go index fa55742..befb4b4 100644 --- a/cloudsettings.go +++ b/cloudsettings.go @@ -36,4 +36,5 @@ const ( // Storage bucket names. Can be anything unique in S3. const ( storageWip = "rescribeinprogress" + storageTest = "rescribetest" ) diff --git a/cmd/booktopipeline/main.go b/cmd/booktopipeline/main.go index b4f4d99..1b16cee 100644 --- a/cmd/booktopipeline/main.go +++ b/cmd/booktopipeline/main.go @@ -112,7 +112,7 @@ func main() { } verboselog.Println("Uploading all images are valid in", bookdir) - err = pipeline.UploadImages(bookdir, bookname, conn) + err = pipeline.UploadImages(bookdir, bookname, conn, conn.WIPStorageId()) if err != nil { log.Fatalln(err) } diff --git a/cmd/rescribe/main.go b/cmd/rescribe/main.go index 251c333..e249138 100644 --- a/cmd/rescribe/main.go +++ b/cmd/rescribe/main.go @@ -362,7 +362,7 @@ func uploadbook(dir string, name string, conn Pipeliner) error { if err != nil { return fmt.Errorf("Error with images in %s: %v", dir, err) } - err = pipeline.UploadImages(dir, name, conn) + err = pipeline.UploadImages(dir, name, conn, conn.WIPStorageId()) if err != nil { return fmt.Errorf("Error saving images to process from %s: %v", dir, err) } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 73e8223..fc223b3 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -33,6 +33,7 @@ type PipelineTester interface { TestInit() error DeleteObjects(bucket string, keys []string) error TestQueueId() string + TestStorageId() string } type connection struct { diff --git a/internal/pipeline/put.go b/internal/pipeline/put.go index c08df53..26d581c 100644 --- a/internal/pipeline/put.go +++ b/internal/pipeline/put.go @@ -77,7 +77,10 @@ func DetectQueueType(dir string, conn Queuer) string { } } -func UploadImages(dir string, bookname string, conn Uploader) error { +// UploadImages uploads all files (except those which start with a ".") +// from a directory (recursively) into a given storage id, prefixed with +// the given bookname and a slash +func UploadImages(dir string, bookname string, conn Uploader, storageId string) error { walker := make(fileWalk) go func() { _ = filepath.Walk(dir, walker.Walk) @@ -86,7 +89,7 @@ func UploadImages(dir string, bookname string, conn Uploader) error { for path := range walker { name := filepath.Base(path) - err := conn.Upload(conn.WIPStorageId(), filepath.Join(bookname, name), path) + err := conn.Upload(storageId, bookname + "/" + name, path) if err != nil { return fmt.Errorf("Failed to upload %s: %v", path, err) } diff --git a/internal/pipeline/put_test.go b/internal/pipeline/put_test.go index af18ab4..912a4ef 100644 --- a/internal/pipeline/put_test.go +++ b/internal/pipeline/put_test.go @@ -6,6 +6,7 @@ package pipeline import ( "errors" + "log" "os" "rescribe.xyz/bookpipeline" "testing" @@ -73,3 +74,38 @@ func Test_DetectQueueType(t *testing.T) { }) } } + +func Test_UploadImages(t *testing.T) { + var slog StrLog + vlog := log.New(&slog, "", 0) + var conns []connection + + conns = append(conns, connection{name: "local", c: &bookpipeline.LocalConn{Logger: vlog}}) + + if !testing.Short() { + conns = append(conns, connection{name: "aws", c: &bookpipeline.AwsConn{Logger: vlog}}) + } + + for _, conn := range conns { + t.Run(conn.name, func(t *testing.T) { + err := conn.c.Init() + if err != nil { + t.Fatalf("Could not initialise %s connection: %v\nLog: %s", conn.name, err, slog.log) + } + err = conn.c.TestInit() + if err != nil { + t.Fatalf("Failed in test initialisalisation for %s: %v\nLog: %s", conn.name, err, slog.log) + } + slog.log = "" + + err = UploadImages("testdata/good", "good", conn.c, conn.c.TestStorageId()) + if err != nil { + t.Fatalf("Error in UploadImages for %s: %v\nLog: %s", conn.name, err, slog.log) + } + + // TODO: decide on whether to use TestStorageId or just the WIPStorageId as with other tests, and align other tests to this if needed + // TODO: download all files and test that they match + // TODO: remove test files from conn storage + }) + } +} diff --git a/local.go b/local.go index 615f9a6..58830c7 100644 --- a/local.go +++ b/local.go @@ -21,6 +21,7 @@ const qidOCR = "queueOCR" const qidAnalyse = "queueAnalyse" const qidTest = "queueTest" const storageId = "storage" +const testStorageId = "test" // LocalConn is a simple implementation of the pipeliner interface // that doesn't rely on any "cloud" services, instead doing everything @@ -66,6 +67,11 @@ func (a *LocalConn) Init() error { // TestInit does nothing for local connections func (a *LocalConn) TestInit() error { + err := os.Mkdir(filepath.Join(a.TempDir, testStorageId), 0700) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("Error creating test storage directory: %v", err) + } + return nil } @@ -134,6 +140,11 @@ func (a *LocalConn) WIPStorageId() string { return storageId } +func (a *LocalConn) TestStorageId() string { + return testStorageId +} + + func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkFunc { return func(path string, info os.FileInfo, err error) error { if err != nil { -- cgit v1.2.1-24-ge1ad