diff options
| author | Nick White <git@njw.name> | 2019-08-20 16:26:02 +0100 |
|---|---|---|
| committer | Nick White <git@njw.name> | 2019-08-20 16:26:02 +0100 |
| commit | 522e1481f5544362027b006d5fe34609f3d366bc (patch) | |
| tree | 761c8edd1d0440e3f88db9f2c0b6c3d205726fef /pipelinepreprocess/main.go | |
| parent | 9f588a71e9a2d7ad179890d0fc19372fae047b04 (diff) | |
Substantially improve problematic object listing part of API
Switch to regular non-concurrent stuff, concurrency is better handled
by the main program anyway. Now we handle errors properly, and things
are way simpler.
Diffstat (limited to 'pipelinepreprocess/main.go')
| -rw-r--r-- | pipelinepreprocess/main.go | 28 |
1 files changed, 16 insertions, 12 deletions
diff --git a/pipelinepreprocess/main.go b/pipelinepreprocess/main.go index 61dec96..b513f92 100644 --- a/pipelinepreprocess/main.go +++ b/pipelinepreprocess/main.go @@ -32,8 +32,7 @@ const PauseBetweenChecks = 60 * time.Second type Clouder interface { Init() error - //ListObjects(bucket string, prefix string, names chan string) error - ListObjects(bucket string, prefix string, names chan string) + ListObjects(bucket string, prefix string) ([]string, error) Download(bucket string, key string, fn string) error Upload(bucket string, key string, path string) error CheckQueue(url string) (Qmsg, error) @@ -44,8 +43,8 @@ type Clouder interface { type Pipeliner interface { Clouder - ListToPreprocess(bookname string, names chan string) error - ListToOCR(bookname string, names chan string) error + ListToPreprocess(bookname string) ([]string, error) + ListToOCR(bookname string) ([]string, error) DownloadFromInProgress(key string, fn string) error UploadToInProgress(key string, path string) error CheckPreQueue() (Qmsg, error) @@ -149,12 +148,15 @@ func preProcBook(msg Qmsg, conn Pipeliner) { go up(upc, done, conn, bookname) conn.Logger().Println("Getting list of objects to download") - err = conn.ListToPreprocess(bookname, dl) + todl, err := conn.ListToPreprocess(bookname) if err != nil { log.Println("Failed to get list of files for book", bookname, err) t.Stop() return } + for _, d := range todl { + dl <- d + } // wait for the done channel to be posted to <-done @@ -206,13 +208,15 @@ func ocrBook(msg Qmsg, conn Pipeliner) { go up(upc, done, conn, bookname) conn.Logger().Println("Getting list of objects to download") - go conn.ListToOCR(bookname, dl) - //err = conn.ListToOCR(bookname, dl) - //if err != nil { - // log.Println("Failed to get list of files for book", bookname, err) - // t.Stop() - // return - //} + todl, err := conn.ListToOCR(bookname) + if err != nil { + log.Println("Failed to get list of files for book", bookname, err) + t.Stop() + return + } + for _, d := range todl { + dl <- d + } // wait for the done channel to be posted to <-done |