summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2021-08-19 17:50:11 +0100
committerNick White <git@njw.name>2021-08-19 17:50:11 +0100
commit9f3fec3e0982c5b419338f68428f12bbeed4c2bb (patch)
treefdc6168965cf56047bda0bc6fbd5eca505469535
parentbde651bbde9df3a8c33b705dbe33bbcaf4e3e73d (diff)
lspipeline-ng: Limit number of book details requests so we don't run into EC2's rate limitingv0.5.0
-rw-r--r--cmd/lspipeline-ng/main.go22
1 files changed, 18 insertions, 4 deletions
diff --git a/cmd/lspipeline-ng/main.go b/cmd/lspipeline-ng/main.go
index 94b81c2..dbafafa 100644
--- a/cmd/lspipeline-ng/main.go
+++ b/cmd/lspipeline-ng/main.go
@@ -12,6 +12,7 @@ import (
"os/exec"
"sort"
"strings"
+ "sync"
"time"
"rescribe.xyz/bookpipeline"
@@ -126,9 +127,11 @@ func getBookDetails(conn LsPipeliner, key string) (date time.Time, done bool, er
// getBookDetailsChan gets the details for a book putting it into either the
// done or inprogress channels as appropriate, or sending an error to errc
// on failure.
-func getBookDetailsChan(conn LsPipeliner, key string, done chan bookpipeline.ObjMeta, inprogress chan bookpipeline.ObjMeta, errc chan error) {
+func getBookDetailsChan(conn LsPipeliner, wg *sync.WaitGroup, key string, done chan bookpipeline.ObjMeta, inprogress chan bookpipeline.ObjMeta, errc chan error) {
+ fmt.Printf("getting book details for %s\n", key)
date, isdone, err := getBookDetails(conn, key)
if err != nil {
+ wg.Done()
errc <- err
return
}
@@ -138,6 +141,7 @@ func getBookDetailsChan(conn LsPipeliner, key string, done chan bookpipeline.Obj
} else {
inprogress <- meta
}
+ wg.Done()
}
// getBookStatus returns a list of in progress and done books.
@@ -159,9 +163,19 @@ func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err er
inprogressc := make(chan bookpipeline.ObjMeta, 100)
errc := make(chan error)
- for _, p := range prefixes {
- go getBookDetailsChan(conn, p, donec, inprogressc, errc)
- }
+ // This is a bit messy, but it works. We get the book details in blocks of
+ // 30 simultaneous requests, using wait groups to pause once each 20 have
+ // been sent.
+ go func() {
+ var wg sync.WaitGroup
+ for i, p := range prefixes {
+ wg.Add(1)
+ go getBookDetailsChan(conn, &wg, p, donec, inprogressc, errc)
+ if i % 30 == 0 {
+ wg.Wait()
+ }
+ }
+ }()
var inprogressmeta, donemeta ObjMetas