From 9f3fec3e0982c5b419338f68428f12bbeed4c2bb Mon Sep 17 00:00:00 2001
From: Nick White <git@njw.name>
Date: Thu, 19 Aug 2021 17:50:11 +0100
Subject: lspipeline-ng: Limit number of book details requests so we don't run
into EC2's rate limiting
---
cmd/lspipeline-ng/main.go | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
(limited to 'cmd')
diff --git a/cmd/lspipeline-ng/main.go b/cmd/lspipeline-ng/main.go
index 94b81c2..dbafafa 100644
--- a/cmd/lspipeline-ng/main.go
+++ b/cmd/lspipeline-ng/main.go
@@ -12,6 +12,7 @@ import (
"os/exec"
"sort"
"strings"
+ "sync"
"time"
"rescribe.xyz/bookpipeline"
@@ -126,9 +127,11 @@ func getBookDetails(conn LsPipeliner, key string) (date time.Time, done bool, er
// getBookDetailsChan gets the details for a book putting it into either the
// done or inprogress channels as appropriate, or sending an error to errc
// on failure.
-func getBookDetailsChan(conn LsPipeliner, key string, done chan bookpipeline.ObjMeta, inprogress chan bookpipeline.ObjMeta, errc chan error) {
+func getBookDetailsChan(conn LsPipeliner, wg *sync.WaitGroup, key string, done chan bookpipeline.ObjMeta, inprogress chan bookpipeline.ObjMeta, errc chan error) {
+ fmt.Printf("getting book details for %s\n", key)
date, isdone, err := getBookDetails(conn, key)
if err != nil {
+ wg.Done()
errc <- err
return
}
@@ -138,6 +141,7 @@ func getBookDetailsChan(conn LsPipeliner, key string, done chan bookpipeline.Obj
} else {
inprogress <- meta
}
+ wg.Done()
}
// getBookStatus returns a list of in progress and done books.
@@ -159,9 +163,19 @@ func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err er
inprogressc := make(chan bookpipeline.ObjMeta, 100)
errc := make(chan error)
- for _, p := range prefixes {
- go getBookDetailsChan(conn, p, donec, inprogressc, errc)
- }
+ // This is a bit messy, but it works. We get the book details in blocks of
+ // 30 simultaneous requests, using wait groups to pause once each 20 have
+ // been sent.
+ go func() {
+ var wg sync.WaitGroup
+ for i, p := range prefixes {
+ wg.Add(1)
+ go getBookDetailsChan(conn, &wg, p, donec, inprogressc, errc)
+ if i % 30 == 0 {
+ wg.Wait()
+ }
+ }
+ }()
var inprogressmeta, donemeta ObjMetas
--
cgit v1.2.1-24-ge1ad