summaryrefslogtreecommitdiff
path: root/cmd/lspipeline/main.go
diff options
context:
space:
mode:
authorNick White <git@njw.name>2021-02-22 16:02:55 +0000
committerNick White <git@njw.name>2021-02-22 16:02:55 +0000
commit866aa409f8844ab2cb8d578672a703b4ddead30c (patch)
tree1e411decdfbca2218522d7fd872b11222e9ea340 /cmd/lspipeline/main.go
parent44a027984044a55a8a483268ddf0b841e9f33e83 (diff)
lspipeline: Rename to lspipeline-ng, and restore pre concurrency version to lspipeline as there are some hard to debug issues in concurrency version
Diffstat (limited to 'cmd/lspipeline/main.go')
-rw-r--r--cmd/lspipeline/main.go82
1 files changed, 18 insertions, 64 deletions
diff --git a/cmd/lspipeline/main.go b/cmd/lspipeline/main.go
index 131ff12..b649778 100644
--- a/cmd/lspipeline/main.go
+++ b/cmd/lspipeline/main.go
@@ -12,7 +12,6 @@ import (
"os/exec"
"sort"
"strings"
- "time"
"rescribe.xyz/bookpipeline"
)
@@ -36,7 +35,7 @@ type LsPipeliner interface {
AnalyseQueueId() string
GetQueueDetails(url string) (string, string, error)
GetInstanceDetails() ([]bookpipeline.InstanceDetails, error)
- ListObjectWithMeta(bucket string, prefix string) (bookpipeline.ObjMeta, error)
+ ListObjectsWithMeta(bucket string, prefix string) ([]bookpipeline.ObjMeta, error)
ListObjectPrefixes(bucket string) ([]string, error)
WIPStorageId() string
}
@@ -101,88 +100,43 @@ func (o ObjMetas) Less(i, j int) bool {
return o[i].Date.Before(o[j].Date)
}
-// getBookDetails determines whether a book is done and what date
-// it was completed, or if it has not finished, the date of any
-// book file.
-func getBookDetails(conn LsPipeliner, key string) (date time.Time, done bool, err error) {
- // First try to get the graph.png file from the book, which marks
- // it as done
- obj, err := conn.ListObjectWithMeta(conn.WIPStorageId(), key+"graph.png")
- if err == nil {
- return obj.Date, true, nil
- }
-
- // Otherwise get any file from the book to get a date to sort by
- obj, err = conn.ListObjectWithMeta(conn.WIPStorageId(), key)
- if err != nil {
- return time.Time{}, false, err
- }
- return obj.Date, false, nil
-}
-
-// getBookDetailsChan gets the details for a book putting it into either the
-// done or inprogress channels as appropriate, or sending an error to errc
-// on failure.
-func getBookDetailsChan(conn LsPipeliner, key string, done chan bookpipeline.ObjMeta, inprogress chan bookpipeline.ObjMeta, errc chan error) {
- date, isdone, err := getBookDetails(conn, key)
- if err != nil {
- errc <- err
- return
- }
- meta := bookpipeline.ObjMeta{Name: strings.TrimSuffix(key, "/"), Date: date}
- if isdone {
- done <- meta
- } else {
- inprogress <- meta
- }
-}
-
// getBookStatus returns a list of in progress and done books.
// It determines this by finding all prefixes, and splitting them
// into two lists, those which have a 'graph.png' file (the done
// list), and those which do not (the inprogress list). They are
// sorted according to the date of the graph.png file, or the date
// of a random file with the prefix if no graph.png was found.
-// It spins up many goroutines to do query the book status and
-// dates, as it is far faster to do concurrently.
func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err error) {
prefixes, err := conn.ListObjectPrefixes(conn.WIPStorageId())
+ var inprogressmeta, donemeta ObjMetas
if err != nil {
log.Println("Error getting object prefixes:", err)
return
}
-
- donec := make(chan bookpipeline.ObjMeta, 100)
- inprogressc := make(chan bookpipeline.ObjMeta, 100)
- errc := make(chan error)
-
+ // Search for graph.png to determine done books (and save the date of it to sort with)
for _, p := range prefixes {
- go getBookDetailsChan(conn, p, donec, inprogressc, errc)
+ objs, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), p+"graph.png")
+ if err != nil || len(objs) == 0 {
+ inprogressmeta = append(inprogressmeta, bookpipeline.ObjMeta{Name: p})
+ } else {
+ donemeta = append(donemeta, bookpipeline.ObjMeta{Name: p, Date: objs[0].Date})
+ }
}
-
- var inprogressmeta, donemeta ObjMetas
-
- // there will be exactly as many sends to donec or inprogressc
- // as there are prefixes
- for range prefixes {
- select {
- case i := <-donec:
- donemeta = append(donemeta, i)
- case i := <-inprogressc:
- inprogressmeta = append(inprogressmeta, i)
- case err = <-errc:
- return inprogress, done, err
+ // Get a random file from the inprogress list to get a date to sort by
+ for _, i := range inprogressmeta {
+ objs, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), i.Name)
+ if err != nil || len(objs) == 0 {
+ continue
}
+ i.Date = objs[0].Date
}
-
sort.Sort(donemeta)
- sort.Sort(inprogressmeta)
-
for _, i := range donemeta {
- done = append(done, i.Name)
+ done = append(done, strings.TrimSuffix(i.Name, "/"))
}
+ sort.Sort(inprogressmeta)
for _, i := range inprogressmeta {
- inprogress = append(inprogress, i.Name)
+ inprogress = append(inprogress, strings.TrimSuffix(i.Name, "/"))
}
return