summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2019-10-16 17:17:05 +0100
committerNick White <git@njw.name>2019-10-16 17:17:05 +0100
commit7ec2824670bae240d73af63d810e0c946ae6981a (patch)
tree843977fe0e73ca49e03fa6d191a4593f6baa13f2
parentff809ad480061d9efce5ce33cda1e877cf63c936 (diff)
Sort book list in lspipeline by modified date
-rw-r--r--aws.go19
-rw-r--r--cmd/lspipeline/main.go73
2 files changed, 87 insertions, 5 deletions
diff --git a/aws.go b/aws.go
index 0127d6e..5a037ff 100644
--- a/aws.go
+++ b/aws.go
@@ -27,6 +27,11 @@ type InstanceDetails struct {
Id, Name, Ip, Spot, Type, State, LaunchTime string
}
+type ObjMeta struct {
+ Name string
+ Date time.Time
+}
+
type AwsConn struct {
// these need to be set before running Init()
Region string
@@ -231,6 +236,20 @@ func (a *AwsConn) ListObjects(bucket string, prefix string) ([]string, error) {
return names, err
}
+func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta, error) {
+ var objs []ObjMeta
+ err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
+ Bucket: aws.String(bucket),
+ Prefix: aws.String(prefix),
+ }, func(page *s3.ListObjectsV2Output, last bool) bool {
+ for _, r := range page.Contents {
+ objs = append(objs, ObjMeta{Name: *r.Key, Date: *r.LastModified})
+ }
+ return true
+ })
+ return objs, err
+}
+
func (a *AwsConn) AddToQueue(url string, msg string) error {
_, err := a.sqssvc.SendMessage(&sqs.SendMessageInput{
MessageBody: &msg,
diff --git a/cmd/lspipeline/main.go b/cmd/lspipeline/main.go
index 0e1ebb0..0b8ce49 100644
--- a/cmd/lspipeline/main.go
+++ b/cmd/lspipeline/main.go
@@ -1,10 +1,12 @@
package main
import (
+ "errors"
"flag"
"fmt"
"log"
"os/exec"
+ "sort"
"strings"
"rescribe.xyz/bookpipeline"
@@ -29,7 +31,7 @@ type LsPipeliner interface {
AnalyseQueueId() string
GetQueueDetails(url string) (string, string, error)
GetInstanceDetails() ([]bookpipeline.InstanceDetails, error)
- ListObjects(bucket string, prefix string) ([]string, error)
+ ListObjectsWithMeta(bucket string, prefix string) ([]bookpipeline.ObjMeta, error)
WIPStorageId() string
}
@@ -76,18 +78,68 @@ func getQueueDetails(conn LsPipeliner, qdetails chan queueDetails) {
close(qdetails)
}
+type ObjMetas []bookpipeline.ObjMeta
+
+// used by sort.Sort
+func (o ObjMetas) Len() int {
+ return len(o)
+}
+
+// used by sort.Sort
+func (o ObjMetas) Swap(i, j int) {
+ o[i], o[j] = o[j], o[i]
+}
+
+// used by sort.Sort
+func (o ObjMetas) Less(i, j int) bool {
+ return o[i].Date.Before(o[j].Date)
+}
+
+// sortBookList sorts a list of book names by date.
+// It uses a list of filenames and dates in an ObjMeta slice to
+// determine the date for a book name.
+func sortBookList(list []string, fileinfo []bookpipeline.ObjMeta) ([]string, error) {
+ var listinfo ObjMetas
+
+ for _, name := range list {
+ found := false
+ for _, f := range fileinfo {
+ parts := strings.Split(f.Name, "/")
+ prefix := parts[0]
+ if name == prefix {
+ listinfo = append(listinfo, bookpipeline.ObjMeta{Name: name, Date: f.Date})
+ found = true
+ break
+ }
+ }
+ if !found {
+ return list, errors.New("Failed to find metadata for list")
+ }
+ }
+
+ // sort listinfo by date
+ sort.Sort(listinfo)
+
+ var l []string
+ for _, i := range listinfo {
+ l = append(l, i.Name)
+ }
+ return l, nil
+}
+
// getBookStatus returns a list of in progress and done books.
// It determines this by listing all objects, and splitting the
// prefixes into two lists, those which have a 'graph.png' file,
-// which are classed as done, and those which are not.
+// which are classed as done, and those which are not. These are
+// sorted by date according to file metadata.
func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err error) {
- allfiles, err := conn.ListObjects(conn.WIPStorageId(), "")
+ allfiles, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), "")
if err != nil {
log.Println("Error getting list of objects:", err)
return inprogress, done, err
}
for _, f := range allfiles {
- parts := strings.Split(f, "/")
+ parts := strings.Split(f.Name, "/")
if parts[1] != "graph.png" {
continue
}
@@ -105,7 +157,7 @@ func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err er
}
for _, f := range allfiles {
- parts := strings.Split(f, "/")
+ parts := strings.Split(f.Name, "/")
prefix := parts[0]
found := false
for _, i := range done {
@@ -125,6 +177,17 @@ func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err er
}
}
+ inprogress, err = sortBookList(inprogress, allfiles)
+ if err != nil {
+ log.Println("Error sorting list of objects:", err)
+ err = nil
+ }
+ done, err = sortBookList(done, allfiles)
+ if err != nil {
+ log.Println("Error sorting list of objects:", err)
+ err = nil
+ }
+
return inprogress, done, err
}