Sort book list in lspipeline by modified date

2 files changed, 87 insertions, 5 deletions

diff --git a/aws.go b/aws.go
index 0127d6e..5a037ff 100644
--- a/aws.go
+++ b/aws.go
@@ -27,6 +27,11 @@ type InstanceDetails struct {
 	Id, Name, Ip, Spot, Type, State, LaunchTime string
 }
 
+type ObjMeta struct {
+	Name string
+	Date time.Time
+}
+
 type AwsConn struct {
 	// these need to be set before running Init()
 	Region string
@@ -231,6 +236,20 @@ func (a *AwsConn) ListObjects(bucket string, prefix string) ([]string, error) {
 	return names, err
 }
 
+func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta, error) {
+	var objs []ObjMeta
+	err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(prefix),
+	}, func(page *s3.ListObjectsV2Output, last bool) bool {
+		for _, r := range page.Contents {
+			objs = append(objs, ObjMeta{Name: *r.Key, Date: *r.LastModified})
+		}
+		return true
+	})
+	return objs, err
+}
+
 func (a *AwsConn) AddToQueue(url string, msg string) error {
 	_, err := a.sqssvc.SendMessage(&sqs.SendMessageInput{
 		MessageBody: &msg,
diff --git a/cmd/lspipeline/main.go b/cmd/lspipeline/main.go
index 0e1ebb0..0b8ce49 100644
--- a/cmd/lspipeline/main.go
+++ b/cmd/lspipeline/main.go
@@ -1,10 +1,12 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"log"
 	"os/exec"
+	"sort"
 	"strings"
 
 	"rescribe.xyz/bookpipeline"
@@ -29,7 +31,7 @@ type LsPipeliner interface {
 	AnalyseQueueId() string
 	GetQueueDetails(url string) (string, string, error)
 	GetInstanceDetails() ([]bookpipeline.InstanceDetails, error)
-	ListObjects(bucket string, prefix string) ([]string, error)
+	ListObjectsWithMeta(bucket string, prefix string) ([]bookpipeline.ObjMeta, error)
 	WIPStorageId() string
 }
 
@@ -76,18 +78,68 @@ func getQueueDetails(conn LsPipeliner, qdetails chan queueDetails) {
 	close(qdetails)
 }
 
+type ObjMetas []bookpipeline.ObjMeta
+
+// used by sort.Sort
+func (o ObjMetas) Len() int {
+	return len(o)
+}
+
+// used by sort.Sort
+func (o ObjMetas) Swap(i, j int) {
+	o[i], o[j] = o[j], o[i]
+}
+
+// used by sort.Sort
+func (o ObjMetas) Less(i, j int) bool {
+	return o[i].Date.Before(o[j].Date)
+}
+
+// sortBookList sorts a list of book names by date.
+// It uses a list of filenames and dates in an ObjMeta slice to
+// determine the date for a book name.
+func sortBookList(list []string, fileinfo []bookpipeline.ObjMeta) ([]string, error) {
+	var listinfo ObjMetas
+
+	for _, name := range list {
+		found := false
+		for _, f := range fileinfo {
+			parts := strings.Split(f.Name, "/")
+			prefix := parts[0]
+			if name == prefix {
+				listinfo = append(listinfo, bookpipeline.ObjMeta{Name: name, Date: f.Date})
+				found = true
+				break
+			}
+		}
+		if !found {
+			return list, errors.New("Failed to find metadata for list")
+		}
+	}
+
+	// sort listinfo by date
+	sort.Sort(listinfo)
+
+	var l []string
+	for _, i := range listinfo {
+		l = append(l, i.Name)
+	}
+	return l, nil
+}
+
 // getBookStatus returns a list of in progress and done books.
 // It determines this by listing all objects, and splitting the
 // prefixes into two lists, those which have a 'graph.png' file,
-// which are classed as done, and those which are not.
+// which are classed as done, and those which are not. These are
+// sorted by date according to file metadata.
 func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err error) {
-	allfiles, err := conn.ListObjects(conn.WIPStorageId(), "")
+	allfiles, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), "")
 	if err != nil {
 		log.Println("Error getting list of objects:", err)
 		return inprogress, done, err
 	}
 	for _, f := range allfiles {
-		parts := strings.Split(f, "/")
+		parts := strings.Split(f.Name, "/")
 		if parts[1] != "graph.png" {
 			continue
 		}
@@ -105,7 +157,7 @@ func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err er
 	}
 
 	for _, f := range allfiles {
-		parts := strings.Split(f, "/")
+		parts := strings.Split(f.Name, "/")
 		prefix := parts[0]
 		found := false
 		for _, i := range done {
@@ -125,6 +177,17 @@ func getBookStatus(conn LsPipeliner) (inprogress []string, done []string, err er
 		}
 	}
 
+	inprogress, err = sortBookList(inprogress, allfiles)
+	if err != nil {
+		log.Println("Error sorting list of objects:", err)
+		err = nil
+	}
+	done, err = sortBookList(done, allfiles)
+	if err != nil {
+		log.Println("Error sorting list of objects:", err)
+		err = nil
+	}
+
 	return inprogress, done, err
 }