Make ListObjectsWithMeta generic again and create a specialised ListObjectWithMeta for single file listing, so we can still be as fast, but do not have a misleading api

3 files changed, 38 insertions, 11 deletions

diff --git a/aws.go b/aws.go
index dd74a01..035b08a 100644
--- a/aws.go
+++ b/aws.go
@@ -355,16 +355,35 @@ func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta,
 	err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
 		Bucket: aws.String(bucket),
 		Prefix: aws.String(prefix),
-		MaxKeys: aws.Int64(1),
 	}, func(page *s3.ListObjectsV2Output, last bool) bool {
 		for _, r := range page.Contents {
 			objs = append(objs, ObjMeta{Name: *r.Key, Date: *r.LastModified})
 		}
-		return false // only process the first page as that's all we need
+		return true
 	})
 	return objs, err
 }
 
+// ListObjectWithMeta lists the name and last modified date of the
+// first object with the specified prefix.
+func (a *AwsConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) {
+	var obj ObjMeta
+	err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(prefix),
+		MaxKeys: aws.Int64(1),
+	}, func(page *s3.ListObjectsV2Output, last bool) bool {
+		for _, r := range page.Contents {
+			obj = ObjMeta{Name: *r.Key, Date: *r.LastModified}
+		}
+		return false
+	})
+	if obj.Name == "" && obj.Date.IsZero() && err == nil {
+		return obj, fmt.Errorf("No object could be found for %s", prefix)
+	}
+	return obj, err
+}
+
 func (a *AwsConn) ListObjectPrefixes(bucket string) ([]string, error) {
 	var prefixes []string
 	err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
diff --git a/cmd/lspipeline/main.go b/cmd/lspipeline/main.go
index 8980c59..131ff12 100644
--- a/cmd/lspipeline/main.go
+++ b/cmd/lspipeline/main.go
@@ -36,7 +36,7 @@ type LsPipeliner interface {
 	AnalyseQueueId() string
 	GetQueueDetails(url string) (string, string, error)
 	GetInstanceDetails() ([]bookpipeline.InstanceDetails, error)
-	ListObjectsWithMeta(bucket string, prefix string) ([]bookpipeline.ObjMeta, error)
+	ListObjectWithMeta(bucket string, prefix string) (bookpipeline.ObjMeta, error)
 	ListObjectPrefixes(bucket string) ([]string, error)
 	WIPStorageId() string
 }
@@ -107,20 +107,17 @@ func (o ObjMetas) Less(i, j int) bool {
 func getBookDetails(conn LsPipeliner, key string) (date time.Time, done bool, err error) {
 	// First try to get the graph.png file from the book, which marks
 	// it as done
-	objs, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), key+"graph.png")
-	if err == nil && len(objs) > 0 {
-		return objs[0].Date, true, nil
+	obj, err := conn.ListObjectWithMeta(conn.WIPStorageId(), key+"graph.png")
+	if err == nil {
+		return obj.Date, true, nil
 	}
 
 	// Otherwise get any file from the book to get a date to sort by
-	objs, err = conn.ListObjectsWithMeta(conn.WIPStorageId(), key)
+	obj, err = conn.ListObjectWithMeta(conn.WIPStorageId(), key)
 	if err != nil {
 		return time.Time{}, false, err
 	}
-	if len(objs) == 0 {
-		return time.Time{}, false, fmt.Errorf("No files found for book %s", key)
-	}
-	return objs[0].Date, false, nil
+	return obj.Date, false, nil
 }
 
 // getBookDetailsChan gets the details for a book putting it into either the
diff --git a/local.go b/local.go
index 0fceca2..31e44a9 100644
--- a/local.go
+++ b/local.go
@@ -159,6 +159,17 @@ func (a *LocalConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta
 	return list, err
 }
 
+func (a *LocalConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) {
+	list, err := a.ListObjectsWithMeta(bucket, prefix)
+	if err != nil {
+		return ObjMeta{}, err
+	}
+	if len(list) == 0 {
+		return ObjMeta{}, fmt.Errorf("No object found for %s", prefix)
+	}
+	return list[0], nil
+}
+
 // AddToQueue adds a message to a queue
 func (a *LocalConn) AddToQueue(url string, msg string) error {
 	f, err := os.OpenFile(filepath.Join(a.TempDir, url), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)