summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2021-01-26 14:56:10 +0000
committerNick White <git@njw.name>2021-01-26 14:56:10 +0000
commit5c3cee66a90ce6ef87e125b3bf011a6903d38083 (patch)
treebe1347c79389b232150227bd8489db3fabb415ac
parent54150b54cd06e3deba44e73b151070b74a4d8e76 (diff)
Make ListObjectsWithMeta generic again and create a specialised ListObjectWithMeta for single file listing, so we can still be as fast, but do not have a misleading api
-rw-r--r--aws.go23
-rw-r--r--cmd/lspipeline/main.go15
-rw-r--r--local.go11
3 files changed, 38 insertions, 11 deletions
diff --git a/aws.go b/aws.go
index dd74a01..035b08a 100644
--- a/aws.go
+++ b/aws.go
@@ -355,16 +355,35 @@ func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta,
err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
- MaxKeys: aws.Int64(1),
}, func(page *s3.ListObjectsV2Output, last bool) bool {
for _, r := range page.Contents {
objs = append(objs, ObjMeta{Name: *r.Key, Date: *r.LastModified})
}
- return false // only process the first page as that's all we need
+ return true
})
return objs, err
}
+// ListObjectWithMeta lists the name and last modified date of the
+// first object with the specified prefix.
+func (a *AwsConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) {
+ var obj ObjMeta
+ err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
+ Bucket: aws.String(bucket),
+ Prefix: aws.String(prefix),
+ MaxKeys: aws.Int64(1),
+ }, func(page *s3.ListObjectsV2Output, last bool) bool {
+ for _, r := range page.Contents {
+ obj = ObjMeta{Name: *r.Key, Date: *r.LastModified}
+ }
+ return false
+ })
+ if obj.Name == "" && obj.Date.IsZero() && err == nil {
+ return obj, fmt.Errorf("No object could be found for %s", prefix)
+ }
+ return obj, err
+}
+
func (a *AwsConn) ListObjectPrefixes(bucket string) ([]string, error) {
var prefixes []string
err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
diff --git a/cmd/lspipeline/main.go b/cmd/lspipeline/main.go
index 8980c59..131ff12 100644
--- a/cmd/lspipeline/main.go
+++ b/cmd/lspipeline/main.go
@@ -36,7 +36,7 @@ type LsPipeliner interface {
AnalyseQueueId() string
GetQueueDetails(url string) (string, string, error)
GetInstanceDetails() ([]bookpipeline.InstanceDetails, error)
- ListObjectsWithMeta(bucket string, prefix string) ([]bookpipeline.ObjMeta, error)
+ ListObjectWithMeta(bucket string, prefix string) (bookpipeline.ObjMeta, error)
ListObjectPrefixes(bucket string) ([]string, error)
WIPStorageId() string
}
@@ -107,20 +107,17 @@ func (o ObjMetas) Less(i, j int) bool {
func getBookDetails(conn LsPipeliner, key string) (date time.Time, done bool, err error) {
// First try to get the graph.png file from the book, which marks
// it as done
- objs, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), key+"graph.png")
- if err == nil && len(objs) > 0 {
- return objs[0].Date, true, nil
+ obj, err := conn.ListObjectWithMeta(conn.WIPStorageId(), key+"graph.png")
+ if err == nil {
+ return obj.Date, true, nil
}
// Otherwise get any file from the book to get a date to sort by
- objs, err = conn.ListObjectsWithMeta(conn.WIPStorageId(), key)
+ obj, err = conn.ListObjectWithMeta(conn.WIPStorageId(), key)
if err != nil {
return time.Time{}, false, err
}
- if len(objs) == 0 {
- return time.Time{}, false, fmt.Errorf("No files found for book %s", key)
- }
- return objs[0].Date, false, nil
+ return obj.Date, false, nil
}
// getBookDetailsChan gets the details for a book putting it into either the
diff --git a/local.go b/local.go
index 0fceca2..31e44a9 100644
--- a/local.go
+++ b/local.go
@@ -159,6 +159,17 @@ func (a *LocalConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta
return list, err
}
+func (a *LocalConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) {
+ list, err := a.ListObjectsWithMeta(bucket, prefix)
+ if err != nil {
+ return ObjMeta{}, err
+ }
+ if len(list) == 0 {
+ return ObjMeta{}, fmt.Errorf("No object found for %s", prefix)
+ }
+ return list[0], nil
+}
+
// AddToQueue adds a message to a queue
func (a *LocalConn) AddToQueue(url string, msg string) error {
f, err := os.OpenFile(filepath.Join(a.TempDir, url), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)