diff options
| -rw-r--r-- | aws.go | 23 | ||||
| -rw-r--r-- | cmd/lspipeline/main.go | 15 | ||||
| -rw-r--r-- | local.go | 11 |
3 files changed, 38 insertions, 11 deletions
@@ -355,16 +355,35 @@ func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta, err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(prefix), - MaxKeys: aws.Int64(1), }, func(page *s3.ListObjectsV2Output, last bool) bool { for _, r := range page.Contents { objs = append(objs, ObjMeta{Name: *r.Key, Date: *r.LastModified}) } - return false // only process the first page as that's all we need + return true }) return objs, err } +// ListObjectWithMeta lists the name and last modified date of the +// first object with the specified prefix. +func (a *AwsConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) { + var obj ObjMeta + err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + MaxKeys: aws.Int64(1), + }, func(page *s3.ListObjectsV2Output, last bool) bool { + for _, r := range page.Contents { + obj = ObjMeta{Name: *r.Key, Date: *r.LastModified} + } + return false + }) + if obj.Name == "" && obj.Date.IsZero() && err == nil { + return obj, fmt.Errorf("No object could be found for %s", prefix) + } + return obj, err +} + func (a *AwsConn) ListObjectPrefixes(bucket string) ([]string, error) { var prefixes []string err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{ diff --git a/cmd/lspipeline/main.go b/cmd/lspipeline/main.go index 8980c59..131ff12 100644 --- a/cmd/lspipeline/main.go +++ b/cmd/lspipeline/main.go @@ -36,7 +36,7 @@ type LsPipeliner interface { AnalyseQueueId() string GetQueueDetails(url string) (string, string, error) GetInstanceDetails() ([]bookpipeline.InstanceDetails, error) - ListObjectsWithMeta(bucket string, prefix string) ([]bookpipeline.ObjMeta, error) + ListObjectWithMeta(bucket string, prefix string) (bookpipeline.ObjMeta, error) ListObjectPrefixes(bucket string) ([]string, error) WIPStorageId() string } @@ -107,20 +107,17 @@ func (o ObjMetas) Less(i, j int) bool { func getBookDetails(conn LsPipeliner, key string) (date time.Time, done bool, err error) { // First try to get the graph.png file from the book, which marks // it as done - objs, err := conn.ListObjectsWithMeta(conn.WIPStorageId(), key+"graph.png") - if err == nil && len(objs) > 0 { - return objs[0].Date, true, nil + obj, err := conn.ListObjectWithMeta(conn.WIPStorageId(), key+"graph.png") + if err == nil { + return obj.Date, true, nil } // Otherwise get any file from the book to get a date to sort by - objs, err = conn.ListObjectsWithMeta(conn.WIPStorageId(), key) + obj, err = conn.ListObjectWithMeta(conn.WIPStorageId(), key) if err != nil { return time.Time{}, false, err } - if len(objs) == 0 { - return time.Time{}, false, fmt.Errorf("No files found for book %s", key) - } - return objs[0].Date, false, nil + return obj.Date, false, nil } // getBookDetailsChan gets the details for a book putting it into either the @@ -159,6 +159,17 @@ func (a *LocalConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta return list, err } +func (a *LocalConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) { + list, err := a.ListObjectsWithMeta(bucket, prefix) + if err != nil { + return ObjMeta{}, err + } + if len(list) == 0 { + return ObjMeta{}, fmt.Errorf("No object found for %s", prefix) + } + return list[0], nil +} + // AddToQueue adds a message to a queue func (a *LocalConn) AddToQueue(url string, msg string) error { f, err := os.OpenFile(filepath.Join(a.TempDir, url), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) |