diff options
author | Nick White <git@njw.name> | 2021-01-26 13:50:02 +0000 |
---|---|---|
committer | Nick White <git@njw.name> | 2021-01-26 13:50:02 +0000 |
commit | 86cc5d6c921ac05e0d08f66b205b51e1f5adb938 (patch) | |
tree | a32226f1fb3969336f6701b389ee0ce186df6adc /aws.go | |
parent | 670d5c1b74f2fa4683bfe7e2d9b1baee14db9104 (diff) |
Speed up lspipeline by making s3 requests concurrently and only processing single results from ListObjects requests
Diffstat (limited to 'aws.go')
-rw-r--r-- | aws.go | 3 |
1 files changed, 2 insertions, 1 deletions
@@ -355,11 +355,12 @@ func (a *AwsConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta, err := a.s3svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(prefix), + MaxKeys: aws.Int64(1), }, func(page *s3.ListObjectsV2Output, last bool) bool { for _, r := range page.Contents { objs = append(objs, ObjMeta{Name: *r.Key, Date: *r.LastModified}) } - return true + return false // only process the first page as that's all we need }) return objs, err } |