summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2021-02-01 11:45:27 +0000
committerNick White <git@njw.name>2021-02-01 11:45:27 +0000
commit16ea8034794ef030c969d586a7fc945bf4a2873a (patch)
treeba03a27df08ea0b670d64b69fea6597ebd11ce6d
parent5c3cee66a90ce6ef87e125b3bf011a6903d38083 (diff)
Ensure DeleteObjects can handle over 1000 files to delete; fixes rmbook for large books
-rw-r--r--aws.go18
1 files changed, 17 insertions, 1 deletions
diff --git a/aws.go b/aws.go
index 035b08a..65671fa 100644
--- a/aws.go
+++ b/aws.go
@@ -401,9 +401,25 @@ func (a *AwsConn) ListObjectPrefixes(bucket string) ([]string, error) {
// Deletes a list of objects
func (a *AwsConn) DeleteObjects(bucket string, keys []string) error {
objs := []*s3.ObjectIdentifier{}
- for _, v := range keys {
+ for i, v := range keys {
o := s3.ObjectIdentifier{Key: aws.String(v)}
objs = append(objs, &o)
+ // s3.DeleteObjects can only take up to 1000 keys at a time,
+ // so if necessary delete those collected so far and empty
+ // the objs queue
+ if i % 1000 == 1 {
+ _, err := a.s3svc.DeleteObjects(&s3.DeleteObjectsInput{
+ Bucket: aws.String(bucket),
+ Delete: &s3.Delete{
+ Objects: objs,
+ Quiet: aws.Bool(true),
+ },
+ })
+ if err != nil {
+ return err
+ }
+ objs = []*s3.ObjectIdentifier{}
+ }
}
_, err := a.s3svc.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: aws.String(bucket),