1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
|
package main
// TODO: use the bookpipeline package for aws stuff
// TODO: set up iam role and policy needed for ec2 instances to access this stuff;
// see arn:aws:iam::557852942063:policy/pipelinestorageandqueue
// and arn:aws:iam::557852942063:role/pipeliner
// TODO: set up launch template for ec2 instances
// NOTE: potentially use json templates to define things, ala aws cli
import (
"log"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/sqs"
)
func main() {
if len(os.Args) != 1 {
log.Fatal("Usage: mkpipeline\n\nSets up necessary S3 buckets and SQS queues for our AWS pipeline\n")
}
sess, err := session.NewSession(&aws.Config{
Region: aws.String("eu-west-2"),
})
if err != nil {
log.Fatalf("Error: failed to set up aws session: %v\n", err)
}
s3svc := s3.New(sess)
sqssvc := sqs.New(sess)
prefix := "rescribe"
buckets := []string{"inprogress", "done"}
queues := []string{"preprocess", "wipeonly", "ocr", "analyse"}
for _, bucket := range buckets {
bname := prefix + bucket
log.Printf("Creating bucket %s\n", bname)
_, err = s3svc.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bname),
})
if err != nil {
aerr, ok := err.(awserr.Error)
if ok && (aerr.Code() == s3.ErrCodeBucketAlreadyExists || aerr.Code() == s3.ErrCodeBucketAlreadyOwnedByYou) {
log.Printf("Bucket %s already exists\n", bname)
} else {
log.Fatalf("Error creating bucket %s: %v\n", bname, err)
}
}
}
for _, queue := range queues {
qname := prefix + queue
log.Printf("Creating queue %s\n", qname)
_, err = sqssvc.CreateQueue(&sqs.CreateQueueInput{
QueueName: aws.String(qname),
Attributes: map[string]*string{
"VisibilityTimeout": aws.String("120"), // 2 minutes
"MessageRetentionPeriod": aws.String("1209600"), // 14 days; max allowed by sqs
"ReceiveMessageWaitTimeSeconds": aws.String("20"),
},
})
if err != nil {
aerr, ok := err.(awserr.Error)
// Note the QueueAlreadyExists code is only emitted if an existing queue
// has different attributes than the one that was being created. SQS just
// quietly ignores the CreateQueue request if it is identical to an
// existing queue.
if ok && aerr.Code() == sqs.ErrCodeQueueNameExists {
log.Fatalf("Error: Queue %s already exists but has different attributes\n", qname)
} else {
log.Fatalf("Error creating queue %s: %v\n", qname, err)
}
}
}
}
|
