From 22200755d471e77d20f490ffe01ede884b5acfcb Mon Sep 17 00:00:00 2001 From: Nick White Date: Tue, 7 Apr 2020 11:32:25 +0100 Subject: Separate out cloud settings into a separate file; cloudsettings.go --- aws.go | 42 ++++++++++++++++++++++++++++++++---------- cloudsettings.go | 30 ++++++++++++++++++++++++++++++ cmd/mkpipeline/main.go | 31 ++++--------------------------- 3 files changed, 66 insertions(+), 37 deletions(-) create mode 100644 cloudsettings.go diff --git a/aws.go b/aws.go index ecea193..74e6142 100644 --- a/aws.go +++ b/aws.go @@ -74,7 +74,7 @@ func (a *AwsConn) MinimalInit() error { a.downloader = s3manager.NewDownloader(a.sess) a.uploader = s3manager.NewUploader(a.sess) - a.wipstorageid = "rescribeinprogress" + a.wipstorageid = storageWip return nil } @@ -89,7 +89,7 @@ func (a *AwsConn) Init() error { a.Logger.Println("Getting preprocess queue URL") result, err := a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: aws.String("rescribepreprocess"), + QueueName: aws.String(queuePreProc), }) if err != nil { return errors.New(fmt.Sprintf("Error getting preprocess queue URL: %s", err)) @@ -98,7 +98,7 @@ func (a *AwsConn) Init() error { a.Logger.Println("Getting wipeonly queue URL") result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: aws.String("rescribewipeonly"), + QueueName: aws.String(queueWipeOnly), }) if err != nil { return errors.New(fmt.Sprintf("Error getting wipeonly queue URL: %s", err)) @@ -107,7 +107,7 @@ func (a *AwsConn) Init() error { a.Logger.Println("Getting OCR queue URL") result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: aws.String("rescribeocr"), + QueueName: aws.String(queueOcr), }) if err != nil { return errors.New(fmt.Sprintf("Error getting OCR queue URL: %s", err)) @@ -116,7 +116,7 @@ func (a *AwsConn) Init() error { a.Logger.Println("Getting analyse queue URL") result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: aws.String("rescribeanalyse"), + QueueName: aws.String(queueAnalyse), }) if err != nil { return errors.New(fmt.Sprintf("Error getting analyse queue URL: %s", err)) @@ -125,7 +125,7 @@ func (a *AwsConn) Init() error { a.Logger.Println("Getting OCR Page queue URL") result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: aws.String("rescribeocrpage"), + QueueName: aws.String(queueOcrPage), }) if err != nil { return errors.New(fmt.Sprintf("Error getting OCR Page queue URL: %s", err)) @@ -445,12 +445,12 @@ func (a *AwsConn) StartInstances(n int) error { InstanceCount: aws.Int64(int64(n)), LaunchSpecification: &ec2.RequestSpotLaunchSpecification{ IamInstanceProfile: &ec2.IamInstanceProfileSpecification{ - Arn: aws.String("arn:aws:iam::557852942063:instance-profile/pipeliner"), + Arn: aws.String(spotProfile), }, - ImageId: aws.String("ami-0bc6ef6900f6da5d3"), - InstanceType: aws.String("m5.large"), + ImageId: aws.String(spotImage), + InstanceType: aws.String(spotType), SecurityGroupIds: []*string{ - aws.String("sg-0be8a3ab89e7136b9"), + aws.String(spotSg), }, }, Type: aws.String("one-time"), @@ -463,3 +463,25 @@ func (a *AwsConn) StartInstances(n int) error { func (a *AwsConn) Log(v ...interface{}) { a.Logger.Print(v...) } + +// mkpipeline sets up necessary buckets and queues for the pipeline +func (a *AwsConn) MkPipeline() error { + buckets := []string{storageWip} + queues := []string{queuePreProc, queueWipeOnly, queueOcr, queueAnalyse, queueOcrPage} + + for _, bucket := range buckets { + err := a.CreateBucket(bucket) + if err != nil { + return err + } + } + + for _, queue := range queues { + err := a.CreateQueue(queue) + if err != nil { + return err + } + } + + return nil +} diff --git a/cloudsettings.go b/cloudsettings.go new file mode 100644 index 0000000..d627342 --- /dev/null +++ b/cloudsettings.go @@ -0,0 +1,30 @@ +// Copyright 2020 Nick White. +// Use of this source code is governed by the GPLv3 +// license that can be found in the LICENSE file. + +package bookpipeline + +// This file contains various cloud account specific stuff; change this if +// you want to use the cloud functionality on your own site. + +// Spot instance details +const ( + spotProfile = "arn:aws:iam::557852942063:instance-profile/pipeliner" + spotImage = "ami-0bc6ef6900f6da5d3" + spotType = "m5.large" + spotSg = "sg-0be8a3ab89e7136b9" +) + +// Queue names +const ( + queuePreProc = "rescribepreprocess" + queueWipeOnly = "rescribewipeonly" + queueOcr = "rescribeocr" + queueOcrPage = "rescribeocrpage" + queueAnalyse = "rescribeanalyse" +) + +// Storage bucket names +const ( + storageWip = "rescribeinprogress" +) diff --git a/cmd/mkpipeline/main.go b/cmd/mkpipeline/main.go index ab68d9a..b15e9ef 100644 --- a/cmd/mkpipeline/main.go +++ b/cmd/mkpipeline/main.go @@ -4,11 +4,6 @@ package main -// TODO: set up iam role and policy needed for ec2 instances to access this stuff; -// see arn:aws:iam::557852942063:policy/pipelinestorageandqueue -// and arn:aws:iam::557852942063:role/pipeliner -// TODO: set up launch template for ec2 instances - import ( "log" "os" @@ -18,8 +13,7 @@ import ( type MkPipeliner interface { MinimalInit() error - CreateBucket(string) error - CreateQueue(string) error + MkPipeline() error } func main() { @@ -34,25 +28,8 @@ func main() { log.Fatalln("Failed to set up cloud connection:", err) } - prefix := "rescribe" - buckets := []string{"inprogress", "done"} - queues := []string{"preprocess", "wipeonly", "ocr", "analyse", "ocrpage"} - - for _, bucket := range buckets { - bname := prefix + bucket - log.Printf("Creating bucket %s\n", bname) - err = conn.CreateBucket(bname) - if err != nil { - log.Fatalln(err) - } - } - - for _, queue := range queues { - qname := prefix + queue - log.Printf("Creating queue %s\n", qname) - err = conn.CreateQueue(qname) - if err != nil { - log.Fatalln(err) - } + err = conn.MkPipeline() + if err != nil { + log.Fatalln("MkPipeline failed:", err) } } -- cgit v1.2.1-24-ge1ad