Separate out cloud settings into a separate file; cloudsettings.go

3 files changed, 66 insertions, 37 deletions

diff --git a/aws.go b/aws.go
index ecea193..74e6142 100644
--- a/aws.go
+++ b/aws.go
@@ -74,7 +74,7 @@ func (a *AwsConn) MinimalInit() error {
 	a.downloader = s3manager.NewDownloader(a.sess)
 	a.uploader = s3manager.NewUploader(a.sess)
 
-	a.wipstorageid = "rescribeinprogress"
+	a.wipstorageid = storageWip
 
 	return nil
 }
@@ -89,7 +89,7 @@ func (a *AwsConn) Init() error {
 
 	a.Logger.Println("Getting preprocess queue URL")
 	result, err := a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{
-		QueueName: aws.String("rescribepreprocess"),
+		QueueName: aws.String(queuePreProc),
 	})
 	if err != nil {
 		return errors.New(fmt.Sprintf("Error getting preprocess queue URL: %s", err))
@@ -98,7 +98,7 @@ func (a *AwsConn) Init() error {
 
 	a.Logger.Println("Getting wipeonly queue URL")
 	result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{
-		QueueName: aws.String("rescribewipeonly"),
+		QueueName: aws.String(queueWipeOnly),
 	})
 	if err != nil {
 		return errors.New(fmt.Sprintf("Error getting wipeonly queue URL: %s", err))
@@ -107,7 +107,7 @@ func (a *AwsConn) Init() error {
 
 	a.Logger.Println("Getting OCR queue URL")
 	result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{
-		QueueName: aws.String("rescribeocr"),
+		QueueName: aws.String(queueOcr),
 	})
 	if err != nil {
 		return errors.New(fmt.Sprintf("Error getting OCR queue URL: %s", err))
@@ -116,7 +116,7 @@ func (a *AwsConn) Init() error {
 
 	a.Logger.Println("Getting analyse queue URL")
 	result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{
-		QueueName: aws.String("rescribeanalyse"),
+		QueueName: aws.String(queueAnalyse),
 	})
 	if err != nil {
 		return errors.New(fmt.Sprintf("Error getting analyse queue URL: %s", err))
@@ -125,7 +125,7 @@ func (a *AwsConn) Init() error {
 
 	a.Logger.Println("Getting OCR Page queue URL")
 	result, err = a.sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{
-		QueueName: aws.String("rescribeocrpage"),
+		QueueName: aws.String(queueOcrPage),
 	})
 	if err != nil {
 		return errors.New(fmt.Sprintf("Error getting OCR Page queue URL: %s", err))
@@ -445,12 +445,12 @@ func (a *AwsConn) StartInstances(n int) error {
 		InstanceCount: aws.Int64(int64(n)),
 		LaunchSpecification: &ec2.RequestSpotLaunchSpecification{
 			IamInstanceProfile: &ec2.IamInstanceProfileSpecification{
-				Arn: aws.String("arn:aws:iam::557852942063:instance-profile/pipeliner"),
+				Arn: aws.String(spotProfile),
 			},
-			ImageId: aws.String("ami-0bc6ef6900f6da5d3"),
-			InstanceType: aws.String("m5.large"),
+			ImageId: aws.String(spotImage),
+			InstanceType: aws.String(spotType),
 			SecurityGroupIds: []*string{
-				aws.String("sg-0be8a3ab89e7136b9"),
+				aws.String(spotSg),
 			},
 		},
 		Type: aws.String("one-time"),
@@ -463,3 +463,25 @@ func (a *AwsConn) StartInstances(n int) error {
 func (a *AwsConn) Log(v ...interface{}) {
 	a.Logger.Print(v...)
 }
+
+// mkpipeline sets up necessary buckets and queues for the pipeline
+func (a *AwsConn) MkPipeline() error {
+	buckets := []string{storageWip}
+	queues := []string{queuePreProc, queueWipeOnly, queueOcr, queueAnalyse, queueOcrPage}
+
+	for _, bucket := range buckets {
+		err := a.CreateBucket(bucket)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, queue := range queues {
+		err := a.CreateQueue(queue)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
diff --git a/cloudsettings.go b/cloudsettings.go
new file mode 100644
index 0000000..d627342
--- /dev/null
+++ b/cloudsettings.go
@@ -0,0 +1,30 @@
+// Copyright 2020 Nick White.
+// Use of this source code is governed by the GPLv3
+// license that can be found in the LICENSE file.
+
+package bookpipeline
+
+// This file contains various cloud account specific stuff; change this if
+// you want to use the cloud functionality on your own site.
+
+// Spot instance details
+const (
+	spotProfile = "arn:aws:iam::557852942063:instance-profile/pipeliner"
+	spotImage   = "ami-0bc6ef6900f6da5d3"
+	spotType    = "m5.large"
+	spotSg      = "sg-0be8a3ab89e7136b9"
+)
+
+// Queue names
+const (
+	queuePreProc  = "rescribepreprocess"
+	queueWipeOnly = "rescribewipeonly"
+	queueOcr      = "rescribeocr"
+	queueOcrPage  = "rescribeocrpage"
+	queueAnalyse  = "rescribeanalyse"
+)
+
+// Storage bucket names
+const (
+	storageWip = "rescribeinprogress"
+)
diff --git a/cmd/mkpipeline/main.go b/cmd/mkpipeline/main.go
index ab68d9a..b15e9ef 100644
--- a/cmd/mkpipeline/main.go
+++ b/cmd/mkpipeline/main.go
@@ -4,11 +4,6 @@
 
 package main
 
-// TODO: set up iam role and policy needed for ec2 instances to access this stuff;
-//       see arn:aws:iam::557852942063:policy/pipelinestorageandqueue
-//       and arn:aws:iam::557852942063:role/pipeliner
-// TODO: set up launch template for ec2 instances
-
 import (
 	"log"
 	"os"
@@ -18,8 +13,7 @@ import (
 
 type MkPipeliner interface {
 	MinimalInit() error
-	CreateBucket(string) error
-	CreateQueue(string) error
+	MkPipeline() error
 }
 
 func main() {
@@ -34,25 +28,8 @@ func main() {
 		log.Fatalln("Failed to set up cloud connection:", err)
 	}
 
-	prefix := "rescribe"
-	buckets := []string{"inprogress", "done"}
-	queues := []string{"preprocess", "wipeonly", "ocr", "analyse", "ocrpage"}
-
-	for _, bucket := range buckets {
-		bname := prefix + bucket
-		log.Printf("Creating bucket %s\n", bname)
-		err = conn.CreateBucket(bname)
-		if err != nil {
-			log.Fatalln(err)
-		}
-	}
-
-	for _, queue := range queues {
-		qname := prefix + queue
-		log.Printf("Creating queue %s\n", qname)
-		err = conn.CreateQueue(qname)
-		if err != nil {
-			log.Fatalln(err)
-		}
+	err = conn.MkPipeline()
+	if err != nil {
+		log.Fatalln("MkPipeline failed:", err)
 	}
 }