From dc9d2911f67d8c7da4d6b761afc7cf882f21b8a7 Mon Sep 17 00:00:00 2001 From: Nick White Date: Tue, 24 Sep 2019 15:05:02 +0100 Subject: Move ec2 stuff out of lspipeline and into aws.go --- bookpipeline/aws.go | 41 +++++++++++++++++++ bookpipeline/cmd/lspipeline/main.go | 78 ++++++++----------------------------- 2 files changed, 57 insertions(+), 62 deletions(-) diff --git a/bookpipeline/aws.go b/bookpipeline/aws.go index f3cdbfa..063bc9f 100644 --- a/bookpipeline/aws.go +++ b/bookpipeline/aws.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/aws/aws-sdk-go/service/sqs" @@ -22,6 +23,10 @@ type Qmsg struct { Id, Handle, Body string } +type InstanceDetails struct { + Id, Name, Ip, Spot, Type, State, LaunchTime string +} + type AwsConn struct { // these need to be set before running Init() Region string @@ -29,6 +34,7 @@ type AwsConn struct { // these are used internally sess *session.Session + ec2svc *ec2.EC2 s3svc *s3.S3 sqssvc *sqs.SQS downloader *s3manager.Downloader @@ -37,6 +43,7 @@ type AwsConn struct { wipstorageid string } +// TODO: split this up, as not everything is needed for different uses func (a *AwsConn) Init() error { if a.Region == "" { return errors.New("No Region set") @@ -52,6 +59,7 @@ func (a *AwsConn) Init() error { if err != nil { return errors.New(fmt.Sprintf("Failed to set up aws session: %s", err)) } + a.ec2svc = ec2.New(a.sess) a.s3svc = s3.New(a.sess) a.sqssvc = sqs.New(a.sess) a.downloader = s3manager.NewDownloader(a.sess) @@ -259,3 +267,36 @@ func (a *AwsConn) Upload(bucket string, key string, path string) error { func (a *AwsConn) GetLogger() *log.Logger { return a.Logger } + +// TODO: split pages function so it can be encapsulated by +// downstream and to feed a channel +func (a *AwsConn) GetInstanceDetails() ([]InstanceDetails, error) { + var details []InstanceDetails + err := a.ec2svc.DescribeInstancesPages(&ec2.DescribeInstancesInput{}, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { + for _, r := range page.Reservations { + for _, i := range r.Instances { + var d InstanceDetails + + for _, t := range i.Tags { + if *t.Key == "Name" { + d.Name = *t.Value + } + } + if i.PublicIpAddress != nil { + d.Ip = *i.PublicIpAddress + } + if i.SpotInstanceRequestId != nil { + d.Spot = *i.SpotInstanceRequestId + } + d.Type = *i.InstanceType + d.Id = *i.InstanceId + d.LaunchTime = i.LaunchTime.String() + d.State = *i.State.Name + + details = append(details, d) + } + } + return !lastPage + }) + return details, err +} diff --git a/bookpipeline/cmd/lspipeline/main.go b/bookpipeline/cmd/lspipeline/main.go index d49b933..3cbc893 100644 --- a/bookpipeline/cmd/lspipeline/main.go +++ b/bookpipeline/cmd/lspipeline/main.go @@ -7,12 +7,6 @@ import ( "os" "rescribe.xyz/go.git/bookpipeline" - - // TODO: abstract out the aws stuff into aws.go in due course - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - //"github.com/aws/aws-sdk-go/service/s3" ) const usage = `Usage: lspipeline [-v] @@ -32,6 +26,7 @@ type LsPipeliner interface { OCRQueueId() string AnalyseQueueId() string GetQueueDetails(url string) (string, string, error) + GetInstanceDetails() ([]bookpipeline.InstanceDetails, error) } // NullWriter is used so non-verbose logging may be discarded @@ -41,52 +36,19 @@ func (w NullWriter) Write(p []byte) (n int, err error) { return len(p), nil } -type instanceDetails struct { - id, name, ip, spot, iType, state, launchTime string -} - type queueDetails struct { name, numAvailable, numInProgress string } -func ec2getInstances(svc *ec2.EC2, instances chan instanceDetails) { - err := svc.DescribeInstancesPages(&ec2.DescribeInstancesInput{}, parseInstances(instances)) +func getInstances(conn LsPipeliner, detailsc chan bookpipeline.InstanceDetails) { + details, err := conn.GetInstanceDetails() if err != nil { - close(instances) - log.Println("Error with ec2 DescribeInstancePages call:", err) + log.Println("Error getting instance details:", err) } -} - -func parseInstances(details chan instanceDetails) (func(*ec2.DescribeInstancesOutput, bool) bool) { - return func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { - for _, r := range page.Reservations { - for _, i := range r.Instances { - var d instanceDetails - - for _, t := range i.Tags { - if *t.Key == "Name" { - d.name = *t.Value - } - } - if i.PublicIpAddress != nil { - d.ip = *i.PublicIpAddress - } - if i.SpotInstanceRequestId != nil { - d.spot = *i.SpotInstanceRequestId - } - d.iType = *i.InstanceType - d.id = *i.InstanceId - d.launchTime = i.LaunchTime.String() - d.state = *i.State.Name - - details <- d - } - } - if lastPage { - close(details) - } - return !lastPage + for _, d := range details { + detailsc <- d } + close(detailsc) } func getQueueDetails(conn LsPipeliner, qdetails chan queueDetails) { @@ -132,31 +94,23 @@ func main() { log.Fatalln("Failed to set up cloud connection:", err) } - sess, err := session.NewSession(&aws.Config{ - Region: aws.String("eu-west-2"), - }) - if err != nil { - log.Fatalln("Failed to set up aws session", err) - } - ec2svc := ec2.New(sess) - - instances := make(chan instanceDetails, 100) + instances := make(chan bookpipeline.InstanceDetails, 100) queues := make(chan queueDetails) - go ec2getInstances(ec2svc, instances) + go getInstances(conn, instances) go getQueueDetails(conn, queues) fmt.Println("# Instances") for i := range instances { - fmt.Printf("ID: %s, Type: %s, LaunchTime: %s, State: %s", i.id, i.iType, i.launchTime, i.state) - if i.name != "" { - fmt.Printf(", Name: %s", i.name) + fmt.Printf("ID: %s, Type: %s, LaunchTime: %s, State: %s", i.Id, i.Type, i.LaunchTime, i.State) + if i.Name != "" { + fmt.Printf(", Name: %s", i.Name) } - if i.ip != "" { - fmt.Printf(", IP: %s", i.ip) + if i.Ip != "" { + fmt.Printf(", IP: %s", i.Ip) } - if i.spot != "" { - fmt.Printf(", SpotRequest: %s", i.spot) + if i.Spot != "" { + fmt.Printf(", SpotRequest: %s", i.Spot) } fmt.Printf("\n") } -- cgit v1.2.1-24-ge1ad