summaryrefslogtreecommitdiff
path: root/bookpipeline/cmd/booktopipeline
diff options
context:
space:
mode:
authorNick White <git@njw.name>2019-10-08 15:49:52 +0100
committerNick White <git@njw.name>2019-10-08 15:49:52 +0100
commit69aae6b93dcadd9e4895f86fe661ee80e79dcf9e (patch)
treebfc42b832a6d6e3051631eb6cf530e1400b2c080 /bookpipeline/cmd/booktopipeline
parentd43c11bf653bfe3c1ad1ed277f1ec08bf155cf98 (diff)
Remove parts that have been moved elsewhere, and rename to rescribe.xyz/utils
bookpipeline is now at rescribe.xyz/bookpipeline preproc is now at rescribe.xyz/preproc integralimg is now at rescribe.xyz/preproc/integralimg
Diffstat (limited to 'bookpipeline/cmd/booktopipeline')
-rw-r--r--bookpipeline/cmd/booktopipeline/main.go140
1 files changed, 0 insertions, 140 deletions
diff --git a/bookpipeline/cmd/booktopipeline/main.go b/bookpipeline/cmd/booktopipeline/main.go
deleted file mode 100644
index 6d9f146..0000000
--- a/bookpipeline/cmd/booktopipeline/main.go
+++ /dev/null
@@ -1,140 +0,0 @@
-package main
-
-// TODO: use bookpipeline package to do aws stuff
-
-import (
- "flag"
- "fmt"
- "log"
- "os"
- "path/filepath"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/session"
- "github.com/aws/aws-sdk-go/service/s3/s3manager"
- "github.com/aws/aws-sdk-go/service/sqs"
-)
-
-const usage = `Usage: booktopipeline [-prebinarised] [-v] bookdir [bookname]
-
-Uploads the book in bookdir to the S3 'inprogress' bucket and adds it
-to the 'preprocess' SQS queue, or the 'wipeonly' queue if the
-prebinarised flag is set.
-
-If bookname is omitted the last part of the bookdir is used.
-`
-
-// null writer to enable non-verbose logging to be discarded
-type NullWriter bool
-
-func (w NullWriter) Write(p []byte) (n int, err error) {
- return len(p), nil
-}
-
-var verboselog *log.Logger
-
-type fileWalk chan string
-
-func (f fileWalk) Walk(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if !info.IsDir() {
- f <- path
- }
- return nil
-}
-
-func main() {
- verbose := flag.Bool("v", false, "Verbose")
- wipeonly := flag.Bool("prebinarised", false, "Prebinarised: only preprocessing will be to wipe")
-
- flag.Usage = func() {
- fmt.Fprintf(flag.CommandLine.Output(), usage)
- flag.PrintDefaults()
- }
- flag.Parse()
- if flag.NArg() < 1 {
- flag.Usage()
- return
- }
-
- bookdir := flag.Arg(0)
- var bookname string
- if flag.NArg() > 2 {
- bookname = flag.Arg(1)
- } else {
- bookname = filepath.Base(bookdir)
- }
-
- if *verbose {
- verboselog = log.New(os.Stdout, "", log.LstdFlags)
- } else {
- var n NullWriter
- verboselog = log.New(n, "", log.LstdFlags)
- }
-
- verboselog.Println("Setting up AWS session")
- sess, err := session.NewSession(&aws.Config{
- Region: aws.String("eu-west-2"),
- })
- if err != nil {
- log.Fatalln("Error: failed to set up aws session:", err)
- }
- sqssvc := sqs.New(sess)
- uploader := s3manager.NewUploader(sess)
-
- var qname string
- if *wipeonly {
- qname = "rescribewipeonly"
- } else {
- qname = "rescribepreprocess"
- }
- verboselog.Println("Getting Queue URL for", qname)
- result, err := sqssvc.GetQueueUrl(&sqs.GetQueueUrlInput{
- QueueName: aws.String(qname),
- })
- if err != nil {
- log.Fatalln("Error getting queue URL for", qname, ":", err)
- }
- qurl := *result.QueueUrl
-
- // concurrent walking upload based on example at
- // https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sdk-utilities.html
- verboselog.Println("Walking", bookdir)
- walker := make(fileWalk)
- go func() {
- err = filepath.Walk(bookdir, walker.Walk)
- if err != nil {
- log.Fatalln("Filesystem walk failed:", err)
- }
- close(walker)
- }()
-
- for path := range walker {
- verboselog.Println("Uploading", path)
- name := filepath.Base(path)
- file, err := os.Open(path)
- if err != nil {
- log.Fatalln("Open file", path, "failed:", err)
- }
- defer file.Close()
- _, err = uploader.Upload(&s3manager.UploadInput{
- Bucket: aws.String("rescribeinprogress"),
- Key: aws.String(filepath.Join(bookname, name)),
- Body: file,
- })
- if err != nil {
- log.Fatalln("Failed to upload", path, err)
- }
- }
-
- verboselog.Println("Sending message", bookname, "to queue", qurl)
- _, err = sqssvc.SendMessage(&sqs.SendMessageInput{
- MessageBody: aws.String(bookname),
- QueueUrl: &qurl,
- })
- if err != nil {
- log.Fatalln("Error adding book to queue:", err)
- }
-}