From 4c0553fa86e1cf453302b01558e7bfdb6f97215f Mon Sep 17 00:00:00 2001 From: Nick White Date: Tue, 14 Apr 2020 12:07:20 +0100 Subject: Remove getbests; it belongs with bookpipeline (and putting it there removes an annoying circular dependency) --- cmd/getbests/main.go | 72 ---------------------------------------------------- 1 file changed, 72 deletions(-) delete mode 100644 cmd/getbests/main.go (limited to 'cmd/getbests') diff --git a/cmd/getbests/main.go b/cmd/getbests/main.go deleted file mode 100644 index 9eca0d8..0000000 --- a/cmd/getbests/main.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020 Nick White. -// Use of this source code is governed by the GPLv3 -// license that can be found in the LICENSE file. - -// getbests downloads every 'best' file from a set of OCRed books -// stored on cloud infrastructure -package main - -import ( - "flag" - "fmt" - "log" - "strings" - - "rescribe.xyz/bookpipeline" -) - -const usage = `Usage: getbests - -Downloads every 'best' file from a set of OCRed books. This is -useful for statistics. -` - -// null writer to enable non-verbose logging to be discarded -type NullWriter bool - -func (w NullWriter) Write(p []byte) (n int, err error) { - return len(p), nil -} - -type Pipeliner interface { - Init() error - ListObjects(bucket string, prefix string) ([]string, error) - Download(bucket string, key string, fn string) error - WIPStorageId() string -} - -func main() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), usage) - flag.PrintDefaults() - } - flag.Parse() - - var n NullWriter - verboselog := log.New(n, "", log.LstdFlags) - - var conn Pipeliner - conn = &bookpipeline.AwsConn{Region: "eu-west-2", Logger: verboselog} - - err := conn.Init() - if err != nil { - log.Fatalln("Error setting up cloud connection:", err) - } - - log.Println("Getting list of all available objects to filter through") - objs, err := conn.ListObjects(conn.WIPStorageId(), "") - if err != nil { - log.Fatalln("Failed to get list of files", err) - } - - log.Println("Downloading all best files found") - for _, i := range objs { - parts := strings.Split(i, "/") - if parts[len(parts) - 1] == "best" { - err = conn.Download(conn.WIPStorageId(), i, parts[0] + "-best") - if err != nil { - log.Fatalln("Failed to download file", i, err) - } - } - } -} -- cgit v1.2.1-24-ge1ad