From ca5014f0ad35ad475860ff817a2ebed121bc1f27 Mon Sep 17 00:00:00 2001 From: Nick White Date: Fri, 13 Mar 2020 16:52:05 +0000 Subject: Add simple "getbests" utility, useful for statistics gathering --- cmd/getbests/main.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 cmd/getbests/main.go (limited to 'cmd') diff --git a/cmd/getbests/main.go b/cmd/getbests/main.go new file mode 100644 index 0000000..2b0c40a --- /dev/null +++ b/cmd/getbests/main.go @@ -0,0 +1,70 @@ +// Copyright 2020 Nick White. +// Use of this source code is governed by the GPLv3 +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "log" + "strings" + + "rescribe.xyz/bookpipeline" +) + +const usage = `Usage: getbests + +Downloads every 'best' file from a set of OCRed books. This is +useful for statistics. +` + +// null writer to enable non-verbose logging to be discarded +type NullWriter bool + +func (w NullWriter) Write(p []byte) (n int, err error) { + return len(p), nil +} + +type Pipeliner interface { + Init() error + ListObjects(bucket string, prefix string) ([]string, error) + Download(bucket string, key string, fn string) error + WIPStorageId() string +} + +func main() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), usage) + flag.PrintDefaults() + } + flag.Parse() + + var n NullWriter + verboselog := log.New(n, "", log.LstdFlags) + + var conn Pipeliner + conn = &bookpipeline.AwsConn{Region: "eu-west-2", Logger: verboselog} + + err := conn.Init() + if err != nil { + log.Fatalln("Error setting up cloud connection:", err) + } + + log.Println("Getting list of all available objects to filter through") + objs, err := conn.ListObjects(conn.WIPStorageId(), "") + if err != nil { + log.Fatalln("Failed to get list of files", err) + } + + log.Println("Downloading all best files found") + for _, i := range objs { + parts := strings.Split(i, "/") + if parts[len(parts) - 1] == "best" { + err = conn.Download(conn.WIPStorageId(), i, parts[0] + "-best") + if err != nil { + log.Fatalln("Failed to download file", i, err) + } + } + } +} -- cgit v1.2.1-24-ge1ad