summaryrefslogtreecommitdiff
path: root/cmd
diff options
context:
space:
mode:
authorNick White <git@njw.name>2020-06-16 16:57:34 +0100
committerNick White <git@njw.name>2020-06-16 16:57:34 +0100
commit629e436f63da8f19fd1654a634edf8e4f1c2bdad (patch)
tree713d6f890bd5e3d22dcc8869815e66cb22c08880 /cmd
parent8aa2d713d511baeaa94472e238e46c6d02ac9332 (diff)
[getallhocrs] Skip files which have already been downloaded
Diffstat (limited to 'cmd')
-rw-r--r--cmd/bookpipeline/main.go9
-rw-r--r--cmd/getallhocrs/main.go7
2 files changed, 15 insertions, 1 deletions
diff --git a/cmd/bookpipeline/main.go b/cmd/bookpipeline/main.go
index 16ce596..0afda16 100644
--- a/cmd/bookpipeline/main.go
+++ b/cmd/bookpipeline/main.go
@@ -667,7 +667,14 @@ func stopTimer(t *time.Timer) {
// TODO: rather than relying on journald, would be nicer to save the logs
// ourselves maybe, so that we weren't relying on a particular systemd
-// setup.
+// setup. this can be done by having the conn.Log also append line
+// to a file (though that would mean everything would have to go through
+// conn.Log, which we're not consistently doing yet). the correct thing
+// to do then would be to implement a new interface that covers the part
+// of log.Logger we use (e.g. Print and Printf), and then have an exported
+// conn struct that implements those, so that we could pass a log.Logger
+// or the new conn struct everywhere (we wouldn't be passing a log.Logger,
+// it's just good to be able to keep the compatibility)
func savelogs(conn Pipeliner, starttime int64, hostname string) error {
cmd := exec.Command("journalctl", "-u", "bookpipeline", "-n", "all")
var stdout, stderr bytes.Buffer
diff --git a/cmd/getallhocrs/main.go b/cmd/getallhocrs/main.go
index 136f07e..696a5fc 100644
--- a/cmd/getallhocrs/main.go
+++ b/cmd/getallhocrs/main.go
@@ -68,6 +68,13 @@ func main() {
if !strings.HasSuffix(o, ".hocr") {
continue
}
+ // skip already downloaded items
+ _, err = os.Stat(o)
+ if err == nil || os.IsExist(err) {
+ log.Println(" Skipping already complete download of", o)
+ continue
+ }
+ log.Println(" Downloading", o)
err = conn.Download(conn.WIPStorageId(), o, o)
if err != nil {
log.Fatalln("Failed to download file", o, err)