summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2020-07-21 16:48:32 +0100
committerNick White <git@njw.name>2020-07-21 16:48:32 +0100
commit6cdfec95a80258f632fa7ce4dce36000cdfeea30 (patch)
tree36952904dec2615367d7b32a8f730b46c1e64438
parent80b49a19f921a6a3742eb7b7ec2937b872fe4d49 (diff)
[bookpipeline] If preprocessing fails, email us and remove the job from the queue
This prevents the current situation where a failed preprocessing job is endlessly repeated, potentially spawning thousands of ocrpage jobs in its wake each time. Note that the email stuff works but requires putting secrets into .go files, so need to rewrite that to read from somewhere more sensible like a dotfile on the host.
-rw-r--r--cmd/bookpipeline/main.go41
-rw-r--r--mailsettings.go18
2 files changed, 55 insertions, 4 deletions
diff --git a/cmd/bookpipeline/main.go b/cmd/bookpipeline/main.go
index cd2d9c7..c0755a6 100644
--- a/cmd/bookpipeline/main.go
+++ b/cmd/bookpipeline/main.go
@@ -12,6 +12,7 @@ import (
"flag"
"fmt"
"log"
+ "net/smtp"
"os"
"os/exec"
"path/filepath"
@@ -617,6 +618,34 @@ func processBook(msg bookpipeline.Qmsg, conn Pipeliner, process func(chan string
case err = <-errc:
t.Stop()
_ = os.RemoveAll(d)
+ // if the error is in preprocessing, chances are that it will never
+ // complete, and will fill the ocrpage queue with parts which succeeded
+ // on each run, so in that case it's better to delete the message from
+ // the queue and notify us.
+ if fromQueue == conn.PreQueueId() {
+ conn.Log("Deleting message from queue due to a bad error", fromQueue)
+ err2 := conn.DelFromQueue(fromQueue, msg.Handle)
+ if err2 != nil {
+ conn.Log("Error deleting message from queue", err2)
+ }
+ if bookpipeline.MailServer != "" {
+ logs, err2 := getlogs()
+ if err2 != nil {
+ conn.Log("Failed to get logs ", err2)
+ logs = ""
+ }
+ msg := fmt.Sprintf("To: %s\r\nFrom: %s\r\n" +
+ "Subject: [bookpipeline] Error in preprocessing queue with %s\r\n\r\n" +
+ " Fail message: %s\r\nFull log:\r\n%s\r\n",
+ bookpipeline.MailTo, bookpipeline.MailFrom, bookname, err, logs)
+ host := fmt.Sprintf("%s:%d", bookpipeline.MailServer, bookpipeline.MailPort)
+ auth := smtp.PlainAuth("", bookpipeline.MailUser, bookpipeline.MailPass, bookpipeline.MailServer)
+ err2 = smtp.SendMail(host, auth, bookpipeline.MailFrom, []string{bookpipeline.MailTo}, []byte(msg))
+ if err2 != nil {
+ conn.Log("!!! Error sending email ", err2)
+ }
+ }
+ }
return err
case <-done:
}
@@ -675,15 +704,19 @@ func stopTimer(t *time.Timer) {
// conn struct that implements those, so that we could pass a log.Logger
// or the new conn struct everywhere (we wouldn't be passing a log.Logger,
// it's just good to be able to keep the compatibility)
-func savelogs(conn Pipeliner, starttime int64, hostname string) error {
+func getlogs() (string, error) {
cmd := exec.Command("journalctl", "-u", "bookpipeline", "-n", "all")
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
+ return stdout.String(), err
+}
+
+func savelogs(conn Pipeliner, starttime int64, hostname string) error {
+ logs, err := getlogs()
if err != nil {
- return fmt.Errorf("Error getting logs, error: %v, stdout: %v, stderr: %v",
- err, stdout.String(), stderr.String())
+ return fmt.Errorf("Error getting logs, error: %v", err)
}
key := fmt.Sprintf("bookpipeline.log.%d.%s", starttime, hostname)
path := filepath.Join(os.TempDir(), key)
@@ -692,7 +725,7 @@ func savelogs(conn Pipeliner, starttime int64, hostname string) error {
return fmt.Errorf("Error creating log file", err)
}
defer f.Close()
- _, err = f.WriteString(stdout.String())
+ _, err = f.WriteString(logs)
if err != nil {
return fmt.Errorf("Error saving log file", err)
}
diff --git a/mailsettings.go b/mailsettings.go
new file mode 100644
index 0000000..b12a3a3
--- /dev/null
+++ b/mailsettings.go
@@ -0,0 +1,18 @@
+// Copyright 2020 Nick White.
+// Use of this source code is governed by the GPLv3
+// license that can be found in the LICENSE file.
+
+package bookpipeline
+
+// This file contains various mail account specific stuff; set this if
+// you want to use the email notification functionality.
+
+// TODO: these should be set in a dotfile on the host, not here where the world can see them
+const (
+ MailServer = ""
+ MailPort = 587
+ MailUser = ""
+ MailPass = ""
+ MailFrom = ""
+ MailTo = ""
+)