From 76d91ea8f65c6ad52efb24ac2c94b22c2908bc5c Mon Sep 17 00:00:00 2001 From: Nick White Date: Mon, 21 Mar 2022 13:51:51 +0000 Subject: Only generate full-size PDF if requested This avoids the issue that large PDFs require a lot of RAM, so there are chances of running out of memory. Plus it's a waste of space and time. --- internal/pipeline/pipeline.go | 90 ++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 44 deletions(-) (limited to 'internal/pipeline') diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index a09a414..d8beeb9 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -330,7 +330,7 @@ func Ocr(training string, tesscmd string) func(context.Context, chan string, cha } } -func Analyse(conn Downloader) func(context.Context, chan string, chan string, chan error, *log.Logger) { +func Analyse(conn Downloader, mkfullpdf bool) func(context.Context, chan string, chan string, chan error, *log.Logger) { return func(ctx context.Context, toanalyse chan string, up chan string, errc chan error, logger *log.Logger) { confs := make(map[string][]*bookpipeline.Conf) bestconfs := make(map[string]*bookpipeline.Conf) @@ -455,12 +455,6 @@ func Analyse(conn Downloader) func(context.Context, chan string, chan string, ch errc <- fmt.Errorf("Failed to set up PDF: %s", err) return } - fullsizepdf := new(bookpipeline.Fpdf) - err = fullsizepdf.Setup() - if err != nil { - errc <- fmt.Errorf("Failed to set up PDF: %s", err) - return - } binhascontent, colourhascontent := false, false select { @@ -583,54 +577,62 @@ func Analyse(conn Downloader) func(context.Context, chan string, chan string, ch up <- fn } - for _, pg := range colourimgs { - select { - case <-ctx.Done(): - errc <- ctx.Err() + if mkfullpdf { + fullsizepdf := new(bookpipeline.Fpdf) + err = fullsizepdf.Setup() + if err != nil { + errc <- fmt.Errorf("Failed to set up PDF: %s", err) return - default: } + for _, pg := range colourimgs { + select { + case <-ctx.Done(): + errc <- ctx.Err() + return + default: + } - logger.Println("Downloading colour page to add to PDF", pg.img) - colourfn := pg.img - err = conn.Download(conn.WIPStorageId(), bookname+"/"+colourfn, filepath.Join(savedir, colourfn)) - if err != nil { - colourfn = strings.Replace(pg.img, ".jpg", ".png", 1) - logger.Println("Download failed; trying", colourfn) + logger.Println("Downloading colour page to add to PDF", pg.img) + colourfn := pg.img err = conn.Download(conn.WIPStorageId(), bookname+"/"+colourfn, filepath.Join(savedir, colourfn)) if err != nil { - logger.Println("Download failed; skipping page", pg.img) + colourfn = strings.Replace(pg.img, ".jpg", ".png", 1) + logger.Println("Download failed; trying", colourfn) + err = conn.Download(conn.WIPStorageId(), bookname+"/"+colourfn, filepath.Join(savedir, colourfn)) + if err != nil { + logger.Println("Download failed; skipping page", pg.img) + } } - } - if err == nil { - err = fullsizepdf.AddPage(filepath.Join(savedir, colourfn), filepath.Join(savedir, pg.hocr), false) - if err != nil { - errc <- fmt.Errorf("Failed to add page %s to PDF: %s", pg.img, err) - return - } - err = os.Remove(filepath.Join(savedir, colourfn)) - if err != nil { - errc <- err - return + if err == nil { + err = fullsizepdf.AddPage(filepath.Join(savedir, colourfn), filepath.Join(savedir, pg.hocr), false) + if err != nil { + errc <- fmt.Errorf("Failed to add page %s to PDF: %s", pg.img, err) + return + } + err = os.Remove(filepath.Join(savedir, colourfn)) + if err != nil { + errc <- err + return + } } } - } - - select { - case <-ctx.Done(): - errc <- ctx.Err() - return - default: - } - if colourhascontent { - fn = filepath.Join(savedir, bookname+".original.pdf") - err = fullsizepdf.Save(fn) - if err != nil { - errc <- fmt.Errorf("Failed to save full size pdf: %s", err) + select { + case <-ctx.Done(): + errc <- ctx.Err() return + default: + } + + if colourhascontent { + fn = filepath.Join(savedir, bookname+".original.pdf") + err = fullsizepdf.Save(fn) + if err != nil { + errc <- fmt.Errorf("Failed to save full size pdf: %s", err) + return + } + up <- fn } - up <- fn } select { -- cgit v1.2.1-24-ge1ad