1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
// Copyright 2021 Nick White.
// Use of this source code is governed by the GPLv3
// license that can be found in the LICENSE file.
package pipeline
import (
"context"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
// null writer to enable non-verbose logging to be discarded
type NullWriter bool
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
type fileWalk chan string
// Walk sends the path of all files to the channel, with the exception of
// any file which starts with "."
func (f fileWalk) Walk(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// skip files starting with . to prevent automatically generated
// files like .DS_Store getting in the way
if strings.HasPrefix(filepath.Base(path), ".") {
return nil
}
if !info.IsDir() {
f <- path
}
return nil
}
// CheckImages checks that all files with a ".jpg" or ".png" suffix
// in a directory are images that can be decoded (skipping dotfiles)
func CheckImages(ctx context.Context, dir string) error {
checker := make(fileWalk)
go func() {
_ = filepath.Walk(dir, checker.Walk)
close(checker)
}()
n := 0
for path := range checker {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
suffix := filepath.Ext(path)
lsuffix := strings.ToLower(suffix)
if lsuffix == ".jpeg" {
lsuffix = ".jpg"
}
if lsuffix != ".jpg" && lsuffix != ".png" {
continue
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("Opening image %s failed: %v", path, err)
}
_, _, err = image.Decode(f)
if err != nil {
return fmt.Errorf("Decoding image %s failed: %v", path, err)
}
n++
}
if n == 0 {
return fmt.Errorf("No images found")
}
return nil
}
// DetectQueueType detects which queue to use based on the preponderance
// of files of a particular extension in a directory
func DetectQueueType(dir string, conn Queuer, nowipe bool) string {
if nowipe {
return conn.PreNoWipeQueueId()
}
pngdirs, _ := filepath.Glob(dir + "/*.png")
jpgdirs, _ := filepath.Glob(dir + "/*.jpg")
pngcount := len(pngdirs)
jpgcount := len(jpgdirs)
if pngcount > jpgcount {
return conn.WipeQueueId()
} else {
return conn.PreQueueId()
}
}
// UploadImages uploads all files with a suffix of ".jpg" or ".png"
// (except those which start with a ".") from a directory (recursively)
// into conn.WIPStorageId(), prefixed with the given bookname and a
// slash. It also appends all file names with sequential numbers, like
// 0001, to ensure they are appropriately named for further processing
// in the pipeline.
func UploadImages(ctx context.Context, dir string, bookname string, conn Uploader) error {
files, err := ioutil.ReadDir(dir)
if err != nil {
fmt.Errorf("Failed to read directory %s: %v", dir, err)
}
filenum := 0
for _, file := range files {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if file.IsDir() {
continue
}
origsuffix := filepath.Ext(file.Name())
lsuffix := strings.ToLower(origsuffix)
if lsuffix == ".jpeg" {
lsuffix = ".jpg"
}
if lsuffix != ".jpg" && lsuffix != ".png" {
continue
}
origname := file.Name()
origbase := strings.TrimSuffix(origname, origsuffix)
origpath := filepath.Join(dir, origname)
newname := fmt.Sprintf("%s_%04d%s", origbase, filenum, lsuffix)
err = conn.Upload(conn.WIPStorageId(), filepath.Join(bookname, newname), origpath)
if err != nil {
return fmt.Errorf("Failed to upload %s: %v", origpath, err)
}
filenum++
}
return nil
}
|
