1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
// Copyright 2021 Nick White.
// Use of this source code is governed by the GPLv3
// license that can be found in the LICENSE file.
package pipeline
import (
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
// null writer to enable non-verbose logging to be discarded
type NullWriter bool
func (w NullWriter) Write(p []byte) (n int, err error) {
return len(p), nil
}
type fileWalk chan string
// Walk sends the path of all files to the channel, with the exception of
// any file which starts with "."
func (f fileWalk) Walk(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// skip files starting with . to prevent automatically generated
// files like .DS_Store getting in the way
if strings.HasPrefix(filepath.Base(path), ".") {
return nil
}
if !info.IsDir() {
f <- path
}
return nil
}
// CheckImages checks that all files with a ".jpg" or ".png" suffix
// in a directory are images that can be decoded (skipping dotfiles)
func CheckImages(dir string) error {
checker := make(fileWalk)
go func() {
_ = filepath.Walk(dir, checker.Walk)
close(checker)
}()
for path := range checker {
suffix := filepath.Ext(path)
lsuffix := strings.ToLower(suffix)
if lsuffix != ".jpg" && lsuffix != ".png" {
continue
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("Opening image %s failed: %v", path, err)
}
_, _, err = image.Decode(f)
if err != nil {
return fmt.Errorf("Decoding image %s failed: %v", path, err)
}
}
return nil
}
// DetectQueueType detects which queue to use based on the preponderance
// of files of a particular extension in a directory
func DetectQueueType(dir string, conn Queuer) string {
pngdirs, _ := filepath.Glob(dir + "/*.png")
jpgdirs, _ := filepath.Glob(dir + "/*.jpg")
pngcount := len(pngdirs)
jpgcount := len(jpgdirs)
if pngcount > jpgcount {
return conn.WipeQueueId()
} else {
return conn.PreQueueId()
}
}
// UploadImages uploads all files with a suffix of ".jpg" or ".png"
// (except those which start with a ".") from a directory (recursively)
// into conn.WIPStorageId(), prefixed with the given bookname and a
// slash. It also appends all file names with sequential numbers, like
// 0001, to ensure they are appropriately named for further processing
// in the pipeline.
func UploadImages(dir string, bookname string, conn Uploader) error {
files, err := ioutil.ReadDir(dir)
if err != nil {
fmt.Errorf("Failed to read directory %s: %v", dir, err)
}
filenum := 0
for _, file := range files {
if file.IsDir() {
continue
}
origsuffix := filepath.Ext(file.Name())
lsuffix := strings.ToLower(origsuffix)
if lsuffix != ".jpg" && lsuffix != ".png" {
continue
}
origname := file.Name()
origbase := strings.TrimSuffix(origname, origsuffix)
origpath := filepath.Join(dir, origname)
newname := fmt.Sprintf("%s_%04d%s", origbase, filenum, origsuffix)
err = conn.Upload(conn.WIPStorageId(), filepath.Join(bookname, newname), origpath)
if err != nil {
return fmt.Errorf("Failed to upload %s: %v", origpath, err)
}
filenum++
}
return nil
}
|
