summaryrefslogtreecommitdiff
path: root/cmd/bucket-lines
diff options
context:
space:
mode:
Diffstat (limited to 'cmd/bucket-lines')
-rw-r--r--cmd/bucket-lines/bucket.go131
-rw-r--r--cmd/bucket-lines/main.go89
2 files changed, 220 insertions, 0 deletions
diff --git a/cmd/bucket-lines/bucket.go b/cmd/bucket-lines/bucket.go
new file mode 100644
index 0000000..7b6fc4f
--- /dev/null
+++ b/cmd/bucket-lines/bucket.go
@@ -0,0 +1,131 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+
+ "rescribe.xyz/utils/pkg/line"
+)
+
+type BucketSpec struct {
+ Min float64
+ Name string
+}
+type BucketSpecs []BucketSpec
+
+func (b BucketSpecs) Len() int { return len(b) }
+func (b BucketSpecs) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b BucketSpecs) Less(i, j int) bool { return b[i].Min < b[j].Min }
+
+type BucketStat struct {
+ name string
+ num int
+}
+type BucketStats []BucketStat
+
+func (b BucketStats) Len() int { return len(b) }
+func (b BucketStats) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b BucketStats) Less(i, j int) bool { return b[i].num < b[j].num }
+
+// Copies the image and text for a line into a directory based on
+// the line confidence, as defined by the buckets struct
+func bucketLine(l line.Detail, buckets BucketSpecs, dirname string) (string, error) {
+ var bucket string
+
+ todir := ""
+ for _, b := range buckets {
+ if l.Avgconf >= b.Min {
+ todir = b.Name
+ bucket = b.Name
+ }
+ }
+
+ if todir == "" {
+ return bucket, nil
+ }
+
+ avgstr := strconv.FormatFloat(l.Avgconf, 'G', -1, 64)
+ if len(avgstr) > 2 {
+ avgstr = avgstr[2:]
+ }
+
+ base := filepath.Join(dirname, todir, l.OcrName+"_"+l.Name+"_"+avgstr)
+
+ err := os.MkdirAll(filepath.Join(dirname, todir), 0700)
+ if err != nil {
+ return bucket, err
+ }
+
+ f, err := os.Create(base + ".png")
+ if err != nil {
+ return bucket, err
+ }
+ defer f.Close()
+
+ err = l.Img.CopyLineTo(f)
+ if err != nil {
+ return bucket, err
+ }
+
+ f, err = os.Create(base + ".txt")
+ if err != nil {
+ return bucket, err
+ }
+ defer f.Close()
+
+ _, err = io.WriteString(f, l.Text)
+ if err != nil {
+ return bucket, err
+ }
+
+ return bucket, err
+}
+
+// Copies line images and text into directories based on their
+// confidence, as defined by the buckets struct, and returns
+// statistics of whire lines went in the process.
+func BucketUp(lines line.Details, buckets BucketSpecs, dirname string) (BucketStats, error) {
+ var all []string
+ var stats BucketStats
+
+ sort.Sort(lines)
+ sort.Sort(buckets)
+ for _, l := range lines {
+ bname, err := bucketLine(l, buckets, dirname)
+ if err != nil {
+ return stats, err
+ }
+ all = append(all, bname)
+ }
+
+ for _, b := range all {
+ i := sort.Search(len(stats), func(i int) bool { return stats[i].name == b })
+ if i == len(stats) {
+ newstat := BucketStat{b, 0}
+ stats = append(stats, newstat)
+ i = len(stats) - 1
+ }
+ stats[i].num++
+ }
+
+ return stats, nil
+}
+
+// Prints statistics of where lines went when bucketing
+func PrintBucketStats(w io.Writer, stats BucketStats) {
+ var total int
+ for _, s := range stats {
+ total += s.num
+ }
+
+ fmt.Fprintf(w, "Copied %d lines\n", total)
+ fmt.Fprintf(w, "---------------------------------\n")
+ sort.Sort(stats)
+ for _, s := range stats {
+ fmt.Fprintf(w, "Lines in %7s: %2d%%\n", s.name, 100*s.num/total)
+ }
+}
diff --git a/cmd/bucket-lines/main.go b/cmd/bucket-lines/main.go
new file mode 100644
index 0000000..af81b44
--- /dev/null
+++ b/cmd/bucket-lines/main.go
@@ -0,0 +1,89 @@
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+
+ "rescribe.xyz/utils/pkg/hocr"
+ "rescribe.xyz/utils/pkg/line"
+ "rescribe.xyz/utils/pkg/prob"
+)
+
+func main() {
+ b := BucketSpecs{
+ // minimum confidence, name
+ {0, "bad"},
+ {0.95, "95to98"},
+ {0.98, "98plus"},
+ }
+
+ flag.Usage = func() {
+ fmt.Fprintf(os.Stderr, "Usage: bucket-lines [-d dir] [-s specs.json] [hocr1] [prob1] [hocr2] [...]\n")
+ fmt.Fprintf(os.Stderr, "Copies image-text line pairs into different directories according\n")
+ fmt.Fprintf(os.Stderr, "to the average character probability for the line.\n")
+ fmt.Fprintf(os.Stderr, "Both .hocr and .prob files can be processed.\n")
+ fmt.Fprintf(os.Stderr, "For .hocr files, the x_wconf data is used to calculate confidence.\n")
+ fmt.Fprintf(os.Stderr, "The .prob files are generated using ocropy-rpred's --probabilities\n")
+ fmt.Fprintf(os.Stderr, "option.\n")
+ fmt.Fprintf(os.Stderr, "The .prob and .hocr files are assumed to be in the same directory\n")
+ fmt.Fprintf(os.Stderr, "as the line's image and text files.\n\n")
+ flag.PrintDefaults()
+ fmt.Fprintf(os.Stderr, "\nAn example specs.json file would be the following:\n")
+ fmt.Fprintf(os.Stderr, "[{\"min\": 0, \"name\": \"terrible\"}, {\"min\": 0.80, \"name\": \"ok\"}, {\"min\": 0.98, \"name\": \"great\"}]\n")
+ }
+ dir := flag.String("d", "buckets", "Directory to store the buckets")
+ specs := flag.String("s", "", "JSON file describing specs to bucket into")
+ flag.Parse()
+ if flag.NArg() < 1 {
+ flag.Usage()
+ os.Exit(1)
+ }
+
+ if *specs != "" {
+ js, err := ioutil.ReadFile(*specs)
+ if err != nil {
+ log.Fatal(err)
+ }
+ err = json.Unmarshal(js, &b)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ var err error
+ lines := make(line.Details, 0)
+
+ for _, f := range flag.Args() {
+ var newlines line.Details
+ switch ext := filepath.Ext(f); ext {
+ case ".prob":
+ newlines, err = prob.GetLineDetails(f)
+ case ".hocr":
+ newlines, err = hocr.GetLineDetails(f)
+ default:
+ log.Printf("Skipping file '%s' as it isn't a .prob or .hocr\n", f)
+ continue
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ for _, l := range newlines {
+ if l.Img != nil {
+ lines = append(lines, l)
+ }
+ }
+ }
+
+ stats, err := BucketUp(lines, b, *dir)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ PrintBucketStats(os.Stdout, stats)
+}