From 3880414bbf2d6f2cd05e208abf919ae5ceabeddc Mon Sep 17 00:00:00 2001 From: Nick White Date: Thu, 27 Feb 2020 17:45:16 +0000 Subject: Reorganise all commands to be behind cmd/ --- cmd/bucket-lines/bucket.go | 131 +++++++++++++++++++++++++++++++++++++++++++++ cmd/bucket-lines/main.go | 89 ++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 cmd/bucket-lines/bucket.go create mode 100644 cmd/bucket-lines/main.go (limited to 'cmd/bucket-lines') diff --git a/cmd/bucket-lines/bucket.go b/cmd/bucket-lines/bucket.go new file mode 100644 index 0000000..7b6fc4f --- /dev/null +++ b/cmd/bucket-lines/bucket.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + + "rescribe.xyz/utils/pkg/line" +) + +type BucketSpec struct { + Min float64 + Name string +} +type BucketSpecs []BucketSpec + +func (b BucketSpecs) Len() int { return len(b) } +func (b BucketSpecs) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b BucketSpecs) Less(i, j int) bool { return b[i].Min < b[j].Min } + +type BucketStat struct { + name string + num int +} +type BucketStats []BucketStat + +func (b BucketStats) Len() int { return len(b) } +func (b BucketStats) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b BucketStats) Less(i, j int) bool { return b[i].num < b[j].num } + +// Copies the image and text for a line into a directory based on +// the line confidence, as defined by the buckets struct +func bucketLine(l line.Detail, buckets BucketSpecs, dirname string) (string, error) { + var bucket string + + todir := "" + for _, b := range buckets { + if l.Avgconf >= b.Min { + todir = b.Name + bucket = b.Name + } + } + + if todir == "" { + return bucket, nil + } + + avgstr := strconv.FormatFloat(l.Avgconf, 'G', -1, 64) + if len(avgstr) > 2 { + avgstr = avgstr[2:] + } + + base := filepath.Join(dirname, todir, l.OcrName+"_"+l.Name+"_"+avgstr) + + err := os.MkdirAll(filepath.Join(dirname, todir), 0700) + if err != nil { + return bucket, err + } + + f, err := os.Create(base + ".png") + if err != nil { + return bucket, err + } + defer f.Close() + + err = l.Img.CopyLineTo(f) + if err != nil { + return bucket, err + } + + f, err = os.Create(base + ".txt") + if err != nil { + return bucket, err + } + defer f.Close() + + _, err = io.WriteString(f, l.Text) + if err != nil { + return bucket, err + } + + return bucket, err +} + +// Copies line images and text into directories based on their +// confidence, as defined by the buckets struct, and returns +// statistics of whire lines went in the process. +func BucketUp(lines line.Details, buckets BucketSpecs, dirname string) (BucketStats, error) { + var all []string + var stats BucketStats + + sort.Sort(lines) + sort.Sort(buckets) + for _, l := range lines { + bname, err := bucketLine(l, buckets, dirname) + if err != nil { + return stats, err + } + all = append(all, bname) + } + + for _, b := range all { + i := sort.Search(len(stats), func(i int) bool { return stats[i].name == b }) + if i == len(stats) { + newstat := BucketStat{b, 0} + stats = append(stats, newstat) + i = len(stats) - 1 + } + stats[i].num++ + } + + return stats, nil +} + +// Prints statistics of where lines went when bucketing +func PrintBucketStats(w io.Writer, stats BucketStats) { + var total int + for _, s := range stats { + total += s.num + } + + fmt.Fprintf(w, "Copied %d lines\n", total) + fmt.Fprintf(w, "---------------------------------\n") + sort.Sort(stats) + for _, s := range stats { + fmt.Fprintf(w, "Lines in %7s: %2d%%\n", s.name, 100*s.num/total) + } +} diff --git a/cmd/bucket-lines/main.go b/cmd/bucket-lines/main.go new file mode 100644 index 0000000..af81b44 --- /dev/null +++ b/cmd/bucket-lines/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + + "rescribe.xyz/utils/pkg/hocr" + "rescribe.xyz/utils/pkg/line" + "rescribe.xyz/utils/pkg/prob" +) + +func main() { + b := BucketSpecs{ + // minimum confidence, name + {0, "bad"}, + {0.95, "95to98"}, + {0.98, "98plus"}, + } + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: bucket-lines [-d dir] [-s specs.json] [hocr1] [prob1] [hocr2] [...]\n") + fmt.Fprintf(os.Stderr, "Copies image-text line pairs into different directories according\n") + fmt.Fprintf(os.Stderr, "to the average character probability for the line.\n") + fmt.Fprintf(os.Stderr, "Both .hocr and .prob files can be processed.\n") + fmt.Fprintf(os.Stderr, "For .hocr files, the x_wconf data is used to calculate confidence.\n") + fmt.Fprintf(os.Stderr, "The .prob files are generated using ocropy-rpred's --probabilities\n") + fmt.Fprintf(os.Stderr, "option.\n") + fmt.Fprintf(os.Stderr, "The .prob and .hocr files are assumed to be in the same directory\n") + fmt.Fprintf(os.Stderr, "as the line's image and text files.\n\n") + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "\nAn example specs.json file would be the following:\n") + fmt.Fprintf(os.Stderr, "[{\"min\": 0, \"name\": \"terrible\"}, {\"min\": 0.80, \"name\": \"ok\"}, {\"min\": 0.98, \"name\": \"great\"}]\n") + } + dir := flag.String("d", "buckets", "Directory to store the buckets") + specs := flag.String("s", "", "JSON file describing specs to bucket into") + flag.Parse() + if flag.NArg() < 1 { + flag.Usage() + os.Exit(1) + } + + if *specs != "" { + js, err := ioutil.ReadFile(*specs) + if err != nil { + log.Fatal(err) + } + err = json.Unmarshal(js, &b) + if err != nil { + log.Fatal(err) + } + } + + var err error + lines := make(line.Details, 0) + + for _, f := range flag.Args() { + var newlines line.Details + switch ext := filepath.Ext(f); ext { + case ".prob": + newlines, err = prob.GetLineDetails(f) + case ".hocr": + newlines, err = hocr.GetLineDetails(f) + default: + log.Printf("Skipping file '%s' as it isn't a .prob or .hocr\n", f) + continue + } + if err != nil { + log.Fatal(err) + } + + for _, l := range newlines { + if l.Img != nil { + lines = append(lines, l) + } + } + } + + stats, err := BucketUp(lines, b, *dir) + if err != nil { + log.Fatal(err) + } + + PrintBucketStats(os.Stdout, stats) +} -- cgit v1.2.1-24-ge1ad