summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick White <git@njw.name>2019-01-25 09:55:55 +0000
committerNick White <git@njw.name>2019-01-25 09:55:55 +0000
commit30c088b90e7b6a25d93cbdad7564ff063e62afd3 (patch)
treec47d7bc086a076cfe5e702628c4e5e3b1eab1aa5
parent1c17766952bdcd6f7d31d0fa1a2e504b1aa4f14a (diff)
Reorganisation and cleanup
-rw-r--r--README1
-rw-r--r--bucket-lines/bucket.go (renamed from parse/bucket.go)32
-rw-r--r--bucket-lines/main.go (renamed from bucket-lines/bucket-lines.go)16
-rw-r--r--lib/hocr/hocr.go79
-rw-r--r--lib/hocr/lines.go107
-rw-r--r--lib/line/line.go (renamed from parse/line.go)43
-rw-r--r--lib/prob/prob.go (renamed from parse/prob/prob.go)26
-rw-r--r--parse/hocr/hocr.go181
8 files changed, 246 insertions, 239 deletions
diff --git a/README b/README
deleted file mode 100644
index 697a1e1..0000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-There will be testing tools here.
diff --git a/parse/bucket.go b/bucket-lines/bucket.go
index 44b1d24..5d9271a 100644
--- a/parse/bucket.go
+++ b/bucket-lines/bucket.go
@@ -1,4 +1,4 @@
-package parse
+package main
import (
"fmt"
@@ -7,6 +7,8 @@ import (
"os"
"sort"
"strconv"
+
+ "git.rescribe.xyz/testingtools/lib/line"
)
type BucketSpec struct {
@@ -18,7 +20,18 @@ func (b BucketSpecs) Len() int { return len(b) }
func (b BucketSpecs) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b BucketSpecs) Less(i, j int) bool { return b[i].Min < b[j].Min }
-func bucketLine(l LineDetail, buckets BucketSpecs, dirname string) (string, error) {
+type BucketStat struct {
+ name string
+ num int
+}
+type BucketStats []BucketStat
+func (b BucketStats) Len() int { return len(b) }
+func (b BucketStats) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b BucketStats) Less(i, j int) bool { return b[i].num < b[j].num }
+
+// Copies the image and text for a line into a directory based on
+// the line confidence, as defined by the buckets struct
+func bucketLine(l line.Detail, buckets BucketSpecs, dirname string) (string, error) {
var bucket string
todir := ""
@@ -70,18 +83,10 @@ func bucketLine(l LineDetail, buckets BucketSpecs, dirname string) (string, erro
return bucket, err
}
-type BucketStat struct {
- name string
- num int
-}
-type BucketStats []BucketStat
-func (b BucketStats) Len() int { return len(b) }
-func (b BucketStats) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
-func (b BucketStats) Less(i, j int) bool { return b[i].num < b[j].num }
-
// Copies line images and text into directories based on their
-// confidence, as defined by the buckets struct
-func BucketUp(lines LineDetails, buckets BucketSpecs, dirname string) (BucketStats, error) {
+// confidence, as defined by the buckets struct, and returns
+// statistics of whire lines went in the process.
+func BucketUp(lines line.Details, buckets BucketSpecs, dirname string) (BucketStats, error) {
var all []string
var stats BucketStats
@@ -108,6 +113,7 @@ func BucketUp(lines LineDetails, buckets BucketSpecs, dirname string) (BucketSta
return stats, nil
}
+// Prints statistics of where lines went when bucketing
func PrintBucketStats(w io.Writer, stats BucketStats) {
var total int
for _, s := range stats {
diff --git a/bucket-lines/bucket-lines.go b/bucket-lines/main.go
index ee81721..b70e358 100644
--- a/bucket-lines/bucket-lines.go
+++ b/bucket-lines/main.go
@@ -9,13 +9,13 @@ import (
"os"
"path/filepath"
- "git.rescribe.xyz/testingtools/parse"
- "git.rescribe.xyz/testingtools/parse/hocr"
- "git.rescribe.xyz/testingtools/parse/prob"
+ "git.rescribe.xyz/testingtools/lib/line"
+ "git.rescribe.xyz/testingtools/lib/hocr"
+ "git.rescribe.xyz/testingtools/lib/prob"
)
func main() {
- b := parse.BucketSpecs{
+ b := BucketSpecs{
// minimum confidence, name
{ 0, "bad" },
{ 0.95, "95to98" },
@@ -56,10 +56,10 @@ func main() {
}
var err error
- lines := make(parse.LineDetails, 0)
+ lines := make(line.Details, 0)
for _, f := range flag.Args() {
- var newlines parse.LineDetails
+ var newlines line.Details
switch ext := filepath.Ext(f); ext {
case ".prob":
newlines, err = prob.GetLineDetails(f)
@@ -78,10 +78,10 @@ func main() {
}
}
- stats, err := parse.BucketUp(lines, b, *dir)
+ stats, err := BucketUp(lines, b, *dir)
if err != nil {
log.Fatal(err)
}
- parse.PrintBucketStats(os.Stdout, stats)
+ PrintBucketStats(os.Stdout, stats)
}
diff --git a/lib/hocr/hocr.go b/lib/hocr/hocr.go
new file mode 100644
index 0000000..0d10819
--- /dev/null
+++ b/lib/hocr/hocr.go
@@ -0,0 +1,79 @@
+package hocr
+
+import (
+ "encoding/xml"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+type Hocr struct {
+ Lines []OcrLine `xml:"body>div>div>p>span"`
+}
+
+type OcrLine struct {
+ Class string `xml:"class,attr"`
+ Id string `xml:"id,attr"`
+ Title string `xml:"title,attr"`
+ Words []OcrWord `xml:"span"`
+ Text string `xml:",chardata"`
+}
+
+type OcrWord struct {
+ Class string `xml:"class,attr"`
+ Id string `xml:"id,attr"`
+ Title string `xml:"title,attr"`
+ Chars []OcrChar `xml:"span"`
+ Text string `xml:",chardata"`
+}
+
+type OcrChar struct {
+ Class string `xml:"class,attr"`
+ Id string `xml:"id,attr"`
+ Title string `xml:"title,attr"`
+ Chars []OcrChar `xml:"span"`
+ Text string `xml:",chardata"`
+}
+
+// Returns the confidence for a word based on its x_wconf value
+func wordConf(s string) (float64, error) {
+ re, err := regexp.Compile(`x_wconf ([0-9.]+)`)
+ if err != nil {
+ return 0.0, err
+ }
+ conf := re.FindStringSubmatch(s)
+ return strconv.ParseFloat(conf[1], 64)
+}
+
+func boxCoords(s string) ([4]int, error) {
+ var coords [4]int
+ re, err := regexp.Compile(`bbox ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+)`)
+ if err != nil {
+ return coords, err
+ }
+ coordstr := re.FindStringSubmatch(s)
+ for i := range coords {
+ c, err := strconv.Atoi(coordstr[i+1])
+ if err != nil {
+ return coords, err
+ }
+ coords[i] = c
+ }
+ return coords, nil
+}
+
+func noText(s string) bool {
+ t := strings.Trim(s, " \n")
+ return len(t) == 0
+}
+
+func Parse(b []byte) (Hocr, error) {
+ var hocr Hocr
+
+ err := xml.Unmarshal(b, &hocr)
+ if err != nil {
+ return hocr, err
+ }
+
+ return hocr, nil
+}
diff --git a/lib/hocr/lines.go b/lib/hocr/lines.go
new file mode 100644
index 0000000..4902b40
--- /dev/null
+++ b/lib/hocr/lines.go
@@ -0,0 +1,107 @@
+package hocr
+
+// TODO: Parse line name to zero pad line numbers, so they can
+// be sorted easily
+// TODO: have same filename format as .prob uses, so include base
+// dirname, and don't include line numbers if there's only
+// one line in the hocr
+
+import (
+ "image"
+ "image/png"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "git.rescribe.xyz/testingtools/lib/line"
+)
+
+func parseLineDetails(h Hocr, i image.Image, name string) (line.Details, error) {
+ lines := make(line.Details, 0)
+
+ for _, l := range h.Lines {
+ totalconf := float64(0)
+ num := 0
+ for _, w := range l.Words {
+ c, err := wordConf(w.Title)
+ if err != nil {
+ return lines, err
+ }
+ num++
+ totalconf += c
+ }
+
+ coords, err := boxCoords(l.Title)
+ if err != nil {
+ return lines, err
+ }
+
+ var ln line.Detail
+ ln.Name = l.Id
+ ln.Avgconf = (totalconf/float64(num)) / 100
+ linetext := ""
+
+ linetext = l.Text
+ if(noText(linetext)) {
+ linetext = ""
+ for _, w := range l.Words {
+ if(w.Class != "ocrx_word") {
+ continue
+ }
+ linetext += w.Text + " "
+ }
+ }
+ if(noText(linetext)) {
+ linetext = ""
+ for _, w := range l.Words {
+ if(w.Class != "ocrx_word") {
+ continue
+ }
+ for _, c := range w.Chars {
+ if(c.Class != "ocrx_cinfo") {
+ continue
+ }
+ linetext += c.Text
+ }
+ linetext += " "
+ }
+ }
+ ln.Text = strings.TrimRight(linetext, " ")
+ ln.Text += "\n"
+ ln.OcrName = name
+ var imgd line.ImgDirect
+ imgd.Img = i.(*image.Gray).SubImage(image.Rect(coords[0], coords[1], coords[2], coords[3]))
+ ln.Img = imgd
+ lines = append(lines, ln)
+ }
+ return lines, nil
+}
+
+func GetLineDetails(hocrfn string) (line.Details, error) {
+ var newlines line.Details
+
+ file, err := ioutil.ReadFile(hocrfn)
+ if err != nil {
+ return newlines, err
+ }
+
+ h, err := Parse(file)
+ if err != nil {
+ return newlines, err
+ }
+
+ pngfn := strings.Replace(hocrfn, ".hocr", ".png", 1)
+ pngf, err := os.Open(pngfn)
+ if err != nil {
+ return newlines, err
+ }
+ defer pngf.Close()
+ img, err := png.Decode(pngf)
+ if err != nil {
+ return newlines, err
+ }
+
+ n := strings.Replace(filepath.Base(hocrfn), ".hocr", "", 1)
+ return parseLineDetails(h, img, n)
+}
diff --git a/parse/line.go b/lib/line/line.go
index 9a2be8e..3adac0a 100644
--- a/parse/line.go
+++ b/lib/line/line.go
@@ -1,9 +1,4 @@
-package parse
-
-// TODO: integrate in line-conf-buckets linedetail
-// TODO: add BucketUp() function here that does what both line-conf-buckets-tess.go
-// and line-conf-buckets.go do
-// TODO: consider naming this package line, and separating it from hocr and prob
+package line
import (
"image"
@@ -12,19 +7,32 @@ import (
"os"
)
-type LineDetail struct {
+type Detail struct {
Name string
Avgconf float64
- Img CopyableLine
+ Img CopyableImg
Text string
OcrName string
}
-type CopyableLine interface {
+type CopyableImg interface {
CopyLineTo(io.Writer) (error)
}
-// This is an implementation of the CopyableLine interface that
+type Details []Detail
+
+// Used by sort.Sort.
+func (l Details) Len() int { return len(l) }
+
+// Used by sort.Sort.
+func (l Details) Less(i, j int) bool {
+ return l[i].Avgconf < l[j].Avgconf
+}
+
+// Used by sort.Sort.
+func (l Details) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
+
+// This is an implementation of the CopyableImg interface that
// stores the image directly as an image.Image
type ImgDirect struct {
Img image.Image
@@ -38,6 +46,8 @@ func (i ImgDirect) CopyLineTo(w io.Writer) (error) {
return nil
}
+// This is an implementation of the CopyableImg interface that
+// stores the path of an image
type ImgPath struct {
Path string
}
@@ -52,16 +62,3 @@ func (i ImgPath) CopyLineTo(w io.Writer) (error) {
_, err = io.Copy(w, f)
return err
}
-
-type LineDetails []LineDetail
-
-// Used by sort.Sort.
-func (l LineDetails) Len() int { return len(l) }
-
-// Used by sort.Sort.
-func (l LineDetails) Less(i, j int) bool {
- return l[i].Avgconf < l[j].Avgconf
-}
-
-// Used by sort.Sort.
-func (l LineDetails) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
diff --git a/parse/prob/prob.go b/lib/prob/prob.go
index 8d01cab..0299a96 100644
--- a/parse/prob/prob.go
+++ b/lib/prob/prob.go
@@ -6,7 +6,7 @@ import (
"strconv"
"strings"
- "git.rescribe.xyz/testingtools/parse"
+ "git.rescribe.xyz/testingtools/lib/line"
)
func getLineAvg(f string) (float64, error) {
@@ -18,8 +18,8 @@ func getLineAvg(f string) (float64, error) {
return 0, err
}
- for _, line := range strings.Split(string(prob), "\n") {
- fields := strings.Fields(line)
+ for _, l := range strings.Split(string(prob), "\n") {
+ fields := strings.Fields(l)
if len(fields) == 2 {
conf, err := strconv.ParseFloat(fields[1], 64)
@@ -38,9 +38,9 @@ func getLineAvg(f string) (float64, error) {
}
// Note this only processes one line at a time
-func GetLineDetails(probfn string) (parse.LineDetails, error) {
- var line parse.LineDetail
- lines := make(parse.LineDetails, 0)
+func GetLineDetails(probfn string) (line.Details, error) {
+ var l line.Detail
+ lines := make(line.Details, 0)
avg, err := getLineAvg(probfn)
if err != nil {
@@ -54,16 +54,16 @@ func GetLineDetails(probfn string) (parse.LineDetails, error) {
return lines, err
}
- line.Name = filepath.Base(filebase)
- line.Avgconf = avg
- line.Text = string(txt)
- line.OcrName = filepath.Dir(filebase)
+ l.Name = filepath.Base(filebase)
+ l.Avgconf = avg
+ l.Text = string(txt)
+ l.OcrName = filepath.Dir(filebase)
- var imgfn parse.ImgPath
+ var imgfn line.ImgPath
imgfn.Path = filebase + ".bin.png"
- line.Img = imgfn
+ l.Img = imgfn
- lines = append(lines, line)
+ lines = append(lines, l)
return lines, nil
}
diff --git a/parse/hocr/hocr.go b/parse/hocr/hocr.go
deleted file mode 100644
index 81250a9..0000000
--- a/parse/hocr/hocr.go
+++ /dev/null
@@ -1,181 +0,0 @@
-package hocr
-
-// TODO: Parse line name to zero pad line numbers, so they can
-// be sorted easily
-// TODO: have same filename format as .prob uses, so include base
-// dirname, and don't include line numbers if there's only
-// one line in the hocr
-
-import (
- "encoding/xml"
- "image"
- "image/png"
- "io/ioutil"
- "os"
- "path/filepath"
- "regexp"
- "strconv"
- "strings"
-
- "git.rescribe.xyz/testingtools/parse"
-)
-
-type Hocr struct {
- Lines []OcrLine `xml:"body>div>div>p>span"`
-}
-
-type OcrLine struct {
- Class string `xml:"class,attr"`
- Id string `xml:"id,attr"`
- Title string `xml:"title,attr"`
- Words []OcrWord `xml:"span"`
- Text string `xml:",chardata"`
-}
-
-type OcrWord struct {
- Class string `xml:"class,attr"`
- Id string `xml:"id,attr"`
- Title string `xml:"title,attr"`
- Chars []OcrChar `xml:"span"`
- Text string `xml:",chardata"`
-}
-
-type OcrChar struct {
- Class string `xml:"class,attr"`
- Id string `xml:"id,attr"`
- Title string `xml:"title,attr"`
- Chars []OcrChar `xml:"span"`
- Text string `xml:",chardata"`
-}
-
-// Returns the confidence for a word based on its x_wconf value
-func wordConf(s string) (float64, error) {
- re, err := regexp.Compile(`x_wconf ([0-9.]+)`)
- if err != nil {
- return 0.0, err
- }
- conf := re.FindStringSubmatch(s)
- return strconv.ParseFloat(conf[1], 64)
-}
-
-func boxCoords(s string) ([4]int, error) {
- var coords [4]int
- re, err := regexp.Compile(`bbox ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+)`)
- if err != nil {
- return coords, err
- }
- coordstr := re.FindStringSubmatch(s)
- for i := range coords {
- c, err := strconv.Atoi(coordstr[i+1])
- if err != nil {
- return coords, err
- }
- coords[i] = c
- }
- return coords, nil
-}
-
-func noText(s string) bool {
- t := strings.Trim(s, " \n")
- return len(t) == 0
-}
-
-func Parse(b []byte) (Hocr, error) {
- var hocr Hocr
-
- err := xml.Unmarshal(b, &hocr)
- if err != nil {
- return hocr, err
- }
-
- return hocr, nil
-}
-
-func parseLineDetails(h Hocr, i image.Image, name string) (parse.LineDetails, error) {
- lines := make(parse.LineDetails, 0)
-
- for _, l := range h.Lines {
- totalconf := float64(0)
- num := 0
- for _, w := range l.Words {
- c, err := wordConf(w.Title)
- if err != nil {
- return lines, err
- }
- num++
- totalconf += c
- }
-
- coords, err := boxCoords(l.Title)
- if err != nil {
- return lines, err
- }
-
- var line parse.LineDetail
- line.Name = l.Id
- line.Avgconf = (totalconf/float64(num)) / 100
- linetext := ""
-
- linetext = l.Text
- if(noText(linetext)) {
- linetext = ""
- for _, w := range l.Words {
- if(w.Class != "ocrx_word") {
- continue
- }
- linetext += w.Text + " "
- }
- }
- if(noText(linetext)) {
- linetext = ""
- for _, w := range l.Words {
- if(w.Class != "ocrx_word") {
- continue
- }
- for _, c := range w.Chars {
- if(c.Class != "ocrx_cinfo") {
- continue
- }
- linetext += c.Text
- }
- linetext += " "
- }
- }
- line.Text = strings.TrimRight(linetext, " ")
- line.Text += "\n"
- line.OcrName = name
- var imgd parse.ImgDirect
- imgd.Img = i.(*image.Gray).SubImage(image.Rect(coords[0], coords[1], coords[2], coords[3]))
- line.Img = imgd
- lines = append(lines, line)
- }
- return lines, nil
-}
-
-func GetLineDetails(hocrfn string) (parse.LineDetails, error) {
- var newlines parse.LineDetails
-
- file, err := ioutil.ReadFile(hocrfn)
- if err != nil {
- return newlines, err
- }
-
- h, err := Parse(file)
- if err != nil {
- return newlines, err
- }
-
- pngfn := strings.Replace(hocrfn, ".hocr", ".png", 1)
- pngf, err := os.Open(pngfn)
- if err != nil {
- return newlines, err
- }
- defer pngf.Close()
- img, err := png.Decode(pngf)
- if err != nil {
- return newlines, err
- }
-
- n := strings.Replace(filepath.Base(hocrfn), ".hocr", "", 1)
- return parseLineDetails(h, img, n)
-}