Reorganisation and cleanup

8 files changed, 246 insertions, 239 deletions

diff --git a/README b/README
deleted file mode 100644
index 697a1e1..0000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-There will be testing tools here.
diff --git a/parse/bucket.go b/bucket-lines/bucket.go
index 44b1d24..5d9271a 100644
--- a/parse/bucket.go
+++ b/bucket-lines/bucket.go
@@ -1,4 +1,4 @@
-package parse
+package main
 
 import (
 	"fmt"
@@ -7,6 +7,8 @@ import (
 	"os"
 	"sort"
 	"strconv"
+
+	"git.rescribe.xyz/testingtools/lib/line"
 )
 
 type BucketSpec struct {
@@ -18,7 +20,18 @@ func (b BucketSpecs) Len() int { return len(b) }
 func (b BucketSpecs) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
 func (b BucketSpecs) Less(i, j int) bool { return b[i].Min < b[j].Min }
 
-func bucketLine(l LineDetail, buckets BucketSpecs, dirname string) (string, error) {
+type BucketStat struct {
+	name string
+	num int
+}
+type BucketStats []BucketStat
+func (b BucketStats) Len() int { return len(b) }
+func (b BucketStats) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b BucketStats) Less(i, j int) bool { return b[i].num < b[j].num }
+
+// Copies the image and text for a line into a directory based on
+// the line confidence, as defined by the buckets struct
+func bucketLine(l line.Detail, buckets BucketSpecs, dirname string) (string, error) {
 	var bucket string
 
 	todir := ""
@@ -70,18 +83,10 @@ func bucketLine(l LineDetail, buckets BucketSpecs, dirname string) (string, erro
 	return bucket, err
 }
 
-type BucketStat struct {
-	name string
-	num int
-}
-type BucketStats []BucketStat
-func (b BucketStats) Len() int { return len(b) }
-func (b BucketStats) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
-func (b BucketStats) Less(i, j int) bool { return b[i].num < b[j].num }
-
 // Copies line images and text into directories based on their
-// confidence, as defined by the buckets struct
-func BucketUp(lines LineDetails, buckets BucketSpecs, dirname string) (BucketStats, error) {
+// confidence, as defined by the buckets struct, and returns
+// statistics of whire lines went in the process.
+func BucketUp(lines line.Details, buckets BucketSpecs, dirname string) (BucketStats, error) {
 	var all []string
 	var stats BucketStats
 
@@ -108,6 +113,7 @@ func BucketUp(lines LineDetails, buckets BucketSpecs, dirname string) (BucketSta
 	return stats, nil
 }
 
+// Prints statistics of where lines went when bucketing
 func PrintBucketStats(w io.Writer, stats BucketStats) {
 	var total int
 	for _, s := range stats {
diff --git a/bucket-lines/bucket-lines.go b/bucket-lines/main.go
index ee81721..b70e358 100644
--- a/bucket-lines/bucket-lines.go
+++ b/bucket-lines/main.go
@@ -9,13 +9,13 @@ import (
 	"os"
 	"path/filepath"
 
-	"git.rescribe.xyz/testingtools/parse"
-	"git.rescribe.xyz/testingtools/parse/hocr"
-	"git.rescribe.xyz/testingtools/parse/prob"
+	"git.rescribe.xyz/testingtools/lib/line"
+	"git.rescribe.xyz/testingtools/lib/hocr"
+	"git.rescribe.xyz/testingtools/lib/prob"
 )
 
 func main() {
-	b := parse.BucketSpecs{
+	b := BucketSpecs{
 		// minimum confidence, name
 		{ 0, "bad" },
 		{ 0.95, "95to98" },
@@ -56,10 +56,10 @@ func main() {
 	}
 
 	var err error
-	lines := make(parse.LineDetails, 0)
+	lines := make(line.Details, 0)
 
 	for _, f := range flag.Args() {
-		var newlines parse.LineDetails
+		var newlines line.Details
 		switch ext := filepath.Ext(f); ext {
 			case ".prob":
 				newlines, err = prob.GetLineDetails(f)
@@ -78,10 +78,10 @@ func main() {
 		}
 	}
 
-	stats, err := parse.BucketUp(lines, b, *dir)
+	stats, err := BucketUp(lines, b, *dir)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	parse.PrintBucketStats(os.Stdout, stats)
+	PrintBucketStats(os.Stdout, stats)
 }
diff --git a/lib/hocr/hocr.go b/lib/hocr/hocr.go
new file mode 100644
index 0000000..0d10819
--- /dev/null
+++ b/lib/hocr/hocr.go
@@ -0,0 +1,79 @@
+package hocr
+
+import (
+	"encoding/xml"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type Hocr struct {
+	Lines []OcrLine `xml:"body>div>div>p>span"`
+}
+
+type OcrLine struct {
+	Class string `xml:"class,attr"`
+	Id string `xml:"id,attr"`
+	Title string `xml:"title,attr"`
+	Words []OcrWord `xml:"span"`
+	Text string `xml:",chardata"`
+}
+
+type OcrWord struct {
+	Class string `xml:"class,attr"`
+	Id string `xml:"id,attr"`
+	Title string `xml:"title,attr"`
+	Chars []OcrChar `xml:"span"`
+	Text string `xml:",chardata"`
+}
+
+type OcrChar struct {
+	Class string `xml:"class,attr"`
+	Id string `xml:"id,attr"`
+	Title string `xml:"title,attr"`
+	Chars []OcrChar `xml:"span"`
+	Text string `xml:",chardata"`
+}
+
+// Returns the confidence for a word based on its x_wconf value
+func wordConf(s string) (float64, error) {
+	re, err := regexp.Compile(`x_wconf ([0-9.]+)`)
+	if err != nil {
+		return 0.0, err
+	}
+	conf := re.FindStringSubmatch(s)
+	return strconv.ParseFloat(conf[1], 64)
+}
+
+func boxCoords(s string) ([4]int, error) {
+	var coords [4]int
+	re, err := regexp.Compile(`bbox ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+)`)
+	if err != nil {
+		return coords, err
+	}
+	coordstr := re.FindStringSubmatch(s)
+	for i := range coords {
+		c, err := strconv.Atoi(coordstr[i+1])
+		if err != nil {
+			return coords, err
+		}
+		coords[i] = c
+	}
+	return coords, nil
+}
+
+func noText(s string) bool {
+	t := strings.Trim(s, " \n")
+	return len(t) == 0
+}
+
+func Parse(b []byte) (Hocr, error) {
+	var hocr Hocr
+
+	err := xml.Unmarshal(b, &hocr)
+	if err != nil {
+		return hocr, err
+	}
+
+	return hocr, nil
+}
diff --git a/lib/hocr/lines.go b/lib/hocr/lines.go
new file mode 100644
index 0000000..4902b40
--- /dev/null
+++ b/lib/hocr/lines.go
@@ -0,0 +1,107 @@
+package hocr
+
+// TODO: Parse line name to zero pad line numbers, so they can
+//       be sorted easily
+// TODO: have same filename format as .prob uses, so include base
+//       dirname, and don't include line numbers if there's only
+//       one line in the hocr
+
+import (
+	"image"
+	"image/png"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.rescribe.xyz/testingtools/lib/line"
+)
+
+func parseLineDetails(h Hocr, i image.Image, name string) (line.Details, error) {
+	lines := make(line.Details, 0)
+
+	for _, l := range h.Lines {
+		totalconf := float64(0)
+		num := 0
+		for _, w := range l.Words {
+			c, err := wordConf(w.Title)
+			if err != nil {
+				return lines, err
+			}
+			num++
+			totalconf += c
+		}
+
+		coords, err := boxCoords(l.Title)
+		if err != nil {
+			return lines, err
+		}
+
+		var ln line.Detail
+		ln.Name = l.Id
+		ln.Avgconf = (totalconf/float64(num)) / 100
+		linetext := ""
+
+		linetext = l.Text
+		if(noText(linetext)) {
+			linetext = ""
+			for _, w := range l.Words {
+				if(w.Class != "ocrx_word") {
+					continue
+				}
+				linetext += w.Text + " "
+			}
+		}
+		if(noText(linetext)) {
+			linetext = ""
+			for _, w := range l.Words {
+				if(w.Class != "ocrx_word") {
+					continue
+				}
+				for _, c := range w.Chars {
+					if(c.Class != "ocrx_cinfo") {
+						continue
+					}
+					linetext += c.Text
+				}
+				linetext += " "
+			}
+		}
+		ln.Text = strings.TrimRight(linetext, " ")
+		ln.Text += "\n"
+		ln.OcrName = name
+		var imgd line.ImgDirect
+		imgd.Img = i.(*image.Gray).SubImage(image.Rect(coords[0], coords[1], coords[2], coords[3]))
+		ln.Img = imgd
+		lines = append(lines, ln)
+	}
+	return lines, nil
+}
+
+func GetLineDetails(hocrfn string) (line.Details, error) {
+	var newlines line.Details
+
+	file, err := ioutil.ReadFile(hocrfn)
+	if err != nil {
+		return newlines, err
+	}
+
+	h, err := Parse(file)
+	if err != nil {
+		return newlines, err
+	}
+
+	pngfn := strings.Replace(hocrfn, ".hocr", ".png", 1)
+	pngf, err := os.Open(pngfn)
+	if err != nil {
+		return newlines, err
+	}
+	defer pngf.Close()
+	img, err := png.Decode(pngf)
+	if err != nil {
+		return newlines, err
+	}
+
+	n := strings.Replace(filepath.Base(hocrfn), ".hocr", "", 1)
+	return parseLineDetails(h, img, n)
+}
diff --git a/parse/line.go b/lib/line/line.go
index 9a2be8e..3adac0a 100644
--- a/parse/line.go
+++ b/lib/line/line.go
@@ -1,9 +1,4 @@
-package parse
-
-// TODO: integrate in line-conf-buckets linedetail
-// TODO: add BucketUp() function here that does what both line-conf-buckets-tess.go
-//       and line-conf-buckets.go do
-// TODO: consider naming this package line, and separating it from hocr and prob
+package line
 
 import (
 	"image"
@@ -12,19 +7,32 @@ import (
 	"os"
 )
 
-type LineDetail struct {
+type Detail struct {
 	Name string
 	Avgconf float64
-	Img CopyableLine
+	Img CopyableImg
 	Text string
 	OcrName string
 }
 
-type CopyableLine interface {
+type CopyableImg interface {
 	CopyLineTo(io.Writer) (error)
 }
 
-// This is an implementation of the CopyableLine interface that
+type Details []Detail
+
+// Used by sort.Sort.
+func (l Details) Len() int { return len(l) }
+
+// Used by sort.Sort.
+func (l Details) Less(i, j int) bool {
+	return l[i].Avgconf < l[j].Avgconf
+}
+
+// Used by sort.Sort.
+func (l Details) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
+
+// This is an implementation of the CopyableImg interface that
 // stores the image directly as an image.Image
 type ImgDirect struct {
 	Img image.Image
@@ -38,6 +46,8 @@ func (i ImgDirect) CopyLineTo(w io.Writer) (error) {
 	return nil
 }
 
+// This is an implementation of the CopyableImg interface that
+// stores the path of an image
 type ImgPath struct {
 	Path string
 }
@@ -52,16 +62,3 @@ func (i ImgPath) CopyLineTo(w io.Writer) (error) {
 	_, err = io.Copy(w, f)
 	return err
 }
-
-type LineDetails []LineDetail
-
-// Used by sort.Sort.
-func (l LineDetails) Len() int { return len(l) }
-
-// Used by sort.Sort.
-func (l LineDetails) Less(i, j int) bool {
-	return l[i].Avgconf < l[j].Avgconf
-}
-
-// Used by sort.Sort.
-func (l LineDetails) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
diff --git a/parse/prob/prob.go b/lib/prob/prob.go
index 8d01cab..0299a96 100644
--- a/parse/prob/prob.go
+++ b/lib/prob/prob.go
@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	"git.rescribe.xyz/testingtools/parse"
+	"git.rescribe.xyz/testingtools/lib/line"
 )
 
 func getLineAvg(f string) (float64, error) {
@@ -18,8 +18,8 @@ func getLineAvg(f string) (float64, error) {
 		return 0, err
 	}
 
-	for _, line := range strings.Split(string(prob), "\n") {
-		fields := strings.Fields(line)
+	for _, l := range strings.Split(string(prob), "\n") {
+		fields := strings.Fields(l)
 
 		if len(fields) == 2 {
 			conf, err := strconv.ParseFloat(fields[1], 64)
@@ -38,9 +38,9 @@ func getLineAvg(f string) (float64, error) {
 }
 
 // Note this only processes one line at a time
-func GetLineDetails(probfn string) (parse.LineDetails, error) {
-	var line parse.LineDetail
-	lines := make(parse.LineDetails, 0)
+func GetLineDetails(probfn string) (line.Details, error) {
+	var l line.Detail
+	lines := make(line.Details, 0)
 
 	avg, err := getLineAvg(probfn)
 	if err != nil {
@@ -54,16 +54,16 @@ func GetLineDetails(probfn string) (parse.LineDetails, error) {
 		return lines, err
 	}
 
-	line.Name = filepath.Base(filebase)
-	line.Avgconf = avg
-	line.Text = string(txt)
-	line.OcrName = filepath.Dir(filebase)
+	l.Name = filepath.Base(filebase)
+	l.Avgconf = avg
+	l.Text = string(txt)
+	l.OcrName = filepath.Dir(filebase)
 
-	var imgfn parse.ImgPath
+	var imgfn line.ImgPath
 	imgfn.Path = filebase + ".bin.png"
-	line.Img = imgfn
+	l.Img = imgfn
 
-	lines = append(lines, line)
+	lines = append(lines, l)
 
 	return lines, nil
 }
diff --git a/parse/hocr/hocr.go b/parse/hocr/hocr.go
deleted file mode 100644
index 81250a9..0000000
--- a/parse/hocr/hocr.go
+++ /dev/null
@@ -1,181 +0,0 @@
-package hocr
-
-// TODO: Parse line name to zero pad line numbers, so they can
-//       be sorted easily
-// TODO: have same filename format as .prob uses, so include base
-//       dirname, and don't include line numbers if there's only
-//       one line in the hocr
-
-import (
-	"encoding/xml"
-	"image"
-	"image/png"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"git.rescribe.xyz/testingtools/parse"
-)
-
-type Hocr struct {
-	Lines []OcrLine `xml:"body>div>div>p>span"`
-}
-
-type OcrLine struct {
-	Class string `xml:"class,attr"`
-	Id string `xml:"id,attr"`
-	Title string `xml:"title,attr"`
-	Words []OcrWord `xml:"span"`
-	Text string `xml:",chardata"`
-}
-
-type OcrWord struct {
-	Class string `xml:"class,attr"`
-	Id string `xml:"id,attr"`
-	Title string `xml:"title,attr"`
-	Chars []OcrChar `xml:"span"`
-	Text string `xml:",chardata"`
-}
-
-type OcrChar struct {
-	Class string `xml:"class,attr"`
-	Id string `xml:"id,attr"`
-	Title string `xml:"title,attr"`
-	Chars []OcrChar `xml:"span"`
-	Text string `xml:",chardata"`
-}
-
-// Returns the confidence for a word based on its x_wconf value
-func wordConf(s string) (float64, error) {
-	re, err := regexp.Compile(`x_wconf ([0-9.]+)`)
-	if err != nil {
-		return 0.0, err
-	}
-	conf := re.FindStringSubmatch(s)
-	return strconv.ParseFloat(conf[1], 64)
-}
-
-func boxCoords(s string) ([4]int, error) {
-	var coords [4]int
-	re, err := regexp.Compile(`bbox ([0-9]+) ([0-9]+) ([0-9]+) ([0-9]+)`)
-	if err != nil {
-		return coords, err
-	}
-	coordstr := re.FindStringSubmatch(s)
-	for i := range coords {
-		c, err := strconv.Atoi(coordstr[i+1])
-		if err != nil {
-			return coords, err
-		}
-		coords[i] = c
-	}
-	return coords, nil
-}
-
-func noText(s string) bool {
-	t := strings.Trim(s, " \n")
-	return len(t) == 0
-}
-
-func Parse(b []byte) (Hocr, error) {
-	var hocr Hocr
-
-	err := xml.Unmarshal(b, &hocr)
-	if err != nil {
-		return hocr, err
-	}
-
-	return hocr, nil
-}
-
-func parseLineDetails(h Hocr, i image.Image, name string) (parse.LineDetails, error) {
-	lines := make(parse.LineDetails, 0)
-
-	for _, l := range h.Lines {
-		totalconf := float64(0)
-		num := 0
-		for _, w := range l.Words {
-			c, err := wordConf(w.Title)
-			if err != nil {
-				return lines, err
-			}
-			num++
-			totalconf += c
-		}
-
-		coords, err := boxCoords(l.Title)
-		if err != nil {
-			return lines, err
-		}
-
-		var line parse.LineDetail
-		line.Name = l.Id
-		line.Avgconf = (totalconf/float64(num)) / 100
-		linetext := ""
-
-		linetext = l.Text
-		if(noText(linetext)) {
-			linetext = ""
-			for _, w := range l.Words {
-				if(w.Class != "ocrx_word") {
-					continue
-				}
-				linetext += w.Text + " "
-			}
-		}
-		if(noText(linetext)) {
-			linetext = ""
-			for _, w := range l.Words {
-				if(w.Class != "ocrx_word") {
-					continue
-				}
-				for _, c := range w.Chars {
-					if(c.Class != "ocrx_cinfo") {
-						continue
-					}
-					linetext += c.Text
-				}
-				linetext += " "
-			}
-		}
-		line.Text = strings.TrimRight(linetext, " ")
-		line.Text += "\n"
-		line.OcrName = name
-		var imgd parse.ImgDirect
-		imgd.Img = i.(*image.Gray).SubImage(image.Rect(coords[0], coords[1], coords[2], coords[3]))
-		line.Img = imgd
-		lines = append(lines, line)
-	}
-	return lines, nil
-}
-
-func GetLineDetails(hocrfn string) (parse.LineDetails, error) {
-	var newlines parse.LineDetails
-
-	file, err := ioutil.ReadFile(hocrfn)
-	if err != nil {
-		return newlines, err
-	}
-
-	h, err := Parse(file)
-	if err != nil {
-		return newlines, err
-	}
-
-	pngfn := strings.Replace(hocrfn, ".hocr", ".png", 1)
-	pngf, err := os.Open(pngfn)
-	if err != nil {
-		return newlines, err
-	}
-	defer pngf.Close()
-	img, err := png.Decode(pngf)
-	if err != nil {
-		return newlines, err
-	}
-
-	n := strings.Replace(filepath.Base(hocrfn), ".hocr", "", 1)
-	return parseLineDetails(h, img, n)
-}