summaryrefslogtreecommitdiff
path: root/cmd/analysestats
diff options
context:
space:
mode:
Diffstat (limited to 'cmd/analysestats')
-rw-r--r--cmd/analysestats/main.go74
1 files changed, 73 insertions, 1 deletions
diff --git a/cmd/analysestats/main.go b/cmd/analysestats/main.go
index 1d6071d..a15f3b5 100644
--- a/cmd/analysestats/main.go
+++ b/cmd/analysestats/main.go
@@ -8,12 +8,14 @@
package main
import (
+ "bufio"
"encoding/csv"
"encoding/xml"
"flag"
"fmt"
"io/ioutil"
"log"
+ "math"
"os"
"path/filepath"
"strings"
@@ -65,6 +67,69 @@ func getTrainingUsed(hocrfn string) (string, error) {
return par.Par[0].Lang, nil
}
+// getMeanStddevOfBest calculates the mean and standard deviation
+// of the confidence values of every page in bestfn, as listed in
+// conffn.
+func getMeanStddevOfBest(bestfn string, conffn string) (float64, float64, error) {
+ f, err := os.Open(conffn)
+ if err != nil {
+ return 0, 0, fmt.Errorf("Failed to open %s: %v", conffn, err)
+ }
+ defer f.Close()
+ s := bufio.NewScanner(f)
+
+ // create a map of confs from the conf file
+ var confs map[string]int
+ confs = make(map[string]int)
+ for s.Scan() {
+ line := s.Text()
+ parts := strings.Fields(line)
+ if len(parts) != 2 {
+ continue
+ }
+ c, err := strconv.Atoi(parts[1])
+ if err != nil {
+ continue
+ }
+ fn := filepath.Base(parts[0])
+ confs[fn] = c
+ }
+
+ f, err = os.Open(bestfn)
+ if err != nil {
+ return 0, 0, fmt.Errorf("Failed to open %s: %v", bestfn, err)
+ }
+ defer f.Close()
+ s = bufio.NewScanner(f)
+
+ var bestConfs []int
+ for s.Scan() {
+ fn := s.Text()
+ c, ok := confs[fn]
+ if !ok {
+ continue
+ }
+ bestConfs = append(bestConfs, c)
+ }
+
+ var sum int
+ for _, v := range bestConfs {
+ sum += v
+ }
+ mean := float64(sum) / float64(len(bestConfs))
+
+ var a, stddev float64
+ if len(bestConfs) > 1 {
+ for _, v := range bestConfs {
+ a += (float64(v) - mean) * (float64(v) - mean)
+ }
+ variance := a / float64(len(bestConfs) - 1)
+ stddev = math.Sqrt(variance)
+ }
+
+ return mean, stddev, nil
+}
+
// walker returns a walkfunc that checks for hocr and best files,
// and uses them to fill the bookstats map & structure. Note that
// the stat file is read when the best file is read, as they need
@@ -112,7 +177,14 @@ func walker(bookstats *Bookstats) filepath.WalkFunc {
}
(*bookstats)[prefix].training = training
case "best":
- // TODO: read conf also and fill in mean and stddev
+ confpath := strings.Replace(fpath, "-best", "-conf", -1)
+ mean, stddev, err := getMeanStddevOfBest(fpath, confpath)
+ if err != nil {
+ log.Printf("Warning: failed to get mean & standard deviation from %s and %s: %v\n", fpath, confpath, err)
+ return nil
+ }
+ (*bookstats)[prefix].mean = mean
+ (*bookstats)[prefix].stddev = stddev
}
return nil