From a1de8862a091f9584220db40671a0d43346c4519 Mon Sep 17 00:00:00 2001 From: Nick White Date: Mon, 9 Nov 2020 18:29:56 +0000 Subject: [rescribe] Local only combo tool basically now working. Testing is still minimal. --- local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'local.go') diff --git a/local.go b/local.go index ebc3611..0ccc761 100644 --- a/local.go +++ b/local.go @@ -36,7 +36,7 @@ func (a *LocalConn) MinimalInit() error { if a.TempDir == "" { a.TempDir = filepath.Join(os.TempDir(), "bookpipeline") } - err = os.Mkdir(a.TempDir, 0700) + err = os.MkdirAll(a.TempDir, 0700) if err != nil && !os.IsExist(err) { return fmt.Errorf("Error creating temporary directory: %v", err) } -- cgit v1.2.1-24-ge1ad From 33f1726a4c9f8013dcde39e644281059d9766bc4 Mon Sep 17 00:00:00 2001 From: Nick White Date: Tue, 10 Nov 2020 12:30:15 +0000 Subject: gofmt --- local.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'local.go') diff --git a/local.go b/local.go index 0ccc761..e5d9bef 100644 --- a/local.go +++ b/local.go @@ -27,7 +27,7 @@ const storageId = "storage" type LocalConn struct { // these should be set before running Init(), or left to defaults TempDir string - Logger *log.Logger + Logger *log.Logger } // MinimalInit does the bare minimum initialisation @@ -184,12 +184,12 @@ func (a *LocalConn) DelFromQueue(url string, handle string) error { // store the joining of part before and part after handle var complete string - if len(s) >= len(handle) + 1 { + if len(s) >= len(handle)+1 { if i > 0 { complete = s[:i] } // the '+1' is for the newline character - complete += s[i + len(handle) + 1:] + complete += s[i+len(handle)+1:] } f, err := os.Create(filepath.Join(a.TempDir, url)) -- cgit v1.2.1-24-ge1ad From cfbb3481368714adcd734906d8a460b873551c90 Mon Sep 17 00:00:00 2001 From: Nick White Date: Mon, 16 Nov 2020 17:43:27 +0000 Subject: Some changes to ensure the pipeline works correctly on Windows There were a couple of places where a file was uploaded while still open, which resulted in an attempt to remove it, which causes an error from Windows. The allOCRed function also included an assumption that the path separator would be a /, which is always correct for AWS, and correct for local on Linux and OSX, but not for local Windows. Fixed by leaving the separator well alone. Also, the local connection was not stripping leading \, like it did /, which caused an issue with Windows local. Windows local is now tested and working, at least through wine. --- local.go | 1 + 1 file changed, 1 insertion(+) (limited to 'local.go') diff --git a/local.go b/local.go index e5d9bef..e66f477 100644 --- a/local.go +++ b/local.go @@ -134,6 +134,7 @@ func prefixwalker(dirpath string, prefix string, list *[]ObjMeta) filepath.WalkF } n := strings.TrimPrefix(path, dirpath) n = strings.TrimPrefix(n, "/") + n = strings.TrimPrefix(n, "\\") o := ObjMeta{Name: n, Date: info.ModTime()} *list = append(*list, o) return nil -- cgit v1.2.1-24-ge1ad From 4fcbfba65689dc5e8ad46ba467343d3da376d92a Mon Sep 17 00:00:00 2001 From: Nick White Date: Fri, 4 Dec 2020 17:12:59 +0000 Subject: Ensure mkdir will succeed in upload --- local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'local.go') diff --git a/local.go b/local.go index e66f477..0fceca2 100644 --- a/local.go +++ b/local.go @@ -222,7 +222,7 @@ func (a *LocalConn) Download(bucket string, key string, path string) error { // Upload just copies the file from path to TempDir/bucket/key func (a *LocalConn) Upload(bucket string, key string, path string) error { d := filepath.Join(a.TempDir, bucket, filepath.Dir(key)) - err := os.Mkdir(d, 0700) + err := os.MkdirAll(d, 0700) if err != nil && !os.IsExist(err) { return fmt.Errorf("Error creating temporary directory: %v", err) } -- cgit v1.2.1-24-ge1ad From 5c3cee66a90ce6ef87e125b3bf011a6903d38083 Mon Sep 17 00:00:00 2001 From: Nick White Date: Tue, 26 Jan 2021 14:56:10 +0000 Subject: Make ListObjectsWithMeta generic again and create a specialised ListObjectWithMeta for single file listing, so we can still be as fast, but do not have a misleading api --- local.go | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'local.go') diff --git a/local.go b/local.go index 0fceca2..31e44a9 100644 --- a/local.go +++ b/local.go @@ -159,6 +159,17 @@ func (a *LocalConn) ListObjectsWithMeta(bucket string, prefix string) ([]ObjMeta return list, err } +func (a *LocalConn) ListObjectWithMeta(bucket string, prefix string) (ObjMeta, error) { + list, err := a.ListObjectsWithMeta(bucket, prefix) + if err != nil { + return ObjMeta{}, err + } + if len(list) == 0 { + return ObjMeta{}, fmt.Errorf("No object found for %s", prefix) + } + return list[0], nil +} + // AddToQueue adds a message to a queue func (a *LocalConn) AddToQueue(url string, msg string) error { f, err := os.OpenFile(filepath.Join(a.TempDir, url), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) -- cgit v1.2.1-24-ge1ad