From 75c3f0d8ec9463d9f5cc3034ab0103862a4a1aaa Mon Sep 17 00:00:00 2001
From: Kurt Jung <kurt.w.jung@gmail.com>
Date: Tue, 28 Apr 2015 17:35:08 -0400
Subject: Tokenize text that does not include HTML tags

---
 htmlbasic.go | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'htmlbasic.go')

diff --git a/htmlbasic.go b/htmlbasic.go
index 2340c23..b7186bc 100644
--- a/htmlbasic.go
+++ b/htmlbasic.go
@@ -85,6 +85,8 @@ func HTMLBasicTokenize(htmlStr string) (list []HTMLBasicSegmentType) {
 			seg.Attr = nil
 			list = append(list, seg)
 		}
+	} else {
+		list = append(list, HTMLBasicSegmentType{Cat: 'T', Str: htmlStr, Attr: nil})
 	}
 	return
 }
-- 
cgit v1.2.1-24-ge1ad