diff options
| author | Kurt Jung <kurt.w.jung@gmail.com> | 2015-04-28 17:35:08 -0400 |
|---|---|---|
| committer | Kurt Jung <kurt.w.jung@gmail.com> | 2015-04-28 17:35:08 -0400 |
| commit | 75c3f0d8ec9463d9f5cc3034ab0103862a4a1aaa (patch) | |
| tree | fc61149bc3b9f3448e75ebfdf236603f7b0b0f67 | |
| parent | 825ddc907e7cfbdda63dfa5ca99ebd5c8ebfb4ff (diff) | |
Tokenize text that does not include HTML tags
| -rw-r--r-- | htmlbasic.go | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/htmlbasic.go b/htmlbasic.go index 2340c23..b7186bc 100644 --- a/htmlbasic.go +++ b/htmlbasic.go @@ -85,6 +85,8 @@ func HTMLBasicTokenize(htmlStr string) (list []HTMLBasicSegmentType) { seg.Attr = nil list = append(list, seg) } + } else { + list = append(list, HTMLBasicSegmentType{Cat: 'T', Str: htmlStr, Attr: nil}) } return } |