Letter-precise html tokenization

webstruct/text_tokenizers.py

@@ -12,9 +12,9 @@ class WordTokenizer(object):
 
     >>> from nltk.tokenize.treebank import TreebankWordTokenizer  # doctest: +SKIP
     >>> s = '''Good muffins cost $3.88\nin New York. Email: muffins@gmail.com'''
-    >>> TreebankWordTokenizer().tokenize(s) # doctest: +SKIP
+    >>> TreebankWordTokenizer().span_tokenize(s) # doctest: +SKIP
     ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York.', 'Email', ':', 'muffins', '@', 'gmail.com']
-    >>> WordTokenizer().tokenize(s)
+    >>> WordTokenizer().span_tokenize(s)
     [TextToken(chars='Good', position=0, length=4),
      TextToken(chars='muffins', position=5, length=7),
      TextToken(chars='cost', position=13, length=4),
@@ -27,25 +27,25 @@ class WordTokenizer(object):
      TextToken(chars='muffins@gmail.com', position=44, length=17)]
 
     >>> s = '''Shelbourne Road,'''
-    >>> WordTokenizer().tokenize(s)
+    >>> WordTokenizer().span_tokenize(s)
     [TextToken(chars='Shelbourne', position=0, length=10),
      TextToken(chars='Road', position=11, length=4),
      TextToken(chars=',', position=15, length=1)]
 
     >>> s = '''population of 100,000'''
-    >>> WordTokenizer().tokenize(s)
+    >>> WordTokenizer().span_tokenize(s)
     [TextToken(chars='population', position=0, length=10),
      TextToken(chars='of', position=11, length=2),
      TextToken(chars='100,000', position=14, length=7)]
 
     >>> s = '''Hello|World'''
-    >>> WordTokenizer().tokenize(s)
+    >>> WordTokenizer().span_tokenize(s)
     [TextToken(chars='Hello', position=0, length=5),
      TextToken(chars='|', position=5, length=1),
      TextToken(chars='World', position=6, length=5)]
 
     >>> s2 = '"We beat some pretty good teams to get here," Slocum said.'
-    >>> WordTokenizer().tokenize(s2)  # doctest: +NORMALIZE_WHITESPACE
+    >>> WordTokenizer().span_tokenize(s2)  # doctest: +NORMALIZE_WHITESPACE
     [TextToken(chars='``', position=0, length=1),
      TextToken(chars='We', position=1, length=2),
      TextToken(chars='beat', position=4, length=4),
@@ -65,7 +65,7 @@ class WordTokenizer(object):
     ... cliche-ridden, \"Touched by an
     ... Angel\" (a show creator John Masius
     ... worked on) wanna-be if she didn't.'''
-    >>> WordTokenizer().tokenize(s3)  # doctest: +NORMALIZE_WHITESPACE
+    >>> WordTokenizer().span_tokenize(s3)  # doctest: +NORMALIZE_WHITESPACE
     [TextToken(chars='Well', position=0, length=4),
      TextToken(chars=',', position=4, length=1),
      TextToken(chars='we', position=6, length=2),
@@ -97,28 +97,28 @@ class WordTokenizer(object):
      TextToken(chars="didn't", position=133, length=6),
      TextToken(chars='.', position=139, length=1)]
 
-    >>> WordTokenizer().tokenize('"')
+    >>> WordTokenizer().span_tokenize('"')
     [TextToken(chars='``', position=0, length=1)]
 
-    >>> WordTokenizer().tokenize('" a')
+    >>> WordTokenizer().span_tokenize('" a')
     [TextToken(chars='``', position=0, length=1),
      TextToken(chars='a', position=2, length=1)]
 
     Some issues:
 
-    >>> WordTokenizer().tokenize("Phone:855-349-1914")  # doctest: +SKIP
+    >>> WordTokenizer().span_tokenize("Phone:855-349-1914")  # doctest: +SKIP
     ['Phone', ':', '855-349-1914']
 
-    >>> WordTokenizer().tokenize("Copyright © 2014 Foo Bar and Buzz Spam. All Rights Reserved.")  # doctest: +SKIP
+    >>> WordTokenizer().span_tokenize("Copyright © 2014 Foo Bar and Buzz Spam. All Rights Reserved.")  # doctest: +SKIP
     ['Copyright', '\xc2\xa9', '2014', 'Wall', 'Decor', 'and', 'Home', 'Accents', '.', 'All', 'Rights', 'Reserved', '.']
 
-    >>> WordTokenizer().tokenize("Powai Campus, Mumbai-400077")  # doctest: +SKIP
+    >>> WordTokenizer().span_tokenize("Powai Campus, Mumbai-400077")  # doctest: +SKIP
     ['Powai', 'Campus', ',', 'Mumbai", "-", "400077']
 
-    >>> WordTokenizer().tokenize("1 5858/ 1800")  # doctest: +SKIP
+    >>> WordTokenizer().span_tokenize("1 5858/ 1800")  # doctest: +SKIP
     ['1', '5858', '/', '1800']
 
-    >>> WordTokenizer().tokenize("Saudi Arabia-")  # doctest: +SKIP
+    >>> WordTokenizer().span_tokenize("Saudi Arabia-")  # doctest: +SKIP
     ['Saudi', 'Arabia', '-']
 
     """
@@ -140,17 +140,17 @@ class WordTokenizer(object):
 
     open_quotes = re.compile(r'(^|[\s(\[{<])"')
 
-    def _tokenize(self, text):
+    def _span_tokenize(self, text):
         # this one cannot be placed in the loop because it requires
         # position check (beginning of the string) or previous char value
         quote = self.open_quotes.search(text)
         if quote is not None:
             end = quote.end() - 1
-            for t in self._tokenize(text[:end]):
+            for t in self._span_tokenize(text[:end]):
                 yield t
             yield TextToken(chars='``', position=end, length=1)
             shift = end + 1
-            for t in self._tokenize(text[shift:]):
+            for t in self._span_tokenize(text[shift:]):
                 yield TextToken(chars=t.chars,
                                 position=t.position + shift,
                                 length=t.length)
@@ -185,13 +185,16 @@ def _tokenize(self, text):
                     break
             i += shift
 
+    def span_tokenize(self, text):
+        return [t for t in self._span_tokenize(text) if t.chars]
+
     def tokenize(self, text):
-        return [t for t in self._tokenize(text) if t.chars]
+        return [t.chars for t in self.span_tokenize(text)]
 
 
 class DefaultTokenizer(WordTokenizer):
-    def tokenize(self, text):
-        tokens = super(DefaultTokenizer, self).tokenize(text)
+    def span_tokenize(self, text):
+        tokens = super(DefaultTokenizer, self).span_tokenize(text)
         # remove standalone commas and semicolons
         # as they broke tag sets,
         # e.g. PERSON->FUNCTION in case "PERSON, FUNCTION"
@@ -205,4 +208,4 @@ def tokenize(self, text):
         return [t for t in tokens if t.chars not in {',', ';'}]
 
 
-tokenize = DefaultTokenizer().tokenize
+tokenize = DefaultTokenizer().span_tokenize