Changeset 74

Timestamp:

12/27/06 18:02:06 (3 years ago)

Author:

rgrp

Message:

Improve ignoring of non words by the concordance/statistic builder and fix a bug introduced by changeset:72.

• src/shakespeare/concordance.py:
  • add ignore_word function to replace simple test in add_text method and improve this using inter alia:
  • is_roman_numeral: new method to test whether a word is a roman numeral
  • non_words: attribute listing non-words
  • remove_text: (bugfix) was not removing associated Statistic only associated Concordance objects
• src/shakespeare/concordance_test.py:
  • test_is_roman_numeral
  • test_ignore_word

Location:

trunk/src/shakespeare

Files:

• concordance.py (modified) (3 diffs)
• concordance_test.py (modified) (1 diff)

trunk/src/shakespeare/concordance.py

@@ -89 +89 @@
 
     words_to_ignore = [ 
-        # 'a', 'the', 'and', 'as', 'are', 'be', 'but', 'd', 'in'
+        # 'a', 'the', 'and', 'as', 'are', 'be', 'but', 'in'
                         ]
+    non_words = [ 
+            'd', # accus'd
+            't',
+            ]
+
+    def is_roman_numeral(self, word):
+        digits = [ 'i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix' ]
+        others = [ 'l', 'x', 'c' ]
+        if word == 'i': return False # exception because this conflicts with I
+        while word[0] in others:
+            if len(word) == 1:
+                return True
+            else:
+                word = word[1:]
+        return word in digits
+
+    def ignore_word(self, word):
+        "Return True if this word should not be added to the concordance."
+        bool1 = word in self.words_to_ignore
+        bool2 = word in self.non_words
+        # do roman numerals
+        bool3 = self.is_roman_numeral(word)
+        return bool1 or bool2 or bool3
 
     def _text_already_done(self, text):
@@ -122 +145 @@
             for match in self.word_regex.finditer(line):
                 word = match.group().lower() # case insensitive
-                if word in self.words_to_ignore:
+                if self.ignore_word(word):
                     continue
                 shakespeare.dm.Concordance(connection=trans,
@@ -164 +187 @@
         for rec in recs:
             shakespeare.dm.Concordance.delete(rec.id)
+        stats = shakespeare.dm.Statistic.select(
+                shakespeare.dm.Statistic.q.textID==dmText.id
+                )
+        for stat in stats:
+            shakespeare.dm.Statistic.delete(stat.id)
 

trunk/src/shakespeare/concordance_test.py

@@ -63 +63 @@
         assert exp == out
 
+    def test_is_roman_numeral(self):
+        testvals = [ 'ii', 'v', 'vi', 'xi', 'xx', 'xxi', 'xlvi', 'c', 'cvi' ]
+        for val in testvals:
+            assert self.builder.is_roman_numeral(val)
+
+    def test_ignore_word(self):
+        testvals = [ 'd', 't' ]
+        for val in testvals:
+            assert self.builder.ignore_word(val)
+
     def test_concordance(self):
         for key, value in self.expConcordance.items():