Changeset 72

Timestamp:

12/27/06 17:09:08 (3 years ago)

Author:

rgrp

Message:

Create Statistic domain object separate from the Concordance object to try and improve performance.

Generating concordance page is very slow. Tracked this down to (a) generation of the word list (which involved iterating over every item in the concordance table) (b) caculating statistics from concorance table using count(*).

By creating a dedicated Statistic object holding occurrences per text and word hoped to improve both of these. With respect to item (a) had a fairly good speed improvement from 23s to ~3s on my mac osx using the sonnets as the corpora. On (b) did not get much improvement as still have to do one db read per word and count(*) is pretty efficient (the cost using SQLObject is all the db reads not the original db query). On my local machine still takes ~ 30s to load the concordance page :( -- looks like caching the html may be the simplest way forward.

• trunk/src/shakespeare/dm.py: add Statistic domain object
• trunk/src/shakespeare/dm_test.py: add relevant tests
• trunk/src/shakespeare/concordance.py:
  • ConcordanceBuilder?.add_text: adapted it to write values into Statistic object
  • ConcordanceBase?.keys(): use Statistic objects to get word lists rather than Concordance objects
  • Statistics.get: use Statistic rather than Concordance (ironically was simpler when using Concordance)
  • make various related changes (_sql_filter -> sqlcc_filter and sqlstat_filter etc)
• trunk/src/shakespeare/concordance_test.py: no new tests but some minor fixes to old ones (*not* related to the other changes though)

Previously had a Statistics object in shakespeare.concordance

Location:

trunk/src/shakespeare

Files:

• concordance.py (modified) (7 diffs)
• concordance_test.py (modified) (4 diffs)
• dm.py (modified) (2 diffs)
• dm_test.py (modified) (2 diffs)

trunk/src/shakespeare/concordance.py

@@ -22 +22 @@
     """
     sqlcc = shakespeare.dm.Concordance
+    sqlstat = shakespeare.dm.Statistic
 
     def __init__(self, filter_names=None):
@@ -29 +30 @@
         """
         self._filter_names = filter_names
-        # piece of sql to use in select to filter texts
-        self._sql_filter = True
+        self.sqlcc_filter = self._make_filter(self.sqlcc)
+        self.sqlstat_filter = self._make_filter(self.sqlstat)
+
+    def _make_filter(self, sqlobj):
+        sql_filter = True
         if self._filter_names is not None:
             arglist = []
             for name in self._filter_names:
-                newarg = self.sqlcc.q.textID == self._name2id(name)
+                newarg = sqlobj.q.textID == self._name2id(name)
                 arglist.append(newarg)
-            self._sql_filter = sqlobject.OR(*arglist)
+            sql_filter = sqlobject.OR(*arglist)
+        return sql_filter
     
     def _name2id(self, name):
@@ -42 +47 @@
 
     def keys(self):
-        """Return list of words in concordance
+        """Return list of *distinct* words in concordance/statistics
         """
-        # distinct does not help us because we need to DISTINCT word
-        # but can't do this with sqlobject
-        all = self.sqlcc.select(self._sql_filter,
-                           orderBy=self.sqlcc.q.word,
-                           distinct=True)
+        all = self.sqlstat.select(self.sqlstat_filter,
+                           orderBy=self.sqlstat.q.word,
+                           )
         words = [ xx.word for xx in list(all) ]
         distinct = list(set(words))
@@ -63 +66 @@
         @return: sqlobject query list 
         """
-        select = self.sqlcc.select(sqlobject.AND(self._sql_filter, self.sqlcc.q.word==word))
+        select = self.sqlcc.select(sqlobject.AND(self.sqlcc_filter, self.sqlcc.q.word==word))
         return select
 
@@ -69 +72 @@
 
     def get(self, word):
-        select = self.sqlcc.select(
-            sqlobject.AND(self._sql_filter, self.sqlcc.q.word==word)
+        select = self.sqlstat.select(
+            sqlobject.AND(self.sqlstat_filter, self.sqlstat.q.word==word)
             )
-        return select.count()
+        total = 0
+        for stat in select:
+            total += stat.occurrences
+        return total
 
 class ConcordanceBuilder(object):
@@ -111 +117 @@
         lineCount = 0
         charIndex = 0
+        stats = {}
         trans = shakespeare.dm.Concordance._connection.transaction()
         for line in text.readlines():
@@ -122 +129 @@
                                            line=lineCount,
                                            char_index=charIndex+match.start())
+                stats[word] = stats.get(word, 0) + 1
             lineCount += 1
             charIndex += len(line)
         trans.commit()
+        trans = shakespeare.dm.Concordance._connection.transaction()
+        for word, value in stats.items():
+            tresults  = shakespeare.dm.Statistic.select(
+                sqlobject.AND(
+                    shakespeare.dm.Statistic.q.textID == dmText.id,
+                    shakespeare.dm.Statistic.q.word == word
+                    ))
+            try:
+                dbstat = list(tresults)[0]
+                dbstat.occurrences += value
+            except:
+                shakespeare.dm.Statistic(
+                        connection=trans,
+                        text=dmText,
+                        word=word,
+                        occurrences=value
+                        )
+        trans.commit()
+
 
     def remove_text(self, name):

trunk/src/shakespeare/concordance_test.py

@@ -15 +15 @@
 I had in charge at my depart for France,
 As procurator to your excellence,
+A fake imperial line.
 """
     name = 'test-concordance'
@@ -20 +21 @@
     
     # ['work_id', 'line-no', 'character-index'] }
+    # incomplete
     expConcordance = {
-        'fake' : [ (name, 0, 2), (name, 0, 7) ],
+        'fake' : [ (name, 0, 2), (name, 0, 7), (name, 5, 136) ],
         'suffolk' : [ (name, 1, 17), ],
         'high' : [ (name, 2, 37), ],
@@ -27 +29 @@
         }
 
+    # incomplete
     expStats = {
-        'fake' : 2,
+        'fake' : 3,
+        'imperial' : 2,
         'suffolk' : 1,
         'high' : 1,
@@ -62 +66 @@
         for key, value in self.expConcordance.items():
             listing = list(self.concordance.get(key))
-            listing.reverse()
-            out = [ (xx.text.name, xx.line, xx.char_index) for xx in listing ]
-            assert out == value
+            assert len(listing) == len(value)
+            for xx in listing:
+                assert (xx.text.name, xx.line, xx.char_index) in value
 
     def test_stats(self):
         for key, value in self.expStats.items():
             out = self.statistics.get(key)
+            print key
             assert out == value
 

trunk/src/shakespeare/dm.py

@@ -20 +20 @@
     Material.createTable(ifNotExists=True)
     Concordance.createTable(ifNotExists=True)
+    Statistic.createTable(ifNotExists=True)
 
 def cleandb():
+    Statistic.dropTable(ifExists=True)
     Concordance.dropTable(ifExists=True)
     Material.dropTable(ifExists=True)
@@ -63 +65 @@
     text_index = sqlobject.DatabaseIndex('text')
 
+class Statistic(sqlobject.SQLObject):
+
+    text = sqlobject.ForeignKey('Material')
+    word = sqlobject.StringCol(length=50)
+    occurrences = sqlobject.IntCol(default=1)
+
+    word_index = sqlobject.DatabaseIndex('word')
+    text_index = sqlobject.DatabaseIndex('text')
+
 
 # auto create db tables on import

trunk/src/shakespeare/dm_test.py

@@ +1 @@
+import sqlobject
+
 import shakespeare.dm
 
@@ -47 +49 @@
         assert self.text == out1.text
 
+class TestStatistic:
+
+    def setup_class(self):
+        self.name = 'test-123'
+        self.title = 'Hamlet'
+        self.text = shakespeare.dm.Material(name=self.name, title=self.title)
+        self.word = 'jones'
+        self.occurrences = 5
+        self.cc1 = shakespeare.dm.Statistic(
+                text=self.text,
+                word=self.word,
+                occurrences=self.occurrences
+                )
+
+    def teardown_class(self):
+        shakespeare.dm.Statistic.delete(self.cc1.id)
+        shakespeare.dm.Material.delete(self.text.id)
+
+    def test1(self):
+        out1 = shakespeare.dm.Statistic.get(self.cc1.id)
+        assert self.text == out1.text
+        assert out1.occurrences == self.occurrences
+
+    def test_select(self):
+        tresults  = shakespeare.dm.Statistic.select(
+            sqlobject.AND(
+                shakespeare.dm.Statistic.q.textID == self.text.id,
+                shakespeare.dm.Statistic.q.word == self.word,
+                ))
+        num = tresults.count()
+        assert num == 1
+