Navigation

← Previous Changeset
Next Changeset →

Changeset 2

Timestamp:

05/09/06 20:01:41 (3 years ago)

Author:

rgrp

Message:

Another major change involving adding a concordance webpage (+ associated
changes) and extensive refactoring of the index.

Index
=====

Rename download.py -> work.py
Refactor all contents into two classes GutenbergIndex? ShakespeareIndex?
Create module variable index = ShakespeareIndex?()
associated changes in all modules that use this one (pretty much all)

Concordance
===========

Create webpage at /concordance
- cherrpy_handler.py
- template/concordance.html
concordancer.py: make_concordance
- Refactor concordancer code to allow one to specify the exact works to add to concordancer and to specify the file to save in
concordancer.py: Concordancer add words_to_ignore class variable listing
words to ignore

Files:

trunk/bin/shakespeare-admin (modified) (1 diff)
trunk/src/shakespeare/cherrypy_handler.py (modified) (1 diff)
trunk/src/shakespeare/concordancer.py (modified) (1 diff)
trunk/src/shakespeare/concordancer_test.py (modified) (1 diff)
trunk/src/shakespeare/format.py (modified) (1 diff)
trunk/src/shakespeare/format_test.py (modified) (1 diff)
trunk/src/shakespeare/template/concordance.html (added)
trunk/src/shakespeare/utils.py (modified) (1 diff)
trunk/src/shakespeare/work.py (moved) (moved from trunk/src/shakespeare/download.py) (1 diff)
trunk/src/shakespeare/work_test.py (moved) (moved from trunk/src/shakespeare/download_test.py) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/bin/shakespeare-admin

 #!/usr/bin/env python
 import cmd
 import StringIO
 from shakespeare.format import GutenbergShakespeare
-from shakespeare.download import download_all_shakespeare
-from shakespeare.download import make_index
 from shakespeare.concordancer import make_concordancer
 import shakespeare.utils as utils
+import shakespeare.work
 class ShakespeareAdmin(cmd.Cmd):
     def __init__(self):
         cmd.Cmd.__init__(self) # cmd.Cmd is not a new style class
-        self._index = make_index()
+        self._index = shakespeare.work.index.all
     def do_format(self, line=None):
         path = sys.argv[1]
         x = GutenbergShakespeare(file(path))
         print x.extract_text()
     def help_format(self, line=None):
         usage = \
 '''Format a raw gutenberg text.
 Take a raw gutenberg text in and return (on stdout) the core text (i.e. strip
 out all the gutenberg bumpf)'''
         print usage
     def do_format_all(self, line):
         index = self._index
         for item in index:
             url = item[1]
             src = utils.get_local_path(url)
             dest = utils.get_local_path(url, 'cleaned')
             infile = file(src)
             if src.endswith('wssnt10.txt'): # if it is the sonnets need a hack
                 # delete last 140 characters
                 tmp1 = infile.read()
                 infile = StringIO.StringIO(tmp1[:-120])
             formatter = GutenbergShakespeare(infile)
             ff = file(dest, 'w')
             out = formatter.extract_text()
             ff.write(out)
             ff.close()
     def do_download_texts(self, line):
-        download_all_shakespeare()
+        for item in self._index:
+            utils.download_url(item[1])
     def help_download_texts(self, line=None):
-        print download_all_shakespeare.__doc__
+        usage = \
+"""
+Download from Project Gutenberg all the shakespeare texts listed in the index."""
+        print usage
     def do_print_index(self, line):
         for row in self._index:
             print row
     def help_print_index(self, line=None):
         usage = \
 '''Print index of Shakespeare texts to stdout'''
         print usage
     def do_make_concordancer(self, line=None):
-        make_concordancer()
+        if line is not None:
+            textsToAdd = []
+            textsUrls = line.split()
+            for item in self._index:
+              if item[1] in textsUrls:
+                  textsToAdd.append(item)
+            make_concordancer(textsToAdd)
+        else:
+            make_concordancer()
     def help_make_concordancer(self, line=None):
         print make_concordance.__doc__
     def do_help(self, line=None):
         cmd.Cmd.do_help(self, line)
     def do_quit(self):
         sys.exit()
     def do_EOF(self, *args):
         print ''
         sys.exit()
 if __name__ == '__main__':
     import sys
     usage = """
 %prog cmd
     format-all: format all gutenberg etexts automatically
     download: download the gutenberg etexts and store them in the cache
     """
     adminCmd = ShakespeareAdmin()
     if len(sys.argv) < 2:
         while 1:
             try:
                 adminCmd.cmdloop()
                 break
             except KeyboardInterrupt:
                 raise
     else:
         args = ' '.join(sys.argv[1:])
         args = args.replace('-','_')
         adminCmd.onecmd(args)

trunk/src/shakespeare/cherrypy_handler.py

 """
-Tutorial - Passing variables
+Web interface to view and analyze shakespeare texts.
+This tutorial shows you how to pass GET/POST variables to methods.
 """
 import cherrypy
 import os
-from shakespeare.download import make_index
+import shakespeare.work
-index = make_index()
+index = shakespeare.work.index.all
 from shakespeare.utils import get_local_path
 import shakespeare.format
 import shakespeare.concordancer
-concordancer = shakespeare.concordancer.get_concordancer()
+cc = shakespeare.concordancer.get_concordancer()
 class WelcomePage:
     def index(self):
         try:
             import kid
             kid.enable_import(suffixes=[".html"])
             import shakespeare.template.index
             template = shakespeare.template.index.Template(works_index=index)
             result = str(template)
             # result = 'test'
             return result
         except Exception, inst:
             return '<p><strong>There was an error: ' +  str(inst) + '</strong></p>'
     index.exposed = True
     def view(self, text_url=None, version='cleaned', format='plain'):
         localPath = get_local_path(text_url, version)
         ff = file(localPath)
         if format == 'plain':
             result = '<pre>' + ff.read() + '</pre>'
         else:
             formatter = shakespeare.format.TextFormatter(ff)
             result = formatter.format(format)
             # import kid
             # kid.enable_import(suffixes=['.html'])
             # module = __import__('shakespeare.template.format_' + format, '', '', '*')
             # template = module.Template(fileobj=ff)
             # result = template.serialize()
         ff.close()
         return result
     view.exposed = True
-#    def concordance(self):
+    def concordance(self):
-#        import kid
+        import kid
-#        kid.enable_import(suffixes=[".html"])
+        kid.enable_import(suffixes=[".html"])
-#        import shakespeare.template.concordance
+        import shakespeare.template.concordance
-#        template = shakespeare.template.concordance.Template(concordancer=concordancer)
+        concordance = cc.concordance
-#        result = template.serialize()
+        words = concordance.keys()
-#        return result
+        words.sort()
-#    concordance.exposed = True
+        template = shakespeare.template.concordance.Template(words=words, stats=cc.stats)
+        result = template.serialize()
+        # result = str(cc)
+        return result
+    concordance.exposed = True
 cherrypy.root = WelcomePage()
 if __name__ == '__main__':
     cherrypy.lowercase_api = True
     # cherrypy.config.update(file = 'tutorial.conf')
     cherrypy.config.update({'server.showTracebacks' : True })
     cherrypy.server.start()
 """
 [global]
 server.socketPort = 8080
 server.threadPool = 10
 server.environment = "production"
 # server.showTracebacks = True
 # server.logToScreen = False
 """

trunk/src/shakespeare/concordancer.py

 import re
 import cPickle
 import utils
-from download import make_index
+import shakespeare.work
-def make_concordancer(showProgress=True):
+def make_concordancer(
+        texts_to_add=shakespeare.work.index.all,
+        out_path=utils.get_local_path('concordance.p'),
+        ):
     """Create Concordancer object and use it to produce concordance and stats
     for all non-folio works.
-    Save resulting object in pickled form to 'concordance.p'.
+    @out_path: where to save the concordance
+    @texts_to_add: index items that should be added to the concordance
     """
-    def _print(msg):
-        if showProgress:
-            print(msg)
-    index = make_index()
     cc = Concordancer()
-    for item in index:
+    for item in texts_to_add:
         url = item[1]
         isfolio = item[2] == 'folio'
         src = utils.get_local_path(url, 'cleaned')
-        if isfolio:
+        cc.add_text(file(src))
-            _print('Is folio so skipping [%s]' % src)
+    ccFile = file(out_path, 'w')
+        else:
+            _print('Adding text [%s]' % src)
+            cc.add_text(file(src))
+    filePath = utils.get_local_path('concordance.p')
+    ccFile = file(filePath, 'w')
     cPickle.dump(cc, ccFile)
 def get_concordancer():
     """Get a concordancer containing concordance and stats by unpickling cached
     copy.
     """
     filePath = utils.get_local_path('concordance.p')
     cc = cPickle.load(file(filePath))
     return cc
 class Concordancer(object):
     """Generate a concordance and associated statistics for a set of texts.
     Concordance and statistics are provided as dictionaries keyed by words.
     NB: all word keys have been lower-cased in order to render them case-insensitive
     """
     # multiline, unicode and ignorecase
-    wordRegex = re.compile(r'\b(\w+)\b', re.U | re.M | re.I)
+    word_regex = re.compile(r'\b(\w+)\b', re.U | re.M | re.I)
+    words_to_ignore = [ 'a', 'the', 'and',
+                        'as', 'are', 'be',
+                        'but', 'd', 'in'
+                        ]
     def __init__(self):
         self.concordance = {}
         self.stats = {}
     def add_text(self, text, textId=None):
         """Add a text to the concordance.
         @text: file like object containing text to add
         """
         lineCount = 0
         charIndex = 0
         for line in text.readlines():
-            for match in self.wordRegex.finditer(line):
+            for match in self.word_regex.finditer(line):
                 word = match.group().lower() # case insensitive
+                if word in self.words_to_ignore:
+                    continue
                 oldValue = self.concordance.get(word, [])
                 oldStat = self.stats.get(word, 0)
-                oldValue.append( (lineCount, charIndex + match.start()) )
+                tloc = (textId, lineCount, charIndex + match.start())
+                oldValue.append(tloc)
                 self.concordance[word] = oldValue
                 self.stats[word] = oldStat + 1
             lineCount += 1
             charIndex += len(line)

trunk/src/shakespeare/concordancer_test.py

 import unittest
 import StringIO
 import concordancer
 def test_suite():
     suites = [
         unittest.makeSuite(ConcordancerTest),
+        ]
     return unittest.TestSuite(suites)
 class ConcordancerTest(unittest.TestCase):
     inText = \
 """A fake fake line
 SUFFOLK.
 As by your high imperial Majesty
 I had in charge at my depart for France,
 As procurator to your excellence,
 """
+    textId = 1
     # ['work_id', 'line-no', 'character-index'] }
     expConcordance = {
-        'fake' : [ (0, 2), (0, 7) ],
+        'fake' : [ (textId, 0, 2), (textId, 0, 7) ],
-        'suffolk' : [ (1, 17), ],
+        'suffolk' : [ (textId, 1, 17), ],
-        'high' : [ (2, 37), ],
+        'high' : [ (textId, 2, 37), ],
+        }
     expStats = {
         'fake' : 2,
         'suffolk' : 1,
         'high' : 1,
+        }
     def setUp(self):
         self.cc = concordancer.Concordancer()
-        self.cc.add_text(StringIO.StringIO(self.inText), 'King Henry VI')
+        self.cc.add_text(StringIO.StringIO(self.inText), self.textId)
     def test__process_line(self):
         line = 'the - quick, brown. fox-jumped over$ the_lazy do8g.'
         exp = ['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the_lazy', 'do8g' ]
-        out = self.cc.wordRegex.findall(line)
+        out = self.cc.word_regex.findall(line)
         self.assertEqual(exp, out)
     def test_concordance(self):
         for key, value in self.expConcordance.items():
             out = self.cc.concordance[key]
             self.assertEqual(out, value)
     def test_stats(self):
         for key, value in self.expStats.items():
             out = self.cc.stats[key]
             self.assertEqual(out, value)
+    def test_make_concordancer(self):
+        import tempfile
+        filePath = tempfile.mkstemp()[1]
+        import shakespeare.work
+        index = shakespeare.work.index.all
+        concordancer.make_concordancer(index[2:3], filePath)

trunk/src/shakespeare/format.py

 """
 Clean up Gutenberg texts by removing all the header and footer bumpf
 """
 import re
-import download
 headerEndPhrases = ["Project Gutenberg's Etext of", 'This etext was prepared by']
 notesStartPhrases = ["Executive Director's Notes:"]
 notesEndPhrases = ['David Reed']
 footerStartPhrases = ['End of Project Gutenberg', 'End of The Project Gutenberg'
+    ]
 def make_re_from_phrase(phrase):
     """
     Make a regular expression that matches a phrase and its surrounding
     paragraph, i.e. that look like:
     ... phrase ....
     more text
     [blank]
     [blank]+
     """
     paragraphText = '(.+\S.+\n)*' # need \S to ensure not just whitespace
     # [[TODO: check slowdown due to inclusion of '^.*' at start
     tmp = '^.*' + phrase + '.*\n' + paragraphText + '\s+'
     return re.compile(tmp, re.I | re.M)  # make it case insensitive
 class GutenbergShakespeare(object):
     """
     Process Gutenberg shakespeare texts
     """
     def __init__(self, etext):
         """
         @param etext: file like object containing the etext
         Procedure:
 . strip out header and footer bumpf
 . are there notes? If so strip them out
         """
         self.etext = etext
         self.etextStr = self.etext.read()
         # normalize the line endings to save us grief later
         self.etextStr = self.etextStr.replace('\r\n', '\n')
         self.hasNotes = False
     def _find_max(self, phrase, string):
         maxIndex = 0
         regex = make_re_from_phrase(phrase)
         matches = regex.finditer(string)
         for match in matches:
             maxIndex = max(match.end(), maxIndex)
         return maxIndex
     def _find_min(self, phrase, string):
         minIndex = len(string)
         regex = make_re_from_phrase(phrase)
         matches = regex.finditer(string)
         for match in matches:
             minIndex = min(match.start(), minIndex)
         return minIndex
     def extract_text(self):
         """Extract the core text.
         """
         self.notesEnd = self.get_notes_end()
         self.headerEnd = self.get_header_end()
         self.footerStart = self.get_footer_start()
         startIndex = self.headerEnd
         if self.notesEnd > 0:
             startIndex = self.notesEnd
         return self.etextStr[startIndex : self.footerStart].rstrip()
     def get_notes_end(self):
         "Return 0 if no notes"
         indices = [ self._find_max(phrase, self.etextStr) for phrase in notesEndPhrases]
         index = max(indices)
         return index
     def get_header_end(self):
         indices = [ self._find_max(phrase, self.etextStr) for phrase in headerEndPhrases]
         return max(indices)
     def get_footer_start(self):
         indices = [ self._find_min(phrase, self.etextStr) for phrase in footerStartPhrases]
         return min(indices)
 class TextFormatter(object):
     """Format a provided text in a variety of ways.
     For example: add line numbers, convert to html with line ids etc
     """
     def __init__(self, file):
         """
         @file: file-like object containing a text in plain txt
         """
         self.file = file
     def format(self, format):
         """
         @format: the name specifying the format to use
         """
         if format == 'lineno':
             return self.add_line_numbers()
         else:
             raise ValueError('Unknown format: %s' % format)
     def add_line_numbers(self):
         result = ''
         count = 0
         for line in self.file.readlines():
             tlineno = str(count).ljust(4) # assume line no < 10000
             result += '<pre id="%s">%s %s</pre>\n' % (count, tlineno, line.rstrip())
             count += 1
         return result

trunk/src/shakespeare/format_test.py

 import unittest
-import download
+import utils
 from format import make_re_from_phrase, GutenbergShakespeare
 def test_suite():
     suites = [
         unittest.makeSuite(FormatTest),
         unittest.makeSuite(GutenbergShakespeareTest),
+        ]
     return unittest.TestSuite(suites)
 class FormatTest(unittest.TestCase):
     def test_make_re_from_phrase(self):
         outStr = """blah
             """
         inStr = outStr + 'All is Well that'
         regex = make_re_from_phrase('blah')
         out = regex.search(inStr)
         self.assertEquals(out.group(), outStr)
     def test_makeReFromPhrase2(self):
         outStr = """blah
             joe
             hello
             """
         inStr = outStr + 'All is Well that'
         regex = make_re_from_phrase('blah')
         out = regex.search(inStr)
         self.assertEquals(out.group(), outStr)
 class GutenbergShakespeareTest(unittest.TestCase):
-    etext1 = file(download.get_cache_path('0ws2510.txt'))
+    etext1 = file(utils.get_cache_path('0ws2510.txt'))
-    etext2 = file(download.get_cache_path('2ws2510.txt'))
+    etext2 = file(utils.get_cache_path('2ws2510.txt'))
     gut1 = GutenbergShakespeare(etext1)
     gut2 = GutenbergShakespeare(etext2)
     def test_get_header_end(self):
         out = self.gut1.get_header_end()
         exp = self.gut1.etextStr.index("Executive Director's Notes:")
         self.assertEqual(out, exp)
     def test_get_footer_start(self):
         out = self.gut1.get_footer_start()
         # has no footer
         exp = len(self.gut1.etextStr)
         self.assertEqual(out, exp)
         out = self.gut2.get_footer_start()
         exp = self.gut2.etextStr.index("End of Project Gutenberg Etext of As You Like It by Shakespeare")
         self.assertEqual(out, exp)
     def test_get_notes_end(self):
         out = self.gut1.get_notes_end()
         exp = self.gut1.etextStr.index("As you Like it\n\nActus")
         self.assertEqual(out, exp)
     def test_extract_text(self):
         # [[TODO: run this test on all of the etexts]]
         for gut in [self.gut1, self.gut2]:
             out = gut.extract_text()
             notFound = (out.find('Gutenberg') == -1)
             self.failUnless(notFound)

trunk/src/shakespeare/utils.py

 import os
 import urllib
 import conf
 def get_local_path(remoteUrl, version=''):
     """Get local path to text of remote url.
     @type: string giving version of text (''|'cleaned')
     """
     host,path = urllib.splithost(remoteUrl)
     name = os.path.basename(path)
     name = version + name
     localPath = get_cache_path(name)
     return localPath
 def download_url(url):
     localPath = get_local_path(url)
     urllib.urlretrieve(url, localPath)
 def get_cache_path(offset):
     "Get full path of file in cache given by offset."
     return os.path.join(conf.CACHEDIR, offset)
+def download_gutenberg_index():
+    "Download the Gutenberg Index file GUTINDEX.ALL."
+    utils.download_url(conf.GUTINDEX)

trunk/src/shakespeare/work.py

 import os
-import urllib
-from utils import *
+import utils
 import conf
-def download_gutenberg_index():
-    "Download the Gutenberg Index file GUTINDEX.ALL."
-    download_url(conf.GUTINDEX)
-def download_all_shakespeare():
+class GutenbergIndex(object):
-    """Download from Project Gutenberg all the shakespeare texts listed in
+    """Parse the index of Gutenberg works so as to find Shakespeare works.
+    the index.
     """
     index = make_index()
-    for item in index:
+    def make_url(self, year, idStr):
-        download_url(item[1])
+        return 'http://www.gutenberg.org/dirs/etext%s/%s10.txt' % (year[2:], idStr)
-def make_url(year, idStr):
+    def get_shakespeare_list(self):
-    return 'http://www.gutenberg.org/dirs/etext%s/%s10.txt' % (year[2:], idStr)
+        """Get list of shakespeare works and urls.
+        Results are sorted by work title.
+        """
+        # results have format [ title, url, comments ]
+        # folio in comments indicates it is a first folio
+        results = [ ["Sonnets", 'http://www.gutenberg.org/dirs/etext97/wssnt10.txt', ''] ]
+        plays = self._extract_shakespeare_works()
+        for play in plays:
+            url = self.make_url(play[1], play[2])
+            results.append([play[0], url, play[3]])
+        def compare_list(item1, item2):
+            if item1[0] > item2[0]: return 1
+            else: return -1
+        results.sort(compare_list)
+        return results
+    def _extract_shakespeare_works(self):
+        """Get non-copyrighted Shakespeare works from Gutenberg
+        Results consist of folio and one other 'standard' version.
+        @return: list consisting of tuples in form [title, year, id, comment]
+        """
+        ff = file(utils.get_cache_path('GUTINDEX.ALL'))
+        results = []
+        for line in ff.readlines():
+            result = self.parse_line_for_folio(line)
+            if result:
+                results.append(result + ['folio'])
+            resultNormal = self.parse_line_for_normal(line)
+            if resultNormal:
+                results.append(resultNormal + [''])
+        return results
+    def parse_line_for_normal(self, line):
+        "Parse GUTINDEX line for the 'normal' gutenberg shakespeare versions (i.e. not folio and out of copyright)."
+        if 'by William Shakespeare' in line and '[2' in line:
+            year = line[4:8]
+            tmp = line[9:]
+            endOfTitle = tmp.find(', by')
+            title = tmp[:endOfTitle]
+            startOfId = tmp.find('[2')
+            endOfId = tmp.find(']', startOfId)
+            idStr = tmp[startOfId+1:endOfId]
+            xstart = idStr.find('x')
+            idStr = idStr[:xstart]
+            return [title, year, idStr]
+&