Navigation

← Previous Changeset
Next Changeset →

Changeset 254

Timestamp:

08/25/05 02:12:20 (3 years ago)

Author:

zool

Message:

optionally pass in a model

Files:

bbox/bbox/__init__.py (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

bbox/bbox/init.py

 # bbox - an RSS / RDF aggregator
 # Jo Walsh - Dec 2004 - Mar 2005
 # This code owes heavily to the approach and source in Edd Dumbill's
 # IBM Developerworks article on aggregating RSS with contexts:
 # http://www-106.ibm.com/developerworks/xml/library/x-rdfprov.html
 # It uses Mark Pilgrim's feedparser, at http://feedparser.org/
 # This software has 2000 tests. The code is included in this package.
 # It uses the 'rdfobj', an object interface to the python interface
 # to the redland rdf toolkit. This is also included.
 # redland is at http://www.redland.opensource.ac.uk/
 import feedparser
 import time, datetime
 import rdfobj
 import RDF
 from bbox.politehttp import polite_request
 import bbox.spatialStore
 import bbox.config
 import os
 from warnings import warn
 class BBox:
-    def __init__(self,spatial=None,verbose=None,always_visit=None,db=None):
+    def __init__(self,spatial=None,verbose=None,always_visit=None,db=None,model=None):
         """We initialise a bbox by passing a database to it. If wishing to use an optional spatial index, set spatial = [name of index or database]. setting verbose to a true value turns on BBox's stream of consciousness."""
         if spatial is not None:
             self.spatialStore = bbox.spatialStore.SpatialStore(database=spatial)
         else:
             self.spatialStore = None
         self._verbose = verbose
         self._visit_true = always_visit
         os.chdir(bbox.config.store)
         if db is None:
             db = bbox.config.db
-        self.model = rdfobj.Model(db,db='hash')
+        if model is not None:
-        self.model.load(bbox.config.boot)
+            self.model = model
+        else:
+            self.model = rdfobj.Model(db,db='hash')
+            self.model.load(bbox.config.boot)
         from rdfobj import fbox
         fbox = rdfobj.fbox
         counter = self.model.fetch(fbox.Visit_Count)
         c = counter[fbox.count]
         if c is None: c = 0
         c = int(str(c))+1
         counter[fbox.count] = str(c)
         if counter is None:
             v = self.model.create(fbox.Visit_Count, uri=fbox.Visit_Count)
             v[fbox.count] = 0
     def mention(self,thought):
         """If BBox is constructed with verbose=1, prints to STDOUT (currently) a record of what it's up to."""
         if self._verbose:
             print(thought)
     def read_subscriptions(self):
         """read_subscriptions() picks up the latest RSS feed updates. """
         self.mention("checking subscriptions.")
         subs = self.subscriptions()
         from rdfobj import fbox
         fbox = rdfobj.fbox
         for s in subs:
             self.mention("reading "+str(s[fbox.channel]))
             format = s[fbox.format].uri()
             c = s[fbox.channel]
             # see if we're actually due a visit
             due = self.visit_scheduled(s)
             if due is None:
                 print "nothing due to look at!"
                 subs.next()
             else:
                 if format == fbox.rss:
                     self.read_rss(c.uri(),subscription=s)
                 elif format == fbox.rdf:
                     self.read_rdf(s[fbox.channel].uri(),subscription=s)
     def read_rss(self,uri,context=None,subscription=None,xml=None):
         """Read updates from an RSS feed."""
         #if subscription is None: subscription = {}
         rss = rdfobj.rss
         dc = rdfobj.dc
         ical = rdfobj.ical
         fbox = rdfobj.fbox
         geo = rdfobj.geo
         result = self.politely_get_uri(uri,subscription=subscription)
         channel = self.model.fetch(uri)
         """If we got a feed object back from the request, then create a
         context for this visit to the feed, and store the entries that we
         collected from it."""
         if self._visit_true:
             pass
         elif result['status'] != 200:
             return
         items = []
         feed = feedparser.parse(result['data'])
         if feed.has_key('feed'):
             context = self.visit(uri)
             # existence of exact duplicates?
             for e in feed.entries:
                 link = str(e.link)
                 title = None
                 item = self.model.create( rss.item, uri=link, context = context )
                 if e.has_key('summary'): item[rss.description] = str(e.summary)
                 if e.has_key('content'): item[rss.description] = str(e.content)
                 if e.has_key('title'):
                     item[rss.title] = str(e.title)
                     title = str(e.title)
                 item[fbox.channel] = channel
                 # d.entries[0].modified_parsed is common
                 time_tuple = None
                 if e.has_key('modified_parsed'):
                     time_tuple = e.modified_parsed
                 elif e.has_key('created_parsed'):
                     time_tuple = e.created_parsed
                 # item[ical.datetime] = some process with time_tuple and strftime
                 # d = datetime.datetime(time_tuple)
                 # ical_date = ical_datetime.datetime_to_string(d)
                 # print ical_date
                 # item[ical.datetime] = ical_date
                 # not much use without a timestamp
                 if time_tuple is None:
                     continue
                 ical_enough = time.strftime("%Y%m%dT%H%M%SZ",time_tuple)
                 item[ical.datetime] = ical_enough
                 rdf_type = None
                 if e.has_key('rdf_type'):
                     rdf_type = str(e['rdf_type'])
                     item[rdf.type] = rdf_type
                 if e.has_key('geo_lat'): print "LATT!!! "+str(e['geo_lat'])
                 if e.has_key('geo_lat') and e.has_key('geo_long'):
                     lat = str(e['geo_lat'])
                     long = str(e['geo_long'])
                     item[geo.lat] = lat
                     item[geo.long] = long
                     """Update the spatial index, if we have one."""
                     if self.spatialStore is not None:
                         self.spatialStore.add_or_update_geom(rdf_type=rdf_type,name=title,x=long,y=lat,uri=link)
                 items.append(item)
         return items
     def read_rdf(self,uri,subscription=None,xml=None):
         """Read updates from an RDF url."""
         from rdfobj import geo, dc, rdf
         geo = rdfobj.geo
         dc = rdfobj.dc
         rdf = rdfobj.rdf
         result = self.politely_get_uri(uri,subscription=subscription)
         if self._visit_true:
             pass
         elif result['status'] != 200:
             return
         context = self.visit(uri)
         # we can't just use load() because we want the visit context, and to search for spatial things and index them while we're parsing...
         lats = {}
         longs = {}
         titles = {}
         types = {}
         parser = RDF.Parser('raptor')
         stream = parser.parse_as_stream(RDF.Uri(uri))
         subjects = {}
         if stream:
             while not stream.end():
                 statement = stream.current()
                 subjects[statement.subject] = 1
                 self.model.model.add_statement(statement,context)
                 # pls don't blame me, i just want to get something working fast
                 if self.spatialStore is not None:
                     if statement.predicate == RDF.Node(uri_string=str(geo.lat)):
                         lats[str(statement.subject)] = str(statement.object)
                     elif statement.predicate == RDF.Node(uri_string=str(geo.long)):
                         longs[str(statement.subject)] = str(statement.object)
                     elif statement.predicate == RDF.Node(uri_string=str(dc.title)):
                         titles[str(statement.subject)] = str(statement.object)
                     elif statement.predicate == RDF.Node(uri_string=str(rdf.type)):
                         types[str(statement.subject)] = str(statement.object)
                 stream.next()
         objects = []
         for s in subjects.keys():
             objects.append(self.model.fetch(s))
         if self.spatialStore is not None:
             for k in lats.keys():
                 lat = lats[k]
                 long = longs[k]
                 title = None
                 type = None
                 if titles.has_key(k):
                     title = titles[k]
                 if types.has_key(k):
                     type = types[k]
                 warn("updating "+k)
                 self.spatialStore.add_or_update_geom(uri=k,rdf_type=type,name=title,x=long,y=lat)
         return objects
     def politely_get_uri(self,uri,subscription=None):
         """Request a copy of the document at a url, first checking that it has changed since what we record as last-modified and the last etag that we have for it."""
         # we should deal with etag/last-mod politely here too @@TODO
         #visit = self.visit(uri)
         result = None
         fbox = rdfobj.fbox
         if subscription is None:
             # we might just be using the parser without the context management
             result = polite_request(str(uri))
             subscription = {'fake':1}
         elif self._visit_true is not None:
             # we might always want to read the feed content (for debugging reasons)
             result = polite_request(str(uri))
         elif subscription[fbox.last_etag] is not None:
             result = polite_request(str(uri),etag=str(subscription[fbox.last_etag]))
         elif subscription[fbox.last_modified] is not None:
             result = polite_request(str(uri),last_modified=str(subscription[fbox.last_modified]))
         else: result = polite_request(str(uri))
         if result is None:
             result = {'status':404}
             return result
         if result.has_key('status'):
             # this was a HTTP request
             self.mention("received response: "+str(result['status']))
             """Take actions about other kinds of HTTP statuses.(TODO)"""
             # handling different HTTP statuses.
             if subscription.has_key('fake') is None:
                 warn(type(subscription))
                 subscription[fbox.http_status] = str(result['status'])
                 subscription[fbox.last_etag] = result['etag']
                 subscription[fbox.last_modified] = result['lastmodified']
                 subscription[fbox.last_visited] = time.strftime("%Y%m%dT%H%M%SZ")
         # a 'file:/' uri will only have result['data']
         elif result['data'] is not None:
             # pretend we have a positive HTTP status
             result['status'] = 200
         return result
     def subscriptions(self):
         """Returns a list (Iterator type) of the URLs at which
         there is a feed that we are subscribed to (fbox:Feed type)"""
         from rdfobj import fbox, rdf
         fbox = rdfobj.fbox
         rdf = rdfobj.rdf
         subs = self.model.search(rdf.type,fbox.Feed)
         return subs
     def subscription(self,uri):
         """Given a uri, returns the rdfobj which is the subscription it represents."""
         obj = self.model.fetch(uri)
         return obj
     def items(self,uri,since=None,until=None):
         """Get items from a feed, optionally filtering by date. (not completely implemented)"""
         from rdfobj import fbox, dc, rss
         rss = fbox.rss
         dc = fbox.dc
         fbox = rdfobj.fbox
         s = self.subscription(uri)
         c = s[fbox.channel]
         out = []
         if since is not None:
             for i in c[rss.items]:
                 warn(i[dc.date])
                 if i[dc.date] > since:
                     out.append(i)
         return c.rss_items
     def subscribe(self,feed=None,format=None,interval=None):
         """subscribe() creates a subscription to a uri. format is either 'rss' or 'rdf'. RDF is assumed if none is specified. Interval is the maximum interval in minutes that a feed should be checked at. It sends polite HTTP requests so don't worry about setting it to a bit more often than you might need. A value in minutes - defaults to 100 minutes."""
         from rdfobj import fbox
         fbox = rdfobj.fbox
         if feed is None: return
         f = self.model.search(fbox.channel,feed)
         found = None
         for n in f:
             found = 1
         if found is not None:
             return
         self.mention("subscribing to "+str(feed))
         if format is None:
             format = fbox.rdf
         elif format == 'rss':
             format = fbox.rss
         elif format == 'rdf':
             format = fbox.rdf
         if interval is None: interval = str(100)
         ff = self.model.create( fbox.Feed, uri=None )
         ff[fbox.channel] = str(feed)
         ff[fbox.format] = str(format)
         ff[fbox.interval] = interval
         return ff
     def update(self):
         """Causes all the subscribed URLs to be visited for updates."""
         subs = self.subscriptions()
         from rdfobj import fbox
         fbox = rdfobj.fbox
         while not subs.end():
             s = subs.current()
             self.visit(s[fbox.channel])
             subs.next()
     def visit(self,uri=None):
         """Creates an anonymous object which records a visit that we
         paid to a feed, including a counter of times visited. This object is
         used as a Redland context for all the information collected from a feed
         during this visit."""
         # redland had problems serialising models with bnode context uris
         count = self.counter()
         from rdfobj import fbox
         fbox = rdfobj.fbox
         visit_uri = str(fbox.visit)+'/'+str(count)
         visit = self.model.create( fbox.Visit , visit_uri)
         visit[fbox.source] = uri
         t = time.strftime("%Y%m%dT%H%M%SZ")
         visit[fbox.timestamp] = t
         return RDF.Node(RDF.Uri(str(visit.uri())))
     def user(self,token=None,nick=None,mbox=None):
         """Passed either a user's login token, mbox and name, resolved to mutual exclusion in that order, and returns any corresponding user / foaf:Person object. No security - handle this yourself elsewhere!"""
         from rdfobj import foaf
         foaf = rdfobj.foaf
         if token is not None:
             users = self.model.search(foaf.auth_token,token)
             for u in users:
                 return u[foaf.alias]
         if mbox is not None:
             users = self.model.search(foaf.mbox,mbox)
             for u in users:
                 return u
         if nick is not None:
             o = []
             users = self.model.search(foaf.name,nick)
             for u in users:
                 o.append(u)
             users = self.model.search(foaf.givenName,nick)
             for u in users: o.append(u)
             return o
     def add_user(self,nick=None,mbox=None,password=None):
         """ Create a new user foaf:Person"""
         store = self.store
         from rdfobj import foaf, wlan
         foaf = rdfobj.foaf
         wlan = rdfobj.wlan
         obj = self.model.create(foaf.Person,uri=uri)
         obj[foaf.mbox] = mbox
         obj[foaf.nick] = nick
         if page is not None: obj[foaf.homepage] = page
         self.obj = obj
         """ Create a kind of shadow user where we store the password and the logged-in token, so they won't get serialised accidentally along with the user. """
         auth = store.create(foaf.AuthedPerson)
         auth[foaf.password] = password
         auth[foaf.nick] = nick
         auth[foaf.alias] = obj
         token = self.auth_token()
         auth[foaf.auth_token] = token
         self.model.sync()
         return token
     def auth_token(self):
         """Generate a random auth token."""
         x = ''
         for  n in range(0, 6):
             x = x + chr(65 + random.randint(0, 26))
         return x
     def visit_scheduled(self,sub):
         """Compare the last visited time, if that's applicable, to the interval between events (rather than a schedule? perhaps we'll have to re-think this later."""
         if self._visit_true is not None:
             return 1
         from rdfobj import fbox
         fbox = rdfobj.fbox
         last = sub[fbox.last_visited]
         if last is None:
             return 1
         t = time.time()
         # convert last time simply from ical to epoch?
         return 1
         since = t - float(str(last))
         interval = sub[fbox.interval]
         if interval is None:
             sub[fbox.interval] = str(100)
             interval = sub[fbox.interval]
         secs = int(str(interval))*60
         if since >= secs:
             return 1
         return None
     def counter(self):
         """Update the counter that's used to generate visit context URIs."""
         from rdfobj import fbox
         fbox = rdfobj.fbox
         counter = self.model.fetch(fbox.Visit_Count)
         c = counter[fbox.count]
         c = int(str(c))+1
         counter[fbox.count] = str(c)
         return c
 if __name__ == "__main__":
     bbox = BBox(visit_true = 1)
     bbox.subscribe(feed='http://frot.org/wirelesslondon/bbox.rdf',format=fbox.rss)
     bbox.subscribe(feed='http://frot.org/devlog/index.rss',format=fbox.rss)
     bbox.subscribe(feed='http://zooleika.org.uk/bio/foaf.rdf',format=fbox.rdf)
     bbox.read_subscriptions()

Navigation

Changeset 254

Legend:

bbox/bbox/__init__.py

Download in other formats:

bbox/bbox/init.py