Navigation

← Previous Changeset
Next Changeset →

Changeset 103

Timestamp:

06/12/05 17:39:59 (4 years ago)

Author:

zool

Message:

some older local tweaks to bbox - largely documentation fixes - plus the getitems interface

Files:

bbox/bbox/__init__.py (modified) (1 diff)
bbox/bbox/config.py (modified) (1 diff)
bbox/bbox/spatialStore.py (modified) (1 diff)
bbox/bbox/ui/__init__.py (modified) (1 diff)
bbox/bbox/ui/getitems.py (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

bbox/bbox/init.py

 # bbox - an RSS / RDF aggregator
 # Jo Walsh - Dec 2004 - Mar 2005
 # This code owes heavily to the approach and source in Edd Dumbill's
 # IBM Developerworks article on aggregating RSS with contexts:
 # http://www-106.ibm.com/developerworks/xml/library/x-rdfprov.html
 # It uses Mark Pilgrim's feedparser, at http://feedparser.org/
 # This software has 2000 tests. The code is included in this package.
 # It uses the 'rdfobj', an object interface to the python interface
 # to the redland rdf toolkit. This is also included.
 # redland is at http://www.redland.opensource.ac.uk/
 import feedparser
 import time, datetime
 import rdfobj
 import RDF
 from politehttp import polite_request
 import bbox.config
 import os
+from warnings import warn
 os.chdir(bbox.config.store)
 class BBox:
     def __init__(self,spatialStore=None,verbose=None,always_visit=None):
         """We initialise a bbox by passing a database to it. If wishing to use an optional spatial index, a spatialStore object must be supplied. setting verbose to a true value turns on BBox's stream of consciousness."""
         self.spatialStore = spatialStore
         self._verbose = verbose
         self._visit_true = always_visit
         self.model = rdfobj.Model(bbox.config.db,db='hash')
         self.model.load(bbox.config.boot)
         from rdfobj import fbox
         fbox = rdfobj.fbox
         counter = self.model.fetch(fbox.Visit_Count)
         c = counter[fbox.count]
         if c is None: c = 0
         c = int(str(c))+1
         counter[fbox.count] = str(c)
         if counter is None:
             v = self.model.create(fbox.Visit_Count, uri=fbox.Visit_Count)
             v[fbox.count] = 0
     def mention(self,thought):
         """If BBox is constructed with verbose=1, prints to STDOUT (currently) a record of what it's up to."""
         if self._verbose:
             print(thought)
     def read_subscriptions(self):
         """read_subscriptions() picks up the latest RSS feed updates. """
         self.mention("checking subscriptions.")
         subs = self.subscriptions()
         from rdfobj import fbox
         fbox = rdfobj.fbox
         for s in subs:
             self.mention("reading "+str(s[fbox.channel]))
             format = s[fbox.format].uri()
             c = s[fbox.channel]
             # see if we're actually due a visit
             due = self.visit_scheduled(s)
             if due is None:
                 print "nothing due to look at!"
                 subs.next()
             else:
                 if format == fbox.rss:
                     self.read_rss(c.uri(),subscription=s)
                 elif format == fbox.rdf:
                     self.read_rdf(s[fbox.channel].uri(),subscription=s)
     def read_rss(self,uri,context=None,subscription=None):
+        """Read updates from an RSS feed."""
         #if subscription is None: subscription = {}
         rss = rdfobj.rss
         dc = rdfobj.dc
         ical = rdfobj.ical
         result = self.politely_get_uri(uri,subscription=subscription)
         channel = self.model.fetch(uri)
         """If we got a feed object back from the request, then create a
         context for this visit to the feed, and store the entries that we
         collected from it."""
         if result['status'] != 200: return
         feed = feedparser.parse(result['data'])
         if feed.has_key('feed'):
             context = self.visit(uri)
             # existence of exact duplicates?
             items = []
             for e in feed.entries:
                 item = self.model.create( rss.item, uri=str(e.link), context = context )
                 if e.has_key('summary'): item[rss.description] = str(e.summary)
                 if e.has_key('content'): item[rss.description] = str(e.content.value)
                 # d.entries[0].modified_parsed is common
                 time_tuple = e.modified_parsed
                 # future experiments with pdis and better ical handling
                 # item[ical.datetime] = some process with time_tuple and strftime
                 # d = datetime.datetime(time_tuple)
                 # ical_date = ical_datetime.datetime_to_string(d)
                 # print ical_date
                 # item[ical.datetime] = ical_date
                 ical_enough = time.strftime("%Y%m%dT%H%M%SZ",time_tuple)
                 item[ical.datetime] = ical_enough
                 items.append(item)
-            for i in channel[rss.items]:
+            its = channel[rss.items]
+            for i in its:
                 items.append(i)
             channel[rss.items] = items
     def read_rdf(self,uri,subscription=None):
+        """Read updates from an RDF url."""
         result = self.politely_get_uri(uri,subscription=subscription)
         if result['status'] == 200:
             self.model.load(uri)
     def politely_get_uri(self,uri,subscription=None):
+        """Request a copy of the document at a url, first checking that it has changed since what we record as last-modified and the last etag that we have for it."""
+        """Parse RDF feeds directly into Redland"""
         # we should deal with etag/last-mod politely here too @@TODO
         #visit = self.visit(uri)
         result = None
         fbox = rdfobj.fbox
         if subscription[fbox.last_etag] is not None:
             result = polite_request(str(uri),etag=str(subscription[fbox.last_etag]))
         elif subscription[fbox.last_modified] is not None:
             result = polite_request(str(uri),lastmodified=str(subscription[fbox.last_modified]))
         else: result = polite_request(str(uri))
         self.mention("received response: "+str(result['status']))
         # look to the postgis index
         """Take actions about other kinds of HTTP statuses.(TODO)"""
         # handling different HTTP statuses.
         subscription[fbox.http_status] = str(result['status'])
         subscription[fbox.last_etag] = result['etag']
         subscription[fbox.last_modified] = result['lastmodified']
         subscription[fbox.last_visited] = time.strftime("%Y%m%dT%H%M%SZ")
         return result
     def subscriptions(self):
-        """subscriptions() returns a list (Iterator type) of the URLs at which
+        """Returns a list (Iterator type) of the URLs at which
         there is a feed that we are subscribed to (fbox:Feed type)"""
         from rdfobj import fbox, rdf
         fbox = rdfobj.fbox
         rdf = rdfobj.rdf
         subs = self.model.search(rdf.type,fbox.Feed)
         return subs
     def subscription(self,uri):
+        """Given a uri, returns the rdfobj which is the subscription it represents."""
         obj = self.model.fetch(uri)
         return obj
-    def items(self,uri):
+    def items(self,uri,since=None,until=None):
-        from rdfobj import fbox
+        """Get items from a feed, optionally filtering by date. (not completely implemented)"""
+        from rdfobj import fbox, dc, rss
+        rss = fbox.rss
+        dc = fbox.dc
         fbox = rdfobj.fbox
         s = self.subscription(uri)
         c = s[fbox.channel]
+        out = []
+        if since is not None:
+            for i in c[rss.items]:
+                warn(i[dc.date])
+                if i[dc.date] > since:
+                    out.append(i)
         return c[rss.items]
     def subscribe(self,feed=None,format=None,interval=None):
         """subscribe() creates a subscription to a uri. format is either 'rss' or 'rdf'. RDF is assumed if none is specified. Interval is the maximum interval in minutes that a feed should be checked at. It sends polite HTTP requests so don't worry about setting it to a bit more often than you might need. A value in minutes - defaults to 100 minutes."""
         from rdfobj import fbox
         fbox = rdfobj.fbox
         if feed is None: return
         f = self.model.search(fbox.channel,feed)
         found = None
         for n in f:
             found = 1
         if found is not None:
             return
         self.mention("subscribing to "+str(feed))
         if format is None: format = fbox.rdf
         if interval is None: interval = str(100)
         ff = self.model.create( fbox.Feed, uri=None )
         ff[fbox.channel] = str(feed)
         ff[fbox.format] = str(format)
         ff[fbox.interval] = interval
         return ff
     def update(self):
-        """update() causes all the subscribed URLs to be visited for updates."""
+        """Causes all the subscribed URLs to be visited for updates."""
         subs = self.subscriptions()
         from rdfobj import fbox
         fbox = rdfobj.fbox
         while not subs.end():
             s = subs.current()
             self.visit(s[fbox.channel])
             subs.next()
     def visit(self,uri=None):
-        """visit() creates an anonymous object which records a visit that we
+        """Creates an anonymous object which records a visit that we
         paid to a feed, including a counter of times visited. This object is
         used as a Redland context for all the information collected from a feed
         during this visit."""
         # redland had problems serialising models with bnode context uris
         count = self.counter()
         from rdfobj import fbox
         fbox = rdfobj.fbox
         visit_uri = str(fbox.visit)+'/'+str(count)
         visit = self.model.create( fbox.Visit , visit_uri)
         visit[fbox.source] = uri
         t = time.strftime("%Y%m%dT%H%M%SZ")
         visit[fbox.timestamp] = t
         return RDF.Node(RDF.Uri(str(visit.uri())))
     def user(self,token=None,nick=None,mbox=None):
         """Passed either a user's login token, mbox and name, resolved to mutual exclusion in that order, and returns any corresponding user / foaf:Person object. No security - handle this yourself elsewhere!"""
         from rdfobj import foaf
         foaf = rdfobj.foaf
         if token is not None:
             users = self.model.search(foaf.auth_token,token)
             for u in users:
                 return u[foaf.alias]
         if mbox is not None:
             users = self.model.search(foaf.mbox,mbox)
             for u in users:
                 return u
         if nick is not None:
             o = []
             users = self.model.search(foaf.name,nick)
             for u in users:
                 o.append(u)
             users = self.model.search(foaf.givenName,nick)
             for u in users: o.append(u)
             return o
     def add_user(self,nick=None,mbox=None,password=None):
         """ Create a new user foaf:Person"""
         store = self.store
         from rdfobj import foaf, wlan
         foaf = rdfobj.foaf
         wlan = rdfobj.wlan
         obj = self.model.create(foaf.Person,uri=uri)
         obj[foaf.mbox] = mbox
         obj[foaf.nick] = nick
         if page is not None: obj[foaf.homepage] = page
         self.obj = obj
         """ Create a kind of shadow user where we store the password and the logged-in token, so they won't get serialised accidentally along with the user. """
         auth = store.create(foaf.AuthedPerson)
         auth[foaf.password] = password
         auth[foaf.nick] = nick
         auth[foaf.alias] = obj
         token = self.auth_token()
         auth[foaf.auth_token] = token
         self.model.sync()
         return token
     def auth_token(self):
+        """Generate a random auth token."""
         x = ''
         for  n in range(0, 6):
             x = x + chr(65 + random.randint(0, 26))
         return x
     def visit_scheduled(self,sub):
         """Compare the last visited time, if that's applicable, to the interval between events (rather than a schedule? perhaps we'll have to re-think this later."""
         if self._visit_true is not None:
             return 1
         from rdfobj import fbox
         fbox = rdfobj.fbox
         last = sub[fbox.last_visited]
         if last is None:
             return 1
         t = time.time()
         # convert last time simply from ical to epoch?
         return 1
         since = t - float(str(last))
         interval = sub[fbox.interval]
         if interval is None:
             sub[fbox.interval] = str(100)
             interval = sub[fbox.interval]
         secs = int(str(interval))*60
         if since >= secs:
             return 1
         return None
     def counter(self):
+        """Update the counter that's used to generate visit context URIs."""
         from rdfobj import fbox
         fbox = rdfobj.fbox
         counter = self.model.fetch(fbox.Visit_Count)
         c = counter[fbox.count]
         c = int(str(c))+1
         counter[fbox.count] = str(c)
         return c
 if __name__ == "__main__":
     bbox = BBox(visit_true = 1)
     bbox.subscribe(feed='http://frot.org/wirelesslondon/bbox.rdf',format=fbox.rss)
     bbox.subscribe(feed='http://frot.org/devlog/index.rss',format=fbox.rss)
     bbox.subscribe(feed='http://zooleika.org.uk/bio/foaf.rdf',format=fbox.rdf)
     bbox.read_subscriptions()

bbox/bbox/config.py

-boot='file:/home/wirelesslondon/lib/consumotronic/bbox/boot.rdf'
+boot='file:/home/jo/consumotronic/bbox/boot.rdf'
-store = '/home/wirelesslondon/store/'
+store = '/home/jo/bbox/store/'
 db = 'bbox'
+spatialdb = 'bbox'
+utm_zone = '31'
+srid = '32631'

bbox/bbox/spatialStore.py

-"""provides the unified data store between redland RDF and postGIS index"""
+"""provides the unified data store between redland RDF and spatial (for now PostGIS) index"""
 import pgdb
 import re
 import warnings
 import bbox.config
 _database = bbox.config._spatialdb
 _utm_zone = bbox.config._utm_zone
 _srid = bbox.config._srid
 class SpatialStore:
     def __init__(self,vars = None,database=None):
+        """Create a new store interface to a certain database name, passed in here or defaulting to what's set in bbox.config._spatialdb"""
         self.vars = vars
         if database is None:
             database = _database
         db = pgdb.connect(database=database)
         self.db = db
     def geom(self,uri):
-        """We are passed the uri of something in the DB"""
+        """Passed the uri of something in the DB, returns geometry for it as a dict; if the geom is a point, {'x':x,'y':y}; if the geom is a line, x1,y1,x2,y2; polygon geom isn't really supported yet."""
         # again we have an issue of what kind of geom this is
         select = "select GeometryType(geom) from nodes where node='"+uri+"'"
         db = self.db.cursor()
         db.execute(select)
         g = db.fetchone()
         geom = {}
         if g[0] == 'LINESTRING':
             select = "SELECT name, type, X(StartPoint(geom)) as x, Y(StartPoint(geom)), GeometryFromText(geom) as y, X(EndPoint(geom)), Y(EndPoint(geom)) from nodes where node='"+uri+"'"
             db.execute(select)
             c = db.fetchone()
             geom['x1'] = c[2]
             geom['x2'] = c[4]
             geom['y1'] = c[3]
             geom['y2'] = c[5]
         elif g[0] == 'POINT':
             select = "SELECT name,type,X(geom),Y(geom) from nodes where node='"+uri+"'"
             db.execute(select)
             c = db.fetchone()
             geom['x'] = c[2]
             geom['y'] = c[3]
         elif g[0] == 'POLYGON':
+            #@@TODO support this properly, as vectors, whatever
             select = 'SELECT GeometryAsText(geom) where node = \''+uri+"'"
         return geom
     def add_geom(self,uri=None,name=None,type=None,rdf_type=None,status=None,owner=None,x=None,y=None,z=None,date=None):
         """ Add geometry for a Uri into a spatial index. needs uri and also name=[name] must be supplied."""
         if uri is None: return
         if type is None: type = ''
         if rdf_type is None:
             rdf_type = 'http://www.w3.org/2003/01/geo/wgs84_pos#SpatialThing'
         if status is None: status = ''
         if owner is None: owner = 'wl@frot.org'
         if date is None:
             date = 'now()'
         else:
             date = "'"+str(date)+"'"
         geom = "GeometryFromText('POINT("+str(x)+' '+str(y)+")',"+str(_srid)+")"
         #insert = "INSERT INTO nodes (node,geom,name,status,rdf_type,type,created) values ('"+str(uri)+"',"+geom+",'"+str(name)+"','"+str(status)+"','"+str(rdf_type)+"','"+str(type)+"',"+date+')'
         #print insert
         db = self.db
         db.cursor().execute("INSERT INTO nodes (node,geom,name,status,rdf_type,type,created) values (%s,"+geom+",%s,%s,%s,%s,"+date+")", (str(uri),str(name),str(status),str(rdf_type),str(type)))
         db.commit()
         #print "new geometry for "+str(uri)
     def find_near(self,node=None, lat=None,lon=None,x=None,y=None,r=None,type=None,terse=None):
         if lat is not None:
             thing = wl.spatialThing.SpatialThing(spatialStore = self)
             (x,y) = thing.latlon_to_utm(lat=lat,lon=lon)
         if r is None:
             r = 1600
         found = self.within_box(minx=x-r,miny=y-r,maxx=x+r,maxy=y+r,rdf_type=type,terse=terse)
         return found
     def within_box(self,minx=None,miny=None,maxx=None,maxy=None,type=None,rdf_type=None,terse=None):
         """Accepts a minx,miny,maxx,maxy bounding box; should return a list of spatialThings, unless you specify terse=1, in whih case you just get a array of dicts built from the postgis db contents."""
         select = "SELECT node, name, rdf_type, X(geom) as x, Y(geom) as y FROM nodes where Within(geom,GeometryFromText('POLYGON(("+str(minx)+' '+str(miny)+','+str(minx)+' '+str(maxy)+','+str(maxx)+' '+str(maxy)+','+str(maxx)+' '+str(miny)+','+str(minx)+' '+str(miny)+"))',"+str(_srid)+"))"
         warnings.warn( select )
         """Also optionally filter by type"""
         if type is not None:
             select = select+' AND type=\''+str(type)+'\''
         if rdf_type is not None:
             select = select+' AND rdf_type=\''+str(rdf_type)+'\''
         db = self.db
         c = db.cursor()
         c.execute(select)
         geoms = c.fetchall()
         things = []
         if terse is not None:
             for g in geoms:
                 things.append({'uri':g[0],'name':g[1],'type':g[2],'x':g[3],'y':g[4]})
         else:
             for g in geoms:
                 things.append(g[0])
         """Return a rdfobj node/object for each geometry."""
         return things
     def recent(self,days=None,type=None):
         if days is None: days = 30
         if type is None: type = 'http://xmlns.com/2003/wireless/Node'
         query = "SELECT node from nodes where (now() - interval '"+str(days)+" days') <= created and rdf_type = '"+type+"'"
         db = self.db
         c = db.cursor()
         c.execute(query)
         nodes = c.fetchall()
         urls = []
         for n in nodes:
             urls.append(n[0])
         return urls
     def update_geom(self,x=None,y=None,uri=None):
         if x is None: return
         if uri is None: return
         db = self.db
         c = db.cursor()
         c.execute("UPDATE nodes SET geom = GeometryFromText('POINT("+str(x)+' '+str(y)+")',"+str(_srid)+") where node ='%s'",(str(uri)))
         db.commit()

bbox/bbox/ui/init.py

 from Cheetah.Template import Template
 from bbox import BBox
 #from spatialStore import Store
 import quixote
 import os
 import RDF
 import rdfobj
-_q_exports = ['_q_index','subs','_q_lookup','getitems']
+_q_exports = ['_q_index','subs','_q_lookup','getitems','items']
 def _q_index(r):
     return 'bbox'
 def subs(r):
     """List current subscriptions."""
     bb = BBoxUI()
     return bb.subs(r)
 def read_subs(r):
     """Trigger a read of updates to subscriptions, if found."""
     bb = BBoxUI()
     return bb.read_subs(r)
 def _q_lookup(r,n):
     pass
 #def _q_access(request):
 #    raise UnauthorizedError(realm='Foo Realm')
 class BBoxUI:
     def __init__(self):
         self.bbox = BBox('bbox.db')
-    def subscribe(self,r):
+    def subscribe(self,r,uri=None):
+        """Add a subscription to a uri, either as a query param or passed in as 'uri'."""
         uri = r.get_form_var('uri')
         self.bbox.subscribe(uri=uri)
         context = self.bbox.visit(uri)
         self.bbox.read(uri,context=context)
     def read_subs(self,r):
         self.bbox.read_subscriptions()
     def subs(self,r):
         subs = self.bbox.subscriptions()
         xml = self.serialise(subs)
         return xml
     def getitems(self,r):
         """This is the bloglines-style interface"""
         """See http://www.bloglines.com/services/api/getitems """
         read = r.get_form_var('n')
         """ 'n' parameter - 1 marks downloaded items as read, 0 does not."""
         date = r.get_form_var('d')
         """ 'd' parameter - date to get new entries since. """
         s = r.get_form_var('s')
         """ 'subid'. we will accept an integer id which is a unique key in the node table, but this parameter is optional instead of compulsory; you can otherwise supply a URI-encoded url..."""
         #sub = bbox.subscription(s)
-        items = bbox.items(s)
+        items = self.bbox.items(s)
         out = ''
         for i in items:
             out = out + str(i) + "\n"
         if d:
             pass
         uri = r.get_form_var('uri')
         if uri is None: uri = r.get_form_var('u')
     """HTTP Status Response
     * 200 - normal
     * 304 - the request produced no entries (either there were no unread entries, or no entries after the given date, depending on the parameters passed in)
     * 401 - incorrect email address or password
     * 403 - invalid or missing BloglinesSubId
     * 410 - subscription has been deleted
     """
     def serialise(self,objs):
         """Serialises a list of rdf objects, supplied, as RDF/XML."""
         from rdfobj import dc
         dc = rdfobj.dc
         out = ''
         # proper serialisation
         tmpstore = RDF.MemoryStorage()
         tmpmodel = RDF.Model(tmpstore)
         for o in objs:

Navigation

Changeset 103

Legend:

bbox/bbox/__init__.py

bbox/bbox/config.py

bbox/bbox/spatialStore.py

bbox/bbox/ui/__init__.py

bbox/bbox/init.py

bbox/bbox/ui/init.py