adrianshort
/
planningalerts
mirror of https://github.com/adrianshort/planningalerts.git
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
Browse Source

Removing and regenerating CGI directory

master
duncan.parkes 17 years ago
parent
4a61d9b125
commit
a1ab19b3e7
7 changed files with 32 additions and 733 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +31
    -0
      CGI/Oswestry.cgi
  2. +0
    -101
      CGI/PlanningUtils.py
  3. +0
    -358
      CGI/PublicAccess.py
  4. +0
    -5
      CGI/README
  5. +1
    -1
      CGI/South Staffordshire.cgi
  6. +0
    -20
      CGI/SouthOxfordshire.cgi
  7. +0
    -248
      CGI/SouthOxfordshireParser.py

+ 31
- 0
CGI/Oswestry.cgi View File

@@ -0,0 +1,31 @@
#!/usr/bin/python

# This is the parser for Oswestry Borough Council.
# it is generated from the file CGITemplate

import cgi
import cgitb
#cgitb.enable(display=0, logdir="/tmp")


form = cgi.FieldStorage()
day = form.getfirst('day')
month = form.getfirst('month')
year = form.getfirst('year')


authority_name = "Oswestry Borough Council"
authority_short_name = "Oswestry"
base_url = "http://193.114.205.78/PublicAccess/tdc/"

import PublicAccess

parser = PublicAccess.PublicAccessParser(authority_name,
authority_short_name,
base_url)

xml = parser.getResults(day, month, year)

print "Content-Type: text/xml" # XML is following
print
print xml # print the xml

+ 0
- 101
CGI/PlanningUtils.py View File

@@ -1,101 +0,0 @@
__auth__ = None

import re

date_format = "%d/%m/%Y"


def xmlQuote(text):
# Change &s to &s
# I suspect there is probably some standard python
# function I should be using for this...
return text.replace('&', '&')

def fixNewlines(text):
# This can be used to sort out windows newlines
return text.replace("\r\n","\n")

# So what can a postcode look like then?
# This list of formats comes from http://www.mailsorttechnical.com/frequentlyaskedquestions.cfm
#AN NAA M1 1AA
#ANN NAA M60 1NW
#AAN NAA CR2 6XH
#AANN NAA DN55 1PT
#ANA NAA W1A 1HP
#AANA NAA EC1A 1BB

postcode_regex = re.compile("[A-Z][A-Z]?\d(\d|[A-Z])? ?\d[A-Z][A-Z]")

def getPostcodeFromText(text):
"""This function takes a piece of text and returns the first
bit of it that looks like a postcode."""

postcode_match = postcode_regex.search(text)

if postcode_match is not None:
return postcode_match.group()

class PlanningAuthorityResults:
"""This class represents a set of results of a planning search.

This should probably be separated out so that it can be used for
authorities other than Cherwell.
"""

def __init__(self, authority_name, authority_short_name):
self.authority_name = authority_name
self.authority_short_name = authority_short_name
# this will be a list of PlanningApplication objects
self.planning_applications = []


def addApplication(self, application):
self.planning_applications.append(application)

def __repr__(self):
return self.displayXML()

def displayXML(self):
"""This should display the contents of this object in the planningalerts format.
i.e. in the same format as this one:
http://www.planningalerts.com/lambeth.xml
"""

applications_bit = "".join([x.displayXML() for x in self.planning_applications])

return "<planning>\n" +\
"<authority_name>%s</authority_name>\n" %self.authority_name +\
"<authority_short_name>%s</authority_short_name>\n" %self.authority_short_name +\
"<applications>\n" + applications_bit +\
"</applications>\n" +\
"</planning>\n"



class PlanningApplication:
def __init__(self, no_postcode_default='No postcode'):
self.council_reference = None
self.address = None
self.postcode = no_postcode_default
self.description = None
self.info_url = None
self.comment_url = None

# expecting this as a datetime.date object
self.date_received = None

def __repr__(self):
return self.displayXML()

def displayXML(self):
return "<application>\n" +\
"<council_reference>%s</council_reference>\n" %xmlQuote(self.council_reference) +\
"<address>%s</address>\n" %xmlQuote(self.address) +\
"<postcode>%s</postcode>\n" %self.postcode +\
"<description>%s</description>\n" %xmlQuote(self.description) +\
"<info_url>%s</info_url>\n" %xmlQuote(self.info_url) +\
"<comment_url>%s</comment_url>\n" %xmlQuote(self.comment_url) +\
"<date_received>%s</date_received>\n" %self.date_received.strftime(date_format) +\
"</application>\n"

+ 0
- 358
CGI/PublicAccess.py View File

@@ -1,358 +0,0 @@
#!/usr/bin/python

import urllib, urllib2
import HTMLParser
import urlparse
import datetime, time

import cookielib

cookie_jar = cookielib.CookieJar()


from PlanningUtils import fixNewlines, PlanningAuthorityResults, PlanningApplication


search_form_url_end = "DcApplication/application_searchform.aspx"
search_results_url_end = "DcApplication/application_searchresults.aspx"
comments_url_end = "DcApplication/application_comments_entryform.aspx"

class PublicAccessParser(HTMLParser.HTMLParser):
"""This is the class which parses the PublicAccess search results page.
"""

def __init__(self,
authority_name,
authority_short_name,
base_url,
debug=False):
HTMLParser.HTMLParser.__init__(self)

self.authority_name = authority_name
self.authority_short_name = authority_short_name
self.base_url = base_url

self.debug = debug

# this will change to True when we enter the table of results
self._in_results_table = False

# this will be set to True when we have passed the header row
# in the results table
self._past_header_row = False

# this will be true when we are in a <td> in the results table
self._in_td = False

# For each row, this will say how many tds we have seen so far
self._td_count = 0

# The object which stores our set of planning application results
self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name)

# This will store the planning application we are currently working on.
self._current_application = None

def handle_starttag(self, tag, attrs):
if tag == "table":
self.handle_start_table(attrs)
# we are only interested in tr tags if we are in the results table
elif self._in_results_table and tag == "tr":
self.handle_start_tr(attrs)
# we are only interested in td tags if we are in the results table
elif self._in_results_table and tag == "td":
self.handle_start_td(attrs)
# we are only interested in <a> tags if we are in the 6th td in
# the results table.
elif self._in_td and self._td_count == 6 and tag == "a":
self.handle_start_a(attrs)
# If the tag is not one of these then we aren't interested

def handle_endtag(self, tag):
# we only need to consider end tags if we are in the results table
if self._in_results_table:
if tag == "table":
self.handle_end_table()
if tag == "tr":
self.handle_end_tr()
if tag == "td":
self.handle_end_td()

def handle_start_table(self, attrs):
for attr,value in attrs:
if attr == "class":
if value == "cResultsForm":
self._in_results_table = True
break

def handle_end_table(self):
# If we see an end table tag, then note that we have left the
# results table. This method is only called if we are in that table.
self._in_results_table = False

def handle_start_tr(self, attrs):
# The first tr we meet in the results table is just headers
# We will set a flag at the end of that tr to avoid creating
# a blank PlanningApplication
if self._past_header_row:
# Create a candidate result object
self._current_application = PlanningApplication()
self._td_count = 0

def handle_end_tr(self):
# If we are in the results table, and not finishing the header row
# append the current result to the results list.
if self._past_header_row:
self._results.addApplication(self._current_application)
else:
# The first row of the results table is headers
# We want to do nothing until after it
self._past_header_row = True
def handle_start_td(self, attrs):
# increase the td count by one
self._td_count += 1
# note that we are now in a td
self._in_td = True

def handle_end_td(self):
# note that we are now not in a td
self._in_td = False

def handle_start_a(self, attrs):
# this method is only getting called if we are in the
# 6th td of a non-header row of the results table.

# go through the attributes of the <a> looking for one
# named 'href'
for attr,value in attrs:
if attr == "href":
# the value of this tag is a relative url.
# parse it so we can get the query string from it
parsed_info_url = urlparse.urlparse(value)
# the 4th part of the tuple is the query string
query_string = parsed_info_url[4]

# join this query string to the search URL, and store this as
# the info URL of the current planning application
self._current_application.info_url = urlparse.urljoin(self.base_url, value)

# Join this query string to the comments URL, and store this as
# the comments URL of the current planning application
comments_url = urlparse.urljoin(self.base_url, comments_url_end)
self._current_application.comment_url = urlparse.urljoin(comments_url, query_string)

# while we're here, let's follow some links to find the postcode...
# the postcode is in an input tag in the property page. This page
# can be found by following the info url.
# The newlines in the info page need fixing.
info_file_contents = fixNewlines(urllib2.urlopen(self._current_application.info_url).read())
info_file_parser = PublicAccessInfoPageParser()
info_file_parser.feed(info_file_contents)

property_page_url = urlparse.urljoin(self._current_application.info_url, info_file_parser.property_page_url)
# the newlines in this page need fixing
property_file_contents = fixNewlines(urllib2.urlopen(property_page_url).read())
property_file_parser = PublicAccessPropertyPageParser()
property_file_parser.feed(property_file_contents)

# Set the postcode on the current planning application from the
# one found on the property page
if property_file_parser.postcode is not None:
self._current_application.postcode = property_file_parser.postcode

# There is no need for us to look at any more attributes.
break

def handle_data(self, data):
if self._in_td:
# The first td contains the reference
if self._td_count == 1:
self._current_application.council_reference = data
# The second td contains the date the application was received
elif self._td_count == 2:
year, month, day = time.strptime(data, "%d/%m/%Y")[:3]
received_date = datetime.date(year, month, day)

self._current_application.date_received = received_date
# The third td contains the address
elif self._td_count == 3:
#data = data.replace("^M","\n")
self._current_application.address = data
# The fourth td contains the description
elif self._td_count == 4:
self._current_application.description = data
# 5 is status - we don't need it.
# 6 is a button - this is where we will get our postcode,
# comment_url, and info_url from (when handling the <a> tag).


def getResultsByDayMonthYear(self, day, month, year):
# First download the search form (in order to get a session cookie
search_form_request = urllib2.Request(urlparse.urljoin(self.base_url, search_form_url_end))
search_form_response = urllib2.urlopen(search_form_request)
cookie_jar.extract_cookies(search_form_response, search_form_request)

# We are only doing this first search in order to get a cookie
# The paging on the site doesn't work with cookies turned off.

search_data1 = urllib.urlencode({"searchType":"ADV",
"caseNo":"",
"PPReference":"",
"AltReference":"",
"srchtype":"",
"srchstatus":"",
"srchdecision":"",
"srchapstatus":"",
"srchappealdecision":"",
"srchwardcode":"",
"srchparishcode":"",
"srchagentdetails":"",
"srchDateReceivedStart":"%(day)02d/%(month)02d/%(year)d" %{"day":day ,"month": month ,"year": year},
"srchDateReceivedEnd":"%(day)02d/%(month)02d/%(year)d" %{"day":day, "month":month, "year":year} })

if self.debug:
print search_data1


search_url = urlparse.urljoin(self.base_url, search_results_url_end)
request1 = urllib2.Request(search_url, search_data1)
cookie_jar.add_cookie_header(request1)
response1 = urllib2.urlopen(request1)

# This search is the one we will actually use.
# a maximum of 100 results are returned on this site,
# hence setting "pagesize" to 100. I doubt there will ever
# be more than 100 in one day in PublicAccess...
# "currentpage" = 1 gets us to the first page of results
# (there will only be one anyway, as we are asking for 100 results...)

#http://planning.york.gov.uk/PublicAccess/tdc/DcApplication/application_searchresults.aspx?szSearchDescription=Applications%20received%20between%2022/02/2007%20and%2022/02/2007&searchType=ADV&bccaseno=&currentpage=2&pagesize=10&module=P3

search_data2 = urllib.urlencode((("szSearchDescription","Applications received between %(day)02d/%(month)02d/%(year)d and %(day)02d/%(month)02d/%(year)d"%{"day":day ,"month": month ,"year": year}), ("searchType","ADV"), ("bccaseno",""), ("currentpage","1"), ("pagesize","100"), ("module","P3")))

if self.debug:
print search_data2

# This time we want to do a get request, so add the search data into the url
request2_url = urlparse.urljoin(self.base_url, search_results_url_end + "?" + search_data2)

request2 = urllib2.Request(request2_url)

# add the cookie we stored from our first search
cookie_jar.add_cookie_header(request2)

response2 = urllib2.urlopen(request2)

contents = fixNewlines(response2.read())

if self.debug:
print contents

self.feed(contents)

return self._results


def getResults(self, day, month, year):
return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML()




class PublicAccessInfoPageParser(HTMLParser.HTMLParser):
"""A parser to get the URL for the property details page out of the
info page (this url is needed in order to get the postcode of the
application.
"""

def __init__(self):
HTMLParser.HTMLParser.__init__(self)

self.property_page_url = None

def handle_starttag(self, tag, attrs):
"""The URL of the property details page is contained in an <a> tag in
an attribute with key 'A_btnPropertyDetails'. There is some garbage on
either side of it which we will have to clear up before storing it...

We go through the <a> tags looking for one with an attribute with
key 'id' and value 'A_btnPropertyDetails'. When we find it we go through
its attributes looking for one with key 'href' - the value of this attribute
contains the URL we want, after a bit of tidying up.

Once we have got the URL, there is no need for us to look at any more <a> tags.
"""
if tag == "a" and self.property_page_url is None:
if attrs.count(("id","A_btnPropertyDetails")) > 0:
for attr,value in attrs:
if attr == "href":
the_link = value

# this has some garbage on either side of it...
# let's strip that off

# the garbage on the left is separated by whitespace.
# the garbage on the right is separated by a "'".

self.property_page_url = the_link.split()[1].split("'")[0]




class PublicAccessPropertyPageParser(HTMLParser.HTMLParser):
"""A parser to get the postcode out of the property details page."""
def __init__(self):
HTMLParser.HTMLParser.__init__(self)

self.postcode = None

def handle_starttag(self, tag, attrs):
"""The postcode is contained in an <input> tag.
This tag has an attribute 'name' with value postcode.
It also has an attribute 'value' with value the postcode of this application.

We go through the input tags looking for one with an attribute with
key 'name' and value 'postcode'. When we find one,
we look through its attributes for one with key 'value' - we store the value of this
attribute as self.postcode.

Once we have the postcode, there is no need to look at any more input tags.
"""
if tag == "input" and self.postcode is None:
if attrs.count(("name","postcode")) > 0:
for attr,value in attrs:
if attr == "value":
self.postcode = value


# These still don't work:

# Perthshire
#http://193.63.61.22/publicaccess/tdc/DcApplication/application_searchform.aspx
#"Perth and Kinross Council", "Perthshire", "http://193.63.61.22/publicaccess/tdc/"

# Hambleton
#http://planning.hambleton.gov.uk/publicaccess/tdc/DcApplication/application_searchform.aspx
#"Hambleton District Council", "Hambleton", "http://planning.hambleton.gov.uk/publicaccess/tdc/"

# These use https:

# Chiltern
#https://isa.chiltern.gov.uk/publicaccess/tdc/tdc_home.aspx
# Hinckley-Bosworth
#https://cx.hinckley-bosworth.gov.uk/PublicAccess/dc/DcApplication/application_searchform.aspx

+ 0
- 5
CGI/README View File

@@ -1,5 +0,0 @@

WARNING - this directory is only for generated files
and files which are automatically copied in.
Anything manually added here will be svn deleted.


CGI/South Staffs.cgi → CGI/South Staffordshire.cgi View File

@@ -15,7 +15,7 @@ year = form.getfirst('year')




authority_name = "South Staffordshire Council" authority_name = "South Staffordshire Council"
authority_short_name = "South Staffs"
authority_short_name = "South Staffordshire"
base_url = "https://services.sstaffs.gov.uk/PublicAccess/tdc/" base_url = "https://services.sstaffs.gov.uk/PublicAccess/tdc/"


import PublicAccess import PublicAccess

+ 0
- 20
CGI/SouthOxfordshire.cgi View File

@@ -1,20 +0,0 @@
#!/usr/bin/python

import cgi
import cgitb; cgitb.enable(display=0, logdir="/tmp")


form = cgi.FieldStorage()
day = form.getfirst('day')
month = form.getfirst('month')
year = form.getfirst('year')

from SouthOxfordshireParser import SouthOxfordshireParser

parser = SouthOxfordshireParser()

xml = parser.getResults(day, month, year)

print "Content-Type: text/xml" # XML is following
print
print xml # print the xml

+ 0
- 248
CGI/SouthOxfordshireParser.py View File

@@ -1,248 +0,0 @@

import urllib, urllib2

import HTMLParser
import urlparse
import datetime, time

# This needs a page number inserting
search_url = "http://www.southoxon.gov.uk/ccm/planning/ApplicationList.jsp?PAGE=%d"

# This needs the council reference
comment_url = "https://forms.southoxon.gov.uk/ufs/ufsmain?formid=PLANNINGCOMMENT&PLNGAPPL_REFERENCE=%(reference)s"

authority_name = "South Oxfordshire District Council"
authority_short_name = "South Oxfordshire"


from PlanningUtils import fixNewlines, \
getPostcodeFromText, \
PlanningAuthorityResults, \
PlanningApplication

class SouthOxfordshireParser(HTMLParser.HTMLParser):
"""In this case we'll take the date, so that we can avoid doing dowloads for
the other days in this week's file. This date should be a datetime.date object.
"""
def __init__(self):
HTMLParser.HTMLParser.__init__(self)

self._requested_date = None

# We'll keep a count of the number of tables we have seen.
# All the interesting stuff is in table 3
self._table_count = 0

# While inside table 3, we'll keep a count of the number of
# <td>s we have seen. What is in which numbered <td> is detailed below.
# 1 reference
# 3 place and description
# 5 date received
# 2 and 4 are just padding
self._td_count = 0

# This is just a flag to say that we are now ready to get the reference
# from the next bit of data
self._get_reference = False

self._data = ''

# this will hold the application we are currently working on.
self._current_application = None
# The object which stores our set of planning application results
self._results = PlanningAuthorityResults(authority_name, authority_short_name)

def handle_starttag(self, tag, attrs):
# if we see a table tag, increment the table count.
if tag == 'table':
self._table_count += 1
# we are only interested in other tags if we are in table 3.
if self._table_count == 3:
# If we are starting a <tr>, create a new PlanningApplication object
# for the application currently being processed
if tag == 'tr':
self._current_application = PlanningApplication()

# if we see a td, increment the <td> count.
if tag == 'td':
self._td_count += 1

# if we are in the first <td>, and we see a link,
# then it is to the info page for this applicaion.
if tag == 'a' and self._td_count == 1:
for key, value in attrs:
if key == 'href':
url_end = value
self._current_application.info_url = urlparse.urljoin(search_url,url_end)

# We now know that the next bit of data is the reference
self._get_reference = True
# href is the only attribute we are interested in.
break

def handle_endtag(self, tag):
# There is no need to do anything unless we are in table 3.
if self._table_count == 3:

# The end <tr> indicates that the current application is finished.
# Now we can fetch the info_page to get the address, postcode,
# and description.
# If we don't have a reference, then we are in the header row,
# which we don't want.
# There is no point in doing this if the date is not the requested one.

if tag == 'tr' and \
self._current_application.council_reference is not None and \
self._current_application.date_received == self._requested_date:
info_page_parser = SouthOxfordshireInfoURLParser()
info_page_parser.feed(urllib2.urlopen(self._current_application.info_url).read())

self._current_application.address = info_page_parser.address
self._current_application.postcode = getPostcodeFromText(info_page_parser.address)
self._current_application.description = info_page_parser.description

# Add the current application to the results set
self._results.addApplication(self._current_application)

# At the end of the 5th <td>, self._data should contain
# the received date of the application.
if tag == 'td' and self._td_count == 5:
app_year, app_month, app_day = tuple(time.strptime(self._data, "%d %B %Y")[:3])
self._current_application.date_received = datetime.date(app_year, app_month, app_day)
self._data = ''
self._td_count = 0

def handle_data(self, data):
# There is no need to do anything if we aren't in table 3.
if self._table_count == 3:
# If we are in the first <td>, and the get_reference flag is set,
# then the next data is the reference.
if self._td_count == 1 and self._get_reference:
self._current_application.council_reference = data

# The comment url can now be made, as it depends only on the reference.
# On this site, the link to the comment page is only displayed once
# the planning authority has decided who is handling this application
# and has opened consultations. The link below works straight away,
# and also works for apps for which the consultation period is over.
# I have no idea if anything is actually done with these comments if
# it is followed too early...
self._current_application.comment_url = comment_url %{'reference': self._current_application.council_reference}

# Set the get_reference flag back to False.
self._get_reference = False

# If we are in the 5th <td>, then we need to collect all the data together
# before we can use it. This is actually processed in handle_endtag.
if self._td_count == 5:
self._data += data

def handle_entityref( self, ref ):
# We might have some entity_refs to clear up.
# there is no need to bother with this if we aren't in the results table.
if self._table_count == 3 and self._td_count == 5:
if ref == 'nbsp':
self._data += ' '


def getResultsByDayMonthYear(self, day, month, year):
"""This will return an ApplicationResults object containg the
applications for the date passed in."""

today = datetime.date.today()
self._requested_date = datetime.date(year, month, day)
delta = today - self._requested_date

# to get the correct page, we need
# page ((days mod 7) + 1)
page_number = delta.days/7 + 1

response = urllib2.urlopen(search_url %page_number)

contents = response.read()

self.feed(contents)

return self._results


def getResults(self, day, month, year):
return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML()

class SouthOxfordshireInfoURLParser(HTMLParser.HTMLParser):
"""This parser is to get the description and address out of the info page
for a South Oxfordshire application."""

def __init__(self):
HTMLParser.HTMLParser.__init__(self)

self.address = None
self.description = None

# These two states will be set to:
# 0 - if we haven't yet got that bit
# 1 - if we are currently working on it
# 2 - if we have finished
self._address_state = 0
self._description_state = 0

# We well need to know whether or not we are in a <td>
self._in_td = False

# This is used for collecting together date which comes in several bits.
self._data = ''
def handle_starttag(self, tag, attrs):
# If we see the start of a <td> and we are still interested in some data
# then set the td flag to true, and blank the data
if tag == 'td' and (self._address_state < 2 or self._description_state < 2):
self._in_td = True
self._data = ''

def handle_endtag(self, tag):
if tag == 'td' and (self._address_state < 2 or self._description_state < 2):
# If we are working on the description,
# set description from _data and note that we need to work on it no more.
if self._description_state == 1:
self.description = self._data
self._description_state = 2


# If we are working on the address,
# set address from _data and note that we need to work on it no more.
elif self._address_state == 1:
self.address = self._data
self._address_state = 2

# If we see data which says 'Descripton',
# then set the description state to working.
elif self._data.strip() == 'Description':
self._description_state = 1
# If we see data which says 'Location',
# then set the addresss state to working.
elif self._data.strip() == 'Location':
self._address_state = 1

# Note that we are leaving the <td>
self._in_td = False
def handle_data(self, data):
# if we are in a td, and we are still interested in the data for something,
# append the current bit to self._data
if self._in_td and (self._address_state < 2 or self._description_state < 2):
self._data += data


# TODO

# find out what time of day this is run - does it matter that
# we aren't being careful with daylight saving time etc.

# Can we check that scraped email address really is
# an email address?

Write Preview
Loading…
Cancel
Save