adrianshort
/
planningalerts
mirror of https://github.com/adrianshort/planningalerts.git
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
Browse Source

Birmingham City Council now use Planning Explorer

master
pezholio 13 years ago
parent
0813570a88
commit
5a453414e5
2 changed files with 191 additions and 35 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      SitesToGenerate.csv
  2. +190
    -34
      python_scrapers/PlanningExplorer.py

+ 1
- 1
SitesToGenerate.csv View File

@@ -19,7 +19,7 @@
"Bedford Borough Council","Bedford",,,,,,"http://www.publicaccess.bedford.gov.uk/publicaccess/tdc/","PublicAccess","PublicAccessParser",
"Berwick-upon-Tweed Borough Council","Berwick",,,,,,,"Berwick","BerwickParser",
"Bexley Council","Bexley",,,,,,"http://publicaccess.bexley.gov.uk/publicaccess/tdc/","PublicAccess","PublicAccessParser",
"Birmingham City Council","Birmingham",,,,,,,"Birmingham","BirminghamParser",
"Birmingham City Council","Birmingham",,,,,,"http://eplanning.birmingham.gov.uk/Northgate/","Birmingham","BirminghamParser",
"Blaby District Council","Blaby",,,,,,"http://www.blaby.gov.uk/PublicAccess/tdc/","PublicAccess","PublicAccessParser",
"Blackburn with Darwen Borough Council","Blackburn",,"http://duncan.devel.disruptiveproactivity.com/Blackburn.cgi?day={day}&month={month}&year={year}",1,,,"http://195.8.175.6/","PlanningExplorer","BlackburnParser",
"Blackpool Borough Council","Blackpool",,,,,,"http://www2.blackpool.gov.uk/PlanningApplications/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch","AcolnetParser","BlackpoolParser",


+ 190
- 34
python_scrapers/PlanningExplorer.py View File

@@ -4,7 +4,7 @@ import urlparse
import cgi
import re
import datetime
import BeautifulSoup

import cookielib

@@ -13,9 +13,130 @@ cookie_jar = cookielib.CookieJar()

from BeautifulSoup import BeautifulSoup

from PlanningUtils import PlanningApplication, \
PlanningAuthorityResults, \
getPostcodeFromText
__auth__ = None

import re

date_format = "%d/%m/%Y"

def fixNewlines(text):
# This can be used to sort out windows newlines
return text.replace("\r\n","\n")

# So what can a postcode look like then?
# This list of formats comes from http://www.mailsorttechnical.com/frequentlyaskedquestions.cfm
#AN NAA M1 1AA
#ANN NAA M60 1NW
#AAN NAA CR2 6XH
#AANN NAA DN55 1PT
#ANA NAA W1A 1HP
#AANA NAA EC1A 1BB

postcode_regex = re.compile("[A-Z][A-Z]?\d(\d|[A-Z])? ?\d[A-Z][A-Z]")

def getPostcodeFromText(text, default_postcode="No Postcode"):
"""This function takes a piece of text and returns the first
bit of it that looks like a postcode."""

postcode_match = postcode_regex.search(text)

return postcode_match.group() if postcode_match else default_postcode


class PlanningAuthorityResults:
"""This class represents a set of results of a planning search.

This should probably be separated out so that it can be used for
authorities other than Cherwell.
"""

def __init__(self, authority_name, authority_short_name):
self.authority_name = authority_name
self.authority_short_name = authority_short_name

# this will be a list of PlanningApplication objects
self.planning_applications = []


def addApplication(self, application):
self.planning_applications.append(application)

def __repr__(self):
return self.displayXML()

def displayXML(self):
"""This should display the contents of this object in the planningalerts format.
i.e. in the same format as this one:
http://www.planningalerts.com/lambeth.xml
"""

applications_bit = "".join([x.displayXML() for x in self.planning_applications])

return u"""<?xml version="1.0" encoding="UTF-8"?>\n""" + \
u"<planning>\n" +\
u"<authority_name>%s</authority_name>\n" %self.authority_name +\
u"<authority_short_name>%s</authority_short_name>\n" %self.authority_short_name +\
u"<applications>\n" + applications_bit +\
u"</applications>\n" +\
u"</planning>\n"



class PlanningApplication:
def __init__(self):
self.council_reference = None
self.address = None
self.postcode = None
self.description = None
self.info_url = None
self.comment_url = None

# expecting this as a datetime.date object
self.date_received = None

# If we can get them, we may as well include OSGB.
# These will be the entirely numeric version.
self.osgb_x = None
self.osgb_y = None

def __repr__(self):
return self.displayXML()

def is_ready(self):
# This method tells us if the application is complete
# Because of the postcode default, we can't really
# check the postcode - make sure it is filled in when
# you do the address.
return self.council_reference \
and self.address \
and self.description \
and self.info_url \
and self.comment_url \
and self.date_received


def displayXML(self):
#print self.council_reference, self.address, self.postcode, self.description, self.info_url, self.comment_url, self.date_received

if not self.postcode:
self.postcode = getPostcodeFromText(self.address)

contents = [
u"<council_reference><![CDATA[%s]]></council_reference>" %(self.council_reference),
u"<address><![CDATA[%s]]></address>" %(self.address),
u"<postcode><![CDATA[%s]]></postcode>" %self.postcode,
u"<description><![CDATA[%s]]></description>" %(self.description),
u"<info_url><![CDATA[%s]]></info_url>" %(self.info_url),
u"<comment_url><![CDATA[%s]]></comment_url>" %(self.comment_url),
u"<date_received><![CDATA[%s]]></date_received>" %self.date_received.strftime(date_format),
]
if self.osgb_x:
contents.append(u"<osgb_x>%s</osgb_x>" %(self.osgb_x))
if self.osgb_y:
contents.append(u"<osgb_y>%s</osgb_y>" %(self.osgb_y))

return u"<application>\n%s\n</application>" %('\n'.join(contents))


# Date format to enter into search boxes
date_format = "%d/%m/%Y"
@@ -26,19 +147,19 @@ app_code_regex = re.compile("PARAM0=(\d*)")


class PlanningExplorerParser:
# If this authority doesn't have a comments page,
# then set this email_address to an address for the
# If this authority doesn't have a comments page,
# then set this email_address to an address for the
# planning department, and it will be used in lieu of
# a comments url.
comments_email_address = None

# These are the directories where the info urls, and search urls,
# usually live underneath the base_url.
# usually live underneath the base_url.
# If these are different for a particular
# authority, then they can be overridden in a subclass.
info_url_path = "MVM/Online/Generic/"
search_url_path = "MVM/Online/PL/GeneralSearch.aspx"
# This is the most common place for comments urls to live
# The %s will be filled in with an application code
comments_path = "MVM/Online/PL/PLComments.aspx?pk=%s"
@@ -102,10 +223,10 @@ class PlanningExplorerParser:

def _getHeaders(self):
"""If the authority requires any headers for the post request,
override this method returning a dictionary of header key to
override this method returning a dictionary of header key to
header value."""
headers = {}
if self.use_firefox_user_agent:
headers["User-Agent"] = "Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.8.1.10) Gecko/20071126 Ubuntu/7.10 (gutsy) Firefox/2.0.0.10"

@@ -123,8 +244,8 @@ class PlanningExplorerParser:
for PlanningExplorer sites. It won't work for all of them, so
will sometimes need to be overridden in a subclass.

The parameter edrDateSelection is often not needed.
It is needed by Charnwood though, so I've left it in
The parameter edrDateSelection is often not needed.
It is needed by Charnwood though, so I've left it in
to keep things simple.
"""
year_month_day = search_date.timetuple()[:3]
@@ -138,7 +259,7 @@ class PlanningExplorerParser:
("csbtnSearch", "Search"),
("cboNumRecs", "99999"),
))
return post_data


@@ -150,22 +271,22 @@ class PlanningExplorerParser:
address = address_td.div.string
else:
address = address_td.string
return address


def _getPostCode(self, info_soup):
"""In most cases, the postcode can be got from the address in
"""In most cases, the postcode can be got from the address in
the results table. Some councils put the address there without the
postcode. In this case we will have to go to the info page to get
the postcode. This should be done by overriding this method with
one that parses the info page."""

return getPostcodeFromText(self._current_application.address)
def _getDescription(self, tds, info_soup):
description_td = tds[self.description_td_no]
if description_td.div is not None:
# Mostly this is in a div
# Use the empty string if the description is missing
@@ -190,7 +311,7 @@ class PlanningExplorerParser:

self.search_url = urlparse.urljoin(base_url, self.search_url_path)
self.info_url_base = urlparse.urljoin(self.base_url, self.info_url_path)
self.debug = debug

self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name)
@@ -217,7 +338,7 @@ class PlanningExplorerParser:
# The post data needs to be different for different councils
# so we have a method on each council's scraper to make it.
post_data = self._getPostData(asp_args, search_date)
headers = self._getHeaders()

request = urllib2.Request(self.search_url, post_data, headers)
@@ -250,7 +371,7 @@ class PlanningExplorerParser:
self._current_application = PlanningApplication()

# There is no need to search for the date_received, it's what
# we searched for
# we searched for
self._current_application.date_received = search_date

tds = tr.findAll("td")
@@ -265,7 +386,7 @@ class PlanningExplorerParser:
if self.fetch_info_page:
# We need to quote the spaces in the info url
info_request = urllib2.Request(urllib.quote(self._current_application.info_url, ":/&?="))
info_soup = BeautifulSoup(urllib2.urlopen(info_request))
else:
info_soup = None
@@ -372,7 +493,7 @@ class CreweParser(PlanningExplorerParser):

info_url_path = "Northgate/PlanningExplorer/Generic/"
search_url_path = "northgate/planningexplorer/generalsearch.aspx"
results_table_attrs = {"class": "display_table"}

def _getPostData(self, asp_args, search_date):
@@ -433,13 +554,13 @@ class HackneyParser(PlanningExplorerParser):
real_url_tuple = urlparse.urlsplit(response.geturl())

query_string = real_url_tuple[3]
# Get the query as a list of key, value pairs
parsed_query_list = list(cgi.parse_qsl(query_string))

# Go through the query string replacing any PS parameters
# with PS=99999
for i in range(len(parsed_query_list)):
key, value = parsed_query_list[i]

@@ -448,10 +569,10 @@ class HackneyParser(PlanningExplorerParser):
parsed_query_list[i] = (key, value)

new_query_string = urllib.urlencode(parsed_query_list)
new_url_tuple = real_url_tuple[:3] + (new_query_string,) + real_url_tuple[4:]
new_url = urlparse.urlunsplit(new_url_tuple)
new_url = urlparse.urlunsplit(new_url_tuple)
new_request = urllib2.Request(new_url, None, self._getHeaders())
new_response = urllib2.urlopen(new_request)

@@ -486,13 +607,13 @@ class HackneyParser(PlanningExplorerParser):

class KennetParser(BroadlandLike, PlanningExplorerParser):
comments_path = "Northgate/PlanningExplorer/PLComments.aspx?pk=%s"
class LincolnParser(PlanningExplorerParser):
use_firefox_user_agent = True
use_referer = True

results_table_attrs = {"class": "display_table"}
search_url_path = "northgate/planningexplorer/generalsearch.aspx"
info_url_path = "Northgate/PlanningExplorer/Generic/"

@@ -574,6 +695,40 @@ class MertonParser(PlanningExplorerParser):

class ShrewsburyParser(PlanningExplorerParser):
use_firefox_user_agent = True
class BirminghamParser(PlanningExplorerParser):
search_url_path = "PlanningExplorer/GeneralSearch.aspx"
info_url_path = "PlanningExplorer/Generic/"
comments_path = "PlanningExplorer/PLComments.aspx?pk=%s"
use_firefox_user_agent = True
use_referer = True

results_table_attrs = {"class": "display_table"}

def _getPostData(self, asp_args, search_date):
post_data = urllib.urlencode(asp_args + (
("txtApplicationNumber", ""),
("cboApplicationTypeCode", ""),
("txtSiteAddress", ""),
("txtProposal", ""),
("cboWardCode", ""),
("cboConstituencyCode", ""),
("txtApplicantName", ""),
("txtAgentName", ""),
("cboDevelopmentTypeCode", ""),
("cboSelectDateValue", "DATE_REGISTERED"),
("cboMonths", "1"),
("cboDays", "10"),
("rbGroup", "rbRange"),
("dateStart", search_date.strftime(date_format)),
("dateEnd", search_date.strftime(date_format)),
("edrDateSelection", ""),
("csbtnSearch", "Search"),
)
)

return post_data

class SouthNorfolkParser(PlanningExplorerParser):
use_firefox_user_agent = True
@@ -596,7 +751,7 @@ class SouthShropshireParser(PlanningExplorerParser):
("cboNumRecs", "99999"),
("csbtnSearch", "Search"),
))
return post_data

class SouthTynesideParser(BroadlandLike, PlanningExplorerParser):
@@ -713,11 +868,11 @@ class MendipParser(BroadlandLike, PlanningExplorerParser):
if __name__ == '__main__':
# NOTE - 04/11/2007 is a sunday
# I'm using it to test that the scrapers behave on days with no apps.
# parser = BlackburnParser("Blackburn With Darwen Borough Council", "Blackburn", "http://195.8.175.6/")
# parser = BroadlandParser("Broadland Council", "Broadland", "http://www.broadland.gov.uk/")
# parser = CamdenParser("London Borough of Camden", "Camden", "http://planningrecords.camden.gov.uk/")
parser = CharnwoodParser("Charnwood Borough Council", "Charnwood", "http://portal.charnwoodbc.gov.uk/")
# parser = CharnwoodParser("Charnwood Borough Council", "Charnwood", "http://portal.charnwoodbc.gov.uk/")
# parser = CreweParser("Crewe and Nantwich Borough Council", "Crewe and Nantwich", "http://portal.crewe-nantwich.gov.uk/")
# parser = EastStaffsParser("East Staffordshire Borough Council", "East Staffs", "http://www2.eaststaffsbc.gov.uk/")
# parser = EppingForestParser("Epping Forest District Council", "Epping Forest", "http://plan1.eppingforestdc.gov.uk/")
@@ -738,8 +893,9 @@ if __name__ == '__main__':
# parser = WalthamForestParser("Waltham Forest", "Waltham Forest", "http://planning.walthamforest.gov.uk/")
# parser = ConwyParser("Conwy County Borough Council", "Conwy", "http://www.conwy.gov.uk/")
# parser = MertonParser("London Borough of Merton", "Merton", "http://planning.merton.gov.uk")
parser = MendipParser("Mendip District Council", "Mendip", "http://planning.mendip.gov.uk/")
print parser.getResults(12, 6, 2009)
# parser = MendipParser("Mendip District Council", "Mendip", "http://planning.mendip.gov.uk/")
parser = BirminghamParser("Birmingham City Council", "Birmingham", "http://eplanning.birmingham.gov.uk/Northgate/")
print parser.getResults(27, 4, 2010)

# To Do



Write Preview
Loading…
Cancel
Save