From 8bed7d7ac0e401867df8870fc3111e80a86d27f8 Mon Sep 17 00:00:00 2001 From: "duncan.parkes" Date: Wed, 11 Apr 2007 18:40:39 +0000 Subject: [PATCH] add scrapers for 13 more councils which use ApplicationSearchServlet --- cgi-bin/Allerdale.cgi | 29 + cgi-bin/Alnwick.cgi | 29 + cgi-bin/Angus.cgi | 4 +- cgi-bin/ApplicationSearchServletParser.py | 518 ++++++++++++++++++ cgi-bin/Aylesbury Vale.cgi | 4 +- cgi-bin/Barrow.cgi | 29 + cgi-bin/Basildon.cgi | 4 +- cgi-bin/Bath.cgi | 4 +- cgi-bin/Bexley.cgi | 4 +- cgi-bin/Blaby.cgi | 6 +- cgi-bin/Bristol.cgi | 4 +- cgi-bin/Buckinghamshire.cgi | 4 +- cgi-bin/Chelmsford.cgi | 4 +- cgi-bin/Cherwell.cgi | 4 +- cgi-bin/City of London.cgi | 4 +- cgi-bin/Cornwall.cgi | 4 +- cgi-bin/Coventry.cgi | 29 + cgi-bin/Denbighshire.cgi | 29 + cgi-bin/Doncaster.cgi | 4 +- cgi-bin/Dundee.cgi | 4 +- cgi-bin/Ealing.cgi | 29 + cgi-bin/Easington.cgi | 29 + cgi-bin/East Devon.cgi | 4 +- cgi-bin/Edinburgh.cgi | 4 +- cgi-bin/Epsom and Ewell.cgi | 4 +- cgi-bin/Fenland.cgi | 4 +- cgi-bin/Gedling.cgi | 4 +- cgi-bin/Hammersmith and Fulham.cgi | 4 +- cgi-bin/Haringey.cgi | 29 + cgi-bin/Harrogate.cgi | 4 +- cgi-bin/Hart.cgi | 4 +- cgi-bin/Hartlepool.cgi | 29 + cgi-bin/High Peak.cgi | 29 + cgi-bin/Huntingdonshire.cgi | 4 +- cgi-bin/Kerrier.cgi | 4 +- cgi-bin/Knowsley.cgi | 4 +- cgi-bin/Lancaster.cgi | 4 +- cgi-bin/Luton.cgi | 4 +- cgi-bin/Malvern Hills.cgi | 4 +- cgi-bin/Mid Devon.cgi | 4 +- cgi-bin/Milton Keynes.cgi | 4 +- cgi-bin/North Tyneside.cgi | 4 +- cgi-bin/North Warwickshire.cgi | 29 + cgi-bin/Northumberland.cgi | 4 +- cgi-bin/Oadby and Wigston.cgi | 4 +- cgi-bin/Oswestry.cgi | 4 +- cgi-bin/Peterborough.cgi | 4 +- cgi-bin/PlanningUtils.py | 4 +- cgi-bin/Portsmouth.cgi | 4 +- cgi-bin/Rushmoor.cgi | 4 +- cgi-bin/Scarborough.cgi | 4 +- cgi-bin/Sevenoaks.cgi | 4 +- cgi-bin/South Bucks.cgi | 4 +- cgi-bin/South Ribble.cgi | 4 +- cgi-bin/South Staffordshire.cgi | 4 +- cgi-bin/Southampton.cgi | 4 +- cgi-bin/Spelthorne.cgi | 4 +- cgi-bin/St Helens.cgi | 29 + cgi-bin/Stevenage.cgi | 4 +- cgi-bin/Stockton-On-Tees.cgi | 4 +- cgi-bin/Sunderland.cgi | 4 +- cgi-bin/Test Valley.cgi | 4 +- cgi-bin/Tonbridge.cgi | 4 +- cgi-bin/Torbay.cgi | 4 +- cgi-bin/Waveney.cgi | 4 +- cgi-bin/Wear Valley.cgi | 29 + cgi-bin/Wellingborough.cgi | 29 + cgi-bin/West Lancashire.cgi | 4 +- cgi-bin/West Norfolk.cgi | 4 +- cgi-bin/Woking.cgi | 4 +- cgi-bin/York.cgi | 4 +- .../ApplicationSearchServletParser.py | 518 ++++++++++++++++++ python_scrapers/CGITemplate | 6 +- python_scrapers/OtherFilesToCopy.csv | 2 + python_scrapers/PlanningUtils.py | 4 +- python_scrapers/PublicAccessSites.csv | 126 +++-- python_scrapers/createCGI.sh | 8 +- python_scrapers/generateCGIScripts.py | 4 +- 78 files changed, 1582 insertions(+), 236 deletions(-) create mode 100755 cgi-bin/Allerdale.cgi create mode 100755 cgi-bin/Alnwick.cgi create mode 100644 cgi-bin/ApplicationSearchServletParser.py create mode 100755 cgi-bin/Barrow.cgi create mode 100755 cgi-bin/Coventry.cgi create mode 100755 cgi-bin/Denbighshire.cgi create mode 100755 cgi-bin/Ealing.cgi create mode 100755 cgi-bin/Easington.cgi create mode 100755 cgi-bin/Haringey.cgi create mode 100755 cgi-bin/Hartlepool.cgi create mode 100755 cgi-bin/High Peak.cgi create mode 100755 cgi-bin/North Warwickshire.cgi create mode 100755 cgi-bin/St Helens.cgi create mode 100755 cgi-bin/Wear Valley.cgi create mode 100755 cgi-bin/Wellingborough.cgi create mode 100644 python_scrapers/ApplicationSearchServletParser.py diff --git a/cgi-bin/Allerdale.cgi b/cgi-bin/Allerdale.cgi new file mode 100755 index 0000000..a5fd1be --- /dev/null +++ b/cgi-bin/Allerdale.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Allerdale Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Allerdale Borough Council" +authority_short_name = "Allerdale" +base_url = "http://planning.allerdale.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.AllerdaleSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Alnwick.cgi b/cgi-bin/Alnwick.cgi new file mode 100755 index 0000000..3a0284e --- /dev/null +++ b/cgi-bin/Alnwick.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Alnwick District Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Alnwick District Council" +authority_short_name = "Alnwick" +base_url = "http://services.castlemorpeth.gov.uk:7777/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.AlnwickSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Angus.cgi b/cgi-bin/Angus.cgi index f75292f..0f6cb25 100755 --- a/cgi-bin/Angus.cgi +++ b/cgi-bin/Angus.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.angus.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/ApplicationSearchServletParser.py b/cgi-bin/ApplicationSearchServletParser.py new file mode 100644 index 0000000..b1db56f --- /dev/null +++ b/cgi-bin/ApplicationSearchServletParser.py @@ -0,0 +1,518 @@ + +import urllib, urllib2 + +import HTMLParser +import urlparse +import datetime, time + + +from PlanningUtils import PlanningAuthorityResults, \ + getPostcodeFromText, \ + PlanningApplication + + +# The search results list will give us reference, location, description, +# and info url of each app. + +# The info page gives us the received date, +# and comment_url + +class ApplicationSearchServletParser(HTMLParser.HTMLParser): + """Parser for ApplicationSearchServlet sites. + """ + + + # These indicate the column of the main table containing this + # piece of information. + # They should be overridden in subclasses + + #self._rows_to_ignore_at_start = None + + _reference_col_no = None + _location_col_no = None + _description_col_no = None + + def __init__(self, + authority_name, + authority_short_name, + base_url, + debug=False): + HTMLParser.HTMLParser.__init__(self) + + self.authority_name = authority_name + self.authority_short_name = authority_short_name + self.base_url = base_url + self.debug = debug + + self.search_url = urlparse.urljoin(self.base_url, "portal/servlets/ApplicationSearchServlet") + + self._comment_url = urlparse.urljoin(self.base_url, "portal/servlets/PlanningComments?REFNO=%(council_reference)s") + + self._requested_date = None + + # 0 - no + # 1 - maybe + # 2 - yes + # 3 - finished + self._in_results_table = 0 + self._tr_count = 0 + self._td_count = 0 + self._data_list = [] + + # this will hold the application we are currently working on. + self._current_application = None + + # The object which stores our set of planning application results + self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name) + + def _checkAttrsForResultsTable(self, attrs): + raise SystemError + + def handle_starttag(self, tag, attrs): + if self.debug: + print tag, attrs + if tag == "table" and self._in_results_table == 0: + self._in_results_table = 1 + self._checkAttrsForResultsTable(attrs) + elif tag == "tr" and self._in_results_table == 2: + self._tr_count += 1 + self._td_count = 0 + self._data_list = [] + self._current_application = PlanningApplication() + + elif tag == "td" and self._in_results_table == 2: + self._td_count += 1 + + elif tag == "a" and self._in_results_table == 2 and self._td_count == self._reference_col_no: + # The href attribute contains the link to the info page + for (key, value) in attrs: + if key == "href": + self._current_application.info_url = urlparse.urljoin(self.search_url, value) + + def handle_endtag(self, tag): + if self.debug: + print "ending: " , tag + + if tag == "table" and self._in_results_table == 2: + self._in_results_table = 3 + elif tag == "tr" and self._in_results_table == 2: + if self._current_application.council_reference is not None: + + # get the received date + #info_response = urllib2.urlopen(self._current_application.info_url) + #info_page_parser = InfoPageParser() + #info_page_parser.feed(info_response.read()) + self._current_application.date_received = self._requested_date#info_page_parser.date_received + self._results.addApplication(self._current_application) + elif tag == "td" and self._in_results_table == 2: + if self._td_count == self._location_col_no: + data = ' '.join(self._data_list).strip() + self._current_application.address = data + postcode = getPostcodeFromText(data) + if postcode is not None: + self._current_application.postcode = postcode + self._data_list = [] + elif self._td_count == self._description_col_no: + data = ' '.join(self._data_list).strip() + self._current_application.description = data + self._data_list = [] + elif tag == 'a' and self._in_results_table == 2 and self._td_count == self._reference_col_no: + data = ''.join(self._data_list).strip() + self._current_application.council_reference = data + self._current_application.comment_url = self._comment_url %{"council_reference": data} + self._data_list = [] + + def handle_data(self, data): + if self.debug: + print data + + if self._in_results_table == 2: + if self._td_count == self._reference_col_no or \ + self._td_count == self._location_col_no or \ + self._td_count == self._description_col_no: + self._data_list.append(data.strip()) + + + def getResultsByDayMonthYear(self, day, month, year): + """This will return an ApplicationResults object containg the + applications for the date passed in.""" + + # Were going to need a datetime object for the requested date + self._requested_date = datetime.date(year, month, day) + + required_format = "%d-%m-%Y" + + search_data = urllib.urlencode({"ReceivedDateFrom":self._requested_date.strftime(required_format), + "ReceivedDateTo":self._requested_date.strftime(required_format)}) + + search_request = urllib2.Request(self.search_url, search_data) + search_response = urllib2.urlopen(search_request) + search_contents = search_response.read() + + self.feed(search_contents) + + return self._results + + def getResults(self, day, month, year): + return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML() + + +class CoventrySearchParser(ApplicationSearchServletParser): + # results table spotter + # width="100%" border="0" + + _reference_col_no = 1 + _location_col_no = 5 + _description_col_no = 8 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == '100%': + got_width = True + elif key == 'border' and value == '0': + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + + +class AllerdaleSearchParser(ApplicationSearchServletParser): + # results table spotter +#class="nis_table" summary="Table of planning applications that matched your query, showing reference number, received date, and address" + + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_class = False + got_summary = False + + for key, value in attrs: + if key == 'class' and value == 'nis_table': + got_class = True + elif key == 'summary' and value == 'Table of planning applications that matched your query, showing reference number, received date, and address': + got_summary = True + + if got_class and got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + + +class AlnwickSearchParser(ApplicationSearchServletParser): + # results table spotter + # width="100%" class="niscontent" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 7 + + def _checkAttrsForResultsTable(self, attrs): + got_class = False + + for key, value in attrs: + if key == 'class' and value == 'niscontent': + got_class = True + + if got_class: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class BarrowSearchParser(ApplicationSearchServletParser): + # results table spotter + # width="100%" border="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == '100%': + got_width = True + elif key == 'border' and value == '0': + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class HartlepoolSearchParser(ApplicationSearchServletParser): + # results table spotter + # summary="Table of planning applications that matched your query, showing reference number, received date, and address" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 3 + + def _checkAttrsForResultsTable(self, attrs): + got_summary = False + + for key, value in attrs: + if key == 'summary' and value == "Table of planning applications that matched your query, showing reference number, received date, and address": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class NorthWarksSearchParser(ApplicationSearchServletParser): + # results table spotter + # table width="100%" border="0" cellspacing="0" cellpadding="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 4 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + got_cellspacing = False + got_cellpadding = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == '0': + got_border = True + elif key == 'cellspacing' and value == '0': + got_cellspacing = True + elif key == 'cellpadding' and value == '0': + got_cellpadding = True + + if got_width and got_border and got_cellspacing and got_cellpadding: + self._in_results_table = 2 + else: + self._in_results_table = 0 + +class StHelensSearchParser(ApplicationSearchServletParser): + # results table spotter + # summary="Search Results List" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_summary = False + + for key, value in attrs: + if key == 'summary' and value == "Search Results List": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + +class EasingtonSearchParser(ApplicationSearchServletParser): + # results table spotter + #table width="100%" border="0" cellspacing="0" cellpadding="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + got_cellspacing = False + got_cellpadding = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == '0': + got_border = True + elif key == 'cellspacing' and value == '0': + got_cellspacing = True + elif key == 'cellpadding' and value == '0': + got_cellpadding = True + + if got_width and got_border and got_cellspacing and got_cellpadding: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class HighPeakSearchParser(ApplicationSearchServletParser): + # results table spotter + # table class="data" width="95%" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_class = False + got_width = False + + for key, value in attrs: + if key == 'class' and value == "data": + got_class = True + if key == 'width' and value == "95%": + got_width = True + + if got_class and got_width: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class WearValleySearchParser(ApplicationSearchServletParser): + # results table spotter + # table summary="Table of planning applications that matched your query, showing reference number, received date, and address" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 4 + + def _checkAttrsForResultsTable(self, attrs): + got_summary= False + + for key, value in attrs: + if key == 'summary' and value == "Table of planning applications that matched your query, showing reference number, received date, and address": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class WellingboroughSearchParser(ApplicationSearchServletParser): + # results table spotter + #table width="100%" border="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == "0": + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + +class EalingSearchParser(ApplicationSearchServletParser): + # results table spotter + # table width="100%" cellspacing="0px" border="1px" cellpadding="2px" bordercolor="#FFFFFF" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 4 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_cellspacing = False + got_border = False + got_cellpadding = False + got_bordercolor = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'cellspacing' and value == "0px": + got_cellspacing = True + elif key == 'border' and value == "1px": + got_border = True + elif key == 'cellpadding' and value == "2px": + got_cellpadding = True + elif key == 'bordercolor' and value == "#FFFFFF": + got_bordercolor = True + + if got_width and got_cellspacing and got_border and got_cellpadding and got_bordercolor: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class HaringeySearchParser(ApplicationSearchServletParser): + # results table spotter + # summary="Application Results" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_summary= False + + for key, value in attrs: + if key == 'summary' and value == "Application Results": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class DenbighshireSearchParser(ApplicationSearchServletParser): + # results table spotter + #table width="100%" border="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == "0": + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +if __name__ == "__main__": + #parser = CoventrySearchParser("Coventry", "Coventry", "http://planning.coventry.gov.uk") + #print parser.getResults(28,3,2007) + #parser = AllerdaleSearchParser("Allerdale", "Allerdale", "http://planning.allerdale.gov.uk") + #print parser.getResults(28,3,2007) + #parser = AlnwickSearchParser("Alnwick", "Alnwick", "http://services.castlemorpeth.gov.uk:7777") + #print parser.getResults(28,3,2007) + #parser = BarrowSearchParser("Barrow", "Barrow", "http://localportal.barrowbc.gov.uk") + #print parser.getResults(28,3,2007) + #parser = HartlepoolSearchParser("Hartlepool", "Hartlepool", "http://eforms.hartlepool.gov.uk:7777") + #print parser.getResults(28,3,2007) + #parser = NorthWarksSearchParser("North Warwickshire", "North Warks", "http://planning.northwarks.gov.uk") + #print parser.getResults(28,3,2007) + #parser = StHelensSearchParser("St Helens", "St Helens", "http://212.248.225.150:8080") + #print parser.getResults(28,3,2007) + #parser = EasingtonSearchParser("Easington", "Easington", "http://planning.easington.gov.uk") + #print parser.getResults(28,3,2007) + #parser = HighPeakSearchParser("High Peak", "High Peak", "http://planning.highpeak.gov.uk") + #print parser.getResults(20,3,2007) + #parser = WearValleySearchParser("Wear Valley", "Wear Valley", "http://planning.wearvalley.gov.uk") + #print parser.getResults(20,3,2007) + #parser = WellingboroughSearchParser("Wellingborough", "Wellingborough", "http://planning.wellingborough.gov.uk") + #print parser.getResults(20,3,2007) + #parser = EalingSearchParser("Ealing", "Ealing", "http://www.pam.ealing.gov.uk") + #print parser.getResults(20,3,2007) + #parser = HaringeySearchParser("Haringey", "Haringey", "http://www.planningservices.haringey.gov.uk") + #print parser.getResults(20,3,2007) + #parser = DenbighshireSearchParser("Denbighshire", "Denbighshire", "http://planning.denbighshire.gov.uk") + #print parser.getResults(20,3,2007) + pass diff --git a/cgi-bin/Aylesbury Vale.cgi b/cgi-bin/Aylesbury Vale.cgi index 8ca80b5..fae0a61 100755 --- a/cgi-bin/Aylesbury Vale.cgi +++ b/cgi-bin/Aylesbury Vale.cgi @@ -20,9 +20,7 @@ base_url = "http://eplanning.aylesburyvaledc.gov.uk/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Barrow.cgi b/cgi-bin/Barrow.cgi new file mode 100755 index 0000000..c8232c5 --- /dev/null +++ b/cgi-bin/Barrow.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Barrow Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Barrow Borough Council" +authority_short_name = "Barrow" +base_url = "http://localportal.barrowbc.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.BarrowSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Basildon.cgi b/cgi-bin/Basildon.cgi index 0c71df9..00df2d9 100755 --- a/cgi-bin/Basildon.cgi +++ b/cgi-bin/Basildon.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.basildon.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Bath.cgi b/cgi-bin/Bath.cgi index 85bcd1e..c68eff9 100755 --- a/cgi-bin/Bath.cgi +++ b/cgi-bin/Bath.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.bathnes.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Bexley.cgi b/cgi-bin/Bexley.cgi index 6b3fa65..304d19b 100755 --- a/cgi-bin/Bexley.cgi +++ b/cgi-bin/Bexley.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.bexley.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Blaby.cgi b/cgi-bin/Blaby.cgi index e6d8e7f..0d66595 100755 --- a/cgi-bin/Blaby.cgi +++ b/cgi-bin/Blaby.cgi @@ -1,4 +1,4 @@ -#!/opt/local/bin/python +#!/usr/local/bin/python # This is the parser for Blaby District Council. # it is generated from the file CGITemplate @@ -20,9 +20,7 @@ base_url = "http://www.blaby.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Bristol.cgi b/cgi-bin/Bristol.cgi index 01aa840..3b29e13 100755 --- a/cgi-bin/Bristol.cgi +++ b/cgi-bin/Bristol.cgi @@ -20,9 +20,7 @@ base_url = "http://e2eweb.bristol-city.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Buckinghamshire.cgi b/cgi-bin/Buckinghamshire.cgi index 911663f..3d50799 100755 --- a/cgi-bin/Buckinghamshire.cgi +++ b/cgi-bin/Buckinghamshire.cgi @@ -20,9 +20,7 @@ base_url = "http://www.bucksplanning.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Chelmsford.cgi b/cgi-bin/Chelmsford.cgi index d6c3d7f..4bd1472 100755 --- a/cgi-bin/Chelmsford.cgi +++ b/cgi-bin/Chelmsford.cgi @@ -20,9 +20,7 @@ base_url = "http://web1.chelmsfordbc.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Cherwell.cgi b/cgi-bin/Cherwell.cgi index 3b7b8cf..ac916e7 100755 --- a/cgi-bin/Cherwell.cgi +++ b/cgi-bin/Cherwell.cgi @@ -20,9 +20,7 @@ base_url = "http://cherweb.cherwell-dc.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/City of London.cgi b/cgi-bin/City of London.cgi index 76b8cbf..57a2234 100755 --- a/cgi-bin/City of London.cgi +++ b/cgi-bin/City of London.cgi @@ -20,9 +20,7 @@ base_url = "http://www.planning.cityoflondon.gov.uk/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Cornwall.cgi b/cgi-bin/Cornwall.cgi index b5d57fa..cd5b5a5 100755 --- a/cgi-bin/Cornwall.cgi +++ b/cgi-bin/Cornwall.cgi @@ -20,9 +20,7 @@ base_url = "http://planapps.cornwall.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Coventry.cgi b/cgi-bin/Coventry.cgi new file mode 100755 index 0000000..0f8c44b --- /dev/null +++ b/cgi-bin/Coventry.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Coventry City Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Coventry City Council" +authority_short_name = "Coventry" +base_url = "http://planning.coventry.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.CoventrySearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Denbighshire.cgi b/cgi-bin/Denbighshire.cgi new file mode 100755 index 0000000..9cef9fd --- /dev/null +++ b/cgi-bin/Denbighshire.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Denbighshire County Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Denbighshire County Council" +authority_short_name = "Denbighshire" +base_url = "http://planning.denbighshire.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.DenbighshireSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Doncaster.cgi b/cgi-bin/Doncaster.cgi index a7fa1b4..564f342 100755 --- a/cgi-bin/Doncaster.cgi +++ b/cgi-bin/Doncaster.cgi @@ -20,9 +20,7 @@ base_url = "http://maps.doncaster.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Dundee.cgi b/cgi-bin/Dundee.cgi index d87650e..04a238b 100755 --- a/cgi-bin/Dundee.cgi +++ b/cgi-bin/Dundee.cgi @@ -20,9 +20,7 @@ base_url = "http://bwarrant.dundeecity.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Ealing.cgi b/cgi-bin/Ealing.cgi new file mode 100755 index 0000000..53fb009 --- /dev/null +++ b/cgi-bin/Ealing.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Ealing Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Ealing Council" +authority_short_name = "Ealing" +base_url = "http://www.pam.ealing.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.EalingSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Easington.cgi b/cgi-bin/Easington.cgi new file mode 100755 index 0000000..ef12a77 --- /dev/null +++ b/cgi-bin/Easington.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for District of Easington. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "District of Easington" +authority_short_name = "Easington" +base_url = "http://planning.easington.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.EasingtonSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/East Devon.cgi b/cgi-bin/East Devon.cgi index 3e74149..1171e41 100755 --- a/cgi-bin/East Devon.cgi +++ b/cgi-bin/East Devon.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.eastdevon.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Edinburgh.cgi b/cgi-bin/Edinburgh.cgi index d3686b4..bc94611 100755 --- a/cgi-bin/Edinburgh.cgi +++ b/cgi-bin/Edinburgh.cgi @@ -20,9 +20,7 @@ base_url = "http://citydev-portal.edinburgh.gov.uk/publicaccess/dc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Epsom and Ewell.cgi b/cgi-bin/Epsom and Ewell.cgi index f47a595..3cf0346 100755 --- a/cgi-bin/Epsom and Ewell.cgi +++ b/cgi-bin/Epsom and Ewell.cgi @@ -20,9 +20,7 @@ base_url = "http://eplanning.epsom-ewell.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Fenland.cgi b/cgi-bin/Fenland.cgi index b4eba71..e069dd0 100755 --- a/cgi-bin/Fenland.cgi +++ b/cgi-bin/Fenland.cgi @@ -20,9 +20,7 @@ base_url = "http://www.fenland.gov.uk/publicaccess/dc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Gedling.cgi b/cgi-bin/Gedling.cgi index 70b619b..0a92dad 100755 --- a/cgi-bin/Gedling.cgi +++ b/cgi-bin/Gedling.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.gedling.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Hammersmith and Fulham.cgi b/cgi-bin/Hammersmith and Fulham.cgi index 74739b7..15d6c49 100755 --- a/cgi-bin/Hammersmith and Fulham.cgi +++ b/cgi-bin/Hammersmith and Fulham.cgi @@ -20,9 +20,7 @@ base_url = "http://www.apps.lbhf.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Haringey.cgi b/cgi-bin/Haringey.cgi new file mode 100755 index 0000000..478b564 --- /dev/null +++ b/cgi-bin/Haringey.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Haringey Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Haringey Council" +authority_short_name = "Haringey" +base_url = "http://www.planningservices.haringey.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.HaringeySearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Harrogate.cgi b/cgi-bin/Harrogate.cgi index a5ed82d..9001d50 100755 --- a/cgi-bin/Harrogate.cgi +++ b/cgi-bin/Harrogate.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.harrogate.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Hart.cgi b/cgi-bin/Hart.cgi index 2140a03..4a400b2 100755 --- a/cgi-bin/Hart.cgi +++ b/cgi-bin/Hart.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.hart.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Hartlepool.cgi b/cgi-bin/Hartlepool.cgi new file mode 100755 index 0000000..d3771d4 --- /dev/null +++ b/cgi-bin/Hartlepool.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Hartlepool Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Hartlepool Borough Council" +authority_short_name = "Hartlepool" +base_url = "http://eforms.hartlepool.gov.uk:7777/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.HartlepoolSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/High Peak.cgi b/cgi-bin/High Peak.cgi new file mode 100755 index 0000000..91c05f8 --- /dev/null +++ b/cgi-bin/High Peak.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for High Peak Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "High Peak Borough Council" +authority_short_name = "High Peak" +base_url = "http://planning.highpeak.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.HighPeakSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Huntingdonshire.cgi b/cgi-bin/Huntingdonshire.cgi index d196a38..f3e1aac 100755 --- a/cgi-bin/Huntingdonshire.cgi +++ b/cgi-bin/Huntingdonshire.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.huntsdc.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Kerrier.cgi b/cgi-bin/Kerrier.cgi index 19686a7..0099b24 100755 --- a/cgi-bin/Kerrier.cgi +++ b/cgi-bin/Kerrier.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.kerrier.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Knowsley.cgi b/cgi-bin/Knowsley.cgi index 066d64b..fb4ddef 100755 --- a/cgi-bin/Knowsley.cgi +++ b/cgi-bin/Knowsley.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.knowsley.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Lancaster.cgi b/cgi-bin/Lancaster.cgi index aefa5a3..545b367 100755 --- a/cgi-bin/Lancaster.cgi +++ b/cgi-bin/Lancaster.cgi @@ -20,9 +20,7 @@ base_url = "http://planapps.lancaster.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Luton.cgi b/cgi-bin/Luton.cgi index 468149c..b209e65 100755 --- a/cgi-bin/Luton.cgi +++ b/cgi-bin/Luton.cgi @@ -20,9 +20,7 @@ base_url = "http://www.eplan.luton.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Malvern Hills.cgi b/cgi-bin/Malvern Hills.cgi index 9471114..3d496dd 100755 --- a/cgi-bin/Malvern Hills.cgi +++ b/cgi-bin/Malvern Hills.cgi @@ -20,9 +20,7 @@ base_url = "http://public.malvernhills.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Mid Devon.cgi b/cgi-bin/Mid Devon.cgi index b1f49b7..33ed21c 100755 --- a/cgi-bin/Mid Devon.cgi +++ b/cgi-bin/Mid Devon.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.middevon.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Milton Keynes.cgi b/cgi-bin/Milton Keynes.cgi index f35a337..ea86713 100755 --- a/cgi-bin/Milton Keynes.cgi +++ b/cgi-bin/Milton Keynes.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.milton-keynes.gov.uk/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/North Tyneside.cgi b/cgi-bin/North Tyneside.cgi index 181d8e9..f6218b7 100755 --- a/cgi-bin/North Tyneside.cgi +++ b/cgi-bin/North Tyneside.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.northtyneside.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/North Warwickshire.cgi b/cgi-bin/North Warwickshire.cgi new file mode 100755 index 0000000..9fabe83 --- /dev/null +++ b/cgi-bin/North Warwickshire.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for North Warwickshire Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "North Warwickshire Borough Council" +authority_short_name = "North Warwickshire" +base_url = "http://planning.northwarks.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.NorthWarksSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Northumberland.cgi b/cgi-bin/Northumberland.cgi index 8d103a9..e8d490e 100755 --- a/cgi-bin/Northumberland.cgi +++ b/cgi-bin/Northumberland.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.northumberland.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Oadby and Wigston.cgi b/cgi-bin/Oadby and Wigston.cgi index 0a6ce85..719be02 100755 --- a/cgi-bin/Oadby and Wigston.cgi +++ b/cgi-bin/Oadby and Wigston.cgi @@ -20,9 +20,7 @@ base_url = "http://web.owbc.net/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Oswestry.cgi b/cgi-bin/Oswestry.cgi index 9899dec..dce6990 100755 --- a/cgi-bin/Oswestry.cgi +++ b/cgi-bin/Oswestry.cgi @@ -20,9 +20,7 @@ base_url = "http://193.114.205.78/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Peterborough.cgi b/cgi-bin/Peterborough.cgi index c6ca841..94e4a0a 100755 --- a/cgi-bin/Peterborough.cgi +++ b/cgi-bin/Peterborough.cgi @@ -20,9 +20,7 @@ base_url = "http://194.72.246.15/publicaccess/dc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/PlanningUtils.py b/cgi-bin/PlanningUtils.py index 3430576..8e12412 100644 --- a/cgi-bin/PlanningUtils.py +++ b/cgi-bin/PlanningUtils.py @@ -56,7 +56,7 @@ class PlanningAuthorityResults: def __repr__(self): return self.displayXML() - + def displayXML(self): """This should display the contents of this object in the planningalerts format. i.e. in the same format as this one: @@ -88,7 +88,7 @@ class PlanningApplication: def __repr__(self): return self.displayXML() - + def displayXML(self): return "\n" +\ "%s\n" %xmlQuote(self.council_reference) +\ diff --git a/cgi-bin/Portsmouth.cgi b/cgi-bin/Portsmouth.cgi index 471551c..04b1e73 100755 --- a/cgi-bin/Portsmouth.cgi +++ b/cgi-bin/Portsmouth.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.portsmouth.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Rushmoor.cgi b/cgi-bin/Rushmoor.cgi index 230fd72..3f64c6d 100755 --- a/cgi-bin/Rushmoor.cgi +++ b/cgi-bin/Rushmoor.cgi @@ -20,9 +20,7 @@ base_url = "http://pa-dc.rushmoor.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Scarborough.cgi b/cgi-bin/Scarborough.cgi index 04e0926..267da5e 100755 --- a/cgi-bin/Scarborough.cgi +++ b/cgi-bin/Scarborough.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.scarborough.gov.uk/publicaccess/dc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Sevenoaks.cgi b/cgi-bin/Sevenoaks.cgi index 12fb823..44169f1 100755 --- a/cgi-bin/Sevenoaks.cgi +++ b/cgi-bin/Sevenoaks.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.sevenoaks.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/South Bucks.cgi b/cgi-bin/South Bucks.cgi index cae4e3d..083f42e 100755 --- a/cgi-bin/South Bucks.cgi +++ b/cgi-bin/South Bucks.cgi @@ -20,9 +20,7 @@ base_url = "http://sbdc-paweb.southbucks.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/South Ribble.cgi b/cgi-bin/South Ribble.cgi index a121728..47b1339 100755 --- a/cgi-bin/South Ribble.cgi +++ b/cgi-bin/South Ribble.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.southribble.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/South Staffordshire.cgi b/cgi-bin/South Staffordshire.cgi index 4501389..de2d282 100755 --- a/cgi-bin/South Staffordshire.cgi +++ b/cgi-bin/South Staffordshire.cgi @@ -20,9 +20,7 @@ base_url = "https://services.sstaffs.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Southampton.cgi b/cgi-bin/Southampton.cgi index f975f9d..ef76c19 100755 --- a/cgi-bin/Southampton.cgi +++ b/cgi-bin/Southampton.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.southampton.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Spelthorne.cgi b/cgi-bin/Spelthorne.cgi index 84703d7..09d5a96 100755 --- a/cgi-bin/Spelthorne.cgi +++ b/cgi-bin/Spelthorne.cgi @@ -20,9 +20,7 @@ base_url = "http://phoenix.spelthorne.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/St Helens.cgi b/cgi-bin/St Helens.cgi new file mode 100755 index 0000000..400756d --- /dev/null +++ b/cgi-bin/St Helens.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for St Helens Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "St Helens Council" +authority_short_name = "St Helens" +base_url = "http://212.248.225.150:8080/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.StHelensSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Stevenage.cgi b/cgi-bin/Stevenage.cgi index f62ff60..858bf66 100755 --- a/cgi-bin/Stevenage.cgi +++ b/cgi-bin/Stevenage.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.stevenage.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Stockton-On-Tees.cgi b/cgi-bin/Stockton-On-Tees.cgi index 1500f76..7c06c3c 100755 --- a/cgi-bin/Stockton-On-Tees.cgi +++ b/cgi-bin/Stockton-On-Tees.cgi @@ -20,9 +20,7 @@ base_url = "http://www.developmentcontrol.stockton.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Sunderland.cgi b/cgi-bin/Sunderland.cgi index 0fe8a2b..c4a0a94 100755 --- a/cgi-bin/Sunderland.cgi +++ b/cgi-bin/Sunderland.cgi @@ -20,9 +20,7 @@ base_url = "http://www.sunderland.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Test Valley.cgi b/cgi-bin/Test Valley.cgi index 09c1613..db59e33 100755 --- a/cgi-bin/Test Valley.cgi +++ b/cgi-bin/Test Valley.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.testvalley.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Tonbridge.cgi b/cgi-bin/Tonbridge.cgi index 9eae9f1..0c42fda 100755 --- a/cgi-bin/Tonbridge.cgi +++ b/cgi-bin/Tonbridge.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.tmbc.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Torbay.cgi b/cgi-bin/Torbay.cgi index bb27cde..ff42303 100755 --- a/cgi-bin/Torbay.cgi +++ b/cgi-bin/Torbay.cgi @@ -20,9 +20,7 @@ base_url = "http://www.torbay.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Waveney.cgi b/cgi-bin/Waveney.cgi index f2cbd8a..f8597e9 100755 --- a/cgi-bin/Waveney.cgi +++ b/cgi-bin/Waveney.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.waveney.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Wear Valley.cgi b/cgi-bin/Wear Valley.cgi new file mode 100755 index 0000000..5481f57 --- /dev/null +++ b/cgi-bin/Wear Valley.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Wear Valley District Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Wear Valley District Council" +authority_short_name = "Wear Valley" +base_url = "http://planning.wearvalley.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.WearValleySearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/Wellingborough.cgi b/cgi-bin/Wellingborough.cgi new file mode 100755 index 0000000..fe69ea9 --- /dev/null +++ b/cgi-bin/Wellingborough.cgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +# This is the parser for Wellingborough Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Wellingborough Borough Council" +authority_short_name = "Wellingborough" +base_url = "http://planning.wellingborough.gov.uk/" + +import ApplicationSearchServletParser + +parser = ApplicationSearchServletParser.WellingboroughSearchParser(authority_name, authority_short_name, base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/cgi-bin/West Lancashire.cgi b/cgi-bin/West Lancashire.cgi index 98c5e51..f654d7f 100755 --- a/cgi-bin/West Lancashire.cgi +++ b/cgi-bin/West Lancashire.cgi @@ -20,9 +20,7 @@ base_url = "http://publicaccess.westlancsdc.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/West Norfolk.cgi b/cgi-bin/West Norfolk.cgi index 53f08a6..51ac067 100755 --- a/cgi-bin/West Norfolk.cgi +++ b/cgi-bin/West Norfolk.cgi @@ -20,9 +20,7 @@ base_url = "http://online.west-norfolk.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/Woking.cgi b/cgi-bin/Woking.cgi index c4900de..95ae4b7 100755 --- a/cgi-bin/Woking.cgi +++ b/cgi-bin/Woking.cgi @@ -20,9 +20,7 @@ base_url = "http://caps.woking.gov.uk/publicaccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/cgi-bin/York.cgi b/cgi-bin/York.cgi index 3c23d2c..774cf06 100755 --- a/cgi-bin/York.cgi +++ b/cgi-bin/York.cgi @@ -20,9 +20,7 @@ base_url = "http://planning.york.gov.uk/PublicAccess/tdc/" import PublicAccess -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = PublicAccess.PublicAccessParser(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/python_scrapers/ApplicationSearchServletParser.py b/python_scrapers/ApplicationSearchServletParser.py new file mode 100644 index 0000000..b1db56f --- /dev/null +++ b/python_scrapers/ApplicationSearchServletParser.py @@ -0,0 +1,518 @@ + +import urllib, urllib2 + +import HTMLParser +import urlparse +import datetime, time + + +from PlanningUtils import PlanningAuthorityResults, \ + getPostcodeFromText, \ + PlanningApplication + + +# The search results list will give us reference, location, description, +# and info url of each app. + +# The info page gives us the received date, +# and comment_url + +class ApplicationSearchServletParser(HTMLParser.HTMLParser): + """Parser for ApplicationSearchServlet sites. + """ + + + # These indicate the column of the main table containing this + # piece of information. + # They should be overridden in subclasses + + #self._rows_to_ignore_at_start = None + + _reference_col_no = None + _location_col_no = None + _description_col_no = None + + def __init__(self, + authority_name, + authority_short_name, + base_url, + debug=False): + HTMLParser.HTMLParser.__init__(self) + + self.authority_name = authority_name + self.authority_short_name = authority_short_name + self.base_url = base_url + self.debug = debug + + self.search_url = urlparse.urljoin(self.base_url, "portal/servlets/ApplicationSearchServlet") + + self._comment_url = urlparse.urljoin(self.base_url, "portal/servlets/PlanningComments?REFNO=%(council_reference)s") + + self._requested_date = None + + # 0 - no + # 1 - maybe + # 2 - yes + # 3 - finished + self._in_results_table = 0 + self._tr_count = 0 + self._td_count = 0 + self._data_list = [] + + # this will hold the application we are currently working on. + self._current_application = None + + # The object which stores our set of planning application results + self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name) + + def _checkAttrsForResultsTable(self, attrs): + raise SystemError + + def handle_starttag(self, tag, attrs): + if self.debug: + print tag, attrs + if tag == "table" and self._in_results_table == 0: + self._in_results_table = 1 + self._checkAttrsForResultsTable(attrs) + elif tag == "tr" and self._in_results_table == 2: + self._tr_count += 1 + self._td_count = 0 + self._data_list = [] + self._current_application = PlanningApplication() + + elif tag == "td" and self._in_results_table == 2: + self._td_count += 1 + + elif tag == "a" and self._in_results_table == 2 and self._td_count == self._reference_col_no: + # The href attribute contains the link to the info page + for (key, value) in attrs: + if key == "href": + self._current_application.info_url = urlparse.urljoin(self.search_url, value) + + def handle_endtag(self, tag): + if self.debug: + print "ending: " , tag + + if tag == "table" and self._in_results_table == 2: + self._in_results_table = 3 + elif tag == "tr" and self._in_results_table == 2: + if self._current_application.council_reference is not None: + + # get the received date + #info_response = urllib2.urlopen(self._current_application.info_url) + #info_page_parser = InfoPageParser() + #info_page_parser.feed(info_response.read()) + self._current_application.date_received = self._requested_date#info_page_parser.date_received + self._results.addApplication(self._current_application) + elif tag == "td" and self._in_results_table == 2: + if self._td_count == self._location_col_no: + data = ' '.join(self._data_list).strip() + self._current_application.address = data + postcode = getPostcodeFromText(data) + if postcode is not None: + self._current_application.postcode = postcode + self._data_list = [] + elif self._td_count == self._description_col_no: + data = ' '.join(self._data_list).strip() + self._current_application.description = data + self._data_list = [] + elif tag == 'a' and self._in_results_table == 2 and self._td_count == self._reference_col_no: + data = ''.join(self._data_list).strip() + self._current_application.council_reference = data + self._current_application.comment_url = self._comment_url %{"council_reference": data} + self._data_list = [] + + def handle_data(self, data): + if self.debug: + print data + + if self._in_results_table == 2: + if self._td_count == self._reference_col_no or \ + self._td_count == self._location_col_no or \ + self._td_count == self._description_col_no: + self._data_list.append(data.strip()) + + + def getResultsByDayMonthYear(self, day, month, year): + """This will return an ApplicationResults object containg the + applications for the date passed in.""" + + # Were going to need a datetime object for the requested date + self._requested_date = datetime.date(year, month, day) + + required_format = "%d-%m-%Y" + + search_data = urllib.urlencode({"ReceivedDateFrom":self._requested_date.strftime(required_format), + "ReceivedDateTo":self._requested_date.strftime(required_format)}) + + search_request = urllib2.Request(self.search_url, search_data) + search_response = urllib2.urlopen(search_request) + search_contents = search_response.read() + + self.feed(search_contents) + + return self._results + + def getResults(self, day, month, year): + return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML() + + +class CoventrySearchParser(ApplicationSearchServletParser): + # results table spotter + # width="100%" border="0" + + _reference_col_no = 1 + _location_col_no = 5 + _description_col_no = 8 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == '100%': + got_width = True + elif key == 'border' and value == '0': + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + + +class AllerdaleSearchParser(ApplicationSearchServletParser): + # results table spotter +#class="nis_table" summary="Table of planning applications that matched your query, showing reference number, received date, and address" + + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_class = False + got_summary = False + + for key, value in attrs: + if key == 'class' and value == 'nis_table': + got_class = True + elif key == 'summary' and value == 'Table of planning applications that matched your query, showing reference number, received date, and address': + got_summary = True + + if got_class and got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + + +class AlnwickSearchParser(ApplicationSearchServletParser): + # results table spotter + # width="100%" class="niscontent" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 7 + + def _checkAttrsForResultsTable(self, attrs): + got_class = False + + for key, value in attrs: + if key == 'class' and value == 'niscontent': + got_class = True + + if got_class: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class BarrowSearchParser(ApplicationSearchServletParser): + # results table spotter + # width="100%" border="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == '100%': + got_width = True + elif key == 'border' and value == '0': + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class HartlepoolSearchParser(ApplicationSearchServletParser): + # results table spotter + # summary="Table of planning applications that matched your query, showing reference number, received date, and address" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 3 + + def _checkAttrsForResultsTable(self, attrs): + got_summary = False + + for key, value in attrs: + if key == 'summary' and value == "Table of planning applications that matched your query, showing reference number, received date, and address": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class NorthWarksSearchParser(ApplicationSearchServletParser): + # results table spotter + # table width="100%" border="0" cellspacing="0" cellpadding="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 4 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + got_cellspacing = False + got_cellpadding = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == '0': + got_border = True + elif key == 'cellspacing' and value == '0': + got_cellspacing = True + elif key == 'cellpadding' and value == '0': + got_cellpadding = True + + if got_width and got_border and got_cellspacing and got_cellpadding: + self._in_results_table = 2 + else: + self._in_results_table = 0 + +class StHelensSearchParser(ApplicationSearchServletParser): + # results table spotter + # summary="Search Results List" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_summary = False + + for key, value in attrs: + if key == 'summary' and value == "Search Results List": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + +class EasingtonSearchParser(ApplicationSearchServletParser): + # results table spotter + #table width="100%" border="0" cellspacing="0" cellpadding="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + got_cellspacing = False + got_cellpadding = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == '0': + got_border = True + elif key == 'cellspacing' and value == '0': + got_cellspacing = True + elif key == 'cellpadding' and value == '0': + got_cellpadding = True + + if got_width and got_border and got_cellspacing and got_cellpadding: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class HighPeakSearchParser(ApplicationSearchServletParser): + # results table spotter + # table class="data" width="95%" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_class = False + got_width = False + + for key, value in attrs: + if key == 'class' and value == "data": + got_class = True + if key == 'width' and value == "95%": + got_width = True + + if got_class and got_width: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class WearValleySearchParser(ApplicationSearchServletParser): + # results table spotter + # table summary="Table of planning applications that matched your query, showing reference number, received date, and address" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 4 + + def _checkAttrsForResultsTable(self, attrs): + got_summary= False + + for key, value in attrs: + if key == 'summary' and value == "Table of planning applications that matched your query, showing reference number, received date, and address": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class WellingboroughSearchParser(ApplicationSearchServletParser): + # results table spotter + #table width="100%" border="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 6 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == "0": + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + +class EalingSearchParser(ApplicationSearchServletParser): + # results table spotter + # table width="100%" cellspacing="0px" border="1px" cellpadding="2px" bordercolor="#FFFFFF" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 4 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_cellspacing = False + got_border = False + got_cellpadding = False + got_bordercolor = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'cellspacing' and value == "0px": + got_cellspacing = True + elif key == 'border' and value == "1px": + got_border = True + elif key == 'cellpadding' and value == "2px": + got_cellpadding = True + elif key == 'bordercolor' and value == "#FFFFFF": + got_bordercolor = True + + if got_width and got_cellspacing and got_border and got_cellpadding and got_bordercolor: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class HaringeySearchParser(ApplicationSearchServletParser): + # results table spotter + # summary="Application Results" + _reference_col_no = 1 + _location_col_no = 2 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_summary= False + + for key, value in attrs: + if key == 'summary' and value == "Application Results": + got_summary = True + + if got_summary: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +class DenbighshireSearchParser(ApplicationSearchServletParser): + # results table spotter + #table width="100%" border="0" + _reference_col_no = 1 + _location_col_no = 3 + _description_col_no = 5 + + def _checkAttrsForResultsTable(self, attrs): + got_width = False + got_border = False + + for key, value in attrs: + if key == 'width' and value == "100%": + got_width = True + elif key == 'border' and value == "0": + got_border = True + + if got_width and got_border: + self._in_results_table = 2 + else: + self._in_results_table = 0 + + +if __name__ == "__main__": + #parser = CoventrySearchParser("Coventry", "Coventry", "http://planning.coventry.gov.uk") + #print parser.getResults(28,3,2007) + #parser = AllerdaleSearchParser("Allerdale", "Allerdale", "http://planning.allerdale.gov.uk") + #print parser.getResults(28,3,2007) + #parser = AlnwickSearchParser("Alnwick", "Alnwick", "http://services.castlemorpeth.gov.uk:7777") + #print parser.getResults(28,3,2007) + #parser = BarrowSearchParser("Barrow", "Barrow", "http://localportal.barrowbc.gov.uk") + #print parser.getResults(28,3,2007) + #parser = HartlepoolSearchParser("Hartlepool", "Hartlepool", "http://eforms.hartlepool.gov.uk:7777") + #print parser.getResults(28,3,2007) + #parser = NorthWarksSearchParser("North Warwickshire", "North Warks", "http://planning.northwarks.gov.uk") + #print parser.getResults(28,3,2007) + #parser = StHelensSearchParser("St Helens", "St Helens", "http://212.248.225.150:8080") + #print parser.getResults(28,3,2007) + #parser = EasingtonSearchParser("Easington", "Easington", "http://planning.easington.gov.uk") + #print parser.getResults(28,3,2007) + #parser = HighPeakSearchParser("High Peak", "High Peak", "http://planning.highpeak.gov.uk") + #print parser.getResults(20,3,2007) + #parser = WearValleySearchParser("Wear Valley", "Wear Valley", "http://planning.wearvalley.gov.uk") + #print parser.getResults(20,3,2007) + #parser = WellingboroughSearchParser("Wellingborough", "Wellingborough", "http://planning.wellingborough.gov.uk") + #print parser.getResults(20,3,2007) + #parser = EalingSearchParser("Ealing", "Ealing", "http://www.pam.ealing.gov.uk") + #print parser.getResults(20,3,2007) + #parser = HaringeySearchParser("Haringey", "Haringey", "http://www.planningservices.haringey.gov.uk") + #print parser.getResults(20,3,2007) + #parser = DenbighshireSearchParser("Denbighshire", "Denbighshire", "http://planning.denbighshire.gov.uk") + #print parser.getResults(20,3,2007) + pass diff --git a/python_scrapers/CGITemplate b/python_scrapers/CGITemplate index e72f31e..080d016 100644 --- a/python_scrapers/CGITemplate +++ b/python_scrapers/CGITemplate @@ -16,11 +16,9 @@ authority_name = "%(authority_name)s" authority_short_name = "%(authority_short_name)s" base_url = "%(base_url)s" -import PublicAccess +import %(module)s -parser = PublicAccess.PublicAccessParser(authority_name, - authority_short_name, - base_url) +parser = %(module)s.%(parser)s(authority_name, authority_short_name, base_url) xml = parser.getResults(day, month, year) diff --git a/python_scrapers/OtherFilesToCopy.csv b/python_scrapers/OtherFilesToCopy.csv index ee81dd1..09922e0 100644 --- a/python_scrapers/OtherFilesToCopy.csv +++ b/python_scrapers/OtherFilesToCopy.csv @@ -3,3 +3,5 @@ "PlanningUtils.py", "420" "SouthOxfordshireParser.py", "420" "SouthOxfordshire.cgi", "493" +"ApplicationSearchServletParser.py", "420" + diff --git a/python_scrapers/PlanningUtils.py b/python_scrapers/PlanningUtils.py index 3430576..8e12412 100644 --- a/python_scrapers/PlanningUtils.py +++ b/python_scrapers/PlanningUtils.py @@ -56,7 +56,7 @@ class PlanningAuthorityResults: def __repr__(self): return self.displayXML() - + def displayXML(self): """This should display the contents of this object in the planningalerts format. i.e. in the same format as this one: @@ -88,7 +88,7 @@ class PlanningApplication: def __repr__(self): return self.displayXML() - + def displayXML(self): return "\n" +\ "%s\n" %xmlQuote(self.council_reference) +\ diff --git a/python_scrapers/PublicAccessSites.csv b/python_scrapers/PublicAccessSites.csv index 588ccf0..e1d64de 100644 --- a/python_scrapers/PublicAccessSites.csv +++ b/python_scrapers/PublicAccessSites.csv @@ -1,56 +1,70 @@ -"authority_name", "authority_short_name", "base_url" -"City of York Council", "York", "http://planning.york.gov.uk/PublicAccess/tdc/" -"Cherwell District Council", "Cherwell", "http://cherweb.cherwell-dc.gov.uk/publicaccess/tdc/" -"Angus Council", "Angus", "http://planning.angus.gov.uk/PublicAccess/tdc/" -"Huntingdonshire District Council", "Huntingdonshire", "http://planning.huntsdc.gov.uk/publicaccess/tdc/" -"South Staffordshire Council", "South Staffordshire", "https://services.sstaffs.gov.uk/PublicAccess/tdc/" -"Bexley Council", "Bexley", "http://publicaccess.bexley.gov.uk/publicaccess/tdc/" -"Lancaster City Council", "Lancaster", "http://planapps.lancaster.gov.uk/PublicAccess/tdc/" -"Bristol City Council", "Bristol", "http://e2eweb.bristol-city.gov.uk/publicaccess/tdc/" -"Portsmouth City Council", "Portsmouth", "http://planning.portsmouth.gov.uk/PublicAccess/tdc/" -"The Borough of Oadby and Wigston", "Oadby and Wigston", "http://web.owbc.net/PublicAccess/tdc/" -"Test Valley Borough Council", "Test Valley", "http://publicaccess.testvalley.gov.uk/publicaccess/tdc/" -"Kings Lynn and West Norfolk Borough Council", "West Norfolk", "http://online.west-norfolk.gov.uk/publicaccess/tdc/" -"Sunderland City Council", "Sunderland", "http://www.sunderland.gov.uk/publicaccess/tdc/" -"Southampton City Council", "Southampton", "http://publicaccess.southampton.gov.uk/publicaccess/tdc/" -"Bath and North East Somerset", "Bath", "http://planning.bathnes.gov.uk/publicaccess/tdc/" -"Buckinghamshire County Council", "Buckinghamshire", "http://www.bucksplanning.gov.uk/PublicAccess/tdc/" -"Spelthorne Borough Council", "Spelthorne", "http://phoenix.spelthorne.gov.uk/PublicAccess/tdc/" -"Stevenage Borough Council", "Stevenage", "http://publicaccess.stevenage.gov.uk/publicaccess/tdc/" -"Tonbridge and Malling Borough Council", "Tonbridge", "http://publicaccess.tmbc.gov.uk/publicaccess/tdc/" -"Hart District Council", "Hart", "http://publicaccess.hart.gov.uk/publicaccess/tdc/" -"Luton Borough Council", "Luton", "http://www.eplan.luton.gov.uk/PublicAccess/tdc/" -"Rushmoor Borough Council", "Rushmoor", "http://pa-dc.rushmoor.gov.uk/publicaccess/tdc/" -"Blaby District Council", "Blaby", "http://www.blaby.gov.uk/PublicAccess/tdc/" -"East Devon District Council", "East Devon", "http://planning.eastdevon.gov.uk/PublicAccess/tdc/" -"Mid Devon District Council", "Mid Devon", "http://planning.middevon.gov.uk/publicaccess/tdc/" -"Sevenoaks District Council", "Sevenoaks", "http://publicaccess.sevenoaks.gov.uk/publicaccess/tdc/" -"Woking Borough Council", "Woking", "http://caps.woking.gov.uk/publicaccess/tdc/" -"Basildon District Council", "Basildon", "http://planning.basildon.gov.uk/publicaccess/tdc/" -"The City of Edinburgh Council", "Edinburgh", "http://citydev-portal.edinburgh.gov.uk/publicaccess/dc/" -"Fenland District Council", "Fenland", "http://www.fenland.gov.uk/publicaccess/dc/" -"Scarborough Borough Council", "Scarborough", "http://planning.scarborough.gov.uk/publicaccess/dc/" -"Harrogate Borough Council", "Harrogate", "http://publicaccess.harrogate.gov.uk/publicaccess/tdc/" -"Kerrier District Council", "Kerrier", "http://publicaccess.kerrier.gov.uk/publicaccess/tdc/" -"Oswestry Borough Council", "Oswestry", "http://193.114.205.78/PublicAccess/tdc/" -"Stockton-On-Tees Borough Council", "Stockton-On-Tees", "http://www.developmentcontrol.stockton.gov.uk/publicaccess/tdc/" -"Doncaster Metropolitan Borough Council", "Doncaster", "http://maps.doncaster.gov.uk/publicaccess/tdc/" -"Waveney District Council", "Waveney", "http://publicaccess.waveney.gov.uk/PublicAccess/tdc/" -"Chelmsford Borough Council", "Chelmsford", "http://web1.chelmsfordbc.gov.uk/publicaccess/tdc/" -"Knowsley Council", "Knowsley", "http://publicaccess.knowsley.gov.uk/PublicAccess/tdc/" -"North Tyneside Council", "North Tyneside", "http://publicaccess.northtyneside.gov.uk/PublicAccess/tdc/" -"City of London", "City of London", "http://www.planning.cityoflondon.gov.uk/tdc/" -"London Borough Of Hammersmith and Fulham", "Hammersmith and Fulham", "http://www.apps.lbhf.gov.uk/PublicAccess/tdc/" -"Aylesbury Vale District Council", "Aylesbury Vale", "http://eplanning.aylesburyvaledc.gov.uk/tdc/" -"Epsom and Ewell Borough Council", "Epsom and Ewell", "http://eplanning.epsom-ewell.gov.uk/publicaccess/tdc/" -"Gedling Borough Council", "Gedling", "http://publicaccess.gedling.gov.uk/publicaccess/tdc/" -"Cornwall County Council", "Cornwall", "http://planapps.cornwall.gov.uk/publicaccess/tdc/" -"South Bucks District Council", "South Bucks", "http://sbdc-paweb.southbucks.gov.uk/publicaccess/tdc/" -"Malvern Hills District Council", "Malvern Hills", "http://public.malvernhills.gov.uk/publicaccess/tdc/" -"West Lancashire District Council", "West Lancashire", "http://publicaccess.westlancsdc.gov.uk/PublicAccess/tdc/" -"Torbay Council", "Torbay", "http://www.torbay.gov.uk/publicaccess/tdc/" -"South Ribble Borough Council", "South Ribble", "http://publicaccess.southribble.gov.uk/publicaccess/tdc/" -"Peterborough City Council", "Peterborough", "http://194.72.246.15/publicaccess/dc/" -"Dundee City Council", "Dundee", "http://bwarrant.dundeecity.gov.uk/publicaccess/tdc/" -"Northumberland County Council", "Northumberland", "http://planning.northumberland.gov.uk/publicaccess/tdc/" -"Milton Keynes Council", "Milton Keynes", "http://publicaccess.milton-keynes.gov.uk/tdc/" +"authority_name", "authority_short_name", "base_url", "module", "parser" +"City of York Council", "York", "http://planning.york.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser", "PublicAccess", "PublicAccessParser" +"Cherwell District Council", "Cherwell", "http://cherweb.cherwell-dc.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Angus Council", "Angus", "http://planning.angus.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Huntingdonshire District Council", "Huntingdonshire", "http://planning.huntsdc.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"South Staffordshire Council", "South Staffordshire", "https://services.sstaffs.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Bexley Council", "Bexley", "http://publicaccess.bexley.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Lancaster City Council", "Lancaster", "http://planapps.lancaster.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Bristol City Council", "Bristol", "http://e2eweb.bristol-city.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Portsmouth City Council", "Portsmouth", "http://planning.portsmouth.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"The Borough of Oadby and Wigston", "Oadby and Wigston", "http://web.owbc.net/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Test Valley Borough Council", "Test Valley", "http://publicaccess.testvalley.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Kings Lynn and West Norfolk Borough Council", "West Norfolk", "http://online.west-norfolk.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Sunderland City Council", "Sunderland", "http://www.sunderland.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Southampton City Council", "Southampton", "http://publicaccess.southampton.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Bath and North East Somerset", "Bath", "http://planning.bathnes.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Buckinghamshire County Council", "Buckinghamshire", "http://www.bucksplanning.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Spelthorne Borough Council", "Spelthorne", "http://phoenix.spelthorne.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Stevenage Borough Council", "Stevenage", "http://publicaccess.stevenage.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Tonbridge and Malling Borough Council", "Tonbridge", "http://publicaccess.tmbc.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Hart District Council", "Hart", "http://publicaccess.hart.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Luton Borough Council", "Luton", "http://www.eplan.luton.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Rushmoor Borough Council", "Rushmoor", "http://pa-dc.rushmoor.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Blaby District Council", "Blaby", "http://www.blaby.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"East Devon District Council", "East Devon", "http://planning.eastdevon.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Mid Devon District Council", "Mid Devon", "http://planning.middevon.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Sevenoaks District Council", "Sevenoaks", "http://publicaccess.sevenoaks.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Woking Borough Council", "Woking", "http://caps.woking.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Basildon District Council", "Basildon", "http://planning.basildon.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"The City of Edinburgh Council", "Edinburgh", "http://citydev-portal.edinburgh.gov.uk/publicaccess/dc/", "PublicAccess", "PublicAccessParser" +"Fenland District Council", "Fenland", "http://www.fenland.gov.uk/publicaccess/dc/", "PublicAccess", "PublicAccessParser" +"Scarborough Borough Council", "Scarborough", "http://planning.scarborough.gov.uk/publicaccess/dc/", "PublicAccess", "PublicAccessParser" +"Harrogate Borough Council", "Harrogate", "http://publicaccess.harrogate.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Kerrier District Council", "Kerrier", "http://publicaccess.kerrier.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Oswestry Borough Council", "Oswestry", "http://193.114.205.78/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Stockton-On-Tees Borough Council", "Stockton-On-Tees", "http://www.developmentcontrol.stockton.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Doncaster Metropolitan Borough Council", "Doncaster", "http://maps.doncaster.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Waveney District Council", "Waveney", "http://publicaccess.waveney.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Chelmsford Borough Council", "Chelmsford", "http://web1.chelmsfordbc.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Knowsley Council", "Knowsley", "http://publicaccess.knowsley.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"North Tyneside Council", "North Tyneside", "http://publicaccess.northtyneside.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"City of London", "City of London", "http://www.planning.cityoflondon.gov.uk/tdc/", "PublicAccess", "PublicAccessParser" +"London Borough Of Hammersmith and Fulham", "Hammersmith and Fulham", "http://www.apps.lbhf.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Aylesbury Vale District Council", "Aylesbury Vale", "http://eplanning.aylesburyvaledc.gov.uk/tdc/", "PublicAccess", "PublicAccessParser" +"Epsom and Ewell Borough Council", "Epsom and Ewell", "http://eplanning.epsom-ewell.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Gedling Borough Council", "Gedling", "http://publicaccess.gedling.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Cornwall County Council", "Cornwall", "http://planapps.cornwall.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"South Bucks District Council", "South Bucks", "http://sbdc-paweb.southbucks.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Malvern Hills District Council", "Malvern Hills", "http://public.malvernhills.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"West Lancashire District Council", "West Lancashire", "http://publicaccess.westlancsdc.gov.uk/PublicAccess/tdc/", "PublicAccess", "PublicAccessParser" +"Torbay Council", "Torbay", "http://www.torbay.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"South Ribble Borough Council", "South Ribble", "http://publicaccess.southribble.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Peterborough City Council", "Peterborough", "http://194.72.246.15/publicaccess/dc/", "PublicAccess", "PublicAccessParser" +"Dundee City Council", "Dundee", "http://bwarrant.dundeecity.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Northumberland County Council", "Northumberland", "http://planning.northumberland.gov.uk/publicaccess/tdc/", "PublicAccess", "PublicAccessParser" +"Milton Keynes Council", "Milton Keynes", "http://publicaccess.milton-keynes.gov.uk/tdc/", "PublicAccess", "PublicAccessParser" +"Coventry City Council", "Coventry", "http://planning.coventry.gov.uk/", "ApplicationSearchServletParser", "CoventrySearchParser" +"Alnwick District Council", "Alnwick", "http://services.castlemorpeth.gov.uk:7777/", "ApplicationSearchServletParser", "AlnwickSearchParser" +"Haringey Council", "Haringey", "http://www.planningservices.haringey.gov.uk/", "ApplicationSearchServletParser", "HaringeySearchParser" +"Hartlepool Borough Council", "Hartlepool", "http://eforms.hartlepool.gov.uk:7777/", "ApplicationSearchServletParser", "HartlepoolSearchParser" +"North Warwickshire Borough Council", "North Warwickshire", "http://planning.northwarks.gov.uk/", "ApplicationSearchServletParser", "NorthWarksSearchParser" +"St Helens Council", "St Helens", "http://212.248.225.150:8080/", "ApplicationSearchServletParser", "StHelensSearchParser" +"District of Easington", "Easington", "http://planning.easington.gov.uk/", "ApplicationSearchServletParser", "EasingtonSearchParser" +"High Peak Borough Council", "High Peak", "http://planning.highpeak.gov.uk/", "ApplicationSearchServletParser", "HighPeakSearchParser" +"Wellingborough Borough Council", "Wellingborough", "http://planning.wellingborough.gov.uk/", "ApplicationSearchServletParser", "WellingboroughSearchParser" +"Barrow Borough Council", "Barrow", "http://localportal.barrowbc.gov.uk/", "ApplicationSearchServletParser", "BarrowSearchParser" +"Allerdale Borough Council", "Allerdale", "http://planning.allerdale.gov.uk/", "ApplicationSearchServletParser", "AllerdaleSearchParser" +"Ealing Council", "Ealing", "http://www.pam.ealing.gov.uk/", "ApplicationSearchServletParser", "EalingSearchParser" +"Denbighshire County Council", "Denbighshire", "http://planning.denbighshire.gov.uk/", "ApplicationSearchServletParser", "DenbighshireSearchParser" +"Wear Valley District Council", "Wear Valley", "http://planning.wearvalley.gov.uk/", "ApplicationSearchServletParser", "WearValleySearchParser" diff --git a/python_scrapers/createCGI.sh b/python_scrapers/createCGI.sh index 4bf7f4c..ab1069d 100755 --- a/python_scrapers/createCGI.sh +++ b/python_scrapers/createCGI.sh @@ -1,14 +1,14 @@ #!/bin/bash -echo Removing contents of CGI directory -svn rm --force ../CGI/* +echo Removing contents of directory cgi-bin +svn rm --force ../cgi-bin/* echo Running generateCGIScripts python generateCGIScripts.py -svn add ../CGI/* +svn add ../cgi-bin/* #echo Committing changes to svn -#(cd ../CGI ; svn commit -m "Removing and regenerating CGI directory") +#(cd ../cgi-bin ; svn commit -m "Removing and regenerating directory cgi-bin") echo Done diff --git a/python_scrapers/generateCGIScripts.py b/python_scrapers/generateCGIScripts.py index 3c6129e..b597630 100755 --- a/python_scrapers/generateCGIScripts.py +++ b/python_scrapers/generateCGIScripts.py @@ -3,9 +3,9 @@ list_of_sites_filename = "PublicAccessSites.csv" other_files_to_copy_filename = "OtherFilesToCopy.csv" template_filename = "CGITemplate" -python_location = "/usr/bin/python" +python_location = "/usr/local/bin/python" -cgi_dir = "../CGI/" +cgi_dir = "../cgi-bin/" import csv from os import chmod