From a12d443ccf6a148410e8d64beb588056432ec9a0 Mon Sep 17 00:00:00 2001 From: "duncan.parkes" Date: Wed, 4 Apr 2007 00:20:23 +0000 Subject: [PATCH] add some more publicaccess scrapers --- CGI/Aylesbury Vale.cgi | 31 +++++++++++++++++++++++++++ CGI/Chelmsford.cgi | 31 +++++++++++++++++++++++++++ CGI/City of London.cgi | 31 +++++++++++++++++++++++++++ CGI/Epsom and Ewell.cgi | 31 +++++++++++++++++++++++++++ CGI/Gedling.cgi | 31 +++++++++++++++++++++++++++ CGI/Hammersmith and Fulham.cgi | 31 +++++++++++++++++++++++++++ CGI/Knowsley.cgi | 31 +++++++++++++++++++++++++++ CGI/North Tyneside.cgi | 31 +++++++++++++++++++++++++++ CGI/PublicAccess.py | 22 ++++++++++++++++++- python_scrapers/PublicAccess.py | 22 ++++++++++++++++++- python_scrapers/PublicAccessSites.csv | 8 +++++++ 11 files changed, 298 insertions(+), 2 deletions(-) create mode 100755 CGI/Aylesbury Vale.cgi create mode 100755 CGI/Chelmsford.cgi create mode 100755 CGI/City of London.cgi create mode 100755 CGI/Epsom and Ewell.cgi create mode 100755 CGI/Gedling.cgi create mode 100755 CGI/Hammersmith and Fulham.cgi create mode 100755 CGI/Knowsley.cgi create mode 100755 CGI/North Tyneside.cgi diff --git a/CGI/Aylesbury Vale.cgi b/CGI/Aylesbury Vale.cgi new file mode 100755 index 0000000..2a45dca --- /dev/null +++ b/CGI/Aylesbury Vale.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for Aylesbury Vale District Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Aylesbury Vale District Council" +authority_short_name = "Aylesbury Vale" +base_url = "http://eplanning.aylesburyvaledc.gov.uk/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/Chelmsford.cgi b/CGI/Chelmsford.cgi new file mode 100755 index 0000000..5fe96b9 --- /dev/null +++ b/CGI/Chelmsford.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for Chelmsford Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Chelmsford Borough Council" +authority_short_name = "Chelmsford" +base_url = "http://web1.chelmsfordbc.gov.uk/publicaccess/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/City of London.cgi b/CGI/City of London.cgi new file mode 100755 index 0000000..711469b --- /dev/null +++ b/CGI/City of London.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for City of London. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "City of London" +authority_short_name = "City of London" +base_url = "http://www.planning.cityoflondon.gov.uk/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/Epsom and Ewell.cgi b/CGI/Epsom and Ewell.cgi new file mode 100755 index 0000000..1098835 --- /dev/null +++ b/CGI/Epsom and Ewell.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for Epsom and Ewell Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Epsom and Ewell Borough Council" +authority_short_name = "Epsom and Ewell" +base_url = "http://eplanning.epsom-ewell.gov.uk/publicaccess/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/Gedling.cgi b/CGI/Gedling.cgi new file mode 100755 index 0000000..40b3450 --- /dev/null +++ b/CGI/Gedling.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for Gedling Borough Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Gedling Borough Council" +authority_short_name = "Gedling" +base_url = "http://publicaccess.gedling.gov.uk/publicaccess/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/Hammersmith and Fulham.cgi b/CGI/Hammersmith and Fulham.cgi new file mode 100755 index 0000000..27a2a95 --- /dev/null +++ b/CGI/Hammersmith and Fulham.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for London Borough Of Hammersmith and Fulham. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "London Borough Of Hammersmith and Fulham" +authority_short_name = "Hammersmith and Fulham" +base_url = "http://www.apps.lbhf.gov.uk/PublicAccess/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/Knowsley.cgi b/CGI/Knowsley.cgi new file mode 100755 index 0000000..fc3a86f --- /dev/null +++ b/CGI/Knowsley.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for Knowsley Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "Knowsley Council" +authority_short_name = "Knowsley" +base_url = "http://publicaccess.knowsley.gov.uk/PublicAccess/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/North Tyneside.cgi b/CGI/North Tyneside.cgi new file mode 100755 index 0000000..22cb55f --- /dev/null +++ b/CGI/North Tyneside.cgi @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# This is the parser for North Tyneside Council. +# it is generated from the file CGITemplate + +import cgi +import cgitb +#cgitb.enable(display=0, logdir="/tmp") + + +form = cgi.FieldStorage() +day = form.getfirst('day') +month = form.getfirst('month') +year = form.getfirst('year') + + +authority_name = "North Tyneside Council" +authority_short_name = "North Tyneside" +base_url = "http://publicaccess.northtyneside.gov.uk/PublicAccess/tdc/" + +import PublicAccess + +parser = PublicAccess.PublicAccessParser(authority_name, + authority_short_name, + base_url) + +xml = parser.getResults(day, month, year) + +print "Content-Type: text/xml" # XML is following +print +print xml # print the xml diff --git a/CGI/PublicAccess.py b/CGI/PublicAccess.py index bf1aea0..a264689 100644 --- a/CGI/PublicAccess.py +++ b/CGI/PublicAccess.py @@ -346,9 +346,11 @@ class PublicAccessPropertyPageParser(HTMLParser.HTMLParser): #http://193.63.61.22/publicaccess/tdc/DcApplication/application_searchform.aspx #"Perth and Kinross Council", "Perthshire", "http://193.63.61.22/publicaccess/tdc/" -# Hambleton +# These ones give sensible results in a browser, +# but are always empty for my script... #http://planning.hambleton.gov.uk/publicaccess/tdc/DcApplication/application_searchform.aspx #"Hambleton District Council", "Hambleton", "http://planning.hambleton.gov.uk/publicaccess/tdc/" +"Newcastle City Council", "Newcastle", "http://gispublic.newcastle.gov.uk/publicaccess/dc/" # Bromley # http://83.244.199.114/publicaccess/ @@ -359,6 +361,24 @@ class PublicAccessPropertyPageParser(HTMLParser.HTMLParser): #http://ww2.westberks.gov.uk/publicaccess/tdc/tdc_home.aspx # Displays an error message. +# Newham +# doesn't seem to work currently +# http://pacaps.newham.gov.uk/publicaccess/ + +# Stratford +# http://217.33.198.119/publicaccess/default.aspx +# Doesn't seem to work currently + +# Watford +#http://ww3.watford.gov.uk/publicaccess/system_error.aspx?aspxerrorpath=/publicaccess/tdc/tdc_home.aspx +# Doesn't seem to work currently + +# Teignbridge +#http://213.152.63.161/publicaccess/tdc/tdc_home.aspx +# Doesn't seem to work currently + +# Gravesham seems to be broken +#"Gravesham Borough Council", "Gravesham", "http://195.102.67.4/PublicAccess/tdc/" # These use https: # Chiltern diff --git a/python_scrapers/PublicAccess.py b/python_scrapers/PublicAccess.py index bf1aea0..a264689 100644 --- a/python_scrapers/PublicAccess.py +++ b/python_scrapers/PublicAccess.py @@ -346,9 +346,11 @@ class PublicAccessPropertyPageParser(HTMLParser.HTMLParser): #http://193.63.61.22/publicaccess/tdc/DcApplication/application_searchform.aspx #"Perth and Kinross Council", "Perthshire", "http://193.63.61.22/publicaccess/tdc/" -# Hambleton +# These ones give sensible results in a browser, +# but are always empty for my script... #http://planning.hambleton.gov.uk/publicaccess/tdc/DcApplication/application_searchform.aspx #"Hambleton District Council", "Hambleton", "http://planning.hambleton.gov.uk/publicaccess/tdc/" +"Newcastle City Council", "Newcastle", "http://gispublic.newcastle.gov.uk/publicaccess/dc/" # Bromley # http://83.244.199.114/publicaccess/ @@ -359,6 +361,24 @@ class PublicAccessPropertyPageParser(HTMLParser.HTMLParser): #http://ww2.westberks.gov.uk/publicaccess/tdc/tdc_home.aspx # Displays an error message. +# Newham +# doesn't seem to work currently +# http://pacaps.newham.gov.uk/publicaccess/ + +# Stratford +# http://217.33.198.119/publicaccess/default.aspx +# Doesn't seem to work currently + +# Watford +#http://ww3.watford.gov.uk/publicaccess/system_error.aspx?aspxerrorpath=/publicaccess/tdc/tdc_home.aspx +# Doesn't seem to work currently + +# Teignbridge +#http://213.152.63.161/publicaccess/tdc/tdc_home.aspx +# Doesn't seem to work currently + +# Gravesham seems to be broken +#"Gravesham Borough Council", "Gravesham", "http://195.102.67.4/PublicAccess/tdc/" # These use https: # Chiltern diff --git a/python_scrapers/PublicAccessSites.csv b/python_scrapers/PublicAccessSites.csv index 966d235..8a54dd6 100644 --- a/python_scrapers/PublicAccessSites.csv +++ b/python_scrapers/PublicAccessSites.csv @@ -36,3 +36,11 @@ "Stockton-On-Tees Borough Council", "Stockton-On-Tees", "http://www.developmentcontrol.stockton.gov.uk/publicaccess/tdc/" "Doncaster Metropolitan Borough Council", "Doncaster", "http://maps.doncaster.gov.uk/publicaccess/tdc/" "Waveney District Council", "Waveney", "http://publicaccess.waveney.gov.uk/PublicAccess/tdc/" +"Chelmsford Borough Council", "Chelmsford", "http://web1.chelmsfordbc.gov.uk/publicaccess/tdc/" +"Knowsley Council", "Knowsley", "http://publicaccess.knowsley.gov.uk/PublicAccess/tdc/" +"North Tyneside Council", "North Tyneside", "http://publicaccess.northtyneside.gov.uk/PublicAccess/tdc/" +"City of London", "City of London", "http://www.planning.cityoflondon.gov.uk/tdc/" +"London Borough Of Hammersmith and Fulham", "Hammersmith and Fulham", "http://www.apps.lbhf.gov.uk/PublicAccess/tdc/" +"Aylesbury Vale District Council", "Aylesbury Vale", "http://eplanning.aylesburyvaledc.gov.uk/tdc/" +"Epsom and Ewell Borough Council", "Epsom and Ewell", "http://eplanning.epsom-ewell.gov.uk/publicaccess/tdc/" +"Gedling Borough Council", "Gedling", "http://publicaccess.gedling.gov.uk/publicaccess/tdc/"