Ver código fonte

Update python scrapers to cope with publicaccess sites with dc in the

path.
import/raw
duncan.parkes 17 anos atrás
pai
commit
cd9dd408a8
2 arquivos alterados com 53 adições e 31 exclusões
  1. +20
    -3
      trunk/python_scrapers/PublicAccess.py
  2. +33
    -28
      trunk/python_scrapers/PublicAccessSites.csv

+ 20
- 3
trunk/python_scrapers/PublicAccess.py Ver arquivo

@@ -13,9 +13,9 @@ cookie_jar = cookielib.CookieJar()
from PlanningUtils import fixNewlines, PlanningAuthorityResults, PlanningApplication


search_form_url_end = "tdc/DcApplication/application_searchform.aspx"
search_results_url_end = "tdc/DcApplication/application_searchresults.aspx"
comments_url_end = "tdc/DcApplication/application_comments_entryform.aspx"
search_form_url_end = "DcApplication/application_searchform.aspx"
search_results_url_end = "DcApplication/application_searchresults.aspx"
comments_url_end = "DcApplication/application_comments_entryform.aspx"

class PublicAccessParser(HTMLParser.HTMLParser):
"""This is the class which parses the PublicAccess search results page.
@@ -339,3 +339,20 @@ class PublicAccessPropertyPageParser(HTMLParser.HTMLParser):
if attr == "value":
self.postcode = value


# These still don't work:

# Perthshire
#http://193.63.61.22/publicaccess/tdc/DcApplication/application_searchform.aspx
#"Perth and Kinross Council", "Perthshire", "http://193.63.61.22/publicaccess/tdc/"

# Hambleton
#http://planning.hambleton.gov.uk/publicaccess/tdc/DcApplication/application_searchform.aspx
#"Hambleton District Council", "Hambleton", "http://planning.hambleton.gov.uk/publicaccess/tdc/"

# These use https:

# Chiltern
#https://isa.chiltern.gov.uk/publicaccess/tdc/tdc_home.aspx
# Hinckley-Bosworth
#https://cx.hinckley-bosworth.gov.uk/PublicAccess/dc/DcApplication/application_searchform.aspx

+ 33
- 28
trunk/python_scrapers/PublicAccessSites.csv Ver arquivo

@@ -1,29 +1,34 @@
"authority_name", "authority_short_name", "base_url"
"City of York Council", "York", "http://planning.york.gov.uk/PublicAccess/"
"Cherwell District Council", "Cherwell", "http://cherweb.cherwell-dc.gov.uk/publicaccess/"
"Angus Council", "Angus", "http://planning.angus.gov.uk/PublicAccess/"
"Huntingdonshire District Council", "Huntingdonshire", "http://planning.huntsdc.gov.uk/publicaccess/"
"South Staffordshire Council", "South Staffs", "https://services.sstaffs.gov.uk/PublicAccess/"
"Bexley Council", "Bexley", "http://publicaccess.bexley.gov.uk/publicaccess/"
"Lancaster City Council", "Lancaster", "http://planapps.lancaster.gov.uk/PublicAccess/"
"Bristol City Council", "Bristol", "http://e2eweb.bristol-city.gov.uk/publicaccess/"
"Portsmouth City Council", "Portsmouth", "http://planning.portsmouth.gov.uk/PublicAccess/"
"The Borough of Oadby and Wigston", "Oadby and Wigston", "http://web.owbc.net/PublicAccess/"
"Test Valley Borough Council", "Test Valley", "http://publicaccess.testvalley.gov.uk/publicaccess/"
"Kings Lynn and West Norfolk Borough Council", "West Norfolk", "http://online.west-norfolk.gov.uk/publicaccess/"
"Sunderland City Council", "Sunderland", "http://www.sunderland.gov.uk/publicaccess/"
"Southampton City Council", "Southampton", "http://publicaccess.southampton.gov.uk/publicaccess/"
"Bath and North East Somerset", "Bath", "http://planning.bathnes.gov.uk/publicaccess/"
"Buckinghamshire County Council", "Buckinghamshire", "http://www.bucksplanning.gov.uk/PublicAccess/"
"Spelthorne Borough Council", "Spelthorne", "http://phoenix.spelthorne.gov.uk/PublicAccess/"
"Stevenage Borough Council", "Stevenage", "http://publicaccess.stevenage.gov.uk/publicaccess/"
"Tonbridge and Malling Borough Council", "Tonbridge", "http://publicaccess.tmbc.gov.uk/publicaccess/"
"Hart District Council", "Hart", "http://publicaccess.hart.gov.uk/publicaccess/"
"Luton Borough Council", "Luton", "http://www.eplan.luton.gov.uk/PublicAccess/"
"Rushmoor Borough Council", "Rushmoor", "http://pa-dc.rushmoor.gov.uk/publicaccess/"
"Blaby District Council", "Blaby", "http://www.blaby.gov.uk/PublicAccess/"
"East Devon District Council", "East Devon", "http://planning.eastdevon.gov.uk/PublicAccess/"
"Mid Devon District Council", "Mid Devon", "http://planning.middevon.gov.uk/publicaccess/"
"Sevenoaks District Council", "Sevenoaks", "http://publicaccess.sevenoaks.gov.uk/publicaccess/"
"Woking Borough Council", "Woking", "http://caps.woking.gov.uk/publicaccess/"
"Basildon District Council", "Basildon", "http://planning.basildon.gov.uk/publicaccess/"
"City of York Council", "York", "http://planning.york.gov.uk/PublicAccess/tdc/"
"Cherwell District Council", "Cherwell", "http://cherweb.cherwell-dc.gov.uk/publicaccess/tdc/"
"Angus Council", "Angus", "http://planning.angus.gov.uk/PublicAccess/tdc/"
"Huntingdonshire District Council", "Huntingdonshire", "http://planning.huntsdc.gov.uk/publicaccess/tdc/"
"South Staffordshire Council", "South Staffs", "https://services.sstaffs.gov.uk/PublicAccess/tdc/"
"Bexley Council", "Bexley", "http://publicaccess.bexley.gov.uk/publicaccess/tdc/"
"Lancaster City Council", "Lancaster", "http://planapps.lancaster.gov.uk/PublicAccess/tdc/"
"Bristol City Council", "Bristol", "http://e2eweb.bristol-city.gov.uk/publicaccess/tdc/"
"Portsmouth City Council", "Portsmouth", "http://planning.portsmouth.gov.uk/PublicAccess/tdc/"
"The Borough of Oadby and Wigston", "Oadby and Wigston", "http://web.owbc.net/PublicAccess/tdc/"
"Test Valley Borough Council", "Test Valley", "http://publicaccess.testvalley.gov.uk/publicaccess/tdc/"
"Kings Lynn and West Norfolk Borough Council", "West Norfolk", "http://online.west-norfolk.gov.uk/publicaccess/tdc/"
"Sunderland City Council", "Sunderland", "http://www.sunderland.gov.uk/publicaccess/tdc/"
"Southampton City Council", "Southampton", "http://publicaccess.southampton.gov.uk/publicaccess/tdc/"
"Bath and North East Somerset", "Bath", "http://planning.bathnes.gov.uk/publicaccess/tdc/"
"Buckinghamshire County Council", "Buckinghamshire", "http://www.bucksplanning.gov.uk/PublicAccess/tdc/"
"Spelthorne Borough Council", "Spelthorne", "http://phoenix.spelthorne.gov.uk/PublicAccess/tdc/"
"Stevenage Borough Council", "Stevenage", "http://publicaccess.stevenage.gov.uk/publicaccess/tdc/"
"Tonbridge and Malling Borough Council", "Tonbridge", "http://publicaccess.tmbc.gov.uk/publicaccess/tdc/"
"Hart District Council", "Hart", "http://publicaccess.hart.gov.uk/publicaccess/tdc/"
"Luton Borough Council", "Luton", "http://www.eplan.luton.gov.uk/PublicAccess/tdc/"
"Rushmoor Borough Council", "Rushmoor", "http://pa-dc.rushmoor.gov.uk/publicaccess/tdc/"
"Blaby District Council", "Blaby", "http://www.blaby.gov.uk/PublicAccess/tdc/"
"East Devon District Council", "East Devon", "http://planning.eastdevon.gov.uk/PublicAccess/tdc/"
"Mid Devon District Council", "Mid Devon", "http://planning.middevon.gov.uk/publicaccess/tdc/"
"Sevenoaks District Council", "Sevenoaks", "http://publicaccess.sevenoaks.gov.uk/publicaccess/tdc/"
"Woking Borough Council", "Woking", "http://caps.woking.gov.uk/publicaccess/tdc/"
"Basildon District Council", "Basildon", "http://planning.basildon.gov.uk/publicaccess/tdc/"
"The City of Edinburgh Council", "Edinburgh", "http://citydev-portal.edinburgh.gov.uk/publicaccess/dc/"
"Fenland District Council", "Fenland", "http://www.fenland.gov.uk/publicaccess/dc/"
"Scarborough Borough Council", "Scarborough", "http://planning.scarborough.gov.uk/publicaccess/dc/"



Carregando…
Cancelar
Salvar