From becabcd0a554f7534ad81f2f4133cc2163d90417 Mon Sep 17 00:00:00 2001 From: "duncan.parkes" Date: Tue, 22 Apr 2008 09:32:59 +0000 Subject: [PATCH] Fix problem with Harlow. Council had moved the acolnet site to a different obscure location. --- python_scrapers/AcolnetParser.py | 8 +-- .../ApplicationSearchServletParser.py | 2 +- python_scrapers/PlanningExplorer.py | 55 ++++++++++++++++++- python_scrapers/SitesToGenerate.csv | 2 +- python_scrapers/WAM.py | 9 ++- 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/python_scrapers/AcolnetParser.py b/python_scrapers/AcolnetParser.py index dc1b8ac..1d36093 100644 --- a/python_scrapers/AcolnetParser.py +++ b/python_scrapers/AcolnetParser.py @@ -321,8 +321,8 @@ class BassetlawParser(BoltonLikeParser): if __name__ == '__main__': - day = 22 - month = 1 + day = 4 + month = 4 year = 2008 #parser = AcolnetParser("Babergh", "Babergh", "http://planning.babergh.gov.uk/dcdatav2//acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch") @@ -339,7 +339,7 @@ if __name__ == '__main__': #parser = AcolnetParser("Exeter City Council", "Exeter", "http://pub.exeter.gov.uk/scripts/Acolnet/dataonlineplanning/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = BoltonParser("Fylde", "Fylde", "http://www2.fylde.gov.uk/planning/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = AcolnetParser("Guildford", "Guildford", "http://www.guildford.gov.uk/DLDC_Version_2/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch") - #parser = AcolnetParser("Harlow", "Harlow", "http://planning.harlow.gov.uk/PlanningSearch/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch") + parser = AcolnetParser("Harlow", "Harlow", "http://planning.harlow.gov.uk/DLDC_Version_2/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = AcolnetParser("Havant", "Havant", "http://www3.havant.gov.uk/scripts/planningpages/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = BoltonLikeParser("Hertsmere", "Hertsmere", "http://www2.hertsmere.gov.uk/ACOLNET/DCOnline//acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = LewishamParser("Lewisham", "Lewisham", "http://acolnet.lewisham.gov.uk/lewis-xslpagesdc/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.PgeSearch") @@ -349,7 +349,7 @@ if __name__ == '__main__': #parser = AcolnetParser("North Hertfordshire", "North Herts", "http://www.north-herts.gov.uk/dcdataonline/Pages/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.PgeSearch") #parser = AcolnetParser("North Wiltshire", "North Wilts", "http://planning.northwilts.gov.uk/DCOnline/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = OldhamParser("Oldham", "Oldham", "http://planning.oldham.gov.uk/planning/AcolNetCGI.gov?ACTION=UNWRAP&Root=PgeSearch") - parser = BoltonLikeParser("Renfrewshire", "Renfrewshire", "http://planning.renfrewshire.gov.uk/acolnetDCpages/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.PgeSearch") + #parser = BoltonLikeParser("Renfrewshire", "Renfrewshire", "http://planning.renfrewshire.gov.uk/acolnetDCpages/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.PgeSearch") #parser = AcolnetParser("South Bedfordshire", "South Bedfordshire", "http://planning.southbeds.gov.uk/plantech/DCWebPages/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.PgeSearch") #parser = SouthwarkParser("London Borough of Southwark", "Southwark", "http://planningonline.southwarksites.com/planningonline2/AcolNetCGI.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch") #parser = AcolnetParser("Suffolk Coastal", "Suffolk Coastal", "http://apps3.suffolkcoastal.gov.uk/DCDataV2/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch") diff --git a/python_scrapers/ApplicationSearchServletParser.py b/python_scrapers/ApplicationSearchServletParser.py index 644e042..6cb7f6a 100644 --- a/python_scrapers/ApplicationSearchServletParser.py +++ b/python_scrapers/ApplicationSearchServletParser.py @@ -512,7 +512,7 @@ if __name__ == "__main__": #parser = EalingSearchParser("Ealing", "Ealing", "http://www.pam.ealing.gov.uk") #print parser.getResults(20,3,2007) parser = HaringeySearchParser("Haringey", "Haringey", "http://www.planningservices.haringey.gov.uk") - print parser.getResults(3,1,2008) + print parser.getResults(3,3,2008) #parser = DenbighshireSearchParser("Denbighshire", "Denbighshire", "http://planning.denbighshire.gov.uk") #print parser.getResults(20,3,2007) pass diff --git a/python_scrapers/PlanningExplorer.py b/python_scrapers/PlanningExplorer.py index 208e4ab..0249890 100644 --- a/python_scrapers/PlanningExplorer.py +++ b/python_scrapers/PlanningExplorer.py @@ -542,13 +542,65 @@ class WestOxfordshireParser(PlanningExplorerParser): use_firefox_user_agent = True +class WalthamForestParser(PlanningExplorerParser): + address_td_no = 2 + description_td_no = 3 + + search_url_path = "PlanningExplorer/GeneralSearch.aspx" + info_url_path = "PlanningExplorer/Generic/" + use_firefox_user_agent = True + + def _getPostData(self, asp_args, search_date): + post_data = urllib.urlencode(asp_args + ( + ("txtApplicantName", ""), + ("txtAgentName", ""), + ("cboStreetReferenceNumber", ""), + ("txtProposal", ""), + ("cboWardCode", ""), + ("cboParishCode", ""), + ("cboApplicationTypeCode", ""), + ("cboDevelopmentTypeCode", ""), + ("cboStatusCode", ""), + ("cboSelectDateValue", "DATE_RECEIVED"), + ("cboMonths", "1"), + ("cboDays", "1"), + ("rbGroup", "rbRange"), + ("dateStart", search_date.strftime(date_format)), + ("dateEnd", search_date.strftime(date_format)), + #&dateStart=01%2F03%2F2008&dateEnd=01%2F04%2F2008& + ("edrDateSelection", ""), + ("csbtnSearch", "Search"), + )) + + print post_data + return post_data + + +#txtApplicantName= +#txtAgentName= +#cboStreetReferenceNumber= +#txtProposal= +#cboWardCode= +#cboParishCode= +#cboApplicationTypeCode= +#cboDevelopmentTypeCode= +#cboStatusCode= +#cboSelectDateValue=DATE_RECEIVED +#cboMonths=1 +#cboDays=1 +#rbGroup=rbRange +#dateStart=01%2F03%2F2008 +#dateEnd=01%2F04%2F2008 +#edrDateSelection= +#csbtnSearch=Search + if __name__ == '__main__': # NOTE - 04/11/2007 is a sunday # I'm using it to test that the scrapers behave on days with no apps. #parser = BlackburnParser("Blackburn With Darwen Borough Council", "Blackburn", "http://195.8.175.6/") #parser = BroadlandParser("Broadland Council", "Broadland", "http://www.broadland.gov.uk/") - parser = CamdenParser("London Borough of Camden", "Camden", "http://planningrecords.camden.gov.uk/") + #parser = CamdenParser("London Borough of Camden", "Camden", "http://planningrecords.camden.gov.uk/") #parser = CharnwoodParser("Charnwood Borough Council", "Charnwood", "http://portal.charnwoodbc.gov.uk/") #parser = CreweParser("Crewe and Nantwich Borough Council", "Crewe and Nantwich", "http://portal.crewe-nantwich.gov.uk/") #parser = EastStaffsParser("East Staffordshire Borough Council", "East Staffs", "http://www2.eaststaffsbc.gov.uk/") @@ -567,6 +619,7 @@ if __name__ == '__main__': #parser = TamworthParser("Tamworth Borough Council", "Tamworth", "http://80.1.64.77/") #parser = TraffordParser("Trafford Council", "Trafford", "http://planning.trafford.gov.uk/") #parser = WestOxfordshireParser("West Oxfordshire District Council", "West Oxfordshire", "http://planning.westoxon.gov.uk/") + parser = WalthamForestParser("Waltham Forest", "Waltham Forest", "http://planning.walthamforest.gov.uk/") print parser.getResults(22, 1, 2008) # To Do diff --git a/python_scrapers/SitesToGenerate.csv b/python_scrapers/SitesToGenerate.csv index 7412776..7f64d99 100644 --- a/python_scrapers/SitesToGenerate.csv +++ b/python_scrapers/SitesToGenerate.csv @@ -96,7 +96,7 @@ "East Lindsey District Council", "East Lindsey", "http://www.e-lindsey.gov.uk/planning/AcolnetCGI.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "AcolnetParser" "Fylde Borough Council", "Fylde", "http://www2.fylde.gov.uk/planning/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "BoltonLikeParser" "Guildford Borough Council", "Guildford", "http://www.guildford.gov.uk/DLDC_Version_2/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "AcolnetParser" -"Harlow Council", "Harlow", "http://planning.harlow.gov.uk/PlanningSearch/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "AcolnetParser" +"Harlow Council", "Harlow", "http://planning.harlow.gov.uk/DLDC_Version_2/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "AcolnetParser" "Hertsmere Borough Council", "Hertsmere", "http://www2.hertsmere.gov.uk/ACOLNET/DCOnline//acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "BoltonLikeParser" "London Borough of Lewisham", "Lewisham", "http://acolnet.lewisham.gov.uk/lewis-xslpagesdc/acolnetcgi.exe?ACTION=UNWRAP&RIPNAME=Root.PgeSearch", "AcolnetParser", "LewishamParser" "North Hertfordshire District Council", "North Hertfordshire", "http://www.north-herts.gov.uk/dcdataonline/Pages/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.PgeSearch", "AcolnetParser", "AcolnetParser" diff --git a/python_scrapers/WAM.py b/python_scrapers/WAM.py index 705c1f0..795de3d 100644 --- a/python_scrapers/WAM.py +++ b/python_scrapers/WAM.py @@ -132,7 +132,7 @@ class WAMParser: self._results.addApplication(self._current_application) - except SystemExit: + except: # It seems a shame to miss out on all the apps from an authority just because one breaks... if self._current_application.council_reference: if self.debug: @@ -171,18 +171,17 @@ if __name__ == '__main__': #parser = WAMParser("Castle Point", "Castle Point", "http://wam.castlepoint.gov.uk/WAM/pas/searchApplications.do", debug=True) #Chichester - Done as PublicAccess #parser = BraintreeParser("Colchester", "Colchester", "http://www.planning.colchester.gov.uk/WAM/weeklyApplications.do", debug=True) - #parser = WAMParser("East Lothian", "East Lothian", "http://www.planning.eastlothian.gov.uk/WAM/pas/searchApplications.do", debug=True) + parser = WAMParser("East Lothian", "East Lothian", "http://www.planning.eastlothian.gov.uk/WAM/pas/searchApplications.do", debug=True) #parser = BraintreeParser("North Somerset", "North Somerset", "http://wam.n-somerset.gov.uk/MULTIWAM/weeklyApplications.do", debug=True) #parser = WAMParser("Nottingham", "Nottingham", "http://plan4.nottinghamcity.gov.uk/WAM/pas/searchApplications.do", debug=True) #parser = PooleParser("Poole long", "Poole", "http://wam.boroughofpoole.com/WAM/pas/searchApplications.do", debug=True) #parser = WAMParser("Rother long", "Rother", "http://www.planning.rother.gov.uk/WAM/pas/searchApplications.do", debug=True) #parser = BraintreeParser("South Gloucestershire", "South Gloucestershire", "http://planning.southglos.gov.uk/WAM/pas/WeeklyApplications.do", debug=True) - #parser = BraintreeParser("South Gloucestershire", "South Gloucestershire", "http://planning.southglos.gov.uk/WAM/pas/WeeklyApplications.do", debug=True) #parser = WAMParser("South Norfolk", "South Norfolk", "http://wam.south-norfolk.gov.uk/WAM/pas/searchApplications.do", debug=True) - parser = BraintreeParser("Tower Hamlets", "Tower Hamlets", "http://194.201.98.213/WAM/weeklyApplications.do", debug=True) + #parser = BraintreeParser("Tower Hamlets", "Tower Hamlets", "http://194.201.98.213/WAM/weeklyApplications.do", debug=True) #parser = WAMParser("Westminster", "Westminster", "http://idocs.westminster.gov.uk:8080/WAM/search/pas/index.htm", debug=True) - print parser.getResults(8,2,2008) + print parser.getResults(3,3,2008) # Left to fix