adrianshort
/
planningalerts
mirror of https://github.com/adrianshort/planningalerts.git
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
Automatically exported from code.google.com/p/planningalerts
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
383 Commits
2 Branches
1.3 MiB
 
 
 
 
 
 
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
planningalerts/python_scrapers/SwiftLG.py

236 lines
11 KiB
Raw Permalink Blame History 

  1. 

  2. import urllib2

  3. import urllib

  4. import urlparse

  5. import cgi

  6. import re

  7. import datetime

  8. 

  9. import BeautifulSoup

  10. 

  11. from PlanningUtils import getPostcodeFromText, \

  12.     PlanningAuthorityResults, \

  13.     PlanningApplication

  14. 

  15. # - Browser request: --------------------------

  16. # {POST http://digitalmaidstone.co.uk/swiftlg/apas/run/WPHAPPCRITERIA HTTP/1.0} {Host: digitalmaidstone.co.uk

  17. # Accept: text/html, text/plain, text/css, text/sgml, */*;q=0.01

  18. # Accept-Encoding: gzip

  19. # Accept-Language: en

  20. # Pragma: no-cache

  21. # Cache-Control: no-cache

  22. # User-Agent: Lynx/2.8.6rel.4 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.6.3

  23. # Content-type: application/x-www-form-urlencoded

  24. # Content-length: 638

  25. # } %25.MAINBODY.WPACIS.1.=&APNID.MAINBODY.WPACIS.1.=&JUSTLOCATION.MAINBODY.WPACIS.1.=&JUSTDEVDESC.MAINBODY.WPACIS.1.=&DEVDESC.MAINBODY.WPACIS.1.=&SURNAME.MAINBODY.WPACIS.1.=&REGFROMDATE.MAINBODY.WPACIS.1.=01%2F11%2F2007&REGTODATE.MAINBODY.WPACIS.1.=02%2F11%2F2007&DECFROMDATE.MAINBODY.WPACIS.1.=&DECTODATE.MAINBODY.WPACIS.1.=&FINALGRANTFROM.MAINBODY.WPACIS.1.=&FINALGRANTTO.MAINBODY.WPACIS.1.=&APELDGDATFROM.MAINBODY.WPACIS.1.=&APELDGDATTO.MAINBODY.WPACIS.1.=&APEDECDATFROM.MAINBODY.WPACIS.1.=&APEDECDATTO.MAINBODY.WPACIS.1.=&AREA.MAINBODY.WPACIS.1.=&WARD.MAINBODY.WPACIS.1.=&PARISH.MAINBODY.WPACIS.1.=&SEARCHBUTTON.MAINBODY.WPACIS.1.=Search

  26. # server=[digitalmaidstone.co.uk] , port=[80], script=[/swiftlg/apas/run/WPHAPPCRITERIA]

  27. # request_line=[POST /swiftlg/apas/run/WPHAPPCRITERIA HTTP/1.0]

  28. 

  29. # second page

  30. #http://digitalmaidstone.co.uk/swiftlg/apas/run/WPHAPPSEARCHRES.displayResultsURL?ResultID=243941&

  31. #StartIndex=11&

  32. #SortOrder=APNID:asc&

  33. #DispResultsAs=WPHAPPSEARCHRES&

  34. #BackURL=<a%20href=wphappcriteria.display?paSearchKey=147118>Search%20Criteria

  35. 

  36. # Date format to enter into search boxes

  37. date_format = "%d/%m/%Y"

  38. 

  39. class SwiftLGParser:

  40.     search_path = "WPHAPPCRITERIA"

  41.     info_path = "WPHAPPDETAIL.DisplayUrl?theApnID=%s"

  42.     comment_path ="wphmakerep.displayURL?ApnID=%s"

  43. 

  44.     def _fixHTML(self, html):

  45.         return html

  46. 

  47.     def _findResultsTable(self, soup):

  48.         """Unless there is just one table in the page, the resuts table,

  49.         override this in a subclass."""

  50.         return soup.table

  51. 

  52.     def _findTRs(self, results_table):

  53.         """The usual situation is for the results table to contain

  54.         one row of headers, followed by a row per app.

  55.         If this is not the case, override this in a subclass."""

  56. #        import pdb;pdb.set_trace()

  57.         return results_table.findAll("tr")[1:]

  58. 

  59.     def __init__(self,

  60.                  authority_name,

  61.                  authority_short_name,

  62.                  base_url,

  63.                  debug=False):

  64.         

  65.         self.authority_name = authority_name

  66.         self.authority_short_name = authority_short_name

  67.         self.base_url = base_url

  68. 

  69.         self.search_url = urlparse.urljoin(base_url, self.search_path)

  70.         self.info_url = urlparse.urljoin(base_url, self.info_path)

  71.         self.comment_url = urlparse.urljoin(base_url, self.comment_path)

  72. 

  73.         self.debug = debug

  74. 

  75.         self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name)

  76. 

  77. 

  78.     def getResultsByDayMonthYear(self, day, month, year):

  79.         search_date = datetime.date(year, month, day)

  80.         

  81.         post_data = urllib.urlencode((

  82.                 ("REGFROMDATE.MAINBODY.WPACIS.1.", search_date.strftime(date_format)),

  83.                 ("REGTODATE.MAINBODY.WPACIS.1.", search_date.strftime(date_format)),

  84.                 ("SEARCHBUTTON.MAINBODY.WPACIS.1.", "Search"),

  85.                 ))

  86.         

  87.         response = urllib2.urlopen(self.search_url, post_data)

  88.         contents = response.read()

  89. 

  90.         # Let's give scrapers the change to tidy up any rubbish - I'm looking

  91.         # at you Cannock Chase

  92.         contents = self._fixHTML(contents)

  93. 

  94.         # Check for the no results warning

  95.         if not contents.count("No Matching Applications Found"):

  96.             soup = BeautifulSoup.BeautifulSoup(contents)

  97. 

  98.             # Get the links to later pages of results.

  99.             later_pages = soup.findAll("a", {"href": re.compile("WPHAPPSEARCHRES\.displayResultsURL.*StartIndex=\d*.*")})

  100. 

  101.             for a in ["initial_search"] + later_pages:

  102.                 if a != "initial_search":

  103.                     url = a['href']

  104. 

  105.                     # Example url

  106. 

  107.                     #http://digitalmaidstone.co.uk/swiftlg/apas/run/WPHAPPSEARCHRES.displayResultsURL?ResultID=244037&StartIndex=11&SortOrder=APNID:asc&DispResultsAs=WPHAPPSEARCHRES&BackURL=<a href=wphappcriteria.display?paSearchKey=147170>Search Criteria</a>

  108. 

  109.                     # urllib2 doesn't like this url, to make it happy, we'll

  110.                     # get rid of the BackURL parameter, which we don't need.

  111. 

  112.                     split_url = urlparse.urlsplit(url)

  113.                     qs = split_url[3]

  114. 

  115.                     # This gets us a dictionary of key to lists of values

  116.                     qsl = cgi.parse_qsl(qs)

  117. 

  118.                     # Get rid of BackURL

  119.                     qsl.pop(-1)

  120. 

  121.                     # I think this is safe, as there are no repeats of parameters

  122.                     new_qs = urllib.urlencode(qsl)

  123. 

  124.                     url = urlparse.urlunsplit(split_url[:3] + (new_qs,) + split_url[4:])

  125. 

  126.                     this_page_url = urlparse.urljoin(self.base_url, url)

  127.                     response = urllib2.urlopen(this_page_url)

  128.                     contents = response.read()

  129.                     soup = BeautifulSoup.BeautifulSoup(contents)

  130. 

  131.                 results_table = self._findResultsTable(soup)#.body.find("table", {"class": "apas_tbl"})

  132. 

  133.                 trs = self._findTRs(results_table)

  134. 

  135.                 for tr in trs:

  136.                     self._current_application = PlanningApplication()

  137. 

  138.                     tds = tr.findAll("td")

  139. 

  140.                     # The first td

  141. 

  142.                     #<td class="apas_tblContent"><a href="WPHAPPDETAIL.DisplayUrl?theApnID=07/1884&amp;backURL=&lt;a href=wphappcriteria.display?paSearchKey=147125&gt;Search Criteria&lt;/a&gt; &gt; &lt;a href='wphappsearchres.displayResultsURL?ResultID=243950%26StartIndex=1%26SortOrder=APNID:asc%26DispResultsAs=WPHAPPSEARCHRES%26BackURL=&lt;a href=wphappcriteria.display?paSearchKey=147125&gt;Search Criteria&lt;/a&gt;'&gt;Search Results&lt;/a&gt;"></a><a href="wphappcriteria.display?paSearchKey=147125">Search Criteria</a> > <a href="wphappsearchres.displayResultsURL?ResultID=243950%26StartIndex=1%26SortOrder=APNID:asc%26DispResultsAs=WPHAPPSEARCHRES%26BackURL=&lt;a href=wphappcriteria.display?paSearchKey=147125&gt;Search Criteria&lt;/a&gt;"></a><a href="wphappcriteria.display?paSearchKey=147125">Search Criteria</a>'>Search Results">07/1884</td>

  143. 

  144.                     # The html here is a bit of a mess, and doesn't all get into

  145.                     # the soup.

  146.                     # We can get the reference from the first <a href> in td 0.

  147.                     first_link = tds[0].a['href']

  148. 

  149.                     app_id = cgi.parse_qs(urlparse.urlsplit(first_link)[3])['theApnID'][0]

  150. 

  151.                     self._current_application.date_received = search_date

  152.                     self._current_application.council_reference = app_id

  153.                     self._current_application.info_url = self.info_url %(app_id)

  154.                     self._current_application.comment_url = self.comment_url %(app_id)

  155.                     self._current_application.description = tds[1].string.strip()

  156. 

  157.                     # the second td

  158. 

  159.                     #<td class="apas_tblContent"><input type="HIDDEN" name="ORDERCOUNTER.PAHEADER.PACIS2.1-1." value="1" class="input-box" size="7" />

  160.                     #LAND ADJ. BRAMBLING, HAWKENBURY ROAD, HAWKENBURY, TN120EA

  161.                     #</td>

  162. 

  163.                     address = ' '.join([x for x in tds[2].contents if isinstance(x, BeautifulSoup.NavigableString)]).strip()

  164. 

  165.                     self._current_application.address = address

  166.                     self._current_application.postcode = getPostcodeFromText(address)

  167. 

  168.                     self._results.addApplication(self._current_application)

  169. 

  170.         return self._results

  171. 

  172.     def getResults(self, day, month, year):

  173.         return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML()

  174. 

  175. class EastHertsParser(SwiftLGParser):

  176.     def _findResultsTable(self, soup):

  177.         return soup.findAll("table")[3]

  178. 

  179. class GwyneddParser(SwiftLGParser):

  180.     def _findResultsTable(self, soup):

  181.         return soup.find("table", {"class": "thinBox"})

  182. 

  183. class IslingtonParser(SwiftLGParser):

  184.     def _findResultsTable(self, soup):

  185.         return soup.table.table

  186. 

  187. class MacclesfieldParser(SwiftLGParser):

  188.     def _findResultsTable(self, soup):

  189.         return soup.findAll("table")[6]

  190. 

  191. class MoleValleyParser(SwiftLGParser):

  192.     def _findResultsTable(self, soup):

  193. #        import pdb;pdb.set_trace()

  194.         return soup.findAll("table")[2]

  195. 

  196. class SloughParser(SwiftLGParser):

  197.     def _findResultsTable(self, soup):

  198.         return soup.findAll("table")[1]

  199. 

  200.     def _findTRs(self, results_table):

  201.         return results_table.findAll("tr")[2:]

  202. 

  203. class CannockChaseParser(SwiftLGParser):

  204.     def _fixHTML(self, html):

  205.         return html.replace('</tr class="tablebody">', '</tr>')

  206. 

  207. if __name__ == '__main__':

  208. #    parser = SwiftLGParser("Boston Borough Council", "Boston", "http://195.224.121.199/swiftlg/apas/run/")

  209. #    parser = SwiftLGParser("Dudley", "Dudley", "http://www2.dudley.gov.uk/swiftlg/apas/run/")

  210. #    parser = EastHertsParser("East Hertfordshire", "East Herts", "http://e-services.eastherts.gov.uk/swiftlg/apas/run/")

  211. #    parser = GwyneddParser("Gwynedd", "Gwynedd", "http://www.gwynedd.gov.uk/swiftlg/apas/run/")

  212. #    parser = IslingtonParser("Islington", "Islington", "https://www.islington.gov.uk/onlineplanning/apas/run/")

  213. #    parser = SwiftLGParser("Lake District", "Lake District", "http://www.lake-district.gov.uk/swiftlg/apas/run/")

  214. #    parser = SwiftLGParser("Maidstone Borough Council", "Maidstone", "http://digitalmaidstone.co.uk/swiftlg/apas/run/")

  215. #    parser = MoleValleyParser("Mole Valley", "Mole Valley", "http://www.molevalley.gov.uk/swiftlg/apas/run/")

  216. #    parser = SwiftLGParser("Pembrokeshire County Council", "Pembrokeshire", "http://planning.pembrokeshire.gov.uk/swiftlg/apas/run/")

  217. #    parser = SwiftLGParser("Rochdale Metropolitan Borough Council", "Rochdale", "http://www.rochdale.gov.uk/swiftlg/apas/run/")

  218. #    parser = SloughParser("Slough", "Slough", "http://www2.slough.gov.uk/swiftlg/apas/run/")

  219. #    parser = SwiftLGParser("Snowdonia National Park", "Snowdonia", "http://www.snowdonia-npa.gov.uk/swiftlg/apas/run/")

  220. #    parser = SwiftLGParser("St Edmundsbury", "Bury St Edmunds", "http://www.stedmundsbury.gov.uk/swiftlg/apas/run/")

  221. #    parser = MacclesfieldParser("Macclesfield", "Macclesfield", "http://www.planportal.macclesfield.gov.uk/swiftlg/apas/run/")

  222.     parser = SwiftLGParser("Daventry District Council", "Daventry", "http://212.125.73.214/swiftlg/apas/run/wphappcriteria.display")

  223. #    parser = SwiftLGParser("Warrington Borough Council", "Warrington", "http://212.248.237.123:8080/swiftlg/apas/run/wphappcriteria.display")

  224. #    parser = CannockChaseParser("Cannock Chase District Council", "Cannock Chase", "http://planning.cannockchasedc.com/swiftlg/apas/run/wphappcriteria.display")

  225. #    parser = SwiftLGParser("London Borough of Enfield", "Enfield", "http://forms.enfield.gov.uk/swiftlg/apas/run/wphappcriteria.display")

  226.     print parser.getResults(12,6,2009)

  227. 

  228. 

  229. # To Do:

  230. 

  231. #1) Check out comment url on Maidstone

  232. 

  233. #2) Daventry, when it is back up.

  234. 

  235. #3) Work out what goes wrong with Gwynedd on 06/11/2007