Automatically exported from code.google.com/p/planningalerts
Não pode escolher mais do que 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.

PlanningExplorer.py 29 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748
  1. import urllib2
  2. import urllib
  3. import urlparse
  4. import cgi
  5. import re
  6. import datetime
  7. import cookielib
  8. cookie_jar = cookielib.CookieJar()
  9. from BeautifulSoup import BeautifulSoup
  10. from PlanningUtils import PlanningApplication, \
  11. PlanningAuthorityResults, \
  12. getPostcodeFromText
  13. # Date format to enter into search boxes
  14. date_format = "%d/%m/%Y"
  15. # Regex for getting the application code
  16. # (needed for the comments url, when it exists)
  17. app_code_regex = re.compile("PARAM0=(\d*)")
  18. class PlanningExplorerParser:
  19. # If this authority doesn't have a comments page,
  20. # then set this email_address to an address for the
  21. # planning department, and it will be used in lieu of
  22. # a comments url.
  23. comments_email_address = None
  24. # These are the directories where the info urls, and search urls,
  25. # usually live underneath the base_url.
  26. # If these are different for a particular
  27. # authority, then they can be overridden in a subclass.
  28. info_url_path = "MVM/Online/Generic/"
  29. search_url_path = "MVM/Online/PL/GeneralSearch.aspx"
  30. # This is the most common place for comments urls to live
  31. # The %s will be filled in with an application code
  32. comments_path = "MVM/Online/PL/PLComments.aspx?pk=%s"
  33. # Most authorities don't need the referer header on the post
  34. # request. If one does, override this in the subclass
  35. use_referer = False
  36. # Some authorities won't give us anything back if we use the
  37. # python urllib2 useragent string. In that case, override this
  38. # in a subclass to pretend to be firefox.
  39. use_firefox_user_agent = False
  40. # This is the most common css class of the table containing the
  41. # the search results. If it is different for a particular authority
  42. # it can be overridden in a subclass
  43. results_table_attrs = {"class": "ResultsTable"}
  44. # These are the most common column positions for the
  45. # council reference, the address, and the description
  46. # in the results table.
  47. # They should be overridden in subclasses if they are different
  48. # for a particular authority.
  49. reference_td_no = 0
  50. address_td_no = 1
  51. description_td_no = 2
  52. # In some cases we won't be able to get the full address/description/postcode without getting the info page for each app.
  53. # If fetch_info_page is set to true, then we need to get a copy of the info page and store it as an attribute on current_application (naughty!)
  54. fetch_info_page = False
  55. asp_args_regex = re.compile('<input[^>]*name=\"(__[A-Z]*)\"[^>]*value=\"([^\"]*)\"[^>]*>')
  56. def _modify_response(self, response):
  57. """For most sites, we have managed to get all the apps on a
  58. single page by choosing the right parameters.
  59. If that hasn't been possible, override this method to get a
  60. new response object which has all the apps in one page.
  61. (See, for example, Hackney).
  62. """
  63. return response
  64. def _find_trs(self, results_table):
  65. """Normally, we just want a list of all the trs except the first one
  66. (which is usually a header).
  67. If the authority requires a different list of trs, override this method.
  68. """
  69. return results_table.findAll("tr")[1:]
  70. def _sanitisePostHtml(self, html):
  71. """This method can be overriden in subclasses if the
  72. html that comes back from the post request is bad, and
  73. needs tidying up before giving it to BeautifulSoup."""
  74. return html
  75. def _sanitiseInfoUrl(self, url):
  76. """If an authority has info urls which are for some reason full
  77. of crap (like Broadland does), then this method should be overridden
  78. in order to tidy them up."""
  79. return ''.join(url.split())
  80. def _getHeaders(self):
  81. """If the authority requires any headers for the post request,
  82. override this method returning a dictionary of header key to
  83. header value."""
  84. headers = {}
  85. if self.use_firefox_user_agent:
  86. headers["User-Agent"] = "Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.8.1.10) Gecko/20071126 Ubuntu/7.10 (gutsy) Firefox/2.0.0.10"
  87. if self.use_referer:
  88. headers["Referer"] = self.search_url
  89. return headers
  90. def _getPostData(self, asp_args, search_date):
  91. """Accepts asp_args (a tuple of key value pairs of the pesky ASP
  92. parameters, and search_date, a datetime.date object for the day
  93. we are searching for.
  94. This seems to be the most common set of post data which is needed
  95. for PlanningExplorer sites. It won't work for all of them, so
  96. will sometimes need to be overridden in a subclass.
  97. The parameter edrDateSelection is often not needed.
  98. It is needed by Charnwood though, so I've left it in
  99. to keep things simple.
  100. """
  101. year_month_day = search_date.timetuple()[:3]
  102. post_data = urllib.urlencode(asp_args + (
  103. ("_ctl0", "DATE_REGISTERED"),
  104. ("rbGroup", "_ctl5"),
  105. ("_ctl7_hidden", urllib.quote('<DateChooser Value="%d%%2C%d%%2C%d"><ExpandEffects></ExpandEffects></DateChooser>' %year_month_day)),
  106. ("_ctl8_hidden", urllib.quote('<DateChooser Value="%d%%2C%d%%2C%d"><ExpandEffects></ExpandEffects></DateChooser>' %year_month_day)),
  107. ("edrDateSelection", "1"),
  108. ("csbtnSearch", "Search"),
  109. ("cboNumRecs", "99999"),
  110. ))
  111. return post_data
  112. def _getAddress(self, tds, info_soup):
  113. # If this td contains a div, then the address is the
  114. # string in there - otherwise, use the string in the td.
  115. address_td = tds[self.address_td_no]
  116. if address_td.div is not None:
  117. address = address_td.div.string
  118. else:
  119. address = address_td.string
  120. return address
  121. def _getPostCode(self, info_soup):
  122. """In most cases, the postcode can be got from the address in
  123. the results table. Some councils put the address there without the
  124. postcode. In this case we will have to go to the info page to get
  125. the postcode. This should be done by overriding this method with
  126. one that parses the info page."""
  127. return getPostcodeFromText(self._current_application.address)
  128. def _getDescription(self, tds, info_soup):
  129. description_td = tds[self.description_td_no]
  130. if description_td.div is not None:
  131. # Mostly this is in a div
  132. # Use the empty string if the description is missing
  133. description = description_td.div.string or ""
  134. else:
  135. # But sometimes (eg Crewe) it is directly in the td.
  136. # Use the empty string if the description is missing
  137. description = description_td.string or ""
  138. return description
  139. def __init__(self,
  140. authority_name,
  141. authority_short_name,
  142. base_url,
  143. debug=False):
  144. self.authority_name = authority_name
  145. self.authority_short_name = authority_short_name
  146. self.base_url = base_url
  147. self.search_url = urlparse.urljoin(base_url, self.search_url_path)
  148. self.info_url_base = urlparse.urljoin(self.base_url, self.info_url_path)
  149. self.debug = debug
  150. self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name)
  151. def getResultsByDayMonthYear(self, day, month, year):
  152. search_date = datetime.date(year, month, day)
  153. # First do a get, to get some state
  154. get_request = urllib2.Request(self.search_url)
  155. get_response = urllib2.urlopen(get_request)
  156. cookie_jar.extract_cookies(get_response, get_request)
  157. html = get_response.read()
  158. # We need to find those ASP parameters such as __VIEWSTATE
  159. # so we can use them in the next POST
  160. # re.findall gets us a list of key value pairs.
  161. # We want to concatenate it with a tuple, so we must
  162. # make it a tuple
  163. asp_args = tuple(re.findall(self.asp_args_regex, html))
  164. # The post data needs to be different for different councils
  165. # so we have a method on each council's scraper to make it.
  166. post_data = self._getPostData(asp_args, search_date)
  167. headers = self._getHeaders()
  168. request = urllib2.Request(self.search_url, post_data, headers)
  169. cookie_jar.add_cookie_header(request)
  170. post_response = urllib2.urlopen(request)
  171. # We have actually been returned here by an http302 object
  172. # moved, and the response called post_response is really a get.
  173. # In some cases, we can't get the page size set high
  174. # until now. In that case, override _modify_response
  175. # so that we get back a response with all the apps on one page.
  176. # We pass in headers so that any
  177. post_response = self._modify_response(post_response)
  178. html = self._sanitisePostHtml(post_response.read())
  179. soup = BeautifulSoup(html)
  180. results_table = soup.find("table", attrs=self.results_table_attrs)
  181. # If there is no results table, then there were no apps on that day.
  182. if results_table:
  183. trs = self._find_trs(results_table)
  184. self._current_application = None
  185. # The first tr is just titles, cycle through the trs after that
  186. for tr in trs:
  187. self._current_application = PlanningApplication()
  188. # There is no need to search for the date_received, it's what
  189. # we searched for
  190. self._current_application.date_received = search_date
  191. tds = tr.findAll("td")
  192. self._current_application.council_reference = tds[self.reference_td_no].a.string
  193. relative_info_url = self._sanitiseInfoUrl(tds[self.reference_td_no].a['href'])
  194. self._current_application.info_url = urlparse.urljoin(self.info_url_base, relative_info_url)
  195. # Fetch the info page if we need it, otherwise set it to None
  196. if self.fetch_info_page:
  197. # We need to quote the spaces in the info url
  198. info_request = urllib2.Request(urllib.quote(self._current_application.info_url, ":/&?="))
  199. info_soup = BeautifulSoup(urllib2.urlopen(info_request))
  200. else:
  201. info_soup = None
  202. # What about a comment url?
  203. # There doesn't seem to be one, so we'll use the email address
  204. if self.comments_email_address is not None:
  205. # We're using the email address, as there doesn't seem
  206. # to be a web form for comments
  207. self._current_application.comment_url = self.comments_email_address
  208. else:
  209. # This link contains a code which we need for the comments url
  210. # (on those sites that use it)
  211. application_code = app_code_regex.search(relative_info_url).groups()[0]
  212. relative_comments_url = self.comments_path %(application_code)
  213. self._current_application.comment_url = urlparse.urljoin(self.base_url, relative_comments_url)
  214. self._current_application.address = self._getAddress(tds, info_soup)
  215. self._current_application.postcode = self._getPostCode(info_soup)
  216. self._current_application.description = self._getDescription(tds, info_soup)
  217. self._results.addApplication(self._current_application)
  218. return self._results
  219. def getResults(self, day, month, year):
  220. return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML()
  221. class BroadlandLike:
  222. # FIXME - BroadlandLike authorities don't have postcodes on their site, but
  223. # they do have grid references. We should use these.
  224. results_table_attrs = {"class": "display_table"}
  225. info_url_path = "Northgate/PlanningExplorer/Generic/"
  226. search_url_path = "Northgate/PlanningExplorer/GeneralSearch.aspx"
  227. use_firefox_user_agent = True
  228. use_referer = True
  229. def _getPostData(self, asp_args, search_date):
  230. post_data = urllib.urlencode(asp_args + (
  231. ("cboSelectDateValue", "DATE_RECEIVED"),
  232. ("rbGroup", "rbRange"),
  233. ("dateStart", search_date.strftime(date_format)),
  234. ("dateEnd", search_date.strftime(date_format)),
  235. ("cboNumRecs", "99999"),
  236. ("csbtnSearch", "Search"),
  237. ))
  238. return post_data
  239. def _sanitiseInfoUrl(self, url):
  240. """The broadland info urls arrive full of rubbish. This method tidies
  241. them up."""
  242. # We need to
  243. # 1) Remove whitespace
  244. # 2) Remove &#xA; and &#xD;
  245. ws_re = re.compile("(?:(?:\s)|(?:&#x\w;))*")
  246. return ''.join(ws_re.split(url))
  247. class BlackburnParser(PlanningExplorerParser):
  248. use_firefox_user_agent = True
  249. class BroadlandParser(BroadlandLike, PlanningExplorerParser):
  250. # FIXME - is http://secure.broadland.gov.uk/mvm/Online/PL/GeneralSearch.aspx
  251. # a better url for Broadland?
  252. def _sanitisePostHtml(self, html):
  253. """The page that comes back from the post for the broadland site
  254. has a broken doctype declaration. We need to tidy that up before
  255. giving it to BeautifulSoup."""
  256. # This is what it looks like - note the missing close doublequote
  257. #<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd>
  258. # Split on the broken doctype and join with the doctype with
  259. # closing quote.
  260. html = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'.join(html.split('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd>'))
  261. return html
  262. class CamdenParser(BroadlandLike, PlanningExplorerParser):
  263. comments_path = "Northgate/PlanningExplorer/PLComments.aspx?pk=%s"
  264. class CharnwoodParser(PlanningExplorerParser):
  265. use_firefox_user_agent = True
  266. class CreweParser(PlanningExplorerParser):
  267. use_firefox_user_agent = True
  268. use_referer = True
  269. info_url_path = "Northgate/PlanningExplorer/Generic/"
  270. search_url_path = "northgate/planningexplorer/generalsearch.aspx"
  271. results_table_attrs = {"class": "display_table"}
  272. def _getPostData(self, asp_args, search_date):
  273. year_month_day = search_date.timetuple()[:3]
  274. post_data = urllib.urlencode(asp_args + (
  275. ("txtApplicantName", ""),
  276. ("txtAgentName", ""),
  277. ("cboStreetReferenceNumber", ""),
  278. ("txtProposal", ""),
  279. ("cboWardCode", ""),
  280. ("cboParishCode", ""),
  281. ("cboApplicationTypeCode", ""),
  282. ("cboDevelopmentTypeCode", ""),
  283. ("cboStatusCode", ""),
  284. ("cboSelectDateValue", "DATE_RECEIVED"),
  285. ("cboMonths", "1"),
  286. ("cboDays", "1"),
  287. ("rbGroup", "rbRange"),
  288. ("dateStart", search_date.strftime(date_format)),
  289. ("dateEnd", search_date.strftime(date_format)),
  290. ("edrDateSelection", ""),
  291. ("csbtnSearch", "Search"),
  292. )
  293. )
  294. return post_data
  295. class EastStaffsParser(PlanningExplorerParser):
  296. use_firefox_user_agent = True
  297. address_td_no = 4
  298. description_td_no = 1
  299. class EppingForestParser(PlanningExplorerParser):
  300. use_firefox_user_agent = True
  301. address_td_no = 3
  302. description_td_no = 1
  303. class ForestHeathParser(BroadlandLike, PlanningExplorerParser):
  304. pass
  305. class HackneyParser(PlanningExplorerParser):
  306. # FIXME - This will only get the first ten records on this
  307. # day. Need to deal with paging.
  308. use_firefox_user_agent = True
  309. address_td_no = 6
  310. description_td_no = 5
  311. def _modify_response(self, response):
  312. # In order to make sure we don't have to worry about any paging,
  313. # We'll fetch this url again with PS=99999.
  314. real_url_tuple = urlparse.urlsplit(response.geturl())
  315. query_string = real_url_tuple[3]
  316. # Get the query as a list of key, value pairs
  317. parsed_query_list = list(cgi.parse_qsl(query_string))
  318. # Go through the query string replacing any PS parameters
  319. # with PS=99999
  320. for i in range(len(parsed_query_list)):
  321. key, value = parsed_query_list[i]
  322. if key == "PS":
  323. value = "99999"
  324. parsed_query_list[i] = (key, value)
  325. new_query_string = urllib.urlencode(parsed_query_list)
  326. new_url_tuple = real_url_tuple[:3] + (new_query_string,) + real_url_tuple[4:]
  327. new_url = urlparse.urlunsplit(new_url_tuple)
  328. new_request = urllib2.Request(new_url, None, self._getHeaders())
  329. new_response = urllib2.urlopen(new_request)
  330. return new_response
  331. #txtApplicationNumber=&ctl00=DATE_REGISTERED&ctl01=1&ctl02=1&rbGroup=ctl05&ctl07_hidden=&ctl07_input=28%2F08%2F2008&ctl08_hidden=&ctl08_input=28%2F08%2F2008&edrDateSelection=1&cboApplicationTypeCode=&txtLocality=&txtPostCode=&txtPropertyName=&txtPropertyNumber=&txtSiteAddress=&txtStreetName=&csbtnSearch=Search&
  332. def _getPostData(self, asp_args, search_date):
  333. """Note - using date registered here, not date received. There is too much time taken
  334. between the council 'receiving' an app and 'registering' it for the latter to be useful."""
  335. post_data = urllib.urlencode(asp_args + (
  336. ("txtApplicationNumber", ""),
  337. ("ctl00", "DATE_REGISTERED"),
  338. ("ctl01", "1"),
  339. ("ctl02", "1"),
  340. ("rbGroup", "ctl05"),
  341. ("ctl07_hidden", ""),
  342. ("ctl07_input", search_date.strftime(date_format)),
  343. ("ctl08_hidden", ""),
  344. ("ctl08_input", search_date.strftime(date_format)),
  345. ("edrDateSelection", "1"),
  346. ("cboApplicationTypeCode", ""),
  347. ("txtLocality", ""),
  348. ("txtPostCode", ""),
  349. ("txtPropertyName", ""),
  350. ("txtPropertyNumber", ""),
  351. ("txtSiteAddress", ""),
  352. ("txtStreetName", ""),
  353. ("csbtnSearch", "Search"),
  354. )
  355. )
  356. return post_data
  357. class KennetParser(BroadlandLike, PlanningExplorerParser):
  358. comments_path = "Northgate/PlanningExplorer/PLComments.aspx?pk=%s"
  359. class LincolnParser(PlanningExplorerParser):
  360. use_firefox_user_agent = True
  361. use_referer = True
  362. results_table_attrs = {"class": "display_table"}
  363. search_url_path = "northgate/planningexplorer/generalsearch.aspx"
  364. info_url_path = "Northgate/PlanningExplorer/Generic/"
  365. def _getPostData(self, asp_args, search_date):
  366. post_data = urllib.urlencode(asp_args + (
  367. ("txtApplicationNumber", ""),
  368. ("txtApplicantName", ""),
  369. ("txtAgentName", ""),
  370. ("cboApplicationTypeCode", ""),
  371. ("cboStatusCode", ""),
  372. ("txtPropertyName", ""),
  373. ("txtPropertyNumber", ""),
  374. ("cboStreetReferenceNumber", ""),
  375. ("txtPostCode", ""),
  376. ("cboLocality", ""),
  377. ("txtProposal", ""),
  378. ("cboSelectDateValue", "DATE_REGISTERED"),
  379. ("cboMonths", "1"),
  380. ("rbGroup", "rbDay"),
  381. ("cboDays", "10"),
  382. ("dateStart", search_date.strftime(date_format)),
  383. ("dateEnd", search_date.strftime(date_format)),
  384. ("edrDateSelection", ""),
  385. ("csbtnSearch", "Search"),
  386. )
  387. )
  388. return post_data
  389. class LiverpoolParser(PlanningExplorerParser):
  390. comments_email_address = "planningandbuildingcontrol@liverpool.gov.uk"
  391. use_firefox_user_agent = True
  392. use_referer = True
  393. results_table_attrs = {"xmlns:mvm":"http://www.mvm.co.uk"}
  394. info_url_path = "mvm/"
  395. search_url_path = "mvm/planningsearch.aspx"
  396. def _find_trs(self, results_table):
  397. """In this case we are after all trs except the first two which have a
  398. class attribute row0 or row1."""
  399. return results_table.findAll("tr", {"class":["row0", "row1"]})[3:]
  400. def _getPostData(self, asp_args, search_date):
  401. post_data = urllib.urlencode(asp_args + (
  402. ("dummy", "dummy field\tused for custom\tvalidator"),
  403. ("drReceived$txtStart", search_date.strftime(date_format)),
  404. ("drReceived$txtEnd", search_date.strftime(date_format)),
  405. ("cboNumRecs", "99999"),
  406. ("cmdSearch", "Search"),
  407. ))
  408. return post_data
  409. def _sanitiseInfoUrl(self, url):
  410. """The liverpool info urls arrive full of rubbish. This method tidies
  411. them up."""
  412. # We need to
  413. # 1) Remove whitespace
  414. # 2) Remove &#xA; and &#xD;
  415. ws_re = re.compile("(?:(?:\s)|(?:&#x\w;))*")
  416. return ''.join(ws_re.split(url))
  417. class MertonParser(PlanningExplorerParser):
  418. use_firefox_user_agent = True
  419. fetch_info_page = True
  420. def _getAddress(self, tds, info_soup):
  421. return info_soup.find(text="Site Address").findNext("td").string.strip()
  422. def _getDescription(self, tds, info_soup):
  423. return info_soup.find(text="Development Proposal").findNext("td").string.strip()
  424. class ShrewsburyParser(PlanningExplorerParser):
  425. use_firefox_user_agent = True
  426. class SouthNorfolkParser(PlanningExplorerParser):
  427. use_firefox_user_agent = True
  428. class SouthShropshireParser(PlanningExplorerParser):
  429. comments_email_address = "planning@southshropshire.gov.uk"
  430. use_firefox_user_agent = True
  431. info_url_path = "MVM/Online/PL/"
  432. def _getPostData(self, asp_args, search_date):
  433. local_date_format = "%d-%m-%Y"
  434. year, month, day = search_date.timetuple()[:3]
  435. post_data = urllib.urlencode(asp_args + (
  436. ("edrDateSelection:htxtRange", "radRangeBetween"),
  437. ("cboDateList", "DATE_REGISTERED"),
  438. ("edrDateSelection:txtStart", search_date.strftime(local_date_format)),
  439. ("edrDateSelection:txtEnd", search_date.strftime(local_date_format)),
  440. ("edrDateSelection:txtDateReceived", "%(day)d-%(month)d-%(year)d~%(day)d-%(month)d-%(year)d" %({"day":day, "month":month, "year":year})),
  441. ("cboNumRecs", "99999"),
  442. ("csbtnSearch", "Search"),
  443. ))
  444. return post_data
  445. class SouthTynesideParser(BroadlandLike, PlanningExplorerParser):
  446. # Unlike the other BroadlandLike sites, there are postcodes :-)
  447. pass
  448. class StockportParser(PlanningExplorerParser):
  449. comments_email_address = "admin.dc@stockport.gov.uk"
  450. info_url_path = "MVM/Online/PL/"
  451. def _getPostData(self, asp_args, search_date):
  452. post_data = urllib.urlencode(asp_args + (
  453. ("drDateReceived:txtStart", search_date.strftime(date_format)),
  454. ("drDateReceived:txtEnd", search_date.strftime(date_format)),
  455. ("cboNumRecs", "99999"),
  456. ("csbtnSearch", "Search"),),
  457. )
  458. return post_data
  459. class SwanseaParser(BroadlandLike, PlanningExplorerParser):
  460. # Unlike the other BroadlandLike sites, there are postcodes :-)
  461. pass
  462. class TamworthParser(PlanningExplorerParser):
  463. comments_email_address = "planningadmin@tamworth.gov.uk"
  464. use_firefox_user_agent = True
  465. info_url_path = "MVM/Online/PL/"
  466. class TraffordParser(PlanningExplorerParser):
  467. # There are no postcodes on the Trafford site.
  468. use_firefox_user_agent = True
  469. address_td_no = 3
  470. class WestOxfordshireParser(PlanningExplorerParser):
  471. address_td_no = 3
  472. description_td_no = 1
  473. use_firefox_user_agent = True
  474. class WalthamForestParser(PlanningExplorerParser):
  475. search_url_path = "PlanningExplorer/GeneralSearch.aspx"
  476. info_url_path = "PlanningExplorer/Generic/"
  477. use_firefox_user_agent = True
  478. use_referer = True
  479. # I know - I should change this so that the attribute is not comments_email_address, but
  480. # something more general
  481. comments_email_address = "https://www1.walthamforest.gov.uk/webforms/plan_comments/"
  482. results_table_attrs = {"class": "display_table"}
  483. def _getPostData(self, asp_args, search_date):
  484. post_data = urllib.urlencode(asp_args + (
  485. ("txtApplicantName", ""),
  486. ("txtAgentName", ""),
  487. ("cboStreetReferenceNumber", ""),
  488. ("txtProposal", ""),
  489. ("cboWardCode", ""),
  490. ("cboParishCode", ""),
  491. ("cboApplicationTypeCode", ""),
  492. ("cboDevelopmentTypeCode", ""),
  493. ("cboStatusCode", ""),
  494. ("cboSelectDateValue", "DATE_REGISTERED"),
  495. ("cboMonths", "1"),
  496. ("cboDays", "10"),
  497. ("rbGroup", "rbRange"),
  498. ("dateStart", search_date.strftime(date_format)),
  499. ("dateEnd", search_date.strftime(date_format)),
  500. ("edrDateSelection", ""),
  501. ("csbtnSearch", "Search"),
  502. )
  503. )
  504. return post_data
  505. class ConwyParser(BroadlandLike, PlanningExplorerParser):
  506. search_url_path = "Northgate/planningexplorerenglish/generalsearch.aspx"
  507. info_url_path = "Northgate/PlanningExplorerEnglish/Generic/"
  508. comments_path = "Northgate/PlanningExplorerEnglish/PLComments.aspx?pk=%s"
  509. use_firefox_user_agent = True
  510. #&txtApplicationNumber=&txtProposal=&txtSiteAddress=&cboWardCode=&cboParishCode=&cboApplicationTypeCode=&cboDevelopmentTypeCode=&cboStatusCode=&cboSelectDateValue=DATE_RECEIVED&cboMonths=1&cboDays=1&rbGroup=rbRange&dateStart=10%2F07%2F2008&dateEnd=20%2F07%2F2008&edrDateSelection=&csbtnSearch=Search
  511. #txtApplicantName=
  512. #txtAgentName=
  513. #cboStreetReferenceNumber=
  514. #txtProposal=
  515. #cboWardCode=
  516. #cboParishCode=
  517. #cboApplicationTypeCode=
  518. #cboDevelopmentTypeCode=
  519. #cboStatusCode=
  520. #cboSelectDateValue=DATE_RECEIVED
  521. #cboMonths=1
  522. #cboDays=1
  523. #rbGroup=rbRange
  524. #dateStart=01%2F03%2F2008
  525. #dateEnd=01%2F04%2F2008
  526. #edrDateSelection=
  527. #csbtnSearch=Search
  528. if __name__ == '__main__':
  529. # NOTE - 04/11/2007 is a sunday
  530. # I'm using it to test that the scrapers behave on days with no apps.
  531. # parser = BlackburnParser("Blackburn With Darwen Borough Council", "Blackburn", "http://195.8.175.6/")
  532. # parser = BroadlandParser("Broadland Council", "Broadland", "http://www.broadland.gov.uk/")
  533. # parser = CamdenParser("London Borough of Camden", "Camden", "http://planningrecords.camden.gov.uk/")
  534. # parser = CharnwoodParser("Charnwood Borough Council", "Charnwood", "http://portal.charnwoodbc.gov.uk/")
  535. # parser = CreweParser("Crewe and Nantwich Borough Council", "Crewe and Nantwich", "http://portal.crewe-nantwich.gov.uk/")
  536. # parser = EastStaffsParser("East Staffordshire Borough Council", "East Staffs", "http://www2.eaststaffsbc.gov.uk/")
  537. # parser = EppingForestParser("Epping Forest District Council", "Epping Forest", "http://plan1.eppingforestdc.gov.uk/")
  538. # parser = ForestHeathParser("Forest Heath District Council", "Forest Heath", "http://195.171.177.73/")
  539. # parser = HackneyParser("London Borough of Hackney", "Hackney", "http://www.hackney.gov.uk/servapps/")
  540. # parser = KennetParser("Kennet District Council", "Kennet", "http://mvm-planning.kennet.gov.uk/")
  541. # parser = LincolnParser("Lincoln City Council", "Lincoln", "http://online.lincoln.gov.uk/")
  542. parser = LiverpoolParser("Liverpool City Council", "Liverpool", "http://www.liverpool.gov.uk/")
  543. # parser = ShrewsburyParser("Shrewsbury and Atcham Borough Council", "Shrewsbury", "http://www2.shrewsbury.gov.uk/")
  544. # parser = SouthNorfolkParser("South Norfolk Council", "South Norfolk", "http://planning.south-norfolk.gov.uk/")
  545. # parser = SouthShropshireParser("South Shropshire District Council", "South Shropshire", "http://194.201.44.102/")
  546. # parser = SouthTynesideParser("South Tyneside Council", "South Tyneside", "http://poppy.southtyneside.gov.uk/")
  547. # parser = StockportParser("Stockport Metropolitan District Council", "Stockport", "http://s1.stockport.gov.uk/council/eed/dc/planning/")
  548. # parser = SwanseaParser("Swansea City and County Council", "Swansea", "http://www2.swansea.gov.uk/")
  549. # parser = TamworthParser("Tamworth Borough Council", "Tamworth", "http://80.1.64.77/")
  550. # parser = TraffordParser("Trafford Council", "Trafford", "http://planning.trafford.gov.uk/")
  551. # parser = WestOxfordshireParser("West Oxfordshire District Council", "West Oxfordshire", "http://planning.westoxon.gov.uk/")
  552. # parser = WalthamForestParser("Waltham Forest", "Waltham Forest", "http://planning.walthamforest.gov.uk/")
  553. # parser = ConwyParser("Conwy County Borough Council", "Conwy", "http://www.conwy.gov.uk/")
  554. # parser = MertonParser("London Borough of Merton", "Merton", "http://planning.merton.gov.uk")
  555. print parser.getResults(28, 8, 2008)
  556. # To Do
  557. # Sort out paging:
  558. # South Shropshire - pages on 6
  559. # Investigate catching unavailable message:
  560. # Charnwood
  561. # South Norfolk has no postcodes. I wonder if the postcodes are in the WAM site...
  562. # Notes:
  563. # Since the changed, Liverpool and Crewe look rather similar. They are also a little Broadlandlike. Maybe we can do some consolidation