浏览代码

Add scraper for Barnsley.

master
duncan.parkes 16 年前
父节点
当前提交
8d823ca649
共有 3 个文件被更改,包括 85 次插入0 次删除
  1. +83
    -0
      python_scrapers/Barnsley.py
  2. +1
    -0
      python_scrapers/OtherFilesToCopy.csv
  3. +1
    -0
      python_scrapers/SitesToGenerate.csv

+ 83
- 0
python_scrapers/Barnsley.py 查看文件

@@ -0,0 +1,83 @@
"""
This is the screenscraper for planning apps for
Barnsley Metropolitan Borough Council.

The apps for Barnsley are displayed in html pages one per week, starting on
monday. There is no date_received, so we'll have to use the date of the
start of this week.

There is no comment url, so we'll use the email address.

Developmentcontrol@barnsley.gov.uk

"""

import urllib2
import urllib
import urlparse

import datetime, time
import cgi

from BeautifulSoup import BeautifulSoup

from PlanningUtils import PlanningApplication, \
PlanningAuthorityResults, \
getPostcodeFromText

date_format = "%d/%m/%Y"

class BarnsleyParser:
comments_email_address = "Developmentcontrol@barnsley.gov.uk"

def __init__(self, *args):

self.authority_name = "Barnsley Metropolitan Borough Council"
self.authority_short_name = "Barnsley"
self.base_url = "http://applications.barnsley.gov.uk/service/development/week_compact.asp?AppDate=%s"

self._results = PlanningAuthorityResults(self.authority_name, self.authority_short_name)


def getResultsByDayMonthYear(self, day, month, year):
search_day = datetime.date(year, month, day)

# What we actually need is the monday before the date searched for:
monday_before = search_day - datetime.timedelta(search_day.weekday())

# Now get the search page
response = urllib2.urlopen(self.base_url %(monday_before.strftime(date_format)))
soup = BeautifulSoup(response.read())

result_tables = soup.findAll("table", align="Center", cellpadding="3")

for table in result_tables:
application = PlanningApplication()

# We can set the date received and the comment url straight away.
application.comment_url = self.comments_email_address

trs = table.findAll("tr")

application.council_reference = trs[0].a.string.strip()
relative_info_url = trs[0].a['href']

application.info_url = urlparse.urljoin(self.base_url, relative_info_url)

application.date_received = monday_before

application.address = trs[1].findAll("td")[1].string.strip()
application.postcode = getPostcodeFromText(application.address)
application.description = trs[2].findAll("td")[1].string.strip()

self._results.addApplication(application)

return self._results

def getResults(self, day, month, year):
return self.getResultsByDayMonthYear(int(day), int(month), int(year)).displayXML()

if __name__ == '__main__':
parser = BarnsleyParser()
print parser.getResults(21,5,2008)


+ 1
- 0
python_scrapers/OtherFilesToCopy.csv 查看文件

@@ -22,3 +22,4 @@
"Planet.py", "420"
"Ocella.py", "420"
"IsleOfWight.py", "420"
"Barnsley.py", "420"

+ 1
- 0
python_scrapers/SitesToGenerate.csv 查看文件

@@ -219,3 +219,4 @@
"Mid Bedfordshire District Council", "Mid Beds", "http://www.midbeds.gov.uk/acolnetDC/DCpages/acolnetcgi.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "MidBedsParser"
"Cambridgeshire County Council", "Cambridgeshire", "http://planapps2.cambridgeshire.gov.uk/DCWebPages/AcolNetCGI.gov?ACTION=UNWRAP&RIPNAME=Root.pgesearch", "AcolnetParser", "AcolnetParser"
"Isle of Wight Council", "Isle of Wight", "", "IsleOfWight", "IsleOfWightParser"
"Barnsley Metropolitan Borough Council", "Barnsley", "", "Barnsley", "BarnsleyParser"

正在加载...
取消
保存