diff --git a/lib/uk_planning_scraper/authority_scrape_params.rb b/lib/uk_planning_scraper/authority_scrape_params.rb index 0024732..2ecbbea 100644 --- a/lib/uk_planning_scraper/authority_scrape_params.rb +++ b/lib/uk_planning_scraper/authority_scrape_params.rb @@ -84,6 +84,16 @@ module UKPlanningScraper self end + def include_property + unless system == 'idox' + raise NoMethodError.new("include_property is only implemented for \ + Idox. This authority (#{@name}) is #{system.capitalize}.") + end + + @scrape_params[:include_property] = true + self + end + private # Handle the simple params with this diff --git a/lib/uk_planning_scraper/idox.rb b/lib/uk_planning_scraper/idox.rb index 0e10c08..878a546 100644 --- a/lib/uk_planning_scraper/idox.rb +++ b/lib/uk_planning_scraper/idox.rb @@ -129,6 +129,8 @@ module UKPlanningScraper puts "#{i + 1} of #{apps.size}" parse_info_url(app) if app.info_url + + next unless params[:include_property] parse_property_url(app) if app.property_url parse_property_detail_urls(app) if app.property_detail_urls end # scrape summary tab for apps diff --git a/spec/property_details_spec.rb b/spec/property_details_spec.rb new file mode 100644 index 0000000..eb0d3ab --- /dev/null +++ b/spec/property_details_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe UKPlanningScraper::Authority do + + describe '#include_property' do + + let(:scraper) { UKPlanningScraper::Authority.named(authority_name) } + + context 'for 2 days with property details' do + let(:authority_name) { 'Cardiff' } + + it 'returns apps' do + apps = VCR.use_cassette("#{self.class.description}") { + scraper.include_property + .decided_from(Date.new(2019, 4, 8)) + .decided_to(Date.new(2019, 4, 9)) + .scrape(delay: 0) + } + pp apps + end + end + + end + +end diff --git a/spec/vcr_cassettes/for_2_days_with_property_details.yml b/spec/vcr_cassettes/for_2_days_with_property_details.yml new file mode 100644 index 0000000..bd7b7e9 --- /dev/null +++ b/spec/vcr_cassettes/for_2_days_with_property_details.yml @@ -0,0 +1,7183 @@ +--- +http_interactions: +- request: + method: get + uri: https://planningonline.cardiff.gov.uk/online-applications/search.do?action=advanced + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip,deflate,identity + Accept: + - "*/*" + User-Agent: + - Mechanize/2.7.6 Ruby/2.6.2p47 (http://github.com/sparklemotion/mechanize/) + Accept-Charset: + - ISO-8859-1,utf-8;q=0.7,*;q=0.7 + Accept-Language: + - en-us,en;q=0.5 + Host: + - planningonline.cardiff.gov.uk + Connection: + - keep-alive + Keep-Alive: + - '300' + response: + status: + code: 200 + message: OK + headers: + Cache-Control: + - private, no-store + Content-Type: + - text/html; charset=UTF-8 + Expires: + - Mon, 15 Apr 2019 11:14:04 GMT + X-Frame-Options: + - SAMEORIGIN + X-Ua-Compatible: + - IE=edge + Server: + - Apache + Set-Cookie: + - JSESSIONID=ss9pKoHL32yHmhJQONSykek96zXCSSZ75skxJ5v1.vmidoxweblve; path=/; + HttpOnly + - WSLang-CFC001=en; expires=Mon, 15-Apr-2069 11:14:03 GMT; path=/; HttpOnly + X-Powered-By: + - ASP.NET + Date: + - Mon, 15 Apr 2019 11:14:04 GMT + Content-Length: + - '37801' + body: + encoding: UTF-8 + string: "\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n
\r\n\t\r\n\t\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nSearch + \ for Planning Applications, Appeals and Enforcements by matching at least + one search option in the form below.
\r\n\r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n + \ \r\n\r\n\r\n\r\n \r\n + \ \r\n 47 ELM STREET, ROATH, + CARDIFF, CF24 3QS\r\n
\r\n \r\n\r\n + \ \r\n\r\n \r\n\r\n \r\n\r\n\r\n \r\n + \ \r\n 81 KYLE CRESCENT, WHITCHURCH, + CARDIFF, CF14 1SU\r\n
\r\n \r\n\r\n + \ \r\n\r\n \r\n\r\n \r\n\r\n\r\n \r\n \r\n + \ 23 WOMANBY STREET, CITY CENTRE, CARDIFF, CF10 1BR\r\n + \
\r\n \r\n\r\n \r\n\r\n \r\n\r\n \r\n\r\n\r\n + \ \r\n \r\n 3 + RHYD Y PENAU ROAD, CYNCOED, CARDIFF, CF23 6PX\r\n
\r\n + \ \r\n\r\n \r\n\r\n \r\n\r\n \r\n\r\n\r\n \r\n \r\n + \ MILL FARM HOUSE, MILL FARM, ST MELLONS ROAD, LISVANE, + CARDIFF, CF14 0SH\r\n
\r\n \r\n\r\n + \ \r\n\r\n \r\n\r\n \r\n\r\n\r\n \r\n + \ \r\n BANNATYNE HEALTH CLUB, + PARC TY GLAS, LLANISHEN, CARDIFF, CF14 5DU\r\n
\r\n + \ \r\n\r\n \r\n\r\n \r\n\r\n \r\n\r\n\r\n \r\n + \ \r\n 17 PANTGLAS, PENTYRCH, + CARDIFF, CF15 9TH\r\n
\r\n \r\n\r\n + \ \r\n\r\n \r\n\r\n \r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00552/MNR\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07686229\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Thu + 07 Mar 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Thu + 21 Mar 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ 47 ELM STREET, ROATH, CARDIFF, + CF24 3QS\n \n \n + \ | \n
\n Proposal\n | \n + \\n \n + \ \n \n + \ USE AS A (C4) 6 BEDROOM HOUSE + IN MULTIPLE OCCUPATION\n \n \n + \ | \n
\n Status\n | \n + \\n \n + \ \n Withdrawn\n \n + \ \n \n + \ | \n
\n Decision\n | \n + \\n \n + \ \n \n + \ Withdrawn by Applicant\n \n + \ \n | \n + \
\n Decision + Issued Date\n | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 8 + documents associated with this application.
There + are 0 cases associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n100100058644 | \r\n
---|---|
Full Address: | \r\n47 ELM STREET, + ROATH, CARDIFF, CF24 3QS | \r\n
Property + Number: | \r\n47 | \r\n
Street: | \r\nELM + STREET | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF24 3QS | \r\n
Ward: | \r\nPLASNEWYDD | \r\n
Parish: | \r\n\r\n |
\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00503/DCH\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07678469\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Tue + 05 Mar 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Thu + 07 Mar 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ 81 KYLE CRESCENT, WHITCHURCH, + CARDIFF, CF14 1SU\n \n \n + \ | \n
\n Proposal\n | \n + \\n \n + \ \n \n + \ SINGLE STOREY REAR EXTENSION WITH + PITCH ROOF\n \n \n + \ | \n
\n Status\n | \n + \\n \n + \ \n Decided\n \n + \ \n \n + \ | \n
\n Decision\n | \n + \\n \n + \ \n \n + \ Permission be granted\n \n + \ \n | \n + \
\n Decision + Issued Date\n | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 6 + documents associated with this application.
There + are 0 cases associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n100100033792 | \r\n
---|---|
Full Address: | \r\n81 KYLE CRESCENT, + WHITCHURCH, CARDIFF, CF14 1SU | \r\n
Property Number: | \r\n81 | \r\n
Street: | \r\nKYLE CRESCENT | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF14 1SU | \r\n
Ward: | \r\nHEATH | \r\n
Parish: | \r\n\r\n |
\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00309/MJR\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07606471\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Thu + 14 Feb 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Thu + 14 Feb 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ 23 WOMANBY STREET, CITY CENTRE, + CARDIFF, CF10 1BR\n \n \n + \ | \n
\n Proposal\n | \n + \\n \n + \ \n \n + \ HEATING AND COOLING PLANT TO THE + ROOF AND REAR COURTYARD\n \n \n + \ | \n
\n Status\n | \n + \\n \n + \ \n Decided\n \n + \ \n \n + \ | \n
\n Decision\n | \n + \\n \n + \ \n \n + \ Permission be granted\n \n + \ \n | \n + \
\n Decision + Issued Date\n | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 11 + documents associated with this application.
There + are 0 cases associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n100100890011 | \r\n
---|---|
Full Address: | \r\n23 WOMANBY + STREET, CITY CENTRE, CARDIFF, CF10 1BR | \r\n
Property Number: | \r\n23 | \r\n
Street: | \r\nWOMANBY STREET | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF10 1BR | \r\n
Ward: | \r\nCATHAYS | \r\n
Parish: | \r\n\r\n |
\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00284/DCH\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07622839\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Tue + 12 Feb 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Thu + 21 Feb 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ 3 RHYD Y PENAU ROAD, CYNCOED, + CARDIFF, CF23 6PX\n \n \n + \ | \n
\n Proposal\n | \n + \\n \n + \ \n \n + \ 1ST FLOOR REAR AND GROUND FLOOR + SIDE EXTENSION\n \n \n + \ | \n
\n Status\n | \n + \\n \n + \ \n Decided\n \n + \ \n \n + \ | \n
\n Decision\n | \n + \\n \n + \ \n \n + \ Permission be granted\n \n + \ \n | \n + \
\n Decision + Issued Date\n | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 10 + documents associated with this application.
There + are 0 cases associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n100100044782 | \r\n
---|---|
Full Address: | \r\n3 RHYD Y PENAU + ROAD, CYNCOED, CARDIFF, CF23 6PX | \r\n
Property Number: | \r\n3 | \r\n
Street: | \r\nRHYD Y PENAU ROAD | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF23 6PX | \r\n
Ward: | \r\nCYNCOED | \r\n
Parish: | \r\n\r\n |
\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00263/DCH\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07614385\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Fri + 08 Feb 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Mon + 11 Feb 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ MILL FARM HOUSE, MILL FARM, ST + MELLONS ROAD, LISVANE, CARDIFF, CF14 0SH\n \n + \ \n | \n + \
\n Proposal\n + \ | \n\n + \ \n \n + \ \n VARIATION + OF CONDITION 2 (APPROVED PLANS) OF 17/03122/DCH\n \n + \ \n | \n + \
\n Status\n + \ | \n\n + \ \n \n + \ Decided\n + \ \n \n + \ \n | \n + \
\n Decision\n + \ | \n\n + \ \n \n + \ \n Permission + be granted\n \n \n + \ | \n
\n Decision Issued Date\n + \ | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 10 + documents associated with this application.
There + is 1 + case associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n10023549353 | \r\n
---|---|
Full Address: | \r\nMILL FARM HOUSE, + MILL FARM, ST MELLONS ROAD, LISVANE, CARDIFF, CF14 0SH | \r\n
Property Description: | \r\nMILL + FARM | \r\n
Property Number: | \r\n\r\n |
Street: | \r\nST MELLONS ROAD | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF14 0SH | \r\n
Ward: | \r\n\r\n |
Parish: | \r\n\r\n |
\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00257/MNR\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07613169\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Fri + 08 Feb 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Fri + 08 Feb 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ BANNATYNE HEALTH CLUB, PARC TY + GLAS, LLANISHEN, CARDIFF, CF14 5DU\n \n + \ \n | \n + \
\n Proposal\n + \ | \n\n + \ \n \n + \ \n NEW + BUILD EXTENSION TO EXISTING POOL HALL TO CREATE SMALL RELAX AREA\n \n + \ \n | \n + \
\n Status\n + \ | \n\n + \ \n \n + \ Decided\n + \ \n \n + \ \n | \n + \
\n Decision\n + \ | \n\n + \ \n \n + \ \n Permission + be granted\n \n \n + \ | \n
\n Decision Issued Date\n + \ | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 18 + documents associated with this application.
There + are 0 cases associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n200001097060 | \r\n
---|---|
Full Address: | \r\nBANNATYNE HEALTH + CLUB, PARC TY GLAS, LLANISHEN, CARDIFF, CF14 5DU | \r\n
Property Description: | \r\nBANNATYNE HEALTH CLUB | \r\n
Property Number: | \r\n\r\n |
Street: | \r\nPARC TY GLAS | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF14 5DU | \r\n
Ward: | \r\nLLANISHEN | \r\n
Parish: | \r\n\r\n |
\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\n Reference\n + \ | \n\n + \ \n \n + \ \n 19/00214/DCH\n + \ \n \n + \ | \n
---|---|
\n Alternative Reference\n + \ | \n\n + \ \n \n + \ \n PP-07582302\n + \ \n \n + \ | \n
\n Application Received\n + \ | \n\n + \ \n \n + \ \n Mon + 04 Feb 2019\n \n \n + \ | \n
\n Application Validated\n + \ | \n\n + \ \n \n + \ \n Mon + 11 Feb 2019\n \n \n + \ | \n
\n Address\n | \n + \\n \n + \ \n \n + \ 17 PANTGLAS, PENTYRCH, CARDIFF, + CF15 9TH\n \n \n + \ | \n
\n Proposal\n | \n + \\n \n + \ \n \n + \ REMOVE EXISTING DECKING AT REAR + AND REPLACE WITH GROUND LEVEL AND RAISED PATIOS\n \n + \ \n | \n + \
\n Status\n + \ | \n\n + \ \n \n + \ Decided\n + \ \n \n + \ \n | \n + \
\n Decision\n + \ | \n\n + \ \n \n + \ \n Permission + be granted\n \n \n + \ | \n
\n Decision Issued Date\n + \ | \n\n + \ \n \n + \ \n Mon + 08 Apr 2019\n \n \n + \ | \n
\n Appeal Decision\n | \n + \\n \n + \ \n \n + \ \n \n + \ \n | \n + \
There are 9 + documents associated with this application.
There + are 0 cases associated with this application.
There + is 1 + property associated with this application.
\r\n\n \n + \\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\n\r\n\t\r\n\t\tHelp + with this page\r\n\t (opens in a new window)\r\n\t\r\n
\r\n\r\nUPRN: | \r\n100100140556 | \r\n
---|---|
Full Address: | \r\n17 PANTGLAS, + PENTYRCH, CARDIFF, CF15 9TH | \r\n
Property + Number: | \r\n17 | \r\n
Street: | \r\nPANTGLAS | \r\n
Town: | \r\nCARDIFF | \r\n
Postcode: | \r\nCF15 9TH | \r\n
Ward: | \r\nPENTYRCH | \r\n
Parish: | \r\n\r\n |