Browse Source

Don't scrape properties by default

pull/38/head
Graeme Porteous 5 years ago
parent
commit
97f5eb0c7e
4 changed files with 7220 additions and 0 deletions
  1. +10
    -0
      lib/uk_planning_scraper/authority_scrape_params.rb
  2. +2
    -0
      lib/uk_planning_scraper/idox.rb
  3. +25
    -0
      spec/property_details_spec.rb
  4. +7183
    -0
      spec/vcr_cassettes/for_2_days_with_property_details.yml

+ 10
- 0
lib/uk_planning_scraper/authority_scrape_params.rb View File

@@ -84,6 +84,16 @@ module UKPlanningScraper
self
end

def include_property
unless system == 'idox'
raise NoMethodError.new("include_property is only implemented for \
Idox. This authority (#{@name}) is #{system.capitalize}.")
end

@scrape_params[:include_property] = true
self
end

private

# Handle the simple params with this


+ 2
- 0
lib/uk_planning_scraper/idox.rb View File

@@ -129,6 +129,8 @@ module UKPlanningScraper
puts "#{i + 1} of #{apps.size}"

parse_info_url(app) if app.info_url

next unless params[:include_property]
parse_property_url(app) if app.property_url
parse_property_detail_urls(app) if app.property_detail_urls
end # scrape summary tab for apps


+ 25
- 0
spec/property_details_spec.rb View File

@@ -0,0 +1,25 @@
require 'spec_helper'

describe UKPlanningScraper::Authority do

describe '#include_property' do

let(:scraper) { UKPlanningScraper::Authority.named(authority_name) }

context 'for 2 days with property details' do
let(:authority_name) { 'Cardiff' }

it 'returns apps' do
apps = VCR.use_cassette("#{self.class.description}") {
scraper.include_property
.decided_from(Date.new(2019, 4, 8))
.decided_to(Date.new(2019, 4, 9))
.scrape(delay: 0)
}
pp apps
end
end

end

end

+ 7183
- 0
spec/vcr_cassettes/for_2_days_with_property_details.yml
File diff suppressed because it is too large
View File


Loading…
Cancel
Save