Browse Source

test council_reference

test showed that a search for non-existant reference still returns a 'pa' div. Check for existence of <div class="addressCrumb"> for a successful single hit
pull/37/head
Jez Nicholson 5 years ago
parent
commit
fefef1dd70
6 changed files with 341 additions and 19 deletions
  1. +3
    -0
      lib/uk_planning_scraper/authorities.csv
  2. +2
    -2
      lib/uk_planning_scraper/idox.rb
  3. +18
    -17
      spec/authority_spec.rb
  4. +40
    -0
      spec/council_reference_spec.rb
  5. +139
    -0
      spec/vcr_cassettes/for_a_non-existant_idox_planning_reference.yml
  6. +139
    -0
      spec/vcr_cassettes/for_an_existing_idox_planning_reference.yml

+ 3
- 0
lib/uk_planning_scraper/authorities.csv View File

@@ -27,11 +27,13 @@ Bury,https://planning.bury.gov.uk/online-applications/search.do?action=advanced,
Calderdale,https://portal.calderdale.gov.uk/online-applications/search.do?action=advanced,england westyorkshire Calderdale,https://portal.calderdale.gov.uk/online-applications/search.do?action=advanced,england westyorkshire
Camden,http://planningrecords.camden.gov.uk/Northgate/PlanningExplorer17/GeneralSearch.aspx,londonboroughs london Camden,http://planningrecords.camden.gov.uk/Northgate/PlanningExplorer17/GeneralSearch.aspx,londonboroughs london
Cardiff,https://planningonline.cardiff.gov.uk/online-applications/search.do?action=advanced,wales Cardiff,https://planningonline.cardiff.gov.uk/online-applications/search.do?action=advanced,wales
Carlisle,https://publicaccess.carlisle.gov.uk/online-applications/search.do?action=advanced,england
Cheshire West and Chester,https://pa.cheshirewestandchester.gov.uk/online-applications/search.do?action=advanced,chester cheshire england Cheshire West and Chester,https://pa.cheshirewestandchester.gov.uk/online-applications/search.do?action=advanced,chester cheshire england
City of London,http://www.planning2.cityoflondon.gov.uk/online-applications/search.do?action=advanced,london innerlondon northlondon england City of London,http://www.planning2.cityoflondon.gov.uk/online-applications/search.do?action=advanced,london innerlondon northlondon england
Cornwall,http://planning.cornwall.gov.uk/online-applications/search.do?action=advanced,england Cornwall,http://planning.cornwall.gov.uk/online-applications/search.do?action=advanced,england
Croydon,http://publicaccess2.croydon.gov.uk/online-applications/search.do?action=advanced,londonboroughs london Croydon,http://publicaccess2.croydon.gov.uk/online-applications/search.do?action=advanced,londonboroughs london
Cornwall,http://planning.cornwall.gov.uk/online-applications/search.do?action=advanced,england Cornwall,http://planning.cornwall.gov.uk/online-applications/search.do?action=advanced,england
Comhairle nan Eilean Siar,http://planning.cne-siar.gov.uk/PublicAccess/search.do?action=advanced,scotland
County Durham,https://publicaccess.durham.gov.uk/online-applications/search.do?action=advanced,england County Durham,https://publicaccess.durham.gov.uk/online-applications/search.do?action=advanced,england
Craven,https://publicaccess.cravendc.gov.uk/online-applications/search.do?action=advanced,england Craven,https://publicaccess.cravendc.gov.uk/online-applications/search.do?action=advanced,england
Darlington,https://publicaccess.darlington.gov.uk/online-applications/search.do?action=advanced,england Darlington,https://publicaccess.darlington.gov.uk/online-applications/search.do?action=advanced,england
@@ -99,6 +101,7 @@ Southend-on-Sea,https://publicaccess.southend.gov.uk/online-applications/search.
South Downs,https://planningpublicaccess.southdowns.gov.uk/online-applications/search.do?action=advanced,nationalparks england South Downs,https://planningpublicaccess.southdowns.gov.uk/online-applications/search.do?action=advanced,nationalparks england
Southampton,https://planningpublicaccess.southampton.gov.uk/online-applications/search.do?action=advanced,england Southampton,https://planningpublicaccess.southampton.gov.uk/online-applications/search.do?action=advanced,england
South Gloucestershire,https://developments.southglos.gov.uk/online-applications/search.do?action=advanced,england South Gloucestershire,https://developments.southglos.gov.uk/online-applications/search.do?action=advanced,england
South Lanarkshire,https://publicaccess.southlanarkshire.gov.uk/online-applications/search.do?action=advanced,scotland
South Tyneside,http://planning.southtyneside.info/Northgate/PlanningExplorer/GeneralSearch.aspx,england tyneandwear South Tyneside,http://planning.southtyneside.info/Northgate/PlanningExplorer/GeneralSearch.aspx,england tyneandwear
Southwark,https://planning.southwark.gov.uk/online-applications/search.do?action=advanced,londonboroughs london Southwark,https://planning.southwark.gov.uk/online-applications/search.do?action=advanced,londonboroughs london
St. Helens,https://publicaccess.sthelens.gov.uk/online-applications/search.do?action=advanced,england merseyside liverpoolcityregion St. Helens,https://publicaccess.sthelens.gov.uk/online-applications/search.do?action=advanced,england merseyside liverpoolcityregion


+ 2
- 2
lib/uk_planning_scraper/idox.rb View File

@@ -122,9 +122,9 @@ module UKPlanningScraper
end # if end # if
end # scrape summary tab for apps end # scrape summary tab for apps


if apps == [] && page.search('pa')
if apps == [] && params[:council_reference] && page.at_css('.addressCrumb')
app = Application.new app = Application.new
app.council_reference = params[:council_reference] if params[:council_reference]
app.council_reference = params[:council_reference]
parse_summary(app, page) parse_summary(app, page)
apps << app apps << app
end # direct hit end # direct hit


+ 18
- 17
spec/authority_spec.rb View File

@@ -1,13 +1,11 @@
require 'spec_helper' require 'spec_helper'


describe UKPlanningScraper::Authority do describe UKPlanningScraper::Authority do

describe '#named' do describe '#named' do

let(:authority) { described_class.named(authority_name) }
subject(:authority) { UKPlanningScraper::Authority.named(name) }


context 'when authority exists' do context 'when authority exists' do
let(:authority_name) { 'Westminster' }
let(:name) { 'Westminster' }


it 'returns an authority' do it 'returns an authority' do
expect(authority).to be_a(UKPlanningScraper::Authority) expect(authority).to be_a(UKPlanningScraper::Authority)
@@ -15,7 +13,7 @@ describe UKPlanningScraper::Authority do
end end


context 'when authority does not exist' do context 'when authority does not exist' do
let(:authority_name) { 'Westmonster' }
let(:name) { 'Westmonster' }


it 'raises an error' do it 'raises an error' do
expect { authority }.to raise_error(UKPlanningScraper::AuthorityNotFound) expect { authority }.to raise_error(UKPlanningScraper::AuthorityNotFound)
@@ -24,11 +22,10 @@ describe UKPlanningScraper::Authority do
end end


describe '#all' do describe '#all' do
let(:all) { UKPlanningScraper::Authority.all }


let(:all) { described_class.all }

it 'returns all authorities' do
expect(all.count).to eq(124)
it 'returns more than 100 authorities' do
expect(all.count).to be > 100
end end


it 'returns a list of authorities' do it 'returns a list of authorities' do
@@ -40,18 +37,22 @@ describe UKPlanningScraper::Authority do
end end


describe '#tagged' do describe '#tagged' do
let (:tagged_london) { described_class.tagged('london') }
let (:authority) { UKPlanningScraper::Authority.tagged(tag) }

context 'when tagged london' do
let(:tag) { 'london' }


it 'returns all London authorities' do
expect(tagged_london.count).to eq(35)
it 'returns all 35 London authorities' do
expect(authority.count).to eq(35)
end
end end


let (:tagged_londonboroughs) { described_class.tagged('londonboroughs') }
context 'when tagged londonboroughs' do
let(:tag) { 'londonboroughs' }


it 'returns all London boroughs' do
expect(tagged_londonboroughs.count).to eq(32)
it 'returns all 32 London boroughs' do
expect(authority.count).to eq(32)
end
end end

end end

end end

+ 40
- 0
spec/council_reference_spec.rb View File

@@ -0,0 +1,40 @@
require 'spec_helper'

describe UKPlanningScraper::Authority do
describe 'named+council_reference scrape' do
let(:scraper) { UKPlanningScraper::Authority.named(authority_name).council_reference(council_reference) }

context 'for an existing idox planning reference' do
let(:authority_name) { 'Brighton and Hove' }
let(:council_reference) { 'BH2017/04225' }
subject(:apps) {
VCR.use_cassette("#{self.class.description}") {
scraper.scrape
}
}

it 'returns an app (in the apps array)' do
expect(apps.any?).to be_truthy
end

it 'has a status of Withdrawn' do
expect(apps.first[:status]).to eql('Withdrawn')
end
end

context 'for a non-existant idox planning reference' do
let(:authority_name) { 'Brighton and Hove' }
let(:council_reference) { 'XYZ123' }
subject(:apps) {
VCR.use_cassette("#{self.class.description}") {
scraper.scrape
}
}

it 'returns an empty apps array' do
expect(apps.empty?).to be_truthy
end
end
end

end

+ 139
- 0
spec/vcr_cassettes/for_a_non-existant_idox_planning_reference.yml
File diff suppressed because it is too large
View File


+ 139
- 0
spec/vcr_cassettes/for_an_existing_idox_planning_reference.yml
File diff suppressed because it is too large
View File


Loading…
Cancel
Save