|
@@ -4,13 +4,16 @@ require 'scraperwiki' |
|
|
auths = UKPlanningScraper::Authority.all |
|
|
auths = UKPlanningScraper::Authority.all |
|
|
|
|
|
|
|
|
auths.each_with_index do |auth, i| |
|
|
auths.each_with_index do |auth, i| |
|
|
next if auth.name == 'Bolton' # https://github.com/adrianshort/uk_planning_scraper/issues/7 |
|
|
|
|
|
puts "#{i + 1} of #{auths.size}: #{auth.name}" |
|
|
puts "#{i + 1} of #{auths.size}: #{auth.name}" |
|
|
begin |
|
|
begin |
|
|
apps = auth.scrape({ validated_days: ENV['MORPH_DAYS'].to_i, keywords: 'inlink'}) |
|
|
apps = auth.scrape({ validated_days: ENV['MORPH_DAYS'].to_i, keywords: 'inlink'}) |
|
|
ScraperWiki.save_sqlite([:authority_name, :council_reference], apps, 'applications') |
|
|
|
|
|
|
|
|
apps.each do |app| |
|
|
|
|
|
unless app[:description].match(/chainlink/i) # Backend keyword search is weak |
|
|
|
|
|
ScraperWiki.save_sqlite([:authority_name, :council_reference], app, 'applications') |
|
|
|
|
|
end |
|
|
|
|
|
end |
|
|
puts "#{auth.name}: #{apps.size} application(s) saved." |
|
|
puts "#{auth.name}: #{apps.size} application(s) saved." |
|
|
rescue UKPlanningScraper::SystemNotSupported => e |
|
|
|
|
|
|
|
|
rescue StandardError => e |
|
|
puts "Error: #{e}" |
|
|
puts "Error: #{e}" |
|
|
end |
|
|
end |
|
|
end |
|
|
end |