| @@ -6,5 +6,5 @@ source "https://rubygems.org" | |||||
| ruby "2.3.1" | ruby "2.3.1" | ||||
| gem "uk_planning_scraper", git: "https://github.com/adrianshort/uk_planning_scraper.git", ref: "8a070e1" | |||||
| gem "uk_planning_scraper", git: "https://github.com/adrianshort/uk_planning_scraper.git", ref: "cad5fcd" | |||||
| gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" | gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" | ||||
| @@ -1,9 +1,9 @@ | |||||
| GIT | GIT | ||||
| remote: https://github.com/adrianshort/uk_planning_scraper.git | remote: https://github.com/adrianshort/uk_planning_scraper.git | ||||
| revision: 8a070e17732b3304e7e73a81cb7538795fe604c0 | |||||
| ref: 8a070e1 | |||||
| revision: cad5fcd2b98260572dccf0a4cf1ddfeabc469a43 | |||||
| ref: cad5fcd | |||||
| specs: | specs: | ||||
| uk_planning_scraper (0.2.0) | |||||
| uk_planning_scraper (0.4.0) | |||||
| http | http | ||||
| mechanize (~> 2.7) | mechanize (~> 2.7) | ||||
| @@ -24,7 +24,7 @@ GEM | |||||
| connection_pool (2.2.2) | connection_pool (2.2.2) | ||||
| domain_name (0.5.20180417) | domain_name (0.5.20180417) | ||||
| unf (>= 0.0.5, < 1.0.0) | unf (>= 0.0.5, < 1.0.0) | ||||
| http (3.3.0) | |||||
| http (4.0.0) | |||||
| addressable (~> 2.3) | addressable (~> 2.3) | ||||
| http-cookie (~> 1.0) | http-cookie (~> 1.0) | ||||
| http-form_data (~> 2.0) | http-form_data (~> 2.0) | ||||
| @@ -3,16 +3,13 @@ require 'scraperwiki' | |||||
| auths = UKPlanningScraper::Authority.tagged('london') | auths = UKPlanningScraper::Authority.tagged('london') | ||||
| scrapes = [ | |||||
| { validated_days: ENV['MORPH_DAYS'].to_i }, | |||||
| { decided_days: ENV['MORPH_DAYS'].to_i } | |||||
| ] | |||||
| params = %w(validated_days decided_days) | |||||
| auths.each_with_index do |auth, i| | auths.each_with_index do |auth, i| | ||||
| scrapes.each_with_index do |scrape, j| | |||||
| puts "Authority #{i + 1} of #{auths.size}: Scrape #{j + 1} of #{scrapes.size} for #{auth.name}." | |||||
| params.each_with_index do |param, j| | |||||
| puts "Authority #{i + 1} of #{auths.size}: Scrape #{j + 1} of #{params.size} for #{auth.name}." | |||||
| begin | begin | ||||
| apps = auth.scrape(scrape) | |||||
| apps = auth.send(param, ENV['MORPH_DAYS'].to_i).scrape | |||||
| ScraperWiki.save_sqlite([:authority_name, :council_reference], apps) | ScraperWiki.save_sqlite([:authority_name, :council_reference], apps) | ||||
| puts "#{auth.name}: #{apps.size} application(s) saved." | puts "#{auth.name}: #{apps.size} application(s) saved." | ||||
| rescue StandardError => e | rescue StandardError => e | ||||