From 7bd4cd303ef4ddb36abb9f6316ea61378ddae4bf Mon Sep 17 00:00:00 2001
From: Adrian Short <adrian@adrianshort.org>
Date: Mon, 8 Oct 2018 11:18:44 +0100
Subject: [PATCH] Scrape validated apps as well as decided ones

---
 scraper.rb | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index 99f5a62..81c80f9 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -3,13 +3,20 @@ require 'scraperwiki'
 
 auths = UKPlanningScraper::Authority.tagged('london')
 
+scrapes = [
+  { validated_days: ENV['MORPH_DAYS'].to_i },
+  { decided_days: ENV['MORPH_DAYS'].to_i }
+]
+
 auths.each_with_index do |auth, i|
-  begin
-    puts "#{i + 1} of #{auths.size}: Scraping #{auth.name}"
-    apps = auth.scrape({ decided_days: ENV['MORPH_DAYS'].to_i })
-    ScraperWiki.save_sqlite([:authority_name, :council_reference], apps)
-    puts "#{auth.name}: #{apps.size} application(s) saved."
-  rescue StandardError => e
-    puts e
+  scrapes.each_with_index do |scrape, j|
+    puts "Authority #{i + 1} of #{auths.size}: Scrape #{j + 1} of #{scrapes.size} for #{auth.name}."
+    begin
+      apps = auth.scrape(scrape)
+      ScraperWiki.save_sqlite([:authority_name, :council_reference], apps)
+      puts "#{auth.name}: #{apps.size} application(s) saved."
+    rescue StandardError => e
+      puts e
+    end
   end
 end