From 1f6d37ffc08e11a576c72fe332eb095ec2f7a53c Mon Sep 17 00:00:00 2001
From: Adrian Short <adrian@adrianshort.org>
Date: Wed, 19 Sep 2018 00:36:49 +0100
Subject: [PATCH] First commit

---
 Gemfile      |  4 ++--
 Gemfile.lock | 59 +++++++++++++++++++++++++++++++++++++++-------------
 scraper.rb   | 36 ++++++++++++--------------------
 3 files changed, 59 insertions(+), 40 deletions(-)

diff --git a/Gemfile b/Gemfile
index 6ab45dc..043548b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -4,7 +4,7 @@
 
 source "https://rubygems.org"
 
-ruby "2.0.0"
+ruby "2.3.1"
 
+gem "uk_planning_scraper", git: "https://github.com/adrianshort/uk_planning_scraper.git", ref: "1f9d78e"
 gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults"
-gem "mechanize"
diff --git a/Gemfile.lock b/Gemfile.lock
index 30fb5f3..8587864 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,12 @@
+GIT
+  remote: https://github.com/adrianshort/uk_planning_scraper.git
+  revision: 1f9d78e1bed6bfd08f0589e6e7595dec86d9673f
+  ref: 1f9d78e
+  specs:
+    uk_planning_scraper (0.2.0)
+      http
+      mechanize (~> 2.7)
+
 GIT
   remote: https://github.com/openaustralia/scraperwiki-ruby.git
   revision: fc50176812505e463077d5c673d504a6a234aa78
@@ -10,38 +19,58 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    domain_name (0.5.24)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    connection_pool (2.2.2)
+    domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
-    http-cookie (1.0.2)
+    http (3.3.0)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.0)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
       domain_name (~> 0.5)
+    http-form_data (2.1.1)
+    http_parser.rb (0.6.0)
     httpclient (2.6.0.1)
-    mechanize (2.7.3)
+    mechanize (2.7.6)
       domain_name (~> 0.5, >= 0.5.1)
       http-cookie (~> 1.0)
-      mime-types (~> 2.0)
+      mime-types (>= 1.17.2)
       net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (~> 2.5, >= 2.5.2)
-      nokogiri (~> 1.4)
+      net-http-persistent (>= 2.5.2)
+      nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (>= 0.0.9, < 0.2)
-    mime-types (2.5)
-    mini_portile (0.6.2)
-    net-http-digest_auth (1.4)
-    net-http-persistent (2.9.4)
-    nokogiri (1.6.6.2)
-      mini_portile (~> 0.6.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
+    mini_portile2 (2.3.0)
+    net-http-digest_auth (1.4.1)
+    net-http-persistent (3.0.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.8.4)
+      mini_portile2 (~> 2.3.0)
     ntlm-http (0.1.1)
+    public_suffix (3.0.3)
     sqlite3 (1.3.10)
     sqlite_magic (0.0.3)
       sqlite3
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.1)
-    webrobots (0.1.1)
+    unf_ext (0.0.7.5)
+    webrobots (0.1.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  mechanize
   scraperwiki!
+  uk_planning_scraper!
+
+RUBY VERSION
+   ruby 2.3.1p112
+
+BUNDLED WITH
+   1.16.5
diff --git a/scraper.rb b/scraper.rb
index 5799e98..e9c230e 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -1,25 +1,15 @@
-# This is a template for a Ruby scraper on morph.io (https://morph.io)
-# including some code snippets below that you should find helpful
+require 'uk_planning_scraper'
+require 'scraperwiki'
 
-# require 'scraperwiki'
-# require 'mechanize'
-#
-# agent = Mechanize.new
-#
-# # Read in a page
-# page = agent.get("http://foo.com")
-#
-# # Find somehing on the page using css selectors
-# p page.at('div.content')
-#
-# # Write out to the sqlite database using scraperwiki library
-# ScraperWiki.save_sqlite(["name"], {"name" => "susan", "occupation" => "software developer"})
-#
-# # An arbitrary query against the database
-# ScraperWiki.select("* from data where 'name'='peter'")
+auths = UKPlanningScraper::Authority.tagged('london')
 
-# You don't have to do things with the Mechanize or ScraperWiki libraries.
-# You can use whatever gems you want: https://morph.io/documentation/ruby
-# All that matters is that your final data is written to an SQLite database
-# called "data.sqlite" in the current working directory which has at least a table
-# called "data".
+auths.each_with_index do |auth, i|
+  begin
+    puts "#{i + 1} of #{auths.size}: Scraping #{auth.name}"
+    apps = auth.scrape({ decided_days: ENV['MORPH_DAYS'] })
+    ScraperWiki.save_sqlite([:authority_name, :council_reference], apps)
+    puts "#{auth.name}: #{apps.size} application(s) saved."
+  rescue UKPlanningScraper::SystemNotSupportedError => e
+    puts e
+  end
+end