diff --git a/Gemfile b/Gemfile index c9c4f61..282d3ed 100644 --- a/Gemfile +++ b/Gemfile @@ -8,4 +8,3 @@ source "https://rubygems.org" gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" gem "mechanize" -gem "activesupport" # For date calculations diff --git a/Gemfile.lock b/Gemfile.lock index 7a49ced..05a919e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,18 +10,11 @@ GIT GEM remote: https://rubygems.org/ specs: - activesupport (5.1.2) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (~> 0.7) - minitest (~> 5.1) - tzinfo (~> 1.1) - concurrent-ruby (1.0.5) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) http-cookie (1.0.3) domain_name (~> 0.5) httpclient (2.8.3) - i18n (0.8.4) mechanize (2.7.5) domain_name (~> 0.5, >= 0.5.1) http-cookie (~> 1.0) @@ -35,7 +28,6 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) mini_portile2 (2.2.0) - minitest (5.10.2) net-http-digest_auth (1.4.1) net-http-persistent (2.9.4) nokogiri (1.8.0) @@ -44,9 +36,6 @@ GEM sqlite3 (1.3.13) sqlite_magic (0.0.6) sqlite3 - thread_safe (0.3.6) - tzinfo (1.2.3) - thread_safe (~> 0.1) unf (0.1.4) unf_ext unf_ext (0.0.7.4) @@ -56,7 +45,6 @@ PLATFORMS ruby DEPENDENCIES - activesupport mechanize scraperwiki! diff --git a/scraper.rb b/scraper.rb index a9a4b25..47e3820 100644 --- a/scraper.rb +++ b/scraper.rb @@ -5,7 +5,6 @@ require 'mechanize' require 'pp' require 'time' require 'date' -require 'active_support/all' # Use the column names from planningalerts.org.au: # https://www.planningalerts.org.au/how_to_write_a_scraper @@ -69,7 +68,7 @@ agent.verify_mode = OpenSSL::SSL::VERIFY_NONE d = Date.today 12.times do - d_start = (d - 29.days).strftime("%d/%m/%Y") + d_start = (d - 29).strftime("%d/%m/%Y") d_end = d.strftime("%d/%m/%Y") url = "#{BASEURL}Summary?weekListType=SRCH&recFrom=#{d_start}&recTo=#{d_end}&ward=ALL&appTyp=ALL&wardTxt=All%20Wards&appTypTxt=All%20Application%20Types&limit=500" @@ -78,9 +77,9 @@ d = Date.today page = agent.get(url) apps = page.search("#planningApplication") puts apps.size, '' - + apps.each { |app| parse(app) } - d -= 30.days + d -= 30 sleep 5 end