From 6bd7a9570240eb7498e99e087fe09588549cc6f6 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Tue, 29 Jun 2021 10:52:01 +0100 Subject: [PATCH 1/3] Fix warning by using Integer class rather than Fixnum Fixnum and Bignum have been integrated as a single Integer class. Previously this gave the warning: constant ::Fixnum is deprecated. https://bugs.ruby-lang.org/issues/12005 --- lib/uk_planning_scraper/authority_scrape_params.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/uk_planning_scraper/authority_scrape_params.rb b/lib/uk_planning_scraper/authority_scrape_params.rb index b3170e2..9b012a5 100644 --- a/lib/uk_planning_scraper/authority_scrape_params.rb +++ b/lib/uk_planning_scraper/authority_scrape_params.rb @@ -12,7 +12,7 @@ module UKPlanningScraper def validated_days(n) # Validated within the last n days # Assumes that every scraper/system can do a date range search - check_class(n, Fixnum) + check_class(n, Integer) unless n > 0 raise ArgumentError.new("validated_days must be greater than 0") @@ -26,7 +26,7 @@ module UKPlanningScraper def received_days(n) # received within the last n days # Assumes that every scraper/system can do a date range search - check_class(n, Fixnum) + check_class(n, Integer) unless n > 0 raise ArgumentError.new("received_days must be greater than 0") @@ -40,7 +40,7 @@ module UKPlanningScraper def decided_days(n) # decided within the last n days # Assumes that every scraper/system can do a date range search - check_class(n, Fixnum) + check_class(n, Integer) unless n > 0 raise ArgumentError.new("decided_days must be greater than 0") From d4231545bebfc5752b67d9877c17b909c2387b32 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Mon, 12 Jul 2021 11:23:01 +0100 Subject: [PATCH 2/3] Ignore rvm config --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b7e7725..e54c6c1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /spec/reports/ /tmp/ *.gem +.ruby-* From bcd86c9fcd34661e1ec2acd70d4e19f05d505104 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Mon, 12 Jul 2021 11:25:26 +0100 Subject: [PATCH 3/3] Improve comments --- lib/uk_planning_scraper/authority.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/uk_planning_scraper/authority.rb b/lib/uk_planning_scraper/authority.rb index 29be581..c458a76 100644 --- a/lib/uk_planning_scraper/authority.rb +++ b/lib/uk_planning_scraper/authority.rb @@ -2,7 +2,14 @@ require 'csv' module UKPlanningScraper class Authority - attr_reader :name, :url, :system + # eg "Camden" + attr_reader :name + + # URL of the advanced search page + attr_reader :url + + # eg "idox", "northgate" + attr_reader :system @@authorities = [] @@ -26,7 +33,9 @@ module UKPlanningScraper @system = 'unknownsystem' end end - + + # Scrape this authority's website for applications + def scrape(options = {}) default_options = { delay: 10, @@ -61,6 +70,7 @@ module UKPlanningScraper output # Single point of successful exit end + # Return a sorted list of tags for this authority def tags @tags.sort end @@ -121,6 +131,7 @@ module UKPlanningScraper def self.load # Don't run this method more than once return unless @@authorities.empty? + CSV.foreach(File.join(File.dirname(__dir__), 'uk_planning_scraper', \ 'authorities.csv'), :headers => true) do |line| auth = Authority.new(line['authority_name'], line['url'])