diff --git a/.gitignore b/.gitignore index f17467b..a4b6a6c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /tmp/ *.gem *.swp +.ruby-* diff --git a/lib/uk_planning_scraper/authority.rb b/lib/uk_planning_scraper/authority.rb index 29be581..c458a76 100644 --- a/lib/uk_planning_scraper/authority.rb +++ b/lib/uk_planning_scraper/authority.rb @@ -2,7 +2,14 @@ require 'csv' module UKPlanningScraper class Authority - attr_reader :name, :url, :system + # eg "Camden" + attr_reader :name + + # URL of the advanced search page + attr_reader :url + + # eg "idox", "northgate" + attr_reader :system @@authorities = [] @@ -26,7 +33,9 @@ module UKPlanningScraper @system = 'unknownsystem' end end - + + # Scrape this authority's website for applications + def scrape(options = {}) default_options = { delay: 10, @@ -61,6 +70,7 @@ module UKPlanningScraper output # Single point of successful exit end + # Return a sorted list of tags for this authority def tags @tags.sort end @@ -121,6 +131,7 @@ module UKPlanningScraper def self.load # Don't run this method more than once return unless @@authorities.empty? + CSV.foreach(File.join(File.dirname(__dir__), 'uk_planning_scraper', \ 'authorities.csv'), :headers => true) do |line| auth = Authority.new(line['authority_name'], line['url']) diff --git a/lib/uk_planning_scraper/authority_scrape_params.rb b/lib/uk_planning_scraper/authority_scrape_params.rb index b4083d7..d203f18 100644 --- a/lib/uk_planning_scraper/authority_scrape_params.rb +++ b/lib/uk_planning_scraper/authority_scrape_params.rb @@ -12,7 +12,7 @@ module UKPlanningScraper def validated_days(n) # Validated within the last n days # Assumes that every scraper/system can do a date range search - check_class(n, Fixnum) + check_class(n, Integer) unless n > 0 raise ArgumentError.new("validated_days must be greater than 0") @@ -26,7 +26,7 @@ module UKPlanningScraper def received_days(n) # received within the last n days # Assumes that every scraper/system can do a date range search - check_class(n, Fixnum) + check_class(n, Integer) unless n > 0 raise ArgumentError.new("received_days must be greater than 0") @@ -40,7 +40,7 @@ module UKPlanningScraper def decided_days(n) # decided within the last n days # Assumes that every scraper/system can do a date range search - check_class(n, Fixnum) + check_class(n, Integer) unless n > 0 raise ArgumentError.new("decided_days must be greater than 0")