@@ -9,3 +9,4 @@ | |||||
/tmp/ | /tmp/ | ||||
*.gem | *.gem | ||||
*.swp | *.swp | ||||
.ruby-* |
@@ -2,7 +2,14 @@ require 'csv' | |||||
module UKPlanningScraper | module UKPlanningScraper | ||||
class Authority | class Authority | ||||
attr_reader :name, :url, :system | |||||
# eg "Camden" | |||||
attr_reader :name | |||||
# URL of the advanced search page | |||||
attr_reader :url | |||||
# eg "idox", "northgate" | |||||
attr_reader :system | |||||
@@authorities = [] | @@authorities = [] | ||||
@@ -26,7 +33,9 @@ module UKPlanningScraper | |||||
@system = 'unknownsystem' | @system = 'unknownsystem' | ||||
end | end | ||||
end | end | ||||
# Scrape this authority's website for applications | |||||
def scrape(options = {}) | def scrape(options = {}) | ||||
default_options = { | default_options = { | ||||
delay: 10, | delay: 10, | ||||
@@ -61,6 +70,7 @@ module UKPlanningScraper | |||||
output # Single point of successful exit | output # Single point of successful exit | ||||
end | end | ||||
# Return a sorted list of tags for this authority | |||||
def tags | def tags | ||||
@tags.sort | @tags.sort | ||||
end | end | ||||
@@ -121,6 +131,7 @@ module UKPlanningScraper | |||||
def self.load | def self.load | ||||
# Don't run this method more than once | # Don't run this method more than once | ||||
return unless @@authorities.empty? | return unless @@authorities.empty? | ||||
CSV.foreach(File.join(File.dirname(__dir__), 'uk_planning_scraper', \ | CSV.foreach(File.join(File.dirname(__dir__), 'uk_planning_scraper', \ | ||||
'authorities.csv'), :headers => true) do |line| | 'authorities.csv'), :headers => true) do |line| | ||||
auth = Authority.new(line['authority_name'], line['url']) | auth = Authority.new(line['authority_name'], line['url']) | ||||
@@ -12,7 +12,7 @@ module UKPlanningScraper | |||||
def validated_days(n) | def validated_days(n) | ||||
# Validated within the last n days | # Validated within the last n days | ||||
# Assumes that every scraper/system can do a date range search | # Assumes that every scraper/system can do a date range search | ||||
check_class(n, Fixnum) | |||||
check_class(n, Integer) | |||||
unless n > 0 | unless n > 0 | ||||
raise ArgumentError.new("validated_days must be greater than 0") | raise ArgumentError.new("validated_days must be greater than 0") | ||||
@@ -26,7 +26,7 @@ module UKPlanningScraper | |||||
def received_days(n) | def received_days(n) | ||||
# received within the last n days | # received within the last n days | ||||
# Assumes that every scraper/system can do a date range search | # Assumes that every scraper/system can do a date range search | ||||
check_class(n, Fixnum) | |||||
check_class(n, Integer) | |||||
unless n > 0 | unless n > 0 | ||||
raise ArgumentError.new("received_days must be greater than 0") | raise ArgumentError.new("received_days must be greater than 0") | ||||
@@ -40,7 +40,7 @@ module UKPlanningScraper | |||||
def decided_days(n) | def decided_days(n) | ||||
# decided within the last n days | # decided within the last n days | ||||
# Assumes that every scraper/system can do a date range search | # Assumes that every scraper/system can do a date range search | ||||
check_class(n, Fixnum) | |||||
check_class(n, Integer) | |||||
unless n > 0 | unless n > 0 | ||||
raise ArgumentError.new("decided_days must be greater than 0") | raise ArgumentError.new("decided_days must be greater than 0") | ||||