Browse Source

Merge from main branch

northgate-dates
Adrian Short 1 year ago
parent
commit
8349e29e38
3 changed files with 17 additions and 5 deletions
  1. +1
    -0
      .gitignore
  2. +13
    -2
      lib/uk_planning_scraper/authority.rb
  3. +3
    -3
      lib/uk_planning_scraper/authority_scrape_params.rb

+ 1
- 0
.gitignore View File

@@ -9,3 +9,4 @@
/tmp/
*.gem
*.swp
.ruby-*

+ 13
- 2
lib/uk_planning_scraper/authority.rb View File

@@ -2,7 +2,14 @@ require 'csv'

module UKPlanningScraper
class Authority
attr_reader :name, :url, :system
# eg "Camden"
attr_reader :name
# URL of the advanced search page
attr_reader :url
# eg "idox", "northgate"
attr_reader :system
@@authorities = []

@@ -26,7 +33,9 @@ module UKPlanningScraper
@system = 'unknownsystem'
end
end

# Scrape this authority's website for applications
def scrape(options = {})
default_options = {
delay: 10,
@@ -61,6 +70,7 @@ module UKPlanningScraper
output # Single point of successful exit
end
# Return a sorted list of tags for this authority
def tags
@tags.sort
end
@@ -121,6 +131,7 @@ module UKPlanningScraper
def self.load
# Don't run this method more than once
return unless @@authorities.empty?
CSV.foreach(File.join(File.dirname(__dir__), 'uk_planning_scraper', \
'authorities.csv'), :headers => true) do |line|
auth = Authority.new(line['authority_name'], line['url'])


+ 3
- 3
lib/uk_planning_scraper/authority_scrape_params.rb View File

@@ -12,7 +12,7 @@ module UKPlanningScraper
def validated_days(n)
# Validated within the last n days
# Assumes that every scraper/system can do a date range search
check_class(n, Fixnum)
check_class(n, Integer)

unless n > 0
raise ArgumentError.new("validated_days must be greater than 0")
@@ -26,7 +26,7 @@ module UKPlanningScraper
def received_days(n)
# received within the last n days
# Assumes that every scraper/system can do a date range search
check_class(n, Fixnum)
check_class(n, Integer)

unless n > 0
raise ArgumentError.new("received_days must be greater than 0")
@@ -40,7 +40,7 @@ module UKPlanningScraper
def decided_days(n)
# decided within the last n days
# Assumes that every scraper/system can do a date range search
check_class(n, Fixnum)
check_class(n, Integer)

unless n > 0
raise ArgumentError.new("decided_days must be greater than 0")


Loading…
Cancel
Save