uk_planning_scraper/lib/uk_planning_scraper/authority.rb

require 'csv'

module UKPlanningScraper
  class Authority
    attr_reader :name, :tags, :url
    @@authorities = []

    def initialize(name, url, tags)
      @name = name
      @url = url
      @tags = tags
      @applications = [] # Application objects
    end

    def scrape(params, options = {})
      default_options = {
        delay: 10,
      }
      options = default_options.merge(options) # The user-supplied options override the defaults
      
      # Validated within the last n days
      # Assumes that every scraper/system can do a date range search
      if params[:validated_days]
        params[:validated_to] = Date.today
        params[:validated_from] = Date.today - (params[:validated_days] - 1)
      end
        
      # Received within the last n days
      # Assumes that every scraper/system can do a date range search
      if params[:received_days]
        params[:received_to] = Date.today
        params[:received_from] = Date.today - (params[:received_days] - 1)
      end
      
      # Decided within the last n days
      # Assumes that every scraper/system can do a date range search
      if params[:decided_days]
        params[:decided_to] = Date.today
        params[:decided_from] = Date.today - (params[:decided_days] - 1)
      end
      
      # Select which scraper to use
      case system
      when 'idox'
        @applications = scrape_idox(params, options)
      when 'northgate'
        @applications = scrape_northgate(params, options)
      else
        raise SystemNotSupported.new("Planning system not supported for #{@name} at URL: #{@url}")
      end
      
      # Post processing
      @applications.each do |app|
        app.authority_name = @name
      end

      # Output as an array of hashes
      output = []
      # FIXME - silently ignores invalid apps. How should we handle them?
      @applications.each { |app| output << app.to_hash if app.valid? }
      output  # Single point of successful exit
    end
    
    def tagged?(tag)
      @tags.include?(tag)
    end
    
    def system
      if @url.match(/search\.do\?action=advanced/i)
        s = 'idox'
      elsif @url.match(/generalsearch\.aspx/i)
        s = 'northgate'
      elsif @url.match(/ocellaweb/i)
        s = 'ocellaweb'
      elsif @url.match(/\/apas\//)
        s = 'agileplanning'
      else
        s = 'unknownsystem'
      end
    end

    def self.all
      @@authorities
    end
    
    # List all the tags in use
    def self.tags
      tags = []
      @@authorities.each { |a| tags << a.tags }
      tags.flatten.uniq.sort
    end
    
    def self.named(name)
      authority = @@authorities.find { |a| name == a.name }
      raise AuthorityNotFound if authority.nil?
      authority 
    end

    # Tagged x
    def self.tagged(tag)
      found = []
      @@authorities.each { |a| found << a if a.tags.include?(tag) }
      found
    end

    # Not tagged x
    def self.not_tagged(tag)
      found = []
      @@authorities.each { |a| found << a unless a.tags.include?(tag) }
      found
    end

    # Authorities with no tags
    def self.untagged
      found = []
      @@authorities.each { |a| found << a if a.tags.empty? }
      found
    end

    def self.load
      # Don't run this method more than once
      return unless @@authorities.empty?
      # FIXME hardcoded file path
      CSV.foreach(File.join(File.dirname(__dir__), 'uk_planning_scraper', 'authorities.csv')) do |line|
        auth = Authority.new(
          line[0].strip,
          line[1].strip,
          line[2..-1].map { |e| e.strip })
        auth.tags << auth.system unless auth.tagged?(auth.system)
        auth.tags.sort!
        @@authorities << auth
      end
    end
  end
end

UKPlanningScraper::Authority.load