inlink-scraper/scraper.rb

require 'uk_planning_scraper'
require 'scraperwiki'
require 'date'
require 'time'
require 'csv'

councils = []

CSV.foreach('councils.csv') do |line|
  councils << { name: line[0], url: line[1] } unless line[0][0] == '#'
end

params = {
  validated_from: Date.today - ENV['MORPH_DAYS'].to_i,
  validated_to: Date.today,
  description: 'inlink',
}

councils.each do |council|
  apps = UKPlanningScraper.search(council[:url], params)
  
  apps.map! do |app|
    app.merge(
      {
        la_name: council[:name],
        updated_at: Time.now
      }
    )
  end
  
  ScraperWiki.save_sqlite([:council_reference, :la_name], apps, 'applications')
  puts "#{council[:name]}: #{apps.size}"
end