@@ -1,115 +1,8 @@ | |||||
#!/usr/bin/env ruby | #!/usr/bin/env ruby | ||||
require 'scraperwiki' | |||||
require 'haml' | |||||
require 'pp' | |||||
require 'logger' | |||||
require 'csv' | |||||
require_relative '../lib/helpers' | require_relative '../lib/helpers' | ||||
require_relative '../lib/petrify' | |||||
require_relative '../lib/site' | |||||
OUTPUT_DIR = '_site' | |||||
VIEWS_DIR = File.join('views') | |||||
LAYOUT_FN = File.join(VIEWS_DIR, 'layout.haml') | |||||
def write_page(path_items, template, locals = {}) | |||||
dir = create_path(path_items) | |||||
fn = File.join(dir, 'index.html') | |||||
# https://stackoverflow.com/questions/6125265/using-layouts-in-haml-files-independently-of-rails | |||||
html = Haml::Engine.new(File.read(LAYOUT_FN)).render do | |||||
Haml::Engine.new(File.read(File.join(VIEWS_DIR, "#{template}.haml"))).render(Object.new, locals) | |||||
end | |||||
File.write(fn, html) | |||||
@log.info fn | |||||
@pages += 1 | |||||
# TODO - add page to sitemap.xml or sitemap.txt | |||||
# https://support.google.com/webmasters/answer/183668?hl=en&ref_topic=4581190 | |||||
end | |||||
def write_csv(path_items, filename, data) | |||||
dir = create_path(path_items) | |||||
fn = File.join(dir, filename + '.csv') | |||||
csv_string = CSV.generate do |csv| | |||||
csv << data.first.keys # header row | |||||
data.each { |row| csv << row.values } | |||||
end | |||||
File.write(fn, csv_string) | |||||
@log.info fn | |||||
end | |||||
def create_path(path_items) | |||||
dir = File.join(OUTPUT_DIR, path_items) | |||||
FileUtils.mkdir_p(dir) | |||||
@log.debug dir | |||||
dir | |||||
end | |||||
def create_output_dir | |||||
# Recursively delete working directory to ensure no redundant files are left behind from previous builds. | |||||
# FileUtils.rm_rf(@working_dir) | |||||
Dir.mkdir(@working_dir) unless File.directory?(@working_dir) | |||||
# Dir.chdir(@working_dir) | |||||
# Copy `public` dir to output dir | |||||
FileUtils.copy_entry('public', @working_dir) | |||||
end | |||||
def gen_homepage | |||||
summary = ScraperWiki.select(" | |||||
authority_name, status, decision, appeal_status, appeal_decision, count(*) as applications | |||||
from applications | |||||
group by authority_name, status, decision, appeal_status, appeal_decision | |||||
") | |||||
write_page('.', 'index', { summary: summary }) | |||||
# Summary CSV file | |||||
write_csv('.', 'inlink-summary', summary) | |||||
# Full CSV file | |||||
apps = ScraperWiki.select("* from applications") | |||||
write_csv('.', 'inlink-full', apps) | |||||
end | |||||
def gen_new | |||||
apps = ScraperWiki.select("* from `applications` order by date_received desc limit 30") | |||||
write_page('new-applications', 'new', { apps: apps }) | |||||
end | |||||
def gen_decisions | |||||
apps = ScraperWiki.select("* from `applications` order by date_decision desc limit 30") | |||||
write_page('decisions', 'decisions', { apps: apps }) | |||||
end | |||||
def gen_authorities | |||||
auths = ScraperWiki.select("distinct(authority_name) as authority_name from applications order by authority_name") | |||||
write_page('authorities', 'authorities', { auths: auths }) | |||||
auths.each do |auth| | |||||
summary = ScraperWiki.select(" | |||||
status, decision, appeal_status, appeal_decision, count(*) as qty | |||||
from applications | |||||
where authority_name = '#{auth['authority_name']}' | |||||
group by status, decision, appeal_status, appeal_decision | |||||
") | |||||
apps = ScraperWiki.select("* from applications where authority_name='#{auth['authority_name']}' order by date_received") | |||||
write_page(['authorities', slug(auth['authority_name'])], 'authority', { apps: apps, auth: auth, summary: summary }) | |||||
write_csv(['authorities', slug(auth['authority_name'])], slug(auth['authority_name']), apps) | |||||
end | |||||
end | |||||
@working_dir = File.join(Dir.pwd, OUTPUT_DIR) | |||||
puts @working_dir | |||||
# exit | |||||
@log = Logger.new($stdout) | |||||
@log.level = Logger::INFO | |||||
@pages = 0 | |||||
create_output_dir | |||||
gen_homepage | |||||
gen_new | |||||
gen_decisions | |||||
gen_authorities | |||||
Petrify.setup | |||||
Site.generate |
@@ -0,0 +1,60 @@ | |||||
require 'haml' | |||||
require 'csv' | |||||
require 'logger' | |||||
class Petrify | |||||
@@output_dir = '_site' | |||||
@@working_dir = File.join(Dir.pwd, @@output_dir) | |||||
@@views_dir = 'views' | |||||
@@layout_fn = File.join(@@views_dir, 'layout.haml') | |||||
# https://stackoverflow.com/questions/917566/ruby-share-logger-instance-among-module-classes#6768164 | |||||
@@log = Logger.new($stdout) | |||||
@@log.level = Logger::INFO | |||||
def self.write_page(path_items, template, locals = {}) | |||||
dir = create_path(path_items) | |||||
fn = File.join(dir, 'index.html') | |||||
# https://stackoverflow.com/questions/6125265/using-layouts-in-haml-files-independently-of-rails | |||||
html = Haml::Engine.new(File.read(@@layout_fn)).render do | |||||
Haml::Engine.new(File.read(File.join(@@views_dir, "#{template}.haml"))).render(Object.new, locals) | |||||
end | |||||
File.write(fn, html) | |||||
@@log.info fn | |||||
# TODO - add page to sitemap.xml or sitemap.txt | |||||
# https://support.google.com/webmasters/answer/183668?hl=en&ref_topic=4581190 | |||||
end | |||||
def self.write_csv(path_items, filename, data) | |||||
dir = create_path(path_items) | |||||
fn = File.join(dir, filename + '.csv') | |||||
csv_string = CSV.generate do |csv| | |||||
csv << data.first.keys # header row | |||||
data.each { |row| csv << row.values } | |||||
end | |||||
File.write(fn, csv_string) | |||||
@@log.info fn | |||||
end | |||||
def self.setup | |||||
# Recursively delete working directory to ensure no redundant files are left behind from previous builds. | |||||
# FileUtils.rm_rf(@working_dir) | |||||
Dir.mkdir(@@working_dir) unless File.directory?(@@working_dir) | |||||
# Dir.chdir(@working_dir) | |||||
# Copy `public` dir to output dir | |||||
FileUtils.copy_entry('public', @@working_dir) | |||||
end | |||||
def self.create_path(path_items) | |||||
dir = File.join(@@output_dir, path_items) | |||||
FileUtils.mkdir_p(dir) | |||||
@@log.debug dir | |||||
dir | |||||
end | |||||
end | |||||
@@ -0,0 +1,45 @@ | |||||
require 'scraperwiki' | |||||
class Site | |||||
def self.generate | |||||
# Home page | |||||
summary = ScraperWiki.select(" | |||||
authority_name, status, decision, appeal_status, appeal_decision, count(*) as applications | |||||
from applications | |||||
group by authority_name, status, decision, appeal_status, appeal_decision | |||||
") | |||||
Petrify.write_page('.', 'index', { summary: summary }) | |||||
# Summary CSV file | |||||
Petrify.write_csv('.', 'inlink-summary', summary) | |||||
# Full CSV file | |||||
apps = ScraperWiki.select("* from applications") | |||||
Petrify.write_csv('.', 'inlink-full', apps) | |||||
# New applications page | |||||
apps = ScraperWiki.select("* from `applications` order by date_received desc limit 30") | |||||
Petrify.write_page('new-applications', 'new', { apps: apps }) | |||||
# Latest decisions page | |||||
apps = ScraperWiki.select("* from `applications` order by date_decision desc limit 30") | |||||
Petrify.write_page('decisions', 'decisions', { apps: apps }) | |||||
# Page and CSV file for each authority | |||||
auths = ScraperWiki.select("distinct(authority_name) as authority_name from applications") | |||||
auths.each do |auth| | |||||
summary = ScraperWiki.select(" | |||||
status, decision, appeal_status, appeal_decision, count(*) as qty | |||||
from applications | |||||
where authority_name = ? | |||||
group by status, decision, appeal_status, appeal_decision | |||||
", auth['authority_name']) | |||||
apps = ScraperWiki.select("* from applications where authority_name = ? order by date_received", auth['authority_name']) | |||||
path = ['authorities', slug(auth['authority_name'])] | |||||
Petrify.write_page(path, 'authority', { apps: apps, auth: auth, summary: summary }) | |||||
Petrify.write_csv(path, slug(auth['authority_name']), apps) | |||||
end | |||||
end | |||||
end |