From 6d13e8c7627cd4eb9001a0a315b8282f93962222 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Sat, 22 Sep 2018 23:24:08 +0100 Subject: [PATCH] Refactor bin/build into Petrify and Site classes --- bin/build | 115 ++----------------------------------------------- lib/petrify.rb | 60 ++++++++++++++++++++++++++ lib/site.rb | 45 +++++++++++++++++++ 3 files changed, 109 insertions(+), 111 deletions(-) create mode 100644 lib/petrify.rb create mode 100644 lib/site.rb diff --git a/bin/build b/bin/build index 6986a4d..dc7564d 100755 --- a/bin/build +++ b/bin/build @@ -1,115 +1,8 @@ #!/usr/bin/env ruby -require 'scraperwiki' -require 'haml' -require 'pp' -require 'logger' -require 'csv' require_relative '../lib/helpers' +require_relative '../lib/petrify' +require_relative '../lib/site' -OUTPUT_DIR = '_site' -VIEWS_DIR = File.join('views') -LAYOUT_FN = File.join(VIEWS_DIR, 'layout.haml') - -def write_page(path_items, template, locals = {}) - dir = create_path(path_items) - fn = File.join(dir, 'index.html') - - # https://stackoverflow.com/questions/6125265/using-layouts-in-haml-files-independently-of-rails - html = Haml::Engine.new(File.read(LAYOUT_FN)).render do - Haml::Engine.new(File.read(File.join(VIEWS_DIR, "#{template}.haml"))).render(Object.new, locals) - end - - File.write(fn, html) - @log.info fn - @pages += 1 - # TODO - add page to sitemap.xml or sitemap.txt - # https://support.google.com/webmasters/answer/183668?hl=en&ref_topic=4581190 -end - -def write_csv(path_items, filename, data) - dir = create_path(path_items) - fn = File.join(dir, filename + '.csv') - - csv_string = CSV.generate do |csv| - csv << data.first.keys # header row - data.each { |row| csv << row.values } - end - - File.write(fn, csv_string) - @log.info fn -end - -def create_path(path_items) - dir = File.join(OUTPUT_DIR, path_items) - FileUtils.mkdir_p(dir) - @log.debug dir - dir -end - -def create_output_dir - # Recursively delete working directory to ensure no redundant files are left behind from previous builds. - # FileUtils.rm_rf(@working_dir) - Dir.mkdir(@working_dir) unless File.directory?(@working_dir) - # Dir.chdir(@working_dir) - - # Copy `public` dir to output dir - FileUtils.copy_entry('public', @working_dir) -end - -def gen_homepage - summary = ScraperWiki.select(" - authority_name, status, decision, appeal_status, appeal_decision, count(*) as applications - from applications - group by authority_name, status, decision, appeal_status, appeal_decision - ") - write_page('.', 'index', { summary: summary }) - - # Summary CSV file - write_csv('.', 'inlink-summary', summary) - - # Full CSV file - apps = ScraperWiki.select("* from applications") - write_csv('.', 'inlink-full', apps) -end - -def gen_new - apps = ScraperWiki.select("* from `applications` order by date_received desc limit 30") - write_page('new-applications', 'new', { apps: apps }) -end - -def gen_decisions - apps = ScraperWiki.select("* from `applications` order by date_decision desc limit 30") - write_page('decisions', 'decisions', { apps: apps }) -end - -def gen_authorities - auths = ScraperWiki.select("distinct(authority_name) as authority_name from applications order by authority_name") - write_page('authorities', 'authorities', { auths: auths }) - - auths.each do |auth| - summary = ScraperWiki.select(" - status, decision, appeal_status, appeal_decision, count(*) as qty - from applications - where authority_name = '#{auth['authority_name']}' - group by status, decision, appeal_status, appeal_decision - ") - - apps = ScraperWiki.select("* from applications where authority_name='#{auth['authority_name']}' order by date_received") - - write_page(['authorities', slug(auth['authority_name'])], 'authority', { apps: apps, auth: auth, summary: summary }) - write_csv(['authorities', slug(auth['authority_name'])], slug(auth['authority_name']), apps) - end -end - -@working_dir = File.join(Dir.pwd, OUTPUT_DIR) -puts @working_dir -# exit -@log = Logger.new($stdout) -@log.level = Logger::INFO -@pages = 0 -create_output_dir -gen_homepage -gen_new -gen_decisions -gen_authorities +Petrify.setup +Site.generate diff --git a/lib/petrify.rb b/lib/petrify.rb new file mode 100644 index 0000000..4f05d01 --- /dev/null +++ b/lib/petrify.rb @@ -0,0 +1,60 @@ +require 'haml' +require 'csv' +require 'logger' + +class Petrify + @@output_dir = '_site' + @@working_dir = File.join(Dir.pwd, @@output_dir) + @@views_dir = 'views' + @@layout_fn = File.join(@@views_dir, 'layout.haml') + + # https://stackoverflow.com/questions/917566/ruby-share-logger-instance-among-module-classes#6768164 + @@log = Logger.new($stdout) + @@log.level = Logger::INFO + + def self.write_page(path_items, template, locals = {}) + dir = create_path(path_items) + fn = File.join(dir, 'index.html') + + # https://stackoverflow.com/questions/6125265/using-layouts-in-haml-files-independently-of-rails + html = Haml::Engine.new(File.read(@@layout_fn)).render do + Haml::Engine.new(File.read(File.join(@@views_dir, "#{template}.haml"))).render(Object.new, locals) + end + + File.write(fn, html) + @@log.info fn + # TODO - add page to sitemap.xml or sitemap.txt + # https://support.google.com/webmasters/answer/183668?hl=en&ref_topic=4581190 + end + + def self.write_csv(path_items, filename, data) + dir = create_path(path_items) + fn = File.join(dir, filename + '.csv') + + csv_string = CSV.generate do |csv| + csv << data.first.keys # header row + data.each { |row| csv << row.values } + end + + File.write(fn, csv_string) + @@log.info fn + end + + def self.setup + # Recursively delete working directory to ensure no redundant files are left behind from previous builds. + # FileUtils.rm_rf(@working_dir) + Dir.mkdir(@@working_dir) unless File.directory?(@@working_dir) + # Dir.chdir(@working_dir) + + # Copy `public` dir to output dir + FileUtils.copy_entry('public', @@working_dir) + end + + def self.create_path(path_items) + dir = File.join(@@output_dir, path_items) + FileUtils.mkdir_p(dir) + @@log.debug dir + dir + end +end + diff --git a/lib/site.rb b/lib/site.rb new file mode 100644 index 0000000..129c26b --- /dev/null +++ b/lib/site.rb @@ -0,0 +1,45 @@ +require 'scraperwiki' + +class Site + def self.generate + # Home page + summary = ScraperWiki.select(" + authority_name, status, decision, appeal_status, appeal_decision, count(*) as applications + from applications + group by authority_name, status, decision, appeal_status, appeal_decision + ") + Petrify.write_page('.', 'index', { summary: summary }) + + # Summary CSV file + Petrify.write_csv('.', 'inlink-summary', summary) + + # Full CSV file + apps = ScraperWiki.select("* from applications") + Petrify.write_csv('.', 'inlink-full', apps) + + # New applications page + apps = ScraperWiki.select("* from `applications` order by date_received desc limit 30") + Petrify.write_page('new-applications', 'new', { apps: apps }) + + # Latest decisions page + apps = ScraperWiki.select("* from `applications` order by date_decision desc limit 30") + Petrify.write_page('decisions', 'decisions', { apps: apps }) + + # Page and CSV file for each authority + auths = ScraperWiki.select("distinct(authority_name) as authority_name from applications") + + auths.each do |auth| + summary = ScraperWiki.select(" + status, decision, appeal_status, appeal_decision, count(*) as qty + from applications + where authority_name = ? + group by status, decision, appeal_status, appeal_decision + ", auth['authority_name']) + + apps = ScraperWiki.select("* from applications where authority_name = ? order by date_received", auth['authority_name']) + path = ['authorities', slug(auth['authority_name'])] + Petrify.write_page(path, 'authority', { apps: apps, auth: auth, summary: summary }) + Petrify.write_csv(path, slug(auth['authority_name']), apps) + end + end +end