require 'scraperwiki' require 'petrify' require 'csv' require 'json' class Site def self.generate # Home page summary = ScraperWiki.select(" authority_name, status, decision, appeal_status, appeal_decision, count(*) as applications from applications group by authority_name, status, decision, appeal_status, appeal_decision ") q = ScraperWiki.select(" scraped_at from applications order by scraped_at desc limit 1") last_updated = DateTime.parse(q[0]['scraped_at']) path = '.' Petrify.page(path, 'index', \ { summary: summary, last_updated: last_updated }) Petrify.csv(path, 'inlink-summary', summary) # Generate a JSON file with all the data apps = ScraperWiki.select("* from applications") Petrify.file(path, 'data.json', apps.to_json) # New applications page apps = ScraperWiki.select("* from `applications` order by date_received desc limit 60") Petrify.page('new-applications', 'new-applications', { apps: apps, title: "New applications" }) # Latest decisions page apps = ScraperWiki.select("* from `applications` order by date_decision desc limit 60") path = 'decisions' Petrify.page(path, 'decisions', { apps: apps, title: "Latest decisions" }) Petrify.csv(path, 'inlink-decisions', apps) # Appeals page summary = ScraperWiki.select(" authority_name, status, decision, appeal_status, appeal_decision, count(*) as applications from applications where (appeal_status is not null and appeal_status != 'Unknown') or status like '%appeal%' group by authority_name, appeal_status, appeal_decision collate nocase ") apps = ScraperWiki.select(" * from applications where (appeal_status is not null and appeal_status != 'Unknown') or status like '%appeal%' collate nocase ") path = 'appeals' Petrify.page(path, 'appeals', { summary: summary, apps: apps, title: "Appeals" }) Petrify.csv(path, 'inlink-appeals', apps) # Media page stories = CSV.read('media.csv', :headers => true ) Petrify.page('media', 'media', { stories: stories, title: "Media" }) # Authority pages auths = ScraperWiki.select("distinct(authority_name) as authority_name from applications") auths.each do |auth| summary = ScraperWiki.select(" status, decision, appeal_status, appeal_decision, count(*) as qty from applications where authority_name = ? group by status, decision, appeal_status, appeal_decision ", auth['authority_name']) apps = ScraperWiki.select("* from applications where authority_name = ? order by date_received desc", auth['authority_name']) this_stories = stories.select do |story| if story['authorities'] story['authorities'].match(auth['authority_name']) end end path = ['authorities', slug(auth['authority_name'])] Petrify.page(path, 'authority', \ { apps: apps, auth: auth, summary: summary, stories: this_stories, title: auth['authority_name'] }) Petrify.csv(path, slug(auth['authority_name']), apps) end end end