Move from postgres to sqlite database

Gemfile.lock

@@ -11,7 +11,7 @@ GEM
   specs:
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
-    bcrypt (3.1.11)
+    bcrypt (3.1.12)
     bcrypt-ruby (3.1.5)
       bcrypt (>= 3.1.3)
     data_mapper (1.2.0)
@@ -37,15 +37,15 @@ GEM
       dm-core (~> 1.2.0)
     dm-migrations (1.2.0)
       dm-core (~> 1.2.0)
-    dm-postgres-adapter (1.2.0)
-      dm-do-adapter (~> 1.2.0)
-      do_postgres (~> 0.10.6)
     dm-serializer (1.2.2)
       dm-core (~> 1.2.0)
       fastercsv (~> 1.5)
       json (~> 1.6)
       json_pure (~> 1.6)
       multi_json (~> 1.0)
+    dm-sqlite-adapter (1.2.0)
+      dm-do-adapter (~> 1.2.0)
+      do_sqlite3 (~> 0.10.6)
     dm-timestamps (1.2.0)
       dm-core (~> 1.2.0)
     dm-transactions (1.2.0)
@@ -60,7 +60,7 @@ GEM
       uuidtools (~> 2.1)
     dm-validations (1.2.0)
       dm-core (~> 1.2.0)
-    do_postgres (0.10.17)
+    do_sqlite3 (0.10.17)
       data_objects (= 0.10.17)
     fastercsv (1.5.5)
     haml (5.0.4)
@@ -69,8 +69,7 @@ GEM
     json (1.8.6)
     json_pure (1.8.6)
     multi_json (1.13.1)
-    pg (1.0.0)
-    public_suffix (3.0.2)
+    public_suffix (3.0.3)
     stringex (1.5.1)
     temple (0.8.0)
     tilt (2.0.8)
@@ -81,13 +80,11 @@ PLATFORMS
 
 DEPENDENCIES
   data_mapper
-  dm-postgres-adapter
-  haml
+  dm-sqlite-adapter
   petrify!
-  pg
 
 RUBY VERSION
    ruby 2.3.1p112
 
 BUNDLED WITH
-   1.16.1
+   1.16.6

bin/build

@@ -2,6 +2,7 @@
 require 'petrify'
 require_relative '../models'
 require_relative '../lib/helpers'
+require 'pp'
 
 def gen_info_pages
   Petrify.page('about', 'about')
@@ -23,7 +24,7 @@ def gen_bodies_pages
         e.id,
         e.kind,
         e.d,
-        SUM(p.ballot_papers_issued)::float / SUM(p.electorate) * 100 AS turnout_percent
+        (SUM(p.ballot_papers_issued) * 1.0) / SUM(p.electorate) * 100 AS turnout_percent
       
       FROM elections e
       

bin/to_sqlite.rb

@@ -0,0 +1,32 @@
+# require_relative '../models'
+require 'scraperwiki'
+require 'sequel'
+load '_config.rb'
+require 'pp'
+
+class Object
+  def to_hash
+    h = {}
+
+    instance_variables.each do |var|
+      h[var.to_s.delete('@')] = instance_variable_get(var)
+    end
+
+    # Remove DataMapper-specific keys, which start with an underscore
+    h.keys.each do |k|
+      h.delete(k) if k.match(/^_/)
+    end
+    h
+  end
+end
+
+DB = Sequel.connect(ENV['DATABASE_URL'])
+
+DB[:polls].each { |o| ScraperWiki.save_sqlite([:district_id, :election_id], o.to_hash, 'polls') }
+DB[:candidates].each { |o| ScraperWiki.save_sqlite([:id], o.to_hash, 'candidates') }
+DB[:candidacies].each { |o| ScraperWiki.save_sqlite([:id], o.to_hash, 'candidacies') }
+DB[:deleted_candidates].each { |o| ScraperWiki.save_sqlite([:old_candidate_id], o.to_hash, 'deleted_candidates') }
+DB[:elections].each { |o| ScraperWiki.save_sqlite([:id], o.to_hash, 'elections') }
+DB[:districts].each { |o| ScraperWiki.save_sqlite([:id], o.to_hash, 'districts') }
+DB[:bodies].each { |o| ScraperWiki.save_sqlite([:id], o.to_hash, 'bodies') }
+DB[:parties].each { |o| ScraperWiki.save_sqlite([:id], o.to_hash, 'parties') }

lib/helpers.rb

@@ -37,14 +37,19 @@ def format_percent(num)
 end
 
 def short_date(d)
+  # FIXME wtf - because sometimes we're doing raw sql queries and sometimes it's coming through the DataMapper::Resource class
+  d = Date.parse(d) unless d.class == Date
   d.strftime("%e %b %Y")
 end
 
 def long_date(d)
+  # FIXME wtf - because sometimes we're doing raw sql queries and sometimes it's coming through the DataMapper::Resource class
+  d = Date.parse(d) unless d.class == Date
   d.strftime("%e %B %Y")
 end
 
 # Exception for Labour/Co-operative candidacies
 def party_name(labcoop, party_name)
-  labcoop ? "Labour and Co-operative Party" : party_name
+  # puts labcoop.class # FIXME wtf - because sometimes we're doing raw sql queries and sometimes it's coming through the DataMapper::Resource class
+  labcoop == 1 || labcoop == '1' ? "Labour and Co-operative Party" : party_name
 end

models.rb

@@ -62,73 +62,6 @@ class Poll
   belongs_to :district
 end
 
-class PollingStation
-  include DataMapper::Resource
-
-  property :id,           String, :key => true, :length => 2 # e.g. "KA"
-  property :name,         String, :length => 255#, :required => true
-  property :address,      String, :length => 255#, :required => true
-  property :postcode,     String#, :required => true
-  property :easting,      Float,  :required => true
-  property :northing,     Float,  :required => true
-  property :lat,          Float,  :required => true
-  property :lng,          Float,  :required => true
-
-  has n, :postcodes
-end  
-
-class Postcode
-  include DataMapper::Resource
-
-  # Postcode natural key, uppercase with space, eg. "SM1 1EA"
-  # Column names derived from Ordnance Survey CodePoint Open
-  property :postcode,                     String,   :key => true
-  property :positional_quality_indicator, Integer
-  property :eastings,                     Integer,  :required => true
-  property :northings,                    Integer,  :required => true
-  property :country_code,                 String,   :required => true
-  property :nhs_regional_ha_code,         String,   :required => true
-  property :nhs_ha_code,                  String,   :required => true
-  property :admin_county_code,            String # NULL within Greater London
-  property :admin_district_code,          String,   :required => true # e.g. London Borough of Sutton
-  property :admin_ward_code,              String,   :required => true # e.g. Sutton Central
-  property :lat,                          Float,    :required => true
-  property :lng,                          Float,    :required => true
-  property :ward_id,                      Integer,  :required => true # Sutton Council
-  property :constituency_id,              Integer,  :required => false # UK Parliament
-  property :polling_station_id,           String,   :length => 2
-
-  belongs_to :district, :child_key => [:ward_id]
-  belongs_to :polling_station
-
-  def self.finder(postcode)
-    postcode = postcode.strip.upcase
-    
-    if o = self.get(postcode)
-      return o
-    end
-
-    result = Pat.get(postcode)
-
-    unless result.code == 404
-      # cache API result
-      self.create(
-        :postcode => postcode,
-        :lat => result['geo']['lat'],
-        :lng => result['geo']['lng'],
-        :district_name => result['administrative']['district']['title'],
-        :district_code => result['administrative']['district']['uri'].match(/.+\/(.+)$/)[1],
-        :ward_name => result['administrative']['ward']['title'],
-        :ward_code => result['administrative']['ward']['uri'].match(/.+\/(.+)$/)[1]
-      )
-    else
-      # invalid postcode
-      nil
-    end
-    
-  end
-end
-
 class Candidate
   include DataMapper::Resource
 
@@ -136,7 +69,7 @@ class Candidate
   property  :forenames,       String,   :required => true
   property  :surname,         String,   :required => true, :index => true
 
-  has n, :candidacies
+  has n, :candidacies#, 'Candidacy'
   
   def short_name
     @forenames.split(' ')[0] + ' ' + @surname
@@ -171,7 +104,7 @@ class Candidacy
   property  :postcode,          String
   property  :position,          Integer # Position of this candidate in this district. (1..n)
   property  :seats,             Integer # Number of seats won by this candidacy (0 or 1)
-  property  :labcoop,           Boolean, :default => false # Candidacy is for joint Labour/Co-op party
+  property  :labcoop,           String, :default => '0' # Candidacy is for joint Labour/Co-op party
 
   belongs_to  :election
   belongs_to  :candidate
@@ -179,19 +112,6 @@ class Candidacy
   belongs_to  :district
 end
 
-class Campaign
-  include DataMapper::Resource
-
-  property :party_id,           Integer, :key => true
-  property :election_id,        Integer, :key => true
-  property :party_url,          String, :length => 255
-  property :manifesto_html_url, String, :length => 255
-  property :manifesto_pdf_url,  String, :length => 255
-
-  belongs_to :party
-  belongs_to :election
-end
-
 class Election
   include DataMapper::Resource
 
@@ -204,7 +124,6 @@ class Election
   has n,      :candidacies
   has n,      :polls
   belongs_to  :body
-  has n,      :campaigns
 
   def self.past
     self.all(:d.lt => Time.now.to_s, :order => [ :d.desc ])
@@ -238,7 +157,6 @@ class District
   property  :ons_district_code, String
 
   belongs_to :body
-  has n,    :postcodes, :child_key => [:ward_id]
   has n,    :polls
   
   def self.slugify(name)
@@ -267,8 +185,9 @@ class Party
   property :colour,         String
   
   has n, :candidacies
-  has n, :campaigns
+  # has n, :campaigns
 end
 
 DataMapper.setup(:default, ENV['DATABASE_URL'])
+DataMapper.repository(:default).adapter.resource_naming_convention = DataMapper::NamingConventions::Resource::UnderscoredAndPluralized
 DataMapper.auto_upgrade!

views/candidate.haml

@@ -23,7 +23,7 @@
       %td
         %a{ :href => "/bodies/#{ccy['body_slug']}/elections/#{ccy['d']}" }
           -# Postgres returns ccy['d'] as a Date and sqlite returns it as a String
-          = long_date(ccy['d'].to_time)
+          = long_date(ccy['d'])
       %td= party_name(ccy['labcoop'], ccy['party_name'])
       %td
         %a{ :href => "/bodies/#{ccy['body_slug']}" }

views/electionsummary.haml

@@ -11,7 +11,7 @@
       
       - unless election_index == 0
         - previous_election = elections_for_this_body[election_index - 1]
-        %a{ :href => "/bodies/#{election.body.slug}/elections/#{previous_election.d}", :title => "#{previous_election.kind} #{short_date(previous_election.d)}" }
+        %a{ :href => "/bodies/#{election.body.slug}/elections/#{previous_election.d.to_s}", :title => "#{previous_election.kind} #{short_date(previous_election.d.to_s)}" }
           «
           Previous
           = election.body.name
@@ -20,7 +20,7 @@
     
       - unless election_index == elections_for_this_body.size - 1
         - next_election = elections_for_this_body[election_index + 1]
-        %a{ :href => "/bodies/#{election.body.slug}/elections/#{next_election.d}", :title => "#{next_election.kind} #{short_date(next_election.d)}" }
+        %a{ :href => "/bodies/#{election.body.slug}/elections/#{next_election.d.to_s}", :title => "#{next_election.kind} #{short_date(next_election.d)}" }
           Next
           = election.body.name
           election
@@ -131,7 +131,7 @@
     - election.polls.each do |p|
       %tr
         %td
-          %a{ :href => "/bodies/#{election.body.slug}/elections/#{election.d}/#{election.body.districts_name}/#{p.district.slug}"}
+          %a{ :href => "/bodies/#{election.body.slug}/elections/#{election.d.to_s}/#{election.body.districts_name}/#{p.district.slug}"}
             = p.district.name
         - p.successful_candidacies.each do |sc|
           %td{ :style => "background-color: #{sc.party.colour};" }