Consolidated import scripts. Created drill-down nav from directorates to services.

app.rb

@@ -19,7 +19,7 @@ end
 
 get '/directorates/:slug' do
   @directorate = Directorate.first(:slug => params[:slug])
-  @total = @directorate.payments.sum(:amount)
+#   @total = @directorate.payments.sum(:amount)
   haml :directorate
 end
 
@@ -42,6 +42,10 @@ end
 get '/suppliers/:slug' do
   @supplier = Supplier.first(:slug => params[:slug])
   @total = @supplier.payments.sum(:amount)
+  @count = @supplier.payments.size
+  @avg = @supplier.payments.avg(:amount)
+  @max = @supplier.payments.max(:amount)
+  @min = @supplier.payments.min(:amount)
   haml :supplier
 end
 

import.rb

@@ -0,0 +1,110 @@
+require 'lib/models'
+require 'csv'
+
+# Before running this script with a CSV file, prepare it so:
+#   - There is only a single line of column headings on the first line of the file
+#   - There are no spaces before or after the column headings
+#   - The column headings correspond with the key names in the columns{} hash below
+#   - The data starts on line 2
+
+columns = 
+  { 
+    'Directorate' => nil,
+    'Updated'     => nil,
+    'Service'     => nil,
+    'Supplier'    => nil,
+    'Amount'      => nil
+   }
+
+directorate_replacements =
+  [
+    [ "Childrens Services", "Children's Services" ],
+    [ "Policy,Performance and Planning", "Policy, Performance and Planning" ]
+  ]
+
+service_replacements = 
+  [
+    [ "Corporate Performance and Developmt", "Corporate Performance and Development" ],
+    [ "ISB", "ISB - Individual Schools Budget" ],
+    [ "Library and Information Services", "Libraries and Information Services" ],
+    [ "On Street Parking", "On-Street Parking" ]
+  ]
+  
+count = 0
+
+if ARGV[0].nil?
+  puts "Specify the filename of the CSV file to import on the command line"
+  exit
+end
+
+date_format = ARGV[1].upcase
+
+if date_format != 'DMY' && date_format != 'MDY'
+  puts "Specify the date format as DMY or MDY as the second argument on the command line"
+  exit
+end
+
+CSV::Reader.parse(File.open(ARGV[0], 'rb')) do |row|
+
+    count += 1
+    
+    if (count > 1) # skip first line that doesn't contain data
+      
+      p row
+      
+      directorate_name = row[columns['Directorate']].strip.gsub(/&/, "and")
+      service_name = row[columns['Service']].strip.gsub(/&/, "and")
+      supplier_name = row[columns['Supplier']].strip.gsub(/&/, "and")
+      
+      
+      for replacement in directorate_replacements
+        if directorate_name == replacement[0]
+          directorate_name = replacement[1]
+        end
+      end
+      
+      for replacement in service_replacements
+        if service_name == replacement[0]
+          service_name = replacement[1]
+        end
+      end
+      
+      directorate = Directorate.first_or_create(:name => directorate_name)
+      service = Service.first_or_create(:name => service_name, :directorate => directorate)
+      supplier = Supplier.first_or_create(:name => supplier_name)
+      
+      dt = row[columns['Updated']].strip.split('/')
+
+      # Date.new takes YMD params
+      if date_format == 'DMY'
+        d = Date.new(dt[2].to_i, dt[1].to_i, dt[0].to_i) 
+      elsif date_format == 'MDY'
+        d = Date.new(dt[2].to_i, dt[0].to_i, dt[1].to_i)
+      elsif date_format == 'YMD'
+        d = Date.new(dt[0].to_i, dt[1].to_i, dt[2].to_i)
+      end
+      
+      payment = Payment.first_or_new(
+        'service' =>  service,
+        'supplier' => supplier,
+        'amount' => row[columns['Amount']].strip.gsub(/,/, ''),
+        'd' => d
+      )
+  
+      unless payment.save
+        puts "ERROR: Failed to save payment"
+        payment.errors.each do |e|
+          puts e
+        end
+      end
+  else
+    # Get the column headings
+    position = 0
+
+    for column in row
+      columns[column] = position
+      position += 1
+    end
+    puts columns.inspect
+  end
+end

import2009Q4.rb

@@ -1,45 +0,0 @@
-require 'lib/models'
-require 'csv'
-
-count = 0
-
-CSV::Reader.parse(File.open('data/2009Q4.csv', 'rb')) do |row|
-#     2009Q4 Columns:
-#     0: Directorate
-#     1: Updated
-#     2: TransNo
-#     3: Service
-#     4: Cost Centre
-#     5: Supplier Name
-#     6: Amount excl vat
-#     7: Type
-
-    count += 1
-    
-    if (count > 4) # skip first four lines that don't contain data
-  
-      p row
-      
-      directorate = Directorate.first_or_create(:name => row[0].strip)
-      service = Service.first_or_create(:name => row[3].strip)
-      supplier = Supplier.first_or_create(:name => row[5].strip)
-      
-      payment = Payment.first_or_create(
-        'trans_no' => row[2],
-        'directorate' => directorate,
-        'service' =>  service,
-        'supplier' => supplier,
-        'cost_centre' => row[4].strip,
-        'amount' => row[6].strip.gsub(/,/, ''),
-        'd' => row[1],
-        'tyype' => row[7].strip
-      )
-  
-      unless payment.save
-        puts "ERROR: Failed to save payment"
-        payment.errors.each do |e|
-          puts e
-        end
-      end
-  end
-end

import2010Q1.rb

@@ -1,39 +0,0 @@
-require 'lib/models'
-require 'csv'
-
-count = 0
-
-# 2010Q1: 0-Directorate,1-Updated,2-Service,3-Supplier Name,4-Amount excl vat £,5-Type
-
-CSV::Reader.parse(File.open('data/2010Q1.csv', 'rb')) do |row|
-
-
-    count += 1
-    
-    if (count > 4) # skip first four lines that don't contain data
-  
-      p row
-      
-      directorate = Directorate.first_or_create(:name => row[0].strip)
-      service = Service.first_or_create(:name => row[2].strip)
-      supplier = Supplier.first_or_create(:name => row[3].strip)
-      
-      dt = row[1].strip.split('/')
-      
-      payment = Payment.first_or_create(
-        'directorate' => directorate,
-        'service' =>  service,
-        'supplier' => supplier,
-        'amount' => row[4].strip.gsub(/,/, ''),
-        'd' => Date.new(dt[2].to_i, dt[1].to_i, dt[0].to_i),
-        'tyype' => row[5].strip
-      )
-  
-      unless payment.save
-        puts "ERROR: Failed to save payment"
-        payment.errors.each do |e|
-          puts e
-        end
-      end
-  end
-end

lib/models.rb

@@ -10,19 +10,17 @@ class Payment
   include DataMapper::Resource
   
   property :id,             Serial
-  property :trans_no,       Integer,     :required => false # "TransNo" in RBWM CSV files
-  property :directorate_id, Integer,     :required => true
+  property :created_at,     DateTime
+  property :updated_at,     DateTime
   property :service_id,     Integer,     :required => true
   property :supplier_id,    Integer,     :required => true
-  property :cost_centre,    String,      :required => false
   property :amount,         BigDecimal,  :precision => 10, :scale => 2, :required => true # ex VAT
   property :d,              Date,        :required => true # "Updated" in RBWM CSV files
-  property :tyype,          String,      :required => true # Capital or Revenue
   
-  belongs_to :directorate
   belongs_to :service
   belongs_to :supplier
-  
+  has 1, :directorate, { :through => :service }
+
   def url
     SITE_URL + "payments/" + @id.to_s
   end
@@ -32,44 +30,60 @@ end
 class Directorate
   include DataMapper::Resource
   
-  property :id,   Serial
-  property :name, String, :length => 255, :required => true
-  property :slug, String, :length => 255
+  property :id,           Serial
+  property :created_at,   DateTime
+  property :updated_at,   DateTime
+  property :name,         String, :length => 255, :required => true
+  property :slug,         String, :length => 255
   
-  has n, :payments, :order => ['d']
+  has n, :payments, :through => :services, :order => ['d']
+  has n, :services, :order => ['name']
+  has n, :suppliers, :through => :services, :order => ['name']
 
-  def self.slugify(name)
-    name.gsub(/[^\w\s-]/, '').gsub(/\s+/, '-').downcase
-  end
+  before :save, :slugify
 
+  def slugify
+    @slug = @name.gsub(/[^\w\s-]/, '').gsub(/\s+/, '-').downcase
+  end
 end
 
 class Service
   include DataMapper::Resource
   
-  property :id,   Serial
-  property :name, String, :length => 255, :required => true
-  property :slug, String, :length => 255
+  property :id,             Serial
+  property :created_at,     DateTime
+  property :updated_at,     DateTime
+  property :directorate_id, Integer,  :required => true
+  property :name,           String,   :length => 255, :required => true
+  property :slug,           String,   :length => 255
   
   has n, :payments, :order => ['d']
+  has n, :suppliers, :through => :payments, :order => ['name']
+  belongs_to :directorate
 
-  def self.slugify(name)
-    name.gsub(/[^\w\s-]/, '').gsub(/\s+/, '-').downcase
-  end
-  
+  before :save, :slugify
+
+  def slugify
+    @slug = @name.gsub(/[^\w\s-]/, '').gsub(/\s+/, '-').downcase
+  end 
 end
 
 class Supplier
   include DataMapper::Resource
   
-  property :id,   Serial
-  property :name, String, :length => 255, :required => true
-  property :slug, String, :length => 255
+  property :id,         Serial
+  property :created_at, DateTime
+  property :updated_at, DateTime
+  property :name,       String, :length => 255, :required => true
+  property :slug,       String, :length => 255
   
   has n, :payments, :order => ['d']
+  has n, :services, :through => :payments, :order => ['name']
   
-  def self.slugify(name)
-    name.gsub(/[^\w\s-]/, '').gsub(/\s+/, '-').downcase
+  before :save, :slugify
+
+  def slugify
+    @slug = @name.gsub(/[^\w\s-]/, '').gsub(/\s+/, '-').downcase
   end
 end
 

slugify.rb

@@ -1,16 +0,0 @@
-require 'lib/models'
-
-for supplier in Supplier.all
-  supplier.slug = Supplier.slugify(supplier.name)
-  supplier.save!
-end
-
-for service in Service.all
-  service.slug = Service.slugify(service.name)
-  service.save!
-end
-
-for directorate in Directorate.all
-  directorate.slug = Directorate.slugify(directorate.name)
-  directorate.save!
-end

views/directorate.haml

@@ -1,4 +1,5 @@
 .grid_12
+
   %ul#breadcrumb
     %li.home
       %a{ :href => '/'} Home
@@ -9,33 +10,22 @@
       
   %h2= @page_title = @directorate.name + " Directorate"
   
-  %table
-    %tr
-      %th Date
-      %th Ref.
-      %th Service
-      %th Supplier
-      %th.right £
-        
-    - for payment in @directorate.payments
-      %tr
-        %td= payment.d.strftime("%d %b %Y")
-        %td
-          %a{ :href => "/payments/#{payment.id}" }
-            = payment.id
-        %td
-          %a{ :href => '/services/' + payment.service.slug }
-            = payment.service.name
-        %td
-          %a{ :href => '/suppliers/' + payment.supplier.slug }
-            = payment.supplier.name
-        %td.right= sprintf("%0d", payment.amount)
-    
-    %tr
-      %td
-      %td
-      %td
-        %strong TOTAL
-      %td.right= sprintf("%0d", @total)
-      %td
-  
+.clear
+
+.grid_6  
+
+  %h3 Services
+  
+  - for service in @directorate.services
+    %p
+      %a{ :href => "/services/#{service.slug}" }
+        = service.name
+
+.grid_6
+
+  %h3 Suppliers 
+
+  - for supplier in @directorate.suppliers
+    %p
+      %a{ :href => "/suppliers/#{supplier.slug}" }
+        = supplier.name  

views/payment.haml

@@ -26,8 +26,6 @@
     %tr
       %td Transaction Number
       %td= @payment.trans_no.nil? ? "unknown" : @payment.trans_no
-    %tr
-      %td Type
-      %td= @payment.tyype
+
 
       

views/service.haml

@@ -7,17 +7,40 @@
     %li
       = @service.name
 
-  %h2= @page_title = @service.name + " (Service)"
+  %h2= @page_title = @service.name
+  
+  %p
+    A service in the 
+    %a{ :href => '/directorates/' + @service.directorate.slug }
+      = @service.directorate.name
+      Directorate
+
   
   %p.noprint.download
     %a{ :href => "/services/#{@service.slug}.csv" }
       Download data as CSV
-            
+
+  %h3 Summary
+
+  %table
+    %tr
+      %th.right Payments
+      %th.right Max £
+      %th.right Min £
+      %th.right Average £
+
+    %tr
+      %td.right= @count
+      %td.right= sprintf("%0d", @max)
+      %td.right= sprintf("%0d", @min)
+      %td.right= sprintf("%0d", @avg)            
+
+  %h3 Payments to Suppliers
+
   %table
     %tr
       %th Date
       %th Ref.
-      %th Directorate
       %th Supplier
       %th.right £
         
@@ -27,9 +50,6 @@
         %td.right
           %a{ :href => "/payments/#{payment.id}" }
             = payment.id
-        %td
-          %a{ :href => '/directorates/' + payment.directorate.slug }
-            = payment.directorate.name
         %td
           %a{ :href => '/suppliers/' + payment.supplier.slug }
             = payment.supplier.name
@@ -38,21 +58,9 @@
     %tr
       %td
       %td
-      %td
       %td
         %strong TOTAL
       %td.right= sprintf("%0d", @total)
   
-  %table
-    %tr
-      %th.right Count
-      %th.right Max
-      %th.right Min
-      %th.right Average
 
-    %tr
-      %td.right= @count
-      %td.right= sprintf("%0d", @max)
-      %td.right= sprintf("%0d", @min)
-      %td.right= sprintf("%0d", @avg)
     

views/supplier.haml

@@ -9,9 +9,31 @@
 
   %h2= @page_title = @supplier.name + " (Supplier)"
 
+  %p.noprint
+    %a{ :href => "http://www.google.co.uk/search?q=#{@supplier.name}" }
+      Search Google for
+      = @supplier.name
+
   %p.noprint.download
     %a{ :href => "/suppliers/#{@supplier.slug}.csv" }
       Download data as CSV
+
+  %h3 Summary 
+  
+  %table
+    %tr
+      %th.right Payments
+      %th.right Max £
+      %th.right Min £
+      %th.right Average £
+
+    %tr
+      %td.right= @count
+      %td.right= sprintf("%0d", @max)
+      %td.right= sprintf("%0d", @min)
+      %td.right= sprintf("%0d", @avg) 
+      
+  %h3 Payments from the Council
   
   %table
     %tr