From e47e907b3cb9780dd33dbd376645336849a08137 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Tue, 26 Mar 2013 11:47:44 +0000 Subject: [PATCH 1/7] Install delayed job --- Gemfile | 1 + Gemfile.lock | 6 +++++ .../20130326114555_create_delayed_jobs.rb | 22 +++++++++++++++++++ db/schema.rb | 20 +++++++++++++++-- script/delayed_job | 5 +++++ 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20130326114555_create_delayed_jobs.rb create mode 100755 script/delayed_job diff --git a/Gemfile b/Gemfile index 8a0af10..533fd4a 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,7 @@ gem 'sorcery' gem 'will_paginate', '~> 3.0' gem 'activerecord-postgresql-adapter' gem 'pg' +gem 'delayed_job_active_record' group :production do gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index 7508b69..6b42dda 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -48,6 +48,11 @@ GEM coffee-script-source (1.4.0) curb (0.7.18) debug_inspector (0.0.2) + delayed_job (3.0.5) + activesupport (~> 3.0) + delayed_job_active_record (0.4.3) + activerecord (>= 2.1.0, < 4) + delayed_job (~> 3.0) erubis (2.7.0) execjs (1.4.0) multi_json (~> 1.0) @@ -171,6 +176,7 @@ DEPENDENCIES better_errors binding_of_caller coffee-rails (~> 3.2.1) + delayed_job_active_record feedzirra haml-rails htmlentities diff --git a/db/migrate/20130326114555_create_delayed_jobs.rb b/db/migrate/20130326114555_create_delayed_jobs.rb new file mode 100644 index 0000000..e784160 --- /dev/null +++ b/db/migrate/20130326114555_create_delayed_jobs.rb @@ -0,0 +1,22 @@ +class CreateDelayedJobs < ActiveRecord::Migration + def self.up + create_table :delayed_jobs, :force => true do |table| + table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue + table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually. + table.text :handler # YAML-encoded string of the object that will do work + table.text :last_error # reason for last failure (See Note below) + table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future. + table.datetime :locked_at # Set when a client is working on this object + table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) + table.string :locked_by # Who is working on this object (if locked) + table.string :queue # The name of the queue this job is in + table.timestamps + end + + add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority' + end + + def self.down + drop_table :delayed_jobs + end +end diff --git a/db/schema.rb b/db/schema.rb index 37951d4..dead5cd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,23 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20130320181527) do +ActiveRecord::Schema.define(:version => 20130326114555) do + + create_table "delayed_jobs", :force => true do |t| + t.integer "priority", :default => 0 + t.integer "attempts", :default => 0 + t.text "handler" + t.text "last_error" + t.datetime "run_at" + t.datetime "locked_at" + t.datetime "failed_at" + t.string "locked_by" + t.string "queue" + t.datetime "created_at", :null => false + t.datetime "updated_at", :null => false + end + + add_index "delayed_jobs", ["priority", "run_at"], :name => "delayed_jobs_priority" create_table "feeds", :force => true do |t| t.string "title" @@ -29,7 +45,7 @@ ActiveRecord::Schema.define(:version => 20130320181527) do t.integer "layer_id" end - add_index "feeds_layers", ["feed_id", "layer_id"], :name => "index_feeds_layers_on_feed_id_and_layer_id" + add_index "feeds_layers", ["feed_id", "layer_id"], :name => "index_feeds_layers_on_feed_id_and_layer_id", :unique => true create_table "layers", :force => true do |t| t.string "name" diff --git a/script/delayed_job b/script/delayed_job new file mode 100755 index 0000000..edf1959 --- /dev/null +++ b/script/delayed_job @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) +require 'delayed/command' +Delayed::Command.new(ARGV).daemonize From fe0b445120ac780a88cc6d1f5dd886bafdf6802d Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Tue, 26 Mar 2013 14:37:22 +0000 Subject: [PATCH 2/7] Add daemons gem --- Gemfile | 1 + Gemfile.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Gemfile b/Gemfile index 533fd4a..79d4669 100644 --- a/Gemfile +++ b/Gemfile @@ -36,6 +36,7 @@ gem 'will_paginate', '~> 3.0' gem 'activerecord-postgresql-adapter' gem 'pg' gem 'delayed_job_active_record' +gem 'daemons' group :production do gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index 6b42dda..50eb733 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -47,6 +47,7 @@ GEM execjs coffee-script-source (1.4.0) curb (0.7.18) + daemons (1.1.9) debug_inspector (0.0.2) delayed_job (3.0.5) activesupport (~> 3.0) @@ -176,6 +177,7 @@ DEPENDENCIES better_errors binding_of_caller coffee-rails (~> 3.2.1) + daemons delayed_job_active_record feedzirra haml-rails From 179d5957bb2acd30d58e57020550ab3ceffd899a Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Tue, 26 Mar 2013 14:37:43 +0000 Subject: [PATCH 3/7] Fetch all feeds through delayed job --- app/models/feed.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/feed.rb b/app/models/feed.rb index e07fc45..321c506 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -9,7 +9,7 @@ class Feed < ActiveRecord::Base after_create :fetch def self.fetch_all - Feed.all.each { |f| f.fetch } + Feed.all.each { |f| f.delay.fetch } end # Fetch and parse feed contents from web From 9fb2cecf6790a3a1812c4428e7036edd2ca0bc58 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Tue, 26 Mar 2013 14:37:57 +0000 Subject: [PATCH 4/7] Add worker process for Heroku delayed job --- Procfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Procfile b/Procfile index 3daf4f2..f7b4865 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,2 @@ web: bundle exec unicorn -p $PORT -E $RACK_ENV +worker: bundle exec rake jobs:work From 22f1a06ebb605bf01125e5641aa0e8973605f870 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Tue, 26 Mar 2013 15:11:10 +0000 Subject: [PATCH 5/7] Add heroku_delayed_job_autoscale --- Gemfile | 1 + Gemfile.lock | 26 ++++++++++++++++++++++++++ app/models/feed.rb | 1 + 3 files changed, 28 insertions(+) diff --git a/Gemfile b/Gemfile index 79d4669..c4772f0 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,7 @@ gem 'activerecord-postgresql-adapter' gem 'pg' gem 'delayed_job_active_record' gem 'daemons' +gem 'heroku_delayed_job_autoscale' group :production do gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index 50eb733..a1bd38b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -30,6 +30,7 @@ GEM activesupport (3.2.12) i18n (~> 0.6) multi_json (~> 1.0) + addressable (2.3.3) arel (3.0.2) bcrypt-ruby (3.0.1) better_errors (0.6.0) @@ -55,10 +56,12 @@ GEM activerecord (>= 2.1.0, < 4) delayed_job (~> 3.0) erubis (2.7.0) + excon (0.16.10) execjs (1.4.0) multi_json (~> 1.0) faraday (0.8.5) multipart-post (~> 1.1) + fattr (2.2.1) feedzirra (0.0.31) activesupport (>= 3.0.8) builder (~> 3.0.0) @@ -76,6 +79,18 @@ GEM activesupport (>= 3.1, < 4.1) haml (>= 3.1, < 4.1) railties (>= 3.1, < 4.1) + heroku (2.35.0) + heroku-api (~> 0.3.7) + launchy (>= 0.3.2) + netrc (~> 0.7.7) + rest-client (~> 1.6.1) + rubyzip + heroku-api (0.3.8) + excon (~> 0.16.10) + heroku_delayed_job_autoscale (0.0.7) + delayed_job (>= 2.1) + heroku (>= 1) + rush (>= 0.6) hike (1.2.1) htmlentities (4.3.1) httpauth (0.2.0) @@ -88,6 +103,8 @@ GEM jwt (0.1.5) multi_json (>= 1.0) kgio (2.8.0) + launchy (2.2.0) + addressable (~> 2.3) loofah (1.0.0) nokogiri (>= 1.3.3) mail (2.4.4) @@ -100,6 +117,7 @@ GEM mime-types (1.21) multi_json (1.6.1) multipart-post (1.1.5) + netrc (0.7.7) nokogiri (1.4.7) oauth (0.4.7) oauth2 (0.8.1) @@ -138,6 +156,11 @@ GEM rake (10.0.3) rdoc (3.12.1) json (~> 1.4) + rest-client (1.6.7) + mime-types (>= 1.16) + rubyzip (0.9.9) + rush (0.6.8) + session sass (3.2.5) sass-rails (3.2.6) railties (~> 3.2.0) @@ -145,6 +168,8 @@ GEM tilt (~> 1.3) sax-machine (0.0.20) nokogiri (> 0.0.0) + session (3.1.0) + fattr sorcery (0.8.1) bcrypt-ruby (~> 3.0.0) oauth (~> 0.4.4) @@ -181,6 +206,7 @@ DEPENDENCIES delayed_job_active_record feedzirra haml-rails + heroku_delayed_job_autoscale htmlentities jquery-rails meta_request diff --git a/app/models/feed.rb b/app/models/feed.rb index 321c506..da24cb2 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,4 +1,5 @@ class Feed < ActiveRecord::Base + include HerokuDelayedJobAutoscale::Autoscale has_many :posts, :dependent => :destroy has_and_belongs_to_many :layers attr_accessible :title, :url, :description, :generator, :last_fetched, :feed_url From 9005040bd28049442c485cd986300c3846faf834 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Wed, 27 Mar 2013 15:57:01 +0000 Subject: [PATCH 6/7] Add workless gem and config --- Gemfile | 1 + Gemfile.lock | 14 ++++++++++++++ config/environments/production.rb | 6 +++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 79d4669..1d351ae 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,7 @@ gem 'activerecord-postgresql-adapter' gem 'pg' gem 'delayed_job_active_record' gem 'daemons' +gem 'workless' group :production do gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index 50eb733..86f4a9f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -55,10 +55,12 @@ GEM activerecord (>= 2.1.0, < 4) delayed_job (~> 3.0) erubis (2.7.0) + excon (0.16.10) execjs (1.4.0) multi_json (~> 1.0) faraday (0.8.5) multipart-post (~> 1.1) + fattr (2.2.1) feedzirra (0.0.31) activesupport (>= 3.0.8) builder (~> 3.0.0) @@ -76,6 +78,8 @@ GEM activesupport (>= 3.1, < 4.1) haml (>= 3.1, < 4.1) railties (>= 3.1, < 4.1) + heroku-api (0.3.8) + excon (~> 0.16.10) hike (1.2.1) htmlentities (4.3.1) httpauth (0.2.0) @@ -138,6 +142,8 @@ GEM rake (10.0.3) rdoc (3.12.1) json (~> 1.4) + rush (0.6.8) + session sass (3.2.5) sass-rails (3.2.6) railties (~> 3.2.0) @@ -145,6 +151,8 @@ GEM tilt (~> 1.3) sax-machine (0.0.20) nokogiri (> 0.0.0) + session (3.1.0) + fattr sorcery (0.8.1) bcrypt-ruby (~> 3.0.0) oauth (~> 0.4.4) @@ -168,6 +176,11 @@ GEM rack raindrops (~> 0.7) will_paginate (3.0.4) + workless (1.1.2) + delayed_job (>= 2.0.7) + heroku-api + rails + rush PLATFORMS ruby @@ -191,3 +204,4 @@ DEPENDENCIES uglifier (>= 1.0.3) unicorn will_paginate (~> 3.0) + workless diff --git a/config/environments/production.rb b/config/environments/production.rb index fc1e850..29da7bc 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -60,5 +60,9 @@ Apollo::Application.configure do # Send deprecation notices to registered listeners config.active_support.deprecation = :notify - + + # Workless + config.after_initialize do + Delayed::Job.scaler = :heroku_cedar + end end From f9f0eda173fd5b807638960e8f5072cdaa5b2a53 Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Wed, 27 Mar 2013 16:02:33 +0000 Subject: [PATCH 7/7] Revert "Add heroku_delayed_job_autoscale" This reverts commit 22f1a06ebb605bf01125e5641aa0e8973605f870. --- Gemfile | 1 - Gemfile.lock | 26 -------------------------- app/models/feed.rb | 1 - 3 files changed, 28 deletions(-) diff --git a/Gemfile b/Gemfile index c4772f0..79d4669 100644 --- a/Gemfile +++ b/Gemfile @@ -37,7 +37,6 @@ gem 'activerecord-postgresql-adapter' gem 'pg' gem 'delayed_job_active_record' gem 'daemons' -gem 'heroku_delayed_job_autoscale' group :production do gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index a1bd38b..50eb733 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -30,7 +30,6 @@ GEM activesupport (3.2.12) i18n (~> 0.6) multi_json (~> 1.0) - addressable (2.3.3) arel (3.0.2) bcrypt-ruby (3.0.1) better_errors (0.6.0) @@ -56,12 +55,10 @@ GEM activerecord (>= 2.1.0, < 4) delayed_job (~> 3.0) erubis (2.7.0) - excon (0.16.10) execjs (1.4.0) multi_json (~> 1.0) faraday (0.8.5) multipart-post (~> 1.1) - fattr (2.2.1) feedzirra (0.0.31) activesupport (>= 3.0.8) builder (~> 3.0.0) @@ -79,18 +76,6 @@ GEM activesupport (>= 3.1, < 4.1) haml (>= 3.1, < 4.1) railties (>= 3.1, < 4.1) - heroku (2.35.0) - heroku-api (~> 0.3.7) - launchy (>= 0.3.2) - netrc (~> 0.7.7) - rest-client (~> 1.6.1) - rubyzip - heroku-api (0.3.8) - excon (~> 0.16.10) - heroku_delayed_job_autoscale (0.0.7) - delayed_job (>= 2.1) - heroku (>= 1) - rush (>= 0.6) hike (1.2.1) htmlentities (4.3.1) httpauth (0.2.0) @@ -103,8 +88,6 @@ GEM jwt (0.1.5) multi_json (>= 1.0) kgio (2.8.0) - launchy (2.2.0) - addressable (~> 2.3) loofah (1.0.0) nokogiri (>= 1.3.3) mail (2.4.4) @@ -117,7 +100,6 @@ GEM mime-types (1.21) multi_json (1.6.1) multipart-post (1.1.5) - netrc (0.7.7) nokogiri (1.4.7) oauth (0.4.7) oauth2 (0.8.1) @@ -156,11 +138,6 @@ GEM rake (10.0.3) rdoc (3.12.1) json (~> 1.4) - rest-client (1.6.7) - mime-types (>= 1.16) - rubyzip (0.9.9) - rush (0.6.8) - session sass (3.2.5) sass-rails (3.2.6) railties (~> 3.2.0) @@ -168,8 +145,6 @@ GEM tilt (~> 1.3) sax-machine (0.0.20) nokogiri (> 0.0.0) - session (3.1.0) - fattr sorcery (0.8.1) bcrypt-ruby (~> 3.0.0) oauth (~> 0.4.4) @@ -206,7 +181,6 @@ DEPENDENCIES delayed_job_active_record feedzirra haml-rails - heroku_delayed_job_autoscale htmlentities jquery-rails meta_request diff --git a/app/models/feed.rb b/app/models/feed.rb index da24cb2..321c506 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,5 +1,4 @@ class Feed < ActiveRecord::Base - include HerokuDelayedJobAutoscale::Autoscale has_many :posts, :dependent => :destroy has_and_belongs_to_many :layers attr_accessible :title, :url, :description, :generator, :last_fetched, :feed_url