Browse Source

Use chained scrape parameters

master
Adrian Short 6 years ago
parent
commit
c378dcfd9a
3 changed files with 8 additions and 6 deletions
  1. +1
    -1
      Gemfile
  2. +5
    -4
      Gemfile.lock
  3. +2
    -1
      scraper.rb

+ 1
- 1
Gemfile View File

@@ -7,4 +7,4 @@ source "https://rubygems.org"
ruby "2.3.1" ruby "2.3.1"


gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults"
gem "uk_planning_scraper", git: "https://github.com/adrianshort/uk_planning_scraper.git"
gem "uk_planning_scraper", git: "https://github.com/adrianshort/uk_planning_scraper.git", ref: "cad5fcd"

+ 5
- 4
Gemfile.lock View File

@@ -1,8 +1,9 @@
GIT GIT
remote: https://github.com/adrianshort/uk_planning_scraper.git remote: https://github.com/adrianshort/uk_planning_scraper.git
revision: a342ff53bd3c10e5436335b143318271b5a2b36a
revision: cad5fcd2b98260572dccf0a4cf1ddfeabc469a43
ref: cad5fcd
specs: specs:
uk_planning_scraper (0.2.0)
uk_planning_scraper (0.4.0)
http http
mechanize (~> 2.7) mechanize (~> 2.7)


@@ -23,7 +24,7 @@ GEM
connection_pool (2.2.2) connection_pool (2.2.2)
domain_name (0.5.20180417) domain_name (0.5.20180417)
unf (>= 0.0.5, < 1.0.0) unf (>= 0.0.5, < 1.0.0)
http (3.3.0)
http (4.0.0)
addressable (~> 2.3) addressable (~> 2.3)
http-cookie (~> 1.0) http-cookie (~> 1.0)
http-form_data (~> 2.0) http-form_data (~> 2.0)
@@ -49,7 +50,7 @@ GEM
net-http-digest_auth (1.4.1) net-http-digest_auth (1.4.1)
net-http-persistent (3.0.0) net-http-persistent (3.0.0)
connection_pool (~> 2.2) connection_pool (~> 2.2)
nokogiri (1.8.4)
nokogiri (1.8.5)
mini_portile2 (~> 2.3.0) mini_portile2 (~> 2.3.0)
ntlm-http (0.1.1) ntlm-http (0.1.1)
public_suffix (3.0.3) public_suffix (3.0.3)


+ 2
- 1
scraper.rb View File

@@ -1,5 +1,6 @@
require 'uk_planning_scraper' require 'uk_planning_scraper'
require 'scraperwiki' require 'scraperwiki'


apps = UKPlanningScraper::Authority.named(ENV['MORPH_AUTHORITY_NAME']).scrape({ validated_days: ENV['MORPH_DAYS'].to_i })
apps = UKPlanningScraper::Authority.named(ENV['MORPH_AUTHORITY_NAME']). \
validated_days(ENV['MORPH_DAYS'].to_i).scrape
ScraperWiki.save_sqlite([:authority_name, :council_reference], apps) ScraperWiki.save_sqlite([:authority_name, :council_reference], apps)

Loading…
Cancel
Save