Browse Source

Add coordinates: eastings/northings and lat/lon

master
Adrian Short 7 years ago
parent
commit
dff3001403
3 changed files with 13 additions and 0 deletions
  1. +1
    -0
      Gemfile
  2. +2
    -0
      Gemfile.lock
  3. +10
    -0
      scraper.rb

+ 1
- 0
Gemfile View File

@@ -8,3 +8,4 @@ source "https://rubygems.org"


gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults"
gem "mechanize" gem "mechanize"
gem "breasal"

+ 2
- 0
Gemfile.lock View File

@@ -10,6 +10,7 @@ GIT
GEM GEM
remote: https://rubygems.org/ remote: https://rubygems.org/
specs: specs:
breasal (0.0.1)
domain_name (0.5.20170404) domain_name (0.5.20170404)
unf (>= 0.0.5, < 1.0.0) unf (>= 0.0.5, < 1.0.0)
http-cookie (1.0.3) http-cookie (1.0.3)
@@ -45,6 +46,7 @@ PLATFORMS
ruby ruby


DEPENDENCIES DEPENDENCIES
breasal
mechanize mechanize
scraperwiki! scraperwiki!




+ 10
- 0
scraper.rb View File

@@ -5,6 +5,7 @@ require 'mechanize'
require 'pp' require 'pp'
require 'time' require 'time'
require 'date' require 'date'
require 'breasal'


# Use the column names from planningalerts.org.au: # Use the column names from planningalerts.org.au:
# https://www.planningalerts.org.au/how_to_write_a_scraper # https://www.planningalerts.org.au/how_to_write_a_scraper
@@ -29,6 +30,15 @@ def parse(app)
record['comment_url'] = BASEURL + link['href'].strip if link['href'].match(/PlanningComments/) record['comment_url'] = BASEURL + link['href'].strip if link['href'].match(/PlanningComments/)
end end


if record['map_url']
matches = record['map_url'].match(/x=(\d+)&y=(\d+)/)
record['easting'] = matches[1].to_i
record['northing'] = matches[2].to_i
en = Breasal::EastingNorthing.new(easting: record['easting'], northing: record['northing'], type: :gb)
record['latitude']= en.to_wgs84[:latitude]
record['longitude'] = en.to_wgs84[:longitude]
end

spans = app.search("span") spans = app.search("span")
record['description'] = spans[0].inner_text record['description'] = spans[0].inner_text
record['address'] = spans[1].inner_text record['address'] = spans[1].inner_text


Loading…
Cancel
Save