From dff300140303c151e38b8f786a0a1d5886ed33da Mon Sep 17 00:00:00 2001 From: Adrian Short Date: Fri, 30 Jun 2017 14:52:56 +0100 Subject: [PATCH] Add coordinates: eastings/northings and lat/lon --- Gemfile | 1 + Gemfile.lock | 2 ++ scraper.rb | 10 ++++++++++ 3 files changed, 13 insertions(+) diff --git a/Gemfile b/Gemfile index 282d3ed..a65b38c 100644 --- a/Gemfile +++ b/Gemfile @@ -8,3 +8,4 @@ source "https://rubygems.org" gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" gem "mechanize" +gem "breasal" diff --git a/Gemfile.lock b/Gemfile.lock index 05a919e..ff78fba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,6 +10,7 @@ GIT GEM remote: https://rubygems.org/ specs: + breasal (0.0.1) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) http-cookie (1.0.3) @@ -45,6 +46,7 @@ PLATFORMS ruby DEPENDENCIES + breasal mechanize scraperwiki! diff --git a/scraper.rb b/scraper.rb index c350bc9..5c9ee4f 100644 --- a/scraper.rb +++ b/scraper.rb @@ -5,6 +5,7 @@ require 'mechanize' require 'pp' require 'time' require 'date' +require 'breasal' # Use the column names from planningalerts.org.au: # https://www.planningalerts.org.au/how_to_write_a_scraper @@ -29,6 +30,15 @@ def parse(app) record['comment_url'] = BASEURL + link['href'].strip if link['href'].match(/PlanningComments/) end + if record['map_url'] + matches = record['map_url'].match(/x=(\d+)&y=(\d+)/) + record['easting'] = matches[1].to_i + record['northing'] = matches[2].to_i + en = Breasal::EastingNorthing.new(easting: record['easting'], northing: record['northing'], type: :gb) + record['latitude']= en.to_wgs84[:latitude] + record['longitude'] = en.to_wgs84[:longitude] + end + spans = app.search("span") record['description'] = spans[0].inner_text record['address'] = spans[1].inner_text