| @@ -71,8 +71,8 @@ module UKPlanningScraper | |||||
| '__EVENTVALIDATION' => doc.at('#__EVENTVALIDATION')['value'] | '__EVENTVALIDATION' => doc.at('#__EVENTVALIDATION')['value'] | ||||
| } | } | ||||
| else | else | ||||
| logger.fatal "Bad response from search page. Response code: #{response.code.to_s}. Exiting." | |||||
| exit 1 | |||||
| logger.fatal "Bad response from search page. Response code: #{response.code.to_s}." | |||||
| raise RuntimeError.new("Northgate: Bad response from search page. Response code: #{response.code.to_s}.") | |||||
| end | end | ||||
| cookies = {} | cookies = {} | ||||
| @@ -88,16 +88,14 @@ module UKPlanningScraper | |||||
| # Follow the redirect manually | # Follow the redirect manually | ||||
| # Set the page size (PS) to max so we don't have to page through search results | # Set the page size (PS) to max so we don't have to page through search results | ||||
| logger.debug "Location: #{response2.headers['Location']}" | logger.debug "Location: #{response2.headers['Location']}" | ||||
| # exit | |||||
| results_url = URI::encode(base_url + response2.headers['Location'].gsub!('PS=10', 'PS=99999')) | results_url = URI::encode(base_url + response2.headers['Location'].gsub!('PS=10', 'PS=99999')) | ||||
| logger.debug "GET: " + results_url | logger.debug "GET: " + results_url | ||||
| response3 = HTTP.headers(headers).cookies(cookies).get(results_url) | response3 = HTTP.headers(headers).cookies(cookies).get(results_url) | ||||
| logger.debug "Response code: HTTP " + response3.code.to_s | logger.debug "Response code: HTTP " + response3.code.to_s | ||||
| doc = Nokogiri::HTML(response3.to_s) | doc = Nokogiri::HTML(response3.to_s) | ||||
| else | else | ||||
| logger.fatal "Didn't get redirected from search. Exiting." | |||||
| exit 1 | |||||
| logger.error "Didn't get redirected from search." | |||||
| raise RuntimeError.new("Northgate: didn't get redirected from search.") | |||||
| end | end | ||||
| rows = doc.search("table.display_table tr") | rows = doc.search("table.display_table tr") | ||||