diff --git a/lib/uk_planning_scraper.rb b/lib/uk_planning_scraper.rb index d011469..151390a 100644 --- a/lib/uk_planning_scraper.rb +++ b/lib/uk_planning_scraper.rb @@ -10,4 +10,7 @@ module UKPlanningScraper class AuthorityNotFound < StandardError end + + class TooManySearchResults < StandardError + end end diff --git a/lib/uk_planning_scraper/idox.rb b/lib/uk_planning_scraper/idox.rb index f17d291..84760d1 100644 --- a/lib/uk_planning_scraper/idox.rb +++ b/lib/uk_planning_scraper/idox.rb @@ -51,6 +51,10 @@ module UKPlanningScraper page = form.submit + if page.search('.errors').inner_text.match(/Too many results found/i) + raise TooManySearchResults.new("Scrape in smaller chunks. Use shorter date ranges and/or more search parameters.") + end + loop do # Parse search results items = page.search('li.searchresult')