A Ruby gem to get planning applications data from UK council websites.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

183 lines
7.3 KiB

  1. require 'mechanize'
  2. require 'pp'
  3. module UKPlanningScraper
  4. class Authority
  5. private
  6. def scrape_idox(params, options)
  7. puts "Using Idox scraper."
  8. base_url = @url.match(/(https?:\/\/.+?)\//)[1]
  9. apps = []
  10. agent = Mechanize.new
  11. puts "Getting: #{@url}"
  12. page = agent.get(@url) # load the search form page
  13. # Check that the search form is actually present.
  14. # When Idox has an internal error it returns an error page with HTTP 200.
  15. unless form = page.form('searchCriteriaForm')
  16. puts "Error: Search form page failed to load due to Idox internal error."
  17. return []
  18. end
  19. # form.action = form.action + '&searchCriteria.resultsPerPage=100'
  20. # Fill out and submit search form
  21. # Add expected fields to form if they're not already present so that searches using these terms work
  22. %w{
  23. date(applicationReceivedStart)
  24. date(applicationReceivedEnd)
  25. }.each { |f| form.add_field!(f) unless form.has_field?(f) }
  26. date_format = "%d/%m/%Y"
  27. form.send(:"date(applicationReceivedStart)", params[:received_from].strftime(date_format)) if params[:received_from]
  28. form.send(:"date(applicationReceivedEnd)", params[:received_to].strftime(date_format)) if params[:received_to]
  29. form.send(:"date(applicationValidatedStart)", params[:validated_from].strftime(date_format)) if params[:validated_from]
  30. form.send(:"date(applicationValidatedEnd)", params[:validated_to].strftime(date_format)) if params[:validated_to]
  31. form.send(:"date(applicationDecisionStart)", params[:decided_from].strftime(date_format)) if params[:decided_from]
  32. form.send(:"date(applicationDecisionEnd)", params[:decided_to].strftime(date_format)) if params[:decided_to]
  33. form.send(:"searchCriteria\.description", params[:keywords])
  34. # Some councils don't have the applicant name on their form, eg Bexley
  35. form.send(:"searchCriteria\.applicantName", params[:applicant_name]) if form.has_field? 'searchCriteria.applicantName'
  36. form.send(:"searchCriteria\.caseType", params[:application_type]) if form.has_field? 'searchCriteria.caseType'
  37. # Only some Idox sites (eg Bolton) have a 'searchCriteria.developmentType' parameter
  38. form.send(:"searchCriteria\.developmentType", params[:development_type]) if form.has_field? 'searchCriteria.developmentType'
  39. page = form.submit
  40. if page.search('.errors').inner_text.match(/Too many results found/i)
  41. raise TooManySearchResults.new("Scrape in smaller chunks. Use shorter date ranges and/or more search parameters.")
  42. end
  43. loop do
  44. # Parse search results
  45. items = page.search('li.searchresult')
  46. puts "Found #{items.size} apps on this page."
  47. items.each do |app|
  48. data = Application.new
  49. # Parse info line
  50. info_line = app.at("p.metaInfo").inner_text.strip
  51. bits = info_line.split('|').map { |e| e.strip.delete("\r\n") }
  52. bits.each do |bit|
  53. if matches = bit.match(/Ref\. No:\s+(.+)/)
  54. data.council_reference = matches[1]
  55. end
  56. if matches = bit.match(/(Received|Registered):\s+.*(\d{2}\s\w{3}\s\d{2}\d{2}?)/)
  57. data.date_received = Date.parse(matches[2])
  58. end
  59. if matches = bit.match(/Validated:\s+.*(\d{2}\s\w{3}\s\d{2}\d{2}?)/)
  60. data.date_validated = Date.parse(matches[1])
  61. end
  62. if matches = bit.match(/Status:\s+(.+)/)
  63. data.status = matches[1]
  64. end
  65. end
  66. data.scraped_at = Time.now
  67. data.info_url = base_url + app.at('a')['href']
  68. data.address = app.at('p.address').inner_text.strip
  69. data.description = app.at('a').inner_text.strip
  70. apps << data
  71. end
  72. # Get the Next button from the pager, if there is one
  73. if next_button = page.at('a.next')
  74. next_url = base_url + next_button[:href]# + '&searchCriteria.resultsPerPage=100'
  75. sleep options[:delay]
  76. puts "Getting: #{next_url}"
  77. page = agent.get(next_url)
  78. else
  79. break
  80. end
  81. end
  82. # Scrape the summary tab for each app
  83. apps.each_with_index do |app, i|
  84. sleep options[:delay]
  85. puts "#{i + 1} of #{apps.size}: #{app.info_url}"
  86. res = agent.get(app.info_url)
  87. if res.code == '200' # That's a String not an Integer, ffs
  88. # Parse the summary tab for this app
  89. app.scraped_at = Time.now
  90. # The Documents tab doesn't show if there are no documents (we get li.nodocuments instead)
  91. # Bradford has #tab_documents but without the document count on it
  92. app.documents_count = 0
  93. if documents_link = res.at('.associateddocument a')
  94. if documents_link.inner_text.match(/\d+/)
  95. app.documents_count = documents_link.inner_text.match(/\d+/)[0].to_i
  96. app.documents_url = base_url + documents_link[:href]
  97. end
  98. elsif documents_link = res.at('#tab_documents')
  99. if documents_link.inner_text.match(/\d+/)
  100. app.documents_count = documents_link.inner_text.match(/\d+/)[0].to_i
  101. app.documents_url = base_url + documents_link[:href]
  102. end
  103. end
  104. # We need to find values in the table by using the th labels.
  105. # The row indexes/positions change from site to site (or even app to app) so we can't rely on that.
  106. res.search('#simpleDetailsTable tr').each do |row|
  107. key = row.at('th').inner_text.strip
  108. value = row.at('td').inner_text.strip
  109. case key
  110. when 'Reference'
  111. app.council_reference = value
  112. when 'Alternative Reference'
  113. app.alternative_reference = value unless value.empty?
  114. when 'Planning Portal Reference'
  115. app.alternative_reference = value unless value.empty?
  116. when 'Application Received'
  117. app.date_received = Date.parse(value) if value.match(/\d/)
  118. when 'Application Registered'
  119. app.date_received = Date.parse(value) if value.match(/\d/)
  120. when 'Application Validated'
  121. app.date_validated = Date.parse(value) if value.match(/\d/)
  122. when 'Address'
  123. app.address = value unless value.empty?
  124. when 'Proposal'
  125. app.description = value unless value.empty?
  126. when 'Status'
  127. app.status = value unless value.empty?
  128. when 'Decision'
  129. app.decision = value unless value.empty?
  130. when 'Decision Issued Date'
  131. app.date_decision = Date.parse(value) if value.match(/\d/)
  132. when 'Appeal Status'
  133. app.appeal_status = value unless value.empty?
  134. when 'Appeal Decision'
  135. app.appeal_decision = value unless value.empty?
  136. else
  137. puts "Error: key '#{key}' not found"
  138. end # case
  139. end # each row
  140. else
  141. puts "Error: HTTP #{res.code}"
  142. end # if
  143. end # scrape summary tab for apps
  144. apps
  145. end # scrape_idox
  146. end # class
  147. end