Automatically exported from code.google.com/p/planningalerts
選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

application_parser.php 6.4 KiB

17年前
17年前
17年前
17年前
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. <?php
  2. require_once('tools_ini.php');
  3. require_once('application.php');
  4. require_once('DB.php');
  5. //Initialise
  6. $application_parser = new application_parser();
  7. //Scrape for the last X days (apps already in the database are ignored)
  8. for ($i=0; $i < SCRAPE_DELAY; $i++){
  9. $application_parser->date = getdate(strtotime("-" . $i . " days"));
  10. $application_parser->run();
  11. }
  12. //Send email
  13. $application_parser->email_log();
  14. //Parser class
  15. class application_parser{
  16. //Properties
  17. var $date;
  18. var $log = array();
  19. var $sleep_interval = 5; //how long to wait between scraping each feed
  20. //Constructor
  21. function application_parser (){
  22. //set default date
  23. $this->date = getdate();
  24. }
  25. //Run
  26. function run(){
  27. $db = DB::connect(DB_CONNECTION_STRING);
  28. $sql = "Select authority_id, feed_url, external, disabled, short_name from authority where disabled <> 1";
  29. $results = $db->getAll($sql);
  30. if (sizeof($results) == 0){
  31. //throw new exception("You need to put some authorities to scrape in the database");
  32. }
  33. //log
  34. $this->store_log("Scraping " . sizeof($results) . "authorities");
  35. //Parse & save each feed
  36. foreach($results as $result){
  37. //reset the timeout
  38. set_time_limit(0);
  39. $authority_id = $result[0];
  40. $external = $result[2];
  41. $disabled = $result[3];
  42. if($external != true){
  43. $feed_url = BASE_URL . $feed_url = $result[1];
  44. }else{
  45. $feed_url = $result[1];
  46. }
  47. //replace date wild cards
  48. $feed_url = str_replace("{day}",$this->date['mday'], $feed_url);
  49. $feed_url = str_replace("{month}",$this->date['mon'], $feed_url);
  50. $feed_url = str_replace("{year}",$this->date['year'], $feed_url);
  51. //log
  52. $this->store_log("Scraping authority " . $result[4] . " from " . $feed_url);
  53. //if it isnt disabled parse it
  54. if ($disabled == false){
  55. $applications = $this->parse_applications($feed_url, $authority_id);
  56. //log
  57. $this->store_log("Found " . sizeof($applications) . " applications for " . $result[4]);
  58. //save applications (probably shouldent be saved individually, but sod it for the moment)
  59. foreach ($applications as $application){
  60. if(!$application->exists()){
  61. $application->save();
  62. $this->store_log("Saving application" . $application->council_reference);
  63. }else{
  64. $this->store_log("Application already exists in database" . $application->council_reference);
  65. }
  66. }
  67. }
  68. //wait for a bit so we dont blow anyone's server (mainly tinyurl)
  69. sleep($this->sleep_interval);
  70. }
  71. }
  72. //Turn xml into application objects
  73. function parse_applications($feed_url, $authority_id){
  74. $return_applications = array();
  75. //reset warnings
  76. //Grab the XML
  77. $xml = "";
  78. try{
  79. $xml = safe_scrape_page($feed_url);
  80. }catch (exception $e){
  81. array_push($this->log, "ERROR: problem occured when grabbing feed: " . $feed_url . " ---->>>" . $e);
  82. }
  83. if ($xml == false){
  84. $this->store_log("ERROR: empty feed feed: " . $feed_url);
  85. }
  86. //Turn the xml into an object
  87. $parsed_applications = simplexml_load_string($xml);
  88. //Loop through the applications, add tinyurl / google maps etc and add to array
  89. if(sizeof($parsed_applications) >0){
  90. foreach($parsed_applications->applications->application as $parsed_application){
  91. $application = new application();
  92. //Grab basic data from the xml
  93. $application->authority_id = $authority_id;
  94. $application->council_reference = $parsed_application->council_reference;
  95. $application->date_received = $parsed_application->date_received;
  96. $application->address = $parsed_application->address;
  97. $application->postcode = $parsed_application->postcode;
  98. $application->description = $parsed_application->description;
  99. $application->info_url = $parsed_application->info_url;
  100. $application->comment_url = $parsed_application->comment_url;
  101. $application->date_scraped = mysql_date(time());
  102. //Make the urls
  103. $info_tiny_url = tiny_url($application->info_url);
  104. if ($info_tiny_url == ""){
  105. $this->store_log("ERROR: Created blank info tiny url");
  106. }
  107. $comment_tiny_url = tiny_url($application->comment_url);
  108. if ($comment_tiny_url == ""){
  109. $this->store_log("ERROR: Created blank comment tiny url");
  110. }
  111. $application->info_tinyurl =$info_tiny_url;
  112. $application->comment_tinyurl = $comment_tiny_url;
  113. $application->map_url = googlemap_url_from_postcode($application->postcode);
  114. //Workout the XY location from postcode
  115. $xy = postcode_to_location($application->postcode);
  116. $application->x = $xy[0];
  117. $application->y = $xy[1];
  118. //Add to array
  119. array_push($return_applications, $application);
  120. }
  121. }
  122. return $return_applications;
  123. }
  124. function store_log($text){
  125. array_push($this->log, $text);
  126. print $text . "\n\n";
  127. }
  128. function email_log(){
  129. //Email log
  130. send_text_email(LOG_EMAIL, "parser@" . DOMAIN, "parser@" . DOMAIN, "Planning parser log", print_r($this->log, true));
  131. $this->store_log("Debug email sent to " . LOG_EMAIL);
  132. }
  133. }
  134. ?>