ruby bing get serp

SUBMITTED BY: Guest

DATE: Sept. 28, 2014, 8:03 p.m.

FORMAT: Text only

SIZE: 1.9 kB

Raw Download

HITS: 1000

Report
  1. #!/usr/bin/env ruby
  2. # encoding: UTF-8
  4. require 'rubygems'
  5. require 'time'
  6. require 'date'
  7. require 'hpricot'
  8. require 'htmlentities'
  9. require 'rexml/document'
  10. require 'oauth'
  11. require 'mime/types'
  12. require 'socket'
  13. require 'cgi'
  14. require 'httpclient'
  15. require 'net/ftp'
  16. require 'net/smtp'
  17. require 'digest/sha2.rb'
  18. require 'net/http'
  19. require 'uri'
  20. require 'fileutils'
  21. require 'mechanize'
  22. require 'nokogiri'
  23. require 'open-uri'
  26. module Get_bing_serp
  27. class BingSeeker
  29. def initialize
  30. make_agent
  31. end
  33. def aleat
  34. r = Random.new
  35. return r.rand(15..45)
  36. end
  38. def parse_web(query)
  39. formated_tab = []
  40. query.parser.xpath('//h3').take(10).each do |cite|
  41. formated_tab << URI::extract( cite.to_s, [ 'http' ] )#cite.inner_text
  42. end
  43. return formated_tab
  44. end
  46. def run(query,max)
  47. i = 2
  48. limit = i + max
  49. result = []
  50. make_agent
  51. page = @agent.get('http://www.bing.com/').forms[0].tap{|f| f.q = query}.submit
  52. result << parse_web(page)
  54. next_link = page.links.find { |l| l.text == i.to_s }
  56. while !next_link.nil?
  57. sleep aleat
  58. i += 1
  59. next_page = @agent.get next_link.href
  60. result << parse_web(next_page)
  61. next_link = next_page.links.find { |l| l.text == i.to_s }
  62. end
  63. return result
  64. #return result.uniq
  65. end
  67. private
  69. def make_agent
  70. @agent = Mechanize.new do |a|
  71. a.user_agent_alias = 'Mac Safari'
  72. a.max_history = 1
  73. a.keep_alive=false
  74. a.read_timeout=15
  75. end
  76. end
  77. end
  78. end