#!/usr/bin/env ruby
# encoding: UTF-8

require 'rubygems'
require 'time'
require 'date'
require 'hpricot'
require 'htmlentities'
require 'rexml/document'
require 'oauth'
require 'mime/types'
require 'socket' 
require 'cgi'
require 'httpclient'
require 'net/ftp'
require 'net/smtp'
require 'digest/sha2.rb'
require 'net/http' 
require 'uri'
require 'fileutils'
require 'mechanize'
require 'nokogiri'
require 'open-uri'

  class Seeker
    def aleat
      r = Random.new
      return r.rand(35..50)
    end

    def parse_web(query)    
      formated_tab = []
      query.parser.xpath('//h3[@class="r"]').take(10).each do |cite|
        formated_tab << URI::extract( cite.to_s, [ 'http' ] )#cite.inner_text
      end
      return formated_tab    
    end

    def run(query,max,domain,proxy)
      puts " #{query} #{max} #{domain} #{proxy}"
      i = 2
      limit = i + max
      result = []
      #formated_query = CGI.escape(query)
      formated_query = query
      begin
        addr, port = proxy.split(':')
        make_agent(addr, port) 
        #page = @agent.get 'http://www.google.' << domain << '/search?q='<<  formated_query 

        page = @agent.get 'http://www.google.' << domain
        #puts page.body
        #search_result = page.form_with(:name => 'f') do |search|
        form = page.form_with(:action => '/search') 
        form.fields.each { |f| 
          #puts f.name
          if f.name == 'q'
            form[f.name] = formated_query 
          else
            form[f.name] = ''
          end
        }
        page_res = form.submit
        #puts page_res.body

        result << parse_web(page_res)
        #puts result

        while next_link =  page_res.links.find { |l| l.text == i.to_s }
          #puts i
          if i < limit
            sleep aleat
            next_page = @agent.get next_link.href
            result << parse_web(next_page)
            i += 1
          end
        end
        return result   
      rescue Mechanize::ResponseCodeError => exception
              #puts exception.message + " - invalid " + proxy
              #puts exception.backtrace.inspect  
              @agent.log.info exception.message
              return false 
      rescue Net::HTTPBadResponse , Net::HTTP::Persistent::Error, Errno::ECONNRESET=> e
              #puts "net exception "+ e.to_s
              @agent.log.info "net exception "+ e.to_s
              return false
      rescue Timeout::Error => e
              #puts "timeout: "+e.to_s
              @agent.log.info "timeout: "+e.to_s
              return false
      rescue => e
              #puts  "unknown #{e.to_s}"
              @agent.log.info "unknown #{e.to_s}"
              return false
      end
    end

    private

   def make_agent
     user_agents = {
      'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
      'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
      'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
      'Mac Safari' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10',
      'Mac FireFox' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6',
      'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401',
      'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
      'Linux Firefox' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.1) Gecko/20100122 firefox/3.6.1',
      'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
      'iPhone' => 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3'
     } 
     items = user_agents.keys
      @agent = Mechanize.new do |a|
        a.user_agent_alias = items[rand(items.length)]
        a.max_history = 1
        a.open_timeout = 15
        a.read_timeout = 4
        a.keep_alive = true
      end
    end    
  end