#!/usr/bin/env ruby
# encoding: UTF-8

require 'net/http' 
require 'uri'
require 'fileutils'
require 'logger'
require 'mechanize'
require 'nokogiri'
require './google_get_serp.rb'
require './word1_calculate_density.rb'
require './word2_calculate_density.rb'
require './word3_calculate_density.rb'
require './word4_calculate_density.rb'
require './search_proxy.rb'
require 'page_rankr'
require 'uri'
require 'cgi'

module Page_analysis
  class Page_analyse

    def aleat
      #return rand(0.15..0.75)
      r = Random.new
      return r.rand(15..20)
    end

   # Only parses once
    def get_host_without_www2(url)
      url = "http://#{url}" unless url.start_with?('http')
      uri = URI.parse(url)
      host = uri.host.downcase
      host.start_with?('www.') ? host[4..-1] : host
    end

   # URL always gets parsed twice
    def get_host_without_www(url)
      url = "http://#{url}" if URI.parse(url).scheme.nil?
      host = URI.parse(url).host.downcase
      host.start_with?('www.') ? host[4..-1] : host
    end

   # Only parses twice if url doesn't start with a scheme
    def get_host_without_www(url)
      uri = URI.parse(url)
      uri = URI.parse("http://#{url}") if uri.scheme.nil?
      host = uri.host.downcase
      host.start_with?('www.') ? host[4..-1] : host
    end

   ### content
    def check_h1(body)
     #regex_h1 = '/<\s*h1[^>]*>(.*?)<\s*/\s*h1>/g'
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('h1').map { |n| n.inner_text }
    end

    def check_h2(body)
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('h2').map { |n| n.inner_text }
    end

    def check_h3(body)
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('h3').map { |n| n.inner_text }
    end

    def check_h4(body)
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('h4').map { |n| n.inner_text }
    end

    def check_h5(body)
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('h5').map { |n| n.inner_text }
    end

    def check_h6(body)
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('h6').map { |n| n.inner_text }
    end

    def check_title(body) # ++ longueur entre 10 et 70 char
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('title').map { |n| n.inner_text }
    end

    def check_strong(body) #strong
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('strong').map { |n| n.inner_text }
    end

    def check_italic(body) #em
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search('em').map { |n| n.inner_text }
    end

   ### meta
    def check_meta_description(page) # 
     puts page.at('meta[@name="description"]')[:content]
    end

    def check_meta_keywords(page) #
     puts page.at('meta[@name="keywords"]')[:content]
    end

    def check_meta_content_type(page) #
     puts page.at('meta[@http-equiv="Content-Type"]')[:content]
    end

    def check_meta_content(page) #encodage, indexable, ..
     puts page.at('meta[@name="Robots"]')[:content]
    end

   ### robots.txt & sitemap          => récupérer le domaine
    def check_spider(page)
      page_bot = page << "robots.txt"
      puts @agent.get page_bot
    end

    def check_sitemap(page)
      page_sitemap = page << "sitemap1.xml"
      puts @agent.get page_sitemap
    end


   ### links & imgs   
    def check_links(page)
     page.links.each do |link|
       puts link
     end
    end

    def check_images(page)  
     page.images.each do |img|
       puts img.alt
     end
    end

   ### duplicate content
    def check_duplicate_content(body)
     proxy_addr, proxy_port = proxy.split(':') 
    end

   ### pagerank
    def check_pagerank(url)
     res = PageRankr.ranks(url, :google) 
     puts res[:google]
    end

   ### backlink
    def check_link(body,url) # check si la page contient bien le texte
     doc = Nokogiri::HTML::DocumentFragment.parse(body)
     puts doc.search(url).map { |n| n.inner_text }
    end


    def check_backlink(url,lang)

      myobj = Find_Proxy.new
      #proxies = myobj.find_proxy(6)

      res = []
      exact_query = url

      proxies.each do |proxy|
      begin
        myobj = Find_Proxy.new
        proxy = myobj.find_proxy(1)
        puts proxy
        pp = Seeker.new
        tmp = pp.run(exact_query,100,lang,proxy.first) # fr, com, de, nl, it, es
        if tmp == false
          raise
        end
        res.flatten << tmp 
        puts "fin #{tmp.length}"
      rescue
        puts "retry"
        retry     
      end 
      res = res.flatten
      puts res
      res = res.uniq
      return res.length     
    end


   ### run main
   def run(url)

     begin
       myobj = Find_Proxy.new
       proxy = myobj.find_proxy(1)
       addr, port = proxy.first.split(':')
       make_agent(addr, port)
       #list_lang = ["com","fr","de","es","nl","it"]
       list_lang = ["fr"]

       #page = @agent.get url
       #check_h1(page.body)
       #check_title(page.body)
       #check_strong(page.body)
       #check_italic(page.body)
       #check_meta_description(page)
       #check_meta_keywords(page)
       #check_meta_content_type(page)
       #check_meta_content(page)

       #check_spider(page.uri.to_s )
       #check_sitemap(page.uri.to_s )
       #check_links(page)
       #check_images(page)
       #density_one(page.uri.to_s)
       #density_two(page.uri.to_s)
       #density_three(page.uri.to_s)
       #density_for(page.uri.to_s)
       #check_pagerank(page.uri.to_s)
       how = 0
       #@tab_vide = []
       list_lang.each do |lang|
         #check_backlink(page.uri.to_s)
         how = check_backlink(url,lang)
         puts "number #{how.to_s} for #{lang.to_s}"
       end
     rescue e
       puts e
       #retry
     end
   end

   def make_agent(addr, port)
      @agent = Mechanize.new do |a|
        a.user_agent_alias = 'Mac Safari'
        a.max_history = 1
        a.open_timeout = 15
        a.read_timeout = 5
        a.keep_alive = false
        #a.log = Logger.new(STDOUT)
        a.log = Logger.new('log_analyse_page.txt')
        #a.log.level = Logger::INFO
        a.set_proxy(addr, port)
      end
    end      
    
  end
end

  def main(url)
    include Page_analysis

    tt = Page_analyse.new
    analyse_page = tt.run(url)
  end

main ARGV[0]