proxy scraper

SUBMITTED BY: uuuuuu

DATE: Oct. 24, 2016, 9:35 p.m.

FORMAT: Text only

SIZE: 2.4 kB

Raw Download

HITS: 455

Report
  1. from eventlet.green import urllib2
  2. import eventlet
  3. import re
  5. def getProxies(url):
  6. src = urllib2.urlopen(url).read().split("\n")
  8. css = False
  9. classes = {}
  10. ips = []
  11. proxies = []
  13. for i in xrange(len(src)):
  14. line = src[i]
  16. # Start of style
  17. if "<td><span><style>" in line:
  18. css = True
  19. continue
  21. # Handle the CSS
  22. if css == True:
  23. # End of style
  24. if "</style>" in line:
  25. css = False
  27. if "display:none" in line:
  28. classes[line[1:5]] = "none"
  30. if "display:inline" in line:
  31. classes[line[1:5]] = "inline"
  33. # IP line
  34. if len(classes) > 0 and css == False:
  35. ip = line
  36. linePort = src[i + 2]
  37. lineCountry = src[i + 4]
  38. lineResponseTime = src[i + 7]
  39. lineConnectionTime = src[i + 11]
  40. lineType = src[i + 16]
  41. lineAnonymity = src[i + 17]
  43. # Replace class declarations with style ones
  44. for class_ in classes:
  45. ip = ip.replace("class=\"%s\"" % (class_), "style=\"display:%s\"" % (classes[class_]))
  47. # Remove all unecessary poop :)
  48. ip = re.sub(r"<(div|span) style=\"display:none\">[\.0-9]+</(div|span)>", r"", ip)
  49. ip = re.sub(r"class=\"[0-9]+\"", r"", ip)
  50. ip = re.sub(r"[^0123456789\.]", r"", ip)
  52. # Port
  53. port = linePort.replace("</td>", "")
  55. # Country
  56. country = lineCountry.split("/> ")[1].split("<")[0]
  58. # Response Time Percents
  59. responseTime = lineResponseTime.split(":")[1].split("%")[0]
  61. # Connection Time Percents
  62. connectionTime = lineConnectionTime.split(":")[1].split("%")[0]
  64. # Connection Type
  65. type = lineType.split(">")[1].split("<")[0]
  67. # Anonymity
  68. anonymity = lineAnonymity.split(">")[1].split("<")[0]
  70. proxies.append({"ip":ip, "port":port, "country":country, "responseTime":responseTime, "connectionTime":connectionTime, "type":type, "anonymity":anonymity})
  72. classes = {}
  74. return url, proxies
  76. proxies = []
  78. pool = eventlet.GreenPool(12)
  79. for url, proxyList in pool.imap(getProxies, ["https://hidemyass.com/proxy-list/" + str(i + 1) for i in xrange(12)]):
  80. for proxy in proxyList:
  81. proxies.append(proxy)
  83. for proxy in proxies:
  84. print "%s:%s" % (proxy["ip"], proxy["port"])
  86. print "\nGot %s proxies!" % (len(proxies))
comments powered by Disqus