Python 7 5/8/2017

SUBMITTED BY: canonical

DATE: May 8, 2017, 6 p.m.

FORMAT: Python

SIZE: 3.2 kB

Raw Download

HITS: 405

Report
  1. If you want more of my pastes visit: https://randompaste.000webhostapp.com/index.html
  2. --------------------------------------------------------------------------------------
  3. view my last post at: https://bitbin.it/SGh8LiEJ/
  4. --------------------------------------------------------------------------------------
  6. import random
  7. import requests, json
  8. import _pickle as pickle
  9. import googlemaps
  10. from datetime import datetime
  12. gmaps = googlemaps.Client(key='AIzaSyCZs17oOJSf7i6GyL19gV5sjV8TNL9RguQ')
  13. vec = []
  14. detected_locations = {}
  15. NERs = {}
  16. recognized = {}
  17. outputFormat = 'json'
  20. def apply_developed_rules(x, ln):
  21. i = 0
  22. try:
  23. while i < len(x["sentences"][0]["tokens"]):
  24. if x["sentences"][0]["tokens"][i]["ner"] == "LOCATION" and x["sentences"][0]["tokens"][i-1]["ner"] != "LOCATION":
  25. j = i + 1
  26. loc = x["sentences"][0]["tokens"][i]["word"]
  28. while x["sentences"][0]["tokens"][j]["ner"] == "LOCATION" or x["sentences"][0]["tokens"][j]["pos"] == "NNP":
  29. loc += " " + x["sentences"][0]["tokens"][j]["word"]
  30. j += 1
  31. i = j
  32. try:
  33. NERs[ln].append(loc)
  34. except:
  35. NERs[ln] = []
  36. NERs[ln].append(loc)
  37. #print(ln, loc)
  38. try:
  39. detected_locations[loc] += 1
  40. except KeyError:
  41. detected_locations[loc] = 1
  42. #out_file.write(loc+"\n");
  43. else:
  44. i += 1
  45. except KeyError:
  46. pass
  48. def apply_collection(x,ln):
  49. i = 0
  50. try:
  51. while i < len(x["sentences"][0]["tokens"]):
  52. if x["sentences"][0]["tokens"][i]["ner"] == "LOCATION":
  53. loc = x["sentences"][0]["tokens"][i]["word"]
  54. #print (loc)
  55. try:
  56. NERs[ln].append(loc)
  57. except:
  58. NERs[ln] = []
  59. NERs[ln].append(loc)
  61. try:
  62. detected_locations[loc] += 1
  63. except KeyError:
  64. detected_locations[loc] = 1
  65. #out_file.write(loc+"\n");
  66. i += 1
  67. except KeyError:
  68. pass
  71. def main():
  72. to_annotate = "final_files/hand_annotated_dataset.txt"
  73. hand_annotated = "final_files/hand_annotated_dataset.tsv"
  74. use_rules = True
  76. with open("final_files/entity_database.dat", "rb") as myFile:
  77. recognized = pickle.load(myFile)
  79. with open("final_files/testset.txt", encoding="utf8") as in_file:
  80. for l in in_file:
  81. vec.append(l)
  83. with open(to_annotate, 'r', encoding="utf8") as x_file:
  84. for line in x_file:
  85. ln = line
  86. total = ""
  87. try:
  88. total = recognized[ln]
  89. #print("\n\n\n")
  90. except:
  91. pass
  93. try:
  94. x = json.loads(total)
  95. if use_rules:
  96. apply_developed_rules(x,ln)
  97. else:
  98. apply_collection(x,ln)
  99. except:
  100. pass
  103. #print(detected_locations)
  104. #for i in NERs:
  105. # print(i)
  106. #print((i.replace("\n","") + "\t" + str(NERs[i]).replace("[","").replace("]","").replace(', ',"\t")).split('\t'))
  107. num_correct = 0
  108. with open(hand_annotated, 'r', encoding="utf8") as x_file:
  109. for l in x_file:
  110. v = l.replace('\n','').split('\t')
  111. a = []
  112. for i in range(1, len(v)):
  113. a.append(v[i].replace('\'',''))
  114. try:
  115. print(NERs[v[0]+'\n'], a, NERs[v[0]+'\n'] == a)
  116. if NERs[v[0]+'\n'] == a:
  117. num_correct += 1
  118. except:
  119. pass
  120. print (len(NERs), num_correct, num_correct/len(NERs)*100)
  123. if __name__ == '__main__':
  124. main()
comments powered by Disqus