Etemenanki

SUBMITTED BY: Guest

DATE: Dec. 4, 2013, 1:04 p.m.

FORMAT: Bash

SIZE: 10.2 kB

Raw Download

HITS: 870

Report
  1. 1 ---
  2. 2 Etemenanki is an agent that builds word dictionnaries based on remote and
  3. 3 local (hyper)text repositories.
  4. 4 aym3ric-at-goto10-dot-org
  5. 5 NO WARRANTY - USE IT AT YOUR OWN RISK
  6. 6 feel free to send patches if you hack it.
  7. 7 ---
  8. 8
  9. 9 ./etemenanki.sh -h for help
  10. 10
  11. 11 ---
  12. 12
  13. 13 Some notes ...
  14. 14
  15. 15 The default behaviour of this script is to download (hyper)text content
  16. 16 from website and make a sorted dupechecked list of all the words found.
  17. 17 When pointing etemenanki to a website/domain, the ~/dictionnaries
  18. 18 directory is created and at the end of the process you will end up with
  19. 19 a wordlist file.
  20. 20
  21. 21 By default the downloaded material is cached in ~/dictionnaries/cache
  22. 22 this is useful if you intend to resume or update the downloaded content as
  23. 23 etemenanki will try to only download updated material when a cached
  24. 24 location is given.
  25. 25 Also when playing with the link depth parameter you may download much more
  26. 26 than expected and may need to interrupt and resume later on.
  27. 27 This can be also used for later wlist creation as well, when running in
  28. 28 offline mode, etemenanki will skip the download/update of the cache but
  29. 29 process it directly into a wordlist.
  30. 30
  31. 31 If a worldlist already exist for one given domain/location, etemenanki
  32. 32 will update it, never overwrite it.
  33. 33
  34. 34 To generate more entries in the world list, you can turn on the unique
  35. 35 4-PASS l33tifying engine :)
  36. 36 (can require a few GB of temporary free space if you intend to play with
  37. 37 large wordlist files).
  38. 38
  39. 39 When you have collected a couple of wlist dictionnaries you can merge them
  40. 40 into one, simply execute:
  41. 41 ./etemenanki.sh -m
  42. 42
  43. 43 ---
  44. #!/bin/bash
  45. 2 #
  46. 3 # Etemenanki is an agent that builds word dictionnaries based on remote and
  47. 4 # local (hyper)text repositories.
  48. 5 # aym3ric-at-goto10-dot-org
  49. 6 # NO WARRANTY - USE IT AT YOUR OWN RISK
  50. 7
  51. 8
  52. 9 # Config ----------------------------------------------------------------------
  53. 10 GLOBAL_REPOS=~/dictionnaries/cache
  54. 11 DICOS=~/dictionnaries
  55. 12 PROXY=""
  56. 13 SPOOF="Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.8) Gecko/20050511"
  57. 14 REJECT=ogg,iso,wav,mpg,mpeg,mp3,mov,jpg,jpeg,gif,png,bmp,tif,tga,xml,css,ico,js,gz,tgz,zip,rar,exe,bin,svg,pdf,eps,doc
  58. 15 WGET_RLEVEL=1
  59. 16 L33TIFY=0
  60. 17 OFFLINE=0
  61. 18
  62. 19 # Functions -------------------------------------------------------------------
  63. 20
  64. 21 # Dictionnaries path validation
  65. 22 function validate_paths {
  66. 23
  67. 24 if test -e $DICOS ; then
  68. 25 echo "[!] dictionnaries folder found."
  69. 26 else
  70. 27 echo "[!] dictionnaries folder not found ... creating ..."
  71. 28 mkdir $DICOS
  72. 29 mkdir $GLOBAL_REPOS
  73. 30 if test -e $DICOS ; then
  74. 31 echo "[!] dictionnaries folder created."
  75. 32 else
  76. 33 echo "[!] can't create folder ... bye."
  77. 34 fi
  78. 35 fi
  79. 36
  80. 37 }
  81. 38
  82. 39 # Grab the shit
  83. 40 function suck_and_run {
  84. 41
  85. 42 wget -l $WGET_RLEVEL -nv -nd -r -N -R $REJECT -P $LOCAL_REPOS \
  86. 43 --user-agent="$SPOOF" -erobots=off --random-wait $OPTARG -o /dev/stdout | \
  87. 44 awk '/URL/ {print "[ ] sucking "$2}' ;
  88. 45
  89. 46 }
  90. 47
  91. 48
  92. 49 # Garbage cleaner
  93. 50 function nogarbage {
  94. 51
  95. 52 sed 's/[.,;:)]*$//g' | \
  96. 53 sed '/^http/ d' | sed '/^h77p/ d' |\
  97. 54 sed '/^\// d' | \
  98. 55 sed 's/^[[(]//g'
  99. 56
  100. 57 }
  101. 58
  102. 59 # Generate more words using l33t translations
  103. 60 function l33t {
  104. 61
  105. 62 if [ "$L33TIFY" = "1" ] ; then
  106. 63 mv $DICOS/$OPTARG.wlist $DICOS/$OPTARG.wlist.tmp
  107. 64 echo "[!] 4-PASS l33tifying in progress ..."
  108. 65 echo "[ ] pass 1 in progress"
  109. 66 cat $DICOS/$OPTARG.wlist.tmp | tr '[:upper:]' '[:lower:]' > $DICOS/$OPTARG.wlist.l33tlo
  110. 67 cat $DICOS/$OPTARG.wlist.tmp | tr '[:lower:]' '[:upper:]' > $DICOS/$OPTARG.wlist.l33tup
  111. 68 cat $DICOS/$OPTARG.wlist.l33tlo >> $DICOS/$OPTARG.wlist.tmp
  112. 69 cat $DICOS/$OPTARG.wlist.l33tup >> $DICOS/$OPTARG.wlist.tmp
  113. 70 rm $DICOS/$OPTARG.wlist.l33t*
  114. 71 echo "[ ] pass 2 in progress"
  115. 72 cat $DICOS/$OPTARG.wlist.tmp | tr 'aeiou' 'AEIOU' > $DICOS/$OPTARG.wlist.l33t
  116. 73 cat $DICOS/$OPTARG.wlist.l33t >> $DICOS/$OPTARG.wlist.tmp
  117. 74 rm $DICOS/$OPTARG.wlist.l33t
  118. 75 echo "[ ] pass 3 in progress"
  119. 76 cat $DICOS/$OPTARG.wlist.tmp | tr 'aeost' '43057' > $DICOS/$OPTARG.wlist.l33t
  120. 77 cat $DICOS/$OPTARG.wlist.l33t >> $DICOS/$OPTARG.wlist.tmp
  121. 78 rm $DICOS/$OPTARG.wlist.l33t
  122. 79 echo "[ ] pass 4 in progress"
  123. 80 cat $DICOS/$OPTARG.wlist.tmp | tr 'AEOST' '43057' > $DICOS/$OPTARG.wlist.l33t
  124. 81 cat $DICOS/$OPTARG.wlist.l33t >> $DICOS/$OPTARG.wlist.tmp
  125. 82 rm $DICOS/$OPTARG.wlist.l33t
  126. 83 echo "[!] sorting $OPTARG.wlist - please wait"
  127. 84 sort -u $DICOS/$OPTARG.wlist.tmp -o $DICOS/$OPTARG.wlist
  128. 85 rm $DICOS/$OPTARG.wlist.tmp ;
  129. 86 fi
  130. 87
  131. 88 }
  132. 89
  133. 90 # Extract the words
  134. 91 function wlist_creator {
  135. 92
  136. 93 # pack/parse/clean all the gathered html crap
  137. 94
  138. 95 for i in `ls --sort=time --reverse $LOCAL_REPOS` ; do
  139. 96 echo "[ ] processing $i" ;
  140. 97 WLIST_TMP="$WLIST_TMP `cat $LOCAL_REPOS$i | lynx -verbose off -dump -nolist -stdin | sed 's/*//g'`"
  141. 98 if [ "${#WLIST_TMP}" -ge "20000" ] ; then
  142. 99 echo $WLIST_TMP | sed 's/ /\n/g' | nogarbage > $DICOS/$OPTARG.unsortedchunk
  143. 100 sort -u $DICOS/$OPTARG.unsortedchunk >> $DICOS/$OPTARG.wlist.tmp ;
  144. 101 rm $DICOS/$OPTARG.unsortedchunk
  145. 102 WLIST_TMP="";
  146. 103 fi
  147. 104 done
  148. 105
  149. 106 # add the leftovers
  150. 107 echo $WLIST_TMP | sed 's/ /\n/g' | nogarbage > $DICOS/$OPTARG.unsortedchunk
  151. 108 sort -u $DICOS/$OPTARG.unsortedchunk >> $DICOS/$OPTARG.wlist.tmp ;
  152. 109 rm $DICOS/$OPTARG.unsortedchunk
  153. 110
  154. 111 # create or update the final dictionnary
  155. 112
  156. 113 if test -f $DICOS/$OPTARG.wlist ; then
  157. 114 echo "[!] updating $OPTARG.wlist - please wait"
  158. 115 cat $DICOS/$OPTARG.wlist.tmp >> $DICOS/$OPTARG.wlist
  159. 116 else
  160. 117 echo "[!] creating $OPTARG.wlist - please wait"
  161. 118 cat $DICOS/$OPTARG.wlist.tmp > $DICOS/$OPTARG.wlist
  162. 119 fi
  163. 120
  164. 121 echo "[!] sorting $OPTARG.wlist - please wait"
  165. 122 mv $DICOS/$OPTARG.wlist $DICOS/$OPTARG.wlist.tmp
  166. 123 sort -u $DICOS/$OPTARG.wlist.tmp -o $DICOS/$OPTARG.wlist ;
  167. 124 rm $DICOS/$OPTARG.wlist.tmp ;
  168. 125 l33t
  169. 126
  170. 127 }
  171. 128
  172. 129 # Merge all the wlist into a big dictionnary
  173. 130 function wlist_merge {
  174. 131
  175. 132 # test is there are any wlist around
  176. 133
  177. 134 if [ "`ls -1 $DICOS/*.wlist | wc -l`" -eq "0" ] ; then
  178. 135 echo "[!] you don't have any wlist yet..."
  179. 136 else
  180. 137 echo "[!] wlist found in $DICOS/"
  181. 138 if [ "`ls -1 $DICOS/*.wlist | wc -l`" -eq "1" ] ; then
  182. 139 echo "[!] you only have one though ... aborting."
  183. 140 else
  184. 141 for i in `ls -1 $DICOS/*.wlist | grep -v merge.wlist` ; do
  185. 142 echo "[ ] merging $i"
  186. 143 cat $i >> $DICOS/merge.wlist.tmp
  187. 144 done
  188. 145 echo "[ ] sorting the entries ..."
  189. 146 sort -u $DICOS/merge.wlist.tmp -o $DICOS/merge.wlist
  190. 147 rm $DICOS/merge.wlist.tmp
  191. 148 echo "[!] merge completed - bye."
  192. 149 fi
  193. 150 fi
  194. 151
  195. 152 }
  196. 153
  197. 154
  198. 155
  199. 156
  200. 157 # Options handling ------------------------------------------------------------
  201. 158 OPTSTRING="hd:lot:m"
  202. 159
  203. 160 # Check if any switches have been passed
  204. 161 if [ $# -eq 0 ] ; then
  205. 162 echo "[!] Type -h for help."
  206. 163 exit
  207. 164 fi
  208. 165
  209. 166 # Choose you destiny
  210. 167 while getopts "$OPTSTRING" SWITCH ; do
  211. 168 case $SWITCH in
  212. 169 h) echo "[?] Etemenanki - wordlist builder bot"
  213. 170 echo "[?] aym3ric-at-goto10-dot-org"
  214. 171 echo "[?]"
  215. 172 echo "[?] Usage: etemenanki.sh [-h] (-dlo) -t location"
  216. 173 echo "[?] -h this help"
  217. 174 echo "[?] -d maximum recursion depth (default=1)"
  218. 175 echo "[?] -l l33tify the words "
  219. 176 echo "[?] -o offline mode"
  220. 177 echo "[?] -t target location (http)"
  221. 178 echo "[?] -m merge all the wlist files"
  222. 179 echo "[?] Example: ./etemenanki -d 1 -l -t google.com"
  223. 180 exit 0
  224. 181 ;;
  225. 182 d) WGET_RLEVEL=$OPTARG
  226. 183 ;;
  227. 184 l) L33TIFY=1
  228. 185 ;;
  229. 186 o) OFFLINE=1
  230. 187 ;;
  231. 188 t) LOCAL_REPOS=$GLOBAL_REPOS/$OPTARG/
  232. 189 validate_paths
  233. 190 SUCKSTART=`date +%s`
  234. 191 if [ "$OFFLINE" = "0" ] ; then
  235. 192 suck_and_run
  236. 193 fi
  237. 194 WLISTSTART=`date +%s`
  238. 195 wlist_creator
  239. 196 WLISTSTOP=`date +%s`
  240. 197 echo "[!] $OPTARG.wlist written in $DICOS/"
  241. 198 echo "[!] sucked in `expr $WLISTSTART - $SUCKSTART` s - packed in `expr $WLISTSTOP - $WLISTSTART` s"
  242. 199 echo "[ ] bye..."
  243. 200 exit 0
  244. 201 ;;
  245. 202 m) wlist_merge
  246. 203 exit 0
  247. 204 ;;
  248. 205 *) echo "[!] Unexpected switch or argument"
  249. 206 echo "[!] Type -h for help."
  250. 207 exit 0
  251. 208 ;;
  252. 209 esac
  253. 210 done
comments powered by Disqus