NukeFS Ultimate (NFSU): twin-algorithm shortener-based file cloud storage for Python 2.6+


SUBMITTED BY: Guest

DATE: Oct. 30, 2012, 1 p.m.

FORMAT: Python

SIZE: 4.9 kB

HITS: 2265

  1. #!/usr/bin/env python
  2. '''
  3. NukeFS Ultimate (NFSU): twin-algorithm shortener-based file cloud storage for Python 2.6+
  4. Full library and command-line implementation
  5. by Multiversum
  6. Usage as stand-alone application: see nukefs.py -h
  7. Usage as library:
  8. import nukefs
  9. //for master-chunk algorithm:
  10. key_of_file = nukefs.upload("file.zip")
  11. download(key_of_file, "downoaded.zip")
  12. //for streaming algorithm:
  13. key_of_file = nukefs.upload("file.zip", True)
  14. download(key_of_file, "downoaded.zip", True)
  15. Note: it gets damn slow and somewhat unstable on large files so plz don't upload >1M files
  16. '''
  17. from requests import get
  18. from json import dumps, loads
  19. from urllib import quote_plus, unquote_plus
  20. CHUNKLEN=22000 #current TinyURL effective chunk length limitation
  21. def chunk_upload(localpath):
  22. filekeys = []
  23. with open(localpath, 'rb') as fileobj:
  24. while 1:
  25. data = fileobj.read(CHUNKLEN)
  26. if not data: break
  27. encoded = quote_plus(''.join(data.encode('base64').splitlines()))
  28. key = get(baseurl+encoded).text[19:]
  29. filekeys.append(key)
  30. fileobj.close()
  31. return get(baseurl+quote_plus(dumps(filekeys))).text[19:]
  32. def chunk_download(filekey, localpath):
  33. filekeys = loads(unquote_plus(get('http://tinyurl.com/'+filekey, allow_redirects=False).headers['location']))
  34. with open(localpath, 'wb') as fileobj:
  35. for key in filekeys:
  36. data = get('http://tinyurl.com/'+key, allow_redirects=False).headers['location']
  37. if not data: break
  38. fileobj.write(unquote_plus(data).decode('base64'))
  39. fileobj.close()
  40. def stream_upload(localpath):
  41. chunks = []
  42. with open(localpath, 'rb') as fileobj:
  43. while 1:
  44. data = fileobj.read(CHUNKLEN)
  45. if not data: break
  46. chunks.append(''.join(data.encode('base64').splitlines()))
  47. fileobj.close()
  48. key = None
  49. for i in xrange(len(chunks),0,-1):
  50. chunkobj = {'data' : chunks[i-1], 'next' : key}
  51. encoded = quote_plus(dumps(chunkobj))
  52. key = get(baseurl+encoded).text[19:]
  53. return key
  54. def stream_download(key, localpath):
  55. with open(localpath, 'wb') as fileobj:
  56. while 1:
  57. data = get('http://tinyurl.com/'+key, allow_redirects=False).headers['location']
  58. if not data: break
  59. chunkobj = loads(unquote_plus(data))
  60. if not chunkobj['data']: break
  61. fileobj.write(chunkobj['data'].decode('base64'))
  62. if not chunkobj['next']: break
  63. key = chunkobj['next']
  64. fileobj.close()
  65. #universal functions to be called outside
  66. #algo: False - master-chunk algorithm (default), True - streaming algorithm
  67. def upload(localpath, algo=None):
  68. '''
  69. Upload a file from localpath to cloud and get its access key
  70. algo: False or None - master-chunk algorithm (default), True - streaming algorithm
  71. '''
  72. if algo:
  73. return stream_upload(localpath)
  74. else:
  75. return chunk_upload(localpath)
  76. def download(filekey, localpath, algo=None):
  77. '''
  78. Download a file to localpath by its access key
  79. algo: False or None - master-chunk algorithm (default), True - streaming algorithm
  80. '''
  81. if algo:
  82. stream_download(filekey, localpath)
  83. else:
  84. chunk_download(filekey, localpath)
  85. #stand-alone app code
  86. if __name__ == '__main__':
  87. from optparse import OptionParser
  88. parser = OptionParser()
  89. parser.add_option("-u", dest="upfilename",
  90. help="upload FILE to cloud", metavar="FILE")
  91. parser.add_option("-d", dest="downfilename",
  92. help="download FILE from cloud", metavar="FILE")
  93. parser.add_option("-k", dest="accesskey",
  94. help="Access key to download a file")
  95. parser.add_option("-s", "--stream", action="store_true", dest="algo",
  96. help="Use streaming algorithm instead of chunked")
  97. (options, args) = parser.parse_args()
  98. if options.upfilename and options.downfilename:
  99. parser.error("You can either download or upload at the same time")
  100. if options.downfilename and not options.accesskey:
  101. parser.error("You must specify the access key (with -k option) to download a file")
  102. if options.upfilename:
  103. print 'Uploading...'
  104. accesskey = upload(options.upfilename, options.algo)
  105. print 'Done. File access key is', accesskey
  106. elif options.downfilename:
  107. print 'Downloading...'
  108. download(options.accesskey, options.downfilename, options.algo)
  109. print 'Done.'
  110. else:
  111. print "Please specify either the file to upload or the file to download. See nukefs.py -h for help."