1
0

FilePackPlugin.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194
  1. import os
  2. import re
  3. import gevent
  4. from Plugin import PluginManager
  5. from Config import config
  6. from Debug import Debug
  7. # Keep archive open for faster reponse times for large sites
  8. archive_cache = {}
  9. def closeArchive(archive_path):
  10. if archive_path in archive_cache:
  11. del archive_cache[archive_path]
  12. def openArchive(archive_path, file_obj=None):
  13. if archive_path not in archive_cache:
  14. if archive_path.endswith("tar.gz"):
  15. import tarfile
  16. archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:gz")
  17. elif archive_path.endswith("tar.bz2"):
  18. import tarfile
  19. archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:bz2")
  20. else:
  21. import zipfile
  22. archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path)
  23. gevent.spawn_later(5, lambda: closeArchive(archive_path)) # Close after 5 sec
  24. archive = archive_cache[archive_path]
  25. return archive
  26. def openArchiveFile(archive_path, path_within, file_obj=None):
  27. archive = openArchive(archive_path, file_obj=file_obj)
  28. if archive_path.endswith(".zip"):
  29. return archive.open(path_within)
  30. else:
  31. return archive.extractfile(path_within.encode("utf8"))
  32. @PluginManager.registerTo("UiRequest")
  33. class UiRequestPlugin(object):
  34. def actionSiteMedia(self, path, **kwargs):
  35. if ".zip/" in path or ".tar.gz/" in path:
  36. file_obj = None
  37. path_parts = self.parsePath(path)
  38. file_path = u"%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"].decode("utf8"))
  39. match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path)
  40. archive_path, path_within = match.groups()
  41. if archive_path not in archive_cache:
  42. site = self.server.site_manager.get(path_parts["address"])
  43. if not site:
  44. return self.actionSiteAddPrompt(path)
  45. archive_inner_path = site.storage.getInnerPath(archive_path)
  46. if not os.path.isfile(archive_path):
  47. # Wait until file downloads
  48. result = site.needFile(archive_inner_path, priority=10)
  49. # Send virutal file path download finished event to remove loading screen
  50. site.updateWebsocket(file_done=archive_inner_path)
  51. if not result:
  52. return self.error404(archive_inner_path)
  53. file_obj = site.storage.openBigfile(archive_inner_path)
  54. header_allow_ajax = False
  55. if self.get.get("ajax_key"):
  56. requester_site = self.server.site_manager.get(path_parts["request_address"])
  57. if self.get["ajax_key"] == requester_site.settings["ajax_key"]:
  58. header_allow_ajax = True
  59. else:
  60. return self.error403("Invalid ajax_key")
  61. try:
  62. file = openArchiveFile(archive_path, path_within, file_obj=file_obj)
  63. content_type = self.getContentType(file_path)
  64. self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax)
  65. return self.streamFile(file)
  66. except Exception as err:
  67. self.log.debug("Error opening archive file: %s" % Debug.formatException(err))
  68. return self.error404(path)
  69. return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)
  70. def streamFile(self, file):
  71. for i in range(100): # Read max 6MB
  72. try:
  73. block = file.read(60 * 1024)
  74. if block:
  75. yield block
  76. else:
  77. raise StopIteration
  78. except StopIteration:
  79. file.close()
  80. break
  81. @PluginManager.registerTo("SiteStorage")
  82. class SiteStoragePlugin(object):
  83. def isFile(self, inner_path):
  84. if ".zip/" in inner_path or ".tar.gz/" in inner_path:
  85. match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", inner_path)
  86. archive_inner_path, path_within = match.groups()
  87. return super(SiteStoragePlugin, self).isFile(archive_inner_path)
  88. else:
  89. return super(SiteStoragePlugin, self).isFile(inner_path)
  90. def openArchive(self, inner_path):
  91. archive_path = self.getPath(inner_path)
  92. file_obj = None
  93. if archive_path not in archive_cache:
  94. if not os.path.isfile(archive_path):
  95. result = self.site.needFile(inner_path, priority=10)
  96. self.site.updateWebsocket(file_done=inner_path)
  97. if not result:
  98. raise Exception("Unable to download file")
  99. file_obj = self.site.storage.openBigfile(inner_path)
  100. try:
  101. archive = openArchive(archive_path, file_obj=file_obj)
  102. except Exception as err:
  103. raise Exception("Unable to download file: %s" % err)
  104. return archive
  105. def walk(self, inner_path, *args, **kwags):
  106. if ".zip" in inner_path or ".tar.gz" in inner_path:
  107. match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
  108. archive_inner_path, path_within = match.groups()
  109. archive = self.openArchive(archive_inner_path)
  110. path_within = path_within.lstrip("/")
  111. if archive_inner_path.endswith(".zip"):
  112. namelist = [name for name in archive.namelist() if not name.endswith("/")]
  113. else:
  114. namelist = [item.name for item in archive.getmembers() if not item.isdir()]
  115. namelist_relative = []
  116. for name in namelist:
  117. if not name.startswith(path_within):
  118. continue
  119. name_relative = name.replace(path_within, "", 1).rstrip("/")
  120. namelist_relative.append(name_relative)
  121. return namelist_relative
  122. else:
  123. return super(SiteStoragePlugin, self).walk(inner_path, *args, **kwags)
  124. def list(self, inner_path, *args, **kwags):
  125. if ".zip" in inner_path or ".tar.gz" in inner_path:
  126. match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
  127. archive_inner_path, path_within = match.groups()
  128. archive = self.openArchive(archive_inner_path)
  129. path_within = path_within.lstrip("/")
  130. if archive_inner_path.endswith(".zip"):
  131. namelist = [name for name in archive.namelist()]
  132. else:
  133. namelist = [item.name for item in archive.getmembers()]
  134. namelist_relative = []
  135. for name in namelist:
  136. if not name.startswith(path_within):
  137. continue
  138. name_relative = name.replace(path_within, "", 1).rstrip("/")
  139. if "/" in name_relative: # File is in sub-directory
  140. continue
  141. namelist_relative.append(name_relative)
  142. return namelist_relative
  143. else:
  144. return super(SiteStoragePlugin, self).list(inner_path, *args, **kwags)
  145. def read(self, inner_path, mode="r"):
  146. if ".zip/" in inner_path or ".tar.gz/" in inner_path:
  147. match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
  148. archive_inner_path, path_within = match.groups()
  149. archive = self.openArchive(archive_inner_path)
  150. path_within = path_within.lstrip("/")
  151. print archive, archive_inner_path
  152. if archive_inner_path.endswith(".zip"):
  153. return archive.open(path_within).read()
  154. else:
  155. return archive.extractfile(path_within.encode("utf8")).read()
  156. else:
  157. return super(SiteStoragePlugin, self).read(inner_path, mode)