123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171 |
- import time
- import collections
- from util import helper
- from Plugin import PluginManager
- import ContentDbPlugin
- # We can only import plugin host clases after the plugins are loaded
- @PluginManager.afterLoad
- def importPluginnedClasses():
- global config
- from Config import config
- def processAccessLog():
- if access_log:
- content_db = ContentDbPlugin.content_db
- now = int(time.time())
- num = 0
- for site_id in access_log:
- content_db.execute(
- "UPDATE file_optional SET time_accessed = %s WHERE ?" % now,
- {"site_id": site_id, "inner_path": access_log[site_id].keys()}
- )
- num += len(access_log[site_id])
- access_log.clear()
- def processRequestLog():
- if request_log:
- content_db = ContentDbPlugin.content_db
- cur = content_db.getCursor()
- num = 0
- cur.execute("BEGIN")
- for site_id in request_log:
- for inner_path, uploaded in request_log[site_id].iteritems():
- content_db.execute(
- "UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded,
- {"site_id": site_id, "inner_path": inner_path}
- )
- num += 1
- cur.execute("END")
- request_log.clear()
- if "access_log" not in locals().keys(): # To keep between module reloads
- access_log = collections.defaultdict(dict) # {site_id: {inner_path1: 1, inner_path2: 1...}}
- request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}}
- helper.timer(61, processAccessLog)
- helper.timer(60, processRequestLog)
- @PluginManager.registerTo("ContentManager")
- class ContentManagerPlugin(object):
- def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):
- is_pinned = 0
- if "|" in inner_path: # Big file piece
- file_inner_path, file_range = inner_path.split("|")
- # Auto-pin bigfiles
- if size and config.pin_bigfile and size > 1024 * 1024 * config.pin_bigfile:
- is_pinned = 1
- else:
- file_inner_path = inner_path
- self.contents.db.executeDelayed(
- "UPDATE file_optional SET time_downloaded = :now, is_downloaded = 1, peer = peer + 1, is_pinned = :is_pinned WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 0",
- {"now": int(time.time()), "site_id": self.contents.db.site_ids[self.site.address], "inner_path": file_inner_path, "is_pinned": is_pinned}
- )
- return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own)
- def optionalRemoved(self, inner_path, hash_id, size=None):
- self.contents.db.execute(
- "UPDATE file_optional SET is_downloaded = 0, peer = peer - 1 WHERE site_id = :site_id AND inner_path = :inner_path AND is_downloaded = 1",
- {"site_id": self.contents.db.site_ids[self.site.address], "inner_path": inner_path}
- )
- print "Removed hash_id: %s" % hash_id, self.contents.db.cur.cursor.rowcount
- if self.contents.db.cur.cursor.rowcount > 0:
- back = super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size)
- # Re-add to hashfield if we have other file with the same hash_id
- if self.isDownloaded(hash_id=hash_id, force_check_db=True):
- self.hashfield.appendHashId(hash_id)
- def isDownloaded(self, inner_path=None, hash_id=None, force_check_db=False):
- if hash_id and not force_check_db and hash_id not in self.hashfield:
- return False
- if inner_path:
- res = self.contents.db.execute(
- "SELECT is_downloaded FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1",
- {"site_id": self.contents.db.site_ids[self.site.address], "inner_path": inner_path}
- )
- else:
- res = self.contents.db.execute(
- "SELECT is_downloaded FROM file_optional WHERE site_id = :site_id AND hash_id = :hash_id AND is_downloaded = 1 LIMIT 1",
- {"site_id": self.contents.db.site_ids[self.site.address], "hash_id": hash_id}
- )
- row = res.fetchone()
- if row and row[0]:
- return True
- else:
- return False
- @PluginManager.registerTo("WorkerManager")
- class WorkerManagerPlugin(object):
- def doneTask(self, task):
- super(WorkerManagerPlugin, self).doneTask(task)
- if task["optional_hash_id"] and not self.tasks: # Execute delayed queries immedietly after tasks finished
- ContentDbPlugin.content_db.processDelayed()
- @PluginManager.registerTo("UiRequest")
- class UiRequestPlugin(object):
- def parsePath(self, path):
- global access_log
- path_parts = super(UiRequestPlugin, self).parsePath(path)
- if path_parts:
- site_id = ContentDbPlugin.content_db.site_ids.get(path_parts["request_address"])
- if site_id:
- if ContentDbPlugin.content_db.isOptionalFile(site_id, path_parts["inner_path"]):
- access_log[site_id][path_parts["inner_path"]] = 1
- return path_parts
- @PluginManager.registerTo("FileRequest")
- class FileRequestPlugin(object):
- def actionGetFile(self, params):
- stats = super(FileRequestPlugin, self).actionGetFile(params)
- self.recordFileRequest(params["site"], params["inner_path"], stats)
- return stats
- def actionStreamFile(self, params):
- stats = super(FileRequestPlugin, self).actionStreamFile(params)
- self.recordFileRequest(params["site"], params["inner_path"], stats)
- return stats
- def recordFileRequest(self, site_address, inner_path, stats):
- if not stats:
- # Only track the last request of files
- return False
- site_id = ContentDbPlugin.content_db.site_ids[site_address]
- if site_id and ContentDbPlugin.content_db.isOptionalFile(site_id, inner_path):
- request_log[site_id][inner_path] += stats["bytes_sent"]
- @PluginManager.registerTo("Site")
- class SitePlugin(object):
- def isDownloadable(self, inner_path):
- is_downloadable = super(SitePlugin, self).isDownloadable(inner_path)
- if is_downloadable:
- return is_downloadable
- for path in self.settings.get("optional_help", {}).iterkeys():
- if inner_path.startswith(path):
- return True
- return False
- @PluginManager.registerTo("ConfigPlugin")
- class ConfigPlugin(object):
- def createArguments(self):
- group = self.parser.add_argument_group("OptionalManager plugin")
- group.add_argument('--optional_limit', help='Limit total size of optional files', default="10%", metavar="GB or free space %")
- group.add_argument('--pin_bigfile', help='Automatically pin files larger than this limit', default=20, metavar="MB", type=int)
- return super(ConfigPlugin, self).createArguments()
|