123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926 |
- import json
- import time
- import re
- import os
- import copy
- import gevent
- from Debug import Debug
- from Crypt import CryptHash
- from Config import config
- from util import helper
- from util import Diff
- from util import SafeRe
- from Peer import PeerHashfield
- from ContentDbDict import ContentDbDict
- from Plugin import PluginManager
- class VerifyError(Exception):
- pass
- class SignError(Exception):
- pass
- @PluginManager.acceptPlugins
- class ContentManager(object):
- def __init__(self, site):
- self.site = site
- self.log = self.site.log
- self.contents = ContentDbDict(site)
- self.hashfield = PeerHashfield()
- self.has_optional_files = False
- # Load all content.json files
- def loadContents(self):
- if len(self.contents) == 0:
- self.log.debug("ContentDb not initialized, load files from filesystem")
- self.loadContent(add_bad_files=False, delete_removed_files=False)
- self.site.settings["size"], self.site.settings["size_optional"] = self.getTotalSize()
- # Load hashfield cache
- if "hashfield" in self.site.settings.get("cache", {}):
- self.hashfield.fromstring(self.site.settings["cache"]["hashfield"].decode("base64"))
- del self.site.settings["cache"]["hashfield"]
- elif self.contents.get("content.json") and self.site.settings["size_optional"] > 0:
- self.site.storage.updateBadFiles() # No hashfield cache created yet
- self.has_optional_files = bool(self.hashfield)
- self.contents.db.initSite(self.site)
- # Load content.json to self.content
- # Return: Changed files ["index.html", "data/messages.json"], Deleted files ["old.jpg"]
- def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True, force=False):
- content_inner_path = content_inner_path.strip("/") # Remove / from beginning
- old_content = self.contents.get(content_inner_path)
- content_path = self.site.storage.getPath(content_inner_path)
- content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path))
- content_inner_dir = helper.getDirname(content_inner_path)
- if os.path.isfile(content_path):
- try:
- # Check if file is newer than what we have
- if not force and old_content and not self.site.settings.get("own"):
- for line in open(content_path):
- if '"modified"' not in line:
- continue
- match = re.search("([0-9\.]+),$", line.strip(" \r\n"))
- if match and float(match.group(1)) <= old_content.get("modified", 0):
- self.log.debug("%s loadContent same json file, skipping" % content_inner_path)
- return [], []
- new_content = json.load(open(content_path))
- except Exception, err:
- self.log.warning("%s load error: %s" % (content_path, Debug.formatException(err)))
- return [], []
- else:
- self.log.warning("Content.json not exist: %s" % content_path)
- return [], [] # Content.json not exist
- try:
- # Get the files where the sha512 changed
- changed = []
- deleted = []
- # Check changed
- for relative_path, info in new_content.get("files", {}).iteritems():
- if "sha512" in info:
- hash_type = "sha512"
- else: # Backward compatibility
- hash_type = "sha1"
- new_hash = info[hash_type]
- if old_content and old_content["files"].get(relative_path): # We have the file in the old content
- old_hash = old_content["files"][relative_path].get(hash_type)
- else: # The file is not in the old content
- old_hash = None
- if old_hash != new_hash:
- changed.append(content_inner_dir + relative_path)
- # Check changed optional files
- for relative_path, info in new_content.get("files_optional", {}).iteritems():
- file_inner_path = content_inner_dir + relative_path
- new_hash = info["sha512"]
- if old_content and old_content.get("files_optional", {}).get(relative_path):
- # We have the file in the old content
- old_hash = old_content["files_optional"][relative_path].get("sha512")
- if old_hash != new_hash and self.site.isDownloadable(file_inner_path):
- changed.append(file_inner_path) # Download new file
- elif old_hash != new_hash and self.hashfield.hasHash(old_hash) and not self.site.settings.get("own"):
- try:
- old_hash_id = self.hashfield.getHashId(old_hash)
- self.optionalRemoved(file_inner_path, old_hash_id, old_content["files_optional"][relative_path]["size"])
- self.site.storage.delete(file_inner_path)
- self.log.debug("Deleted changed optional file: %s" % file_inner_path)
- except Exception, err:
- self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
- else: # The file is not in the old content
- if self.site.isDownloadable(file_inner_path):
- changed.append(file_inner_path) # Download new file
- # Check deleted
- if old_content:
- old_files = dict(
- old_content.get("files", {}),
- **old_content.get("files_optional", {})
- )
- new_files = dict(
- new_content.get("files", {}),
- **new_content.get("files_optional", {})
- )
- deleted = [key for key in old_files if key not in new_files]
- if deleted and not self.site.settings.get("own"):
- # Deleting files that no longer in content.json
- for file_relative_path in deleted:
- file_inner_path = content_inner_dir + file_relative_path
- try:
- self.site.storage.delete(file_inner_path)
- # Check if the deleted file is optional
- if old_content.get("files_optional") and old_content["files_optional"].get(file_relative_path):
- old_hash = old_content["files_optional"][file_relative_path].get("sha512")
- if self.hashfield.hasHash(old_hash):
- old_hash_id = self.hashField.getHashid(old_hash)
- self.optionalRemoved(file_inner_path, old_hash_id, old_content["files_optional"][file_relative_path]["size"])
- self.log.debug("Deleted file: %s" % file_inner_path)
- except Exception, err:
- self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
- # Cleanup empty dirs
- tree = {root: [dirs, files] for root, dirs, files in os.walk(self.site.storage.getPath(content_inner_dir))}
- for root in sorted(tree, key=len, reverse=True):
- dirs, files = tree[root]
- if dirs == [] and files == []:
- root_inner_path = self.site.storage.getInnerPath(root.replace("\\", "/"))
- self.log.debug("Empty directory: %s, cleaning up." % root_inner_path)
- try:
- self.site.storage.deleteDir(root_inner_path)
- # Remove from tree dict to reflect changed state
- tree[os.path.dirname(root)][0].remove(os.path.basename(root))
- except Exception, err:
- self.log.debug("Error deleting empty directory %s: %s" % (root_inner_path, err))
- # Check archived
- if old_content and "user_contents" in new_content and "archived" in new_content["user_contents"]:
- old_archived = old_content.get("user_contents", {}).get("archived", {})
- new_archived = new_content.get("user_contents", {}).get("archived", {})
- self.log.debug("old archived: %s, new archived: %s" % (len(old_archived), len(new_archived)))
- archived_changed = {
- key: date_archived
- for key, date_archived in new_archived.iteritems()
- if old_archived.get(key) != new_archived[key]
- }
- if archived_changed:
- self.log.debug("Archived changed: %s" % archived_changed)
- for archived_dirname, date_archived in archived_changed.iteritems():
- archived_inner_path = content_inner_dir + archived_dirname + "/content.json"
- if self.contents.get(archived_inner_path, {}).get("modified", 0) < date_archived:
- self.removeContent(archived_inner_path)
- self.site.settings["size"], self.site.settings["size_optional"] = self.getTotalSize()
- # Load includes
- if load_includes and "includes" in new_content:
- for relative_path, info in new_content["includes"].items():
- include_inner_path = content_inner_dir + relative_path
- if self.site.storage.isFile(include_inner_path): # Content.json exists, load it
- include_changed, include_deleted = self.loadContent(
- include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files
- )
- if include_changed:
- changed += include_changed # Add changed files
- if include_deleted:
- deleted += include_deleted # Add changed files
- else: # Content.json not exist, add to changed files
- self.log.debug("Missing include: %s" % include_inner_path)
- changed += [include_inner_path]
- # Load blind user includes (all subdir)
- if load_includes and "user_contents" in new_content:
- for relative_dir in os.listdir(content_dir):
- include_inner_path = content_inner_dir + relative_dir + "/content.json"
- if not self.site.storage.isFile(include_inner_path):
- continue # Content.json not exist
- include_changed, include_deleted = self.loadContent(
- include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files,
- load_includes=False
- )
- if include_changed:
- changed += include_changed # Add changed files
- if include_deleted:
- deleted += include_deleted # Add changed files
- # Save some memory
- new_content["signs"] = None
- if "cert_sign" in new_content:
- new_content["cert_sign"] = None
- if new_content.get("files_optional"):
- self.has_optional_files = True
- # Update the content
- self.contents[content_inner_path] = new_content
- except Exception, err:
- self.log.warning("%s parse error: %s" % (content_inner_path, Debug.formatException(err)))
- return [], [] # Content.json parse error
- # Add changed files to bad files
- if add_bad_files:
- for inner_path in changed:
- self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1
- for inner_path in deleted:
- if inner_path in self.site.bad_files:
- del self.site.bad_files[inner_path]
- if new_content.get("modified", 0) > self.site.settings.get("modified", 0):
- # Dont store modifications in the far future (more than 10 minute)
- self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"])
- return changed, deleted
- def removeContent(self, inner_path):
- inner_dir = helper.getDirname(inner_path)
- try:
- content = self.contents[inner_path]
- files = dict(
- content.get("files", {}),
- **content.get("files_optional", {})
- )
- except Exception, err:
- self.log.debug("Error loading %s for removeContent: %s" % (inner_path, Debug.formatException(err)))
- files = {}
- files["content.json"] = True
- # Deleting files that no longer in content.json
- for file_relative_path in files:
- file_inner_path = inner_dir + file_relative_path
- try:
- self.site.storage.delete(file_inner_path)
- self.log.debug("Deleted file: %s" % file_inner_path)
- except Exception, err:
- self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
- try:
- self.site.storage.deleteDir(inner_dir)
- except Exception, err:
- self.log.debug("Error deleting dir %s: %s" % (inner_dir, err))
- try:
- del self.contents[inner_path]
- except Exception, err:
- self.log.debug("Error key from contents: %s" % inner_path)
- # Get total size of site
- # Return: 32819 (size of files in kb)
- def getTotalSize(self, ignore=None):
- return self.contents.db.getTotalSize(self.site, ignore)
- def listModified(self, since):
- return self.contents.db.listModified(self.site, since)
- def listContents(self, inner_path="content.json", user_files=False):
- if inner_path not in self.contents:
- return []
- back = [inner_path]
- content_inner_dir = helper.getDirname(inner_path)
- for relative_path in self.contents[inner_path].get("includes", {}).keys():
- include_inner_path = content_inner_dir + relative_path
- back += self.listContents(include_inner_path)
- return back
- # Returns if file with the given modification date is archived or not
- def isArchived(self, inner_path, modified):
- match = re.match("(.*)/(.*?)/", inner_path)
- if not match:
- return False
- user_contents_inner_path = match.group(1) + "/content.json"
- relative_directory = match.group(2)
- file_info = self.getFileInfo(user_contents_inner_path)
- if file_info and file_info.get("archived", {}).get(relative_directory) >= modified:
- return True
- else:
- return False
- # Find the file info line from self.contents
- # Return: { "sha512": "c29d73d...21f518", "size": 41 , "content_inner_path": "content.json"}
- def getFileInfo(self, inner_path, new_file=False):
- dirs = inner_path.split("/") # Parent dirs of content.json
- inner_path_parts = [dirs.pop()] # Filename relative to content.json
- while True:
- content_inner_path = "%s/content.json" % "/".join(dirs)
- content_inner_path = content_inner_path.strip("/")
- content = self.contents.get(content_inner_path)
- # Check in files
- if content and "files" in content:
- back = content["files"].get("/".join(inner_path_parts))
- if back:
- back["content_inner_path"] = content_inner_path
- back["optional"] = False
- back["relative_path"] = "/".join(inner_path_parts)
- return back
- # Check in optional files
- if content and "files_optional" in content: # Check if file in this content.json
- back = content["files_optional"].get("/".join(inner_path_parts))
- if back:
- back["content_inner_path"] = content_inner_path
- back["optional"] = True
- back["relative_path"] = "/".join(inner_path_parts)
- return back
- # Return the rules if user dir
- if content and "user_contents" in content:
- back = content["user_contents"]
- content_inner_path_dir = helper.getDirname(content_inner_path)
- relative_content_path = inner_path[len(content_inner_path_dir):]
- if "/" in relative_content_path:
- user_auth_address = re.match("([A-Za-z0-9]+)/.*", relative_content_path).group(1)
- back["content_inner_path"] = "%s%s/content.json" % (content_inner_path_dir, user_auth_address)
- else:
- back["content_inner_path"] = content_inner_path_dir + "content.json"
- back["optional"] = None
- back["relative_path"] = "/".join(inner_path_parts)
- return back
- if new_file and content:
- back = {}
- back["content_inner_path"] = content_inner_path
- back["relative_path"] = "/".join(inner_path_parts)
- back["optional"] = None
- return back
- # No inner path in this dir, lets try the parent dir
- if dirs:
- inner_path_parts.insert(0, dirs.pop())
- else: # No more parent dirs
- break
- # Not found
- return False
- # Get rules for the file
- # Return: The rules for the file or False if not allowed
- def getRules(self, inner_path, content=None):
- if not inner_path.endswith("content.json"): # Find the files content.json first
- file_info = self.getFileInfo(inner_path)
- if not file_info:
- return False # File not found
- inner_path = file_info["content_inner_path"]
- if inner_path == "content.json": # Root content.json
- rules = {}
- rules["signers"] = self.getValidSigners(inner_path, content)
- return rules
- dirs = inner_path.split("/") # Parent dirs of content.json
- inner_path_parts = [dirs.pop()] # Filename relative to content.json
- inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir
- while True:
- content_inner_path = "%s/content.json" % "/".join(dirs)
- parent_content = self.contents.get(content_inner_path.strip("/"))
- if parent_content and "includes" in parent_content:
- return parent_content["includes"].get("/".join(inner_path_parts))
- elif parent_content and "user_contents" in parent_content:
- return self.getUserContentRules(parent_content, inner_path, content)
- else: # No inner path in this dir, lets try the parent dir
- if dirs:
- inner_path_parts.insert(0, dirs.pop())
- else: # No more parent dirs
- break
- return False
- # Get rules for a user file
- # Return: The rules of the file or False if not allowed
- def getUserContentRules(self, parent_content, inner_path, content):
- user_contents = parent_content["user_contents"]
- # Delivered for directory
- if "inner_path" in parent_content:
- parent_content_dir = helper.getDirname(parent_content["inner_path"])
- user_address = re.match("([A-Za-z0-9]*?)/", inner_path[len(parent_content_dir):]).group(1)
- else:
- user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1)
- try:
- if not content:
- content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
- user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit
- cert_user_id = content["cert_user_id"]
- except Exception: # Content.json not exist
- user_urn = "n-a/n-a"
- cert_user_id = "n-a"
- if user_address in user_contents["permissions"]:
- rules = copy.copy(user_contents["permissions"].get(user_address, {})) # Default rules based on address
- else:
- rules = copy.copy(user_contents["permissions"].get(cert_user_id, {})) # Default rules based on username
- if rules is False:
- banned = True
- rules = {}
- else:
- banned = False
- if "signers" in rules:
- rules["signers"] = rules["signers"][:] # Make copy of the signers
- for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
- if not SafeRe.match(permission_pattern, user_urn):
- continue # Rule is not valid for user
- # Update rules if its better than current recorded ones
- for key, val in permission_rules.iteritems():
- if key not in rules:
- if type(val) is list:
- rules[key] = val[:] # Make copy
- else:
- rules[key] = val
- elif type(val) is int: # Int, update if larger
- if val > rules[key]:
- rules[key] = val
- elif hasattr(val, "startswith"): # String, update if longer
- if len(val) > len(rules[key]):
- rules[key] = val
- elif type(val) is list: # List, append
- rules[key] += val
- rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers
- if "signers" not in rules:
- rules["signers"] = []
- if not banned:
- rules["signers"].append(user_address) # Add user as valid signer
- rules["user_address"] = user_address
- rules["includes_allowed"] = False
- return rules
- # Get diffs for changed files
- def getDiffs(self, inner_path, limit=30 * 1024, update_files=True):
- if inner_path not in self.contents:
- return {}
- diffs = {}
- content_inner_path_dir = helper.getDirname(inner_path)
- for file_relative_path in self.contents[inner_path].get("files", {}):
- file_inner_path = content_inner_path_dir + file_relative_path
- if self.site.storage.isFile(file_inner_path + "-new"): # New version present
- diffs[file_relative_path] = Diff.diff(
- list(self.site.storage.open(file_inner_path)),
- list(self.site.storage.open(file_inner_path + "-new")),
- limit=limit
- )
- if update_files:
- self.site.storage.delete(file_inner_path)
- self.site.storage.rename(file_inner_path + "-new", file_inner_path)
- if self.site.storage.isFile(file_inner_path + "-old"): # Old version present
- diffs[file_relative_path] = Diff.diff(
- list(self.site.storage.open(file_inner_path + "-old")),
- list(self.site.storage.open(file_inner_path)),
- limit=limit
- )
- if update_files:
- self.site.storage.delete(file_inner_path + "-old")
- return diffs
- def hashFile(self, dir_inner_path, file_relative_path, optional=False):
- back = {}
- file_inner_path = dir_inner_path + "/" + file_relative_path
- file_path = self.site.storage.getPath(file_inner_path)
- file_size = os.path.getsize(file_path)
- sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
- if optional and not self.hashfield.hasHash(sha512sum):
- self.optionalDownloaded(file_inner_path, self.hashfield.getHashId(sha512sum), file_size, own=True)
- back[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
- return back
- def isValidRelativePath(self, relative_path):
- if ".." in relative_path:
- return False
- elif len(relative_path) > 255:
- return False
- else:
- return re.match("^[a-z\[\]\(\) A-Z0-9_@=\.\+-/]+$", relative_path)
- def sanitizePath(self, inner_path):
- return re.sub("[^a-z\[\]\(\) A-Z0-9_@=\.\+-/]", "", inner_path)
- # Hash files in directory
- def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None):
- files_node = {}
- files_optional_node = {}
- if dir_inner_path and not self.isValidRelativePath(dir_inner_path):
- ignored = True
- self.log.error("- [ERROR] Only ascii encoded directories allowed: %s" % dir_inner_path)
- for file_relative_path in self.site.storage.walk(dir_inner_path, ignore_pattern):
- file_name = helper.getFilename(file_relative_path)
- ignored = optional = False
- if file_name == "content.json":
- ignored = True
- elif file_name.startswith(".") or file_name.endswith("-old") or file_name.endswith("-new"):
- ignored = True
- elif not self.isValidRelativePath(file_relative_path):
- ignored = True
- self.log.error("- [ERROR] Invalid filename: %s" % file_relative_path)
- elif dir_inner_path == "" and file_relative_path == self.site.storage.getDbFile():
- ignored = True
- elif optional_pattern and SafeRe.match(optional_pattern, file_relative_path):
- optional = True
- if ignored: # Ignore content.json, defined regexp and files starting with .
- self.log.info("- [SKIPPED] %s" % file_relative_path)
- else:
- if optional:
- self.log.info("- [OPTIONAL] %s" % file_relative_path)
- files_optional_node.update(
- self.hashFile(dir_inner_path, file_relative_path, optional=True)
- )
- else:
- self.log.info("- %s" % file_relative_path)
- files_node.update(
- self.hashFile(dir_inner_path, file_relative_path)
- )
- return files_node, files_optional_node
- # Create and sign a content.json
- # Return: The new content if filewrite = False
- def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None, remove_missing_optional=False):
- if not inner_path.endswith("content.json"):
- raise SignError("Invalid file name, you can only sign content.json files")
- if inner_path in self.contents:
- content = self.contents.get(inner_path)
- if content and content.get("cert_sign", False) is None and self.site.storage.isFile(inner_path):
- # Recover cert_sign from file
- content["cert_sign"] = self.site.storage.loadJson(inner_path).get("cert_sign")
- else:
- content = None
- if not content: # Content not exist yet, load default one
- self.log.info("File %s not exist yet, loading default values..." % inner_path)
- if self.site.storage.isFile(inner_path):
- content = self.site.storage.loadJson(inner_path)
- if "files" not in content:
- content["files"] = {}
- if "signs" not in content:
- content["signs"] = {}
- else:
- content = {"files": {}, "signs": {}} # Default content.json
- if inner_path == "content.json": # It's the root content.json, add some more fields
- content["title"] = "%s - ZeroNet_" % self.site.address
- content["description"] = ""
- content["signs_required"] = 1
- content["ignore"] = ""
- if extend:
- # Add extend keys if not exists
- for key, val in extend.items():
- if not content.get(key):
- content[key] = val
- self.log.info("Extending content.json with: %s" % key)
- directory = helper.getDirname(self.site.storage.getPath(inner_path))
- inner_directory = helper.getDirname(inner_path)
- self.log.info("Opening site data directory: %s..." % directory)
- changed_files = [inner_path]
- files_node, files_optional_node = self.hashFiles(
- helper.getDirname(inner_path), content.get("ignore"), content.get("optional")
- )
- if not remove_missing_optional:
- for file_inner_path, file_details in content.get("files_optional", {}).iteritems():
- if file_inner_path not in files_optional_node:
- files_optional_node[file_inner_path] = file_details
- # Find changed files
- files_merged = files_node.copy()
- files_merged.update(files_optional_node)
- for file_relative_path, file_details in files_merged.iteritems():
- old_hash = content.get("files", {}).get(file_relative_path, {}).get("sha512")
- new_hash = files_merged[file_relative_path]["sha512"]
- if old_hash != new_hash:
- changed_files.append(inner_directory + file_relative_path)
- self.log.debug("Changed files: %s" % changed_files)
- if update_changed_files:
- for file_path in changed_files:
- self.site.storage.onUpdated(file_path)
- # Generate new content.json
- self.log.info("Adding timestamp and sha512sums to new content.json...")
- new_content = content.copy() # Create a copy of current content.json
- new_content["files"] = files_node # Add files sha512 hash
- if files_optional_node:
- new_content["files_optional"] = files_optional_node
- elif "files_optional" in new_content:
- del new_content["files_optional"]
- new_content["modified"] = int(time.time()) # Add timestamp
- if inner_path == "content.json":
- new_content["zeronet_version"] = config.version
- new_content["signs_required"] = content.get("signs_required", 1)
- new_content["address"] = self.site.address
- new_content["inner_path"] = inner_path
- # Verify private key
- from Crypt import CryptBitcoin
- self.log.info("Verifying private key...")
- privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)
- valid_signers = self.getValidSigners(inner_path, new_content)
- if privatekey_address not in valid_signers:
- raise SignError(
- "Private key invalid! Valid signers: %s, Private key address: %s" %
- (valid_signers, privatekey_address)
- )
- self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers))
- if inner_path == "content.json" and privatekey_address == self.site.address:
- # If signing using the root key, then sign the valid signers
- signers_data = "%s:%s" % (new_content["signs_required"], ",".join(valid_signers))
- new_content["signers_sign"] = CryptBitcoin.sign(str(signers_data), privatekey)
- if not new_content["signers_sign"]:
- self.log.info("Old style address, signers_sign is none")
- self.log.info("Signing %s..." % inner_path)
- if "signs" in new_content:
- del(new_content["signs"]) # Delete old signs
- if "sign" in new_content:
- del(new_content["sign"]) # Delete old sign (backward compatibility)
- sign_content = json.dumps(new_content, sort_keys=True)
- sign = CryptBitcoin.sign(sign_content, privatekey)
- # new_content["signs"] = content.get("signs", {}) # TODO: Multisig
- if sign: # If signing is successful (not an old address)
- new_content["signs"] = {}
- new_content["signs"][privatekey_address] = sign
- self.verifyContent(inner_path, new_content)
- if filewrite:
- self.log.info("Saving to %s..." % inner_path)
- self.site.storage.writeJson(inner_path, new_content)
- self.contents[inner_path] = new_content
- self.log.info("File %s signed!" % inner_path)
- if filewrite: # Written to file
- return True
- else: # Return the new content
- return new_content
- # The valid signers of content.json file
- # Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"]
- def getValidSigners(self, inner_path, content=None):
- valid_signers = []
- if inner_path == "content.json": # Root content.json
- if "content.json" in self.contents and "signers" in self.contents["content.json"]:
- valid_signers += self.contents["content.json"]["signers"][:]
- else:
- rules = self.getRules(inner_path, content)
- if rules and "signers" in rules:
- valid_signers += rules["signers"]
- if self.site.address not in valid_signers:
- valid_signers.append(self.site.address) # Site address always valid
- return valid_signers
- # Return: The required number of valid signs for the content.json
- def getSignsRequired(self, inner_path, content=None):
- return 1 # Todo: Multisig
- def verifyCert(self, inner_path, content):
- from Crypt import CryptBitcoin
- rules = self.getRules(inner_path, content)
- if not rules.get("cert_signers"):
- return True # Does not need cert
- if "cert_user_id" not in content:
- raise VerifyError("Missing cert_user_id")
- name, domain = content["cert_user_id"].split("@")
- cert_address = rules["cert_signers"].get(domain)
- if not cert_address: # Cert signer not allowed
- raise VerifyError("Invalid cert signer: %s" % domain)
- try:
- cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name)
- result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"])
- except Exception, err:
- raise VerifyError("Certificate verify error: %s" % err)
- return result
- # Checks if the content.json content is valid
- # Return: True or False
- def verifyContent(self, inner_path, content):
- content_size = len(json.dumps(content, indent=1)) + sum([file["size"] for file in content["files"].values() if file["size"] >= 0]) # Size of new content
- # Calculate old content size
- old_content = self.contents.get(inner_path)
- if old_content:
- old_content_size = len(json.dumps(old_content, indent=1)) + sum([file["size"] for file in old_content.get("files", {}).values()])
- old_content_size_optional = sum([file["size"] for file in old_content.get("files_optional", {}).values()])
- else:
- old_content_size = 0
- old_content_size_optional = 0
- # Reset site site on first content.json
- if not old_content and inner_path == "content.json":
- self.site.settings["size"] = 0
- content_size_optional = sum([file["size"] for file in content.get("files_optional", {}).values() if file["size"] >= 0])
- site_size = self.site.settings["size"] - old_content_size + content_size # Site size without old content plus the new
- site_size_optional = self.site.settings["size_optional"] - old_content_size_optional + content_size_optional # Site size without old content plus the new
- site_size_limit = self.site.getSizeLimit() * 1024 * 1024
- # Check site address
- if content.get("address") and content["address"] != self.site.address:
- raise VerifyError("Wrong site address: %s != %s" % (content["address"], self.site.address))
- # Check file inner path
- if content.get("inner_path") and content["inner_path"] != inner_path:
- raise VerifyError("Wrong inner_path: %s" % content["inner_path"])
- # Check total site size limit
- if site_size > site_size_limit:
- if inner_path == "content.json" and self.site.settings["size"] == 0:
- # First content.json download, save site size to display warning
- self.site.settings["size"] = site_size
- task = self.site.worker_manager.findTask(inner_path)
- if task: # Dont try to download from other peers
- self.site.worker_manager.failTask(task)
- raise VerifyError("Content too large %sB > %sB, aborting task..." % (site_size, site_size_limit))
- # Verify valid filenames
- for file_relative_path in content.get("files", {}).keys() + content.get("files_optional", {}).keys():
- if not self.isValidRelativePath(file_relative_path):
- raise VerifyError("Invalid relative path: %s" % file_relative_path)
- if inner_path == "content.json":
- self.site.settings["size"] = site_size
- self.site.settings["size_optional"] = site_size_optional
- return True # Root content.json is passed
- else:
- if self.verifyContentInclude(inner_path, content, content_size, content_size_optional):
- self.site.settings["size"] = site_size
- self.site.settings["size_optional"] = site_size_optional
- return True
- else:
- return False
- def verifyContentInclude(self, inner_path, content, content_size, content_size_optional):
- # Load include details
- rules = self.getRules(inner_path, content)
- if not rules:
- raise VerifyError("No rules")
- # Check include size limit
- if rules.get("max_size") is not None: # Include size limit
- if content_size > rules["max_size"]:
- raise VerifyError("Include too large %sB > %sB" % (content_size, rules["max_size"]))
- if rules.get("max_size_optional") is not None: # Include optional files limit
- if content_size_optional > rules["max_size_optional"]:
- raise VerifyError("Include optional files too large %sB > %sB" % (
- content_size_optional, rules["max_size_optional"])
- )
- # Filename limit
- if rules.get("files_allowed"):
- for file_inner_path in content["files"].keys():
- if not SafeRe.match("^%s$" % rules["files_allowed"], file_inner_path):
- raise VerifyError("File not allowed: %s" % file_inner_path)
- if rules.get("files_allowed_optional"):
- for file_inner_path in content.get("files_optional", {}).keys():
- if not SafeRe.match("^%s$" % rules["files_allowed_optional"], file_inner_path):
- raise VerifyError("Optional file not allowed: %s" % file_inner_path)
- # Check if content includes allowed
- if rules.get("includes_allowed") is False and content.get("includes"):
- raise VerifyError("Includes not allowed")
- return True # All good
- # Verify file validity
- # Return: None = Same as before, False = Invalid, True = Valid
- def verifyFile(self, inner_path, file, ignore_same=True):
- if inner_path.endswith("content.json"): # content.json: Check using sign
- from Crypt import CryptBitcoin
- try:
- if type(file) is dict:
- new_content = file
- else:
- new_content = json.load(file)
- if inner_path in self.contents:
- old_content = self.contents.get(inner_path, {"modified": 0})
- # Checks if its newer the ours
- if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
- return None
- elif old_content["modified"] > new_content["modified"]: # We have newer
- raise VerifyError(
- "We have newer (Our: %s, Sent: %s)" %
- (old_content["modified"], new_content["modified"])
- )
- if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+)
- raise VerifyError("Modify timestamp is in the far future!")
- if self.isArchived(inner_path, new_content["modified"]):
- if inner_path in self.site.bad_files:
- del self.site.bad_files[inner_path]
- raise VerifyError("This file is archived!")
- # Check sign
- sign = new_content.get("sign")
- signs = new_content.get("signs", {})
- if "sign" in new_content:
- del(new_content["sign"]) # The file signed without the sign
- if "signs" in new_content:
- del(new_content["signs"]) # The file signed without the signs
- sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
- # Fix float representation error on Android
- modified = new_content["modified"]
- if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"):
- modified_fixed = "{:.6f}".format(modified).strip("0.")
- sign_content = sign_content.replace(
- '"modified": %s' % repr(modified),
- '"modified": %s' % modified_fixed
- )
- self.verifyContent(inner_path, new_content)
- if signs: # New style signing
- valid_signers = self.getValidSigners(inner_path, new_content)
- signs_required = self.getSignsRequired(inner_path, new_content)
- if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
- signers_data = "%s:%s" % (signs_required, ",".join(valid_signers))
- if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]):
- raise VerifyError("Invalid signers_sign!")
- if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
- raise VerifyError("Invalid cert!")
- valid_signs = 0
- for address in valid_signers:
- if address in signs:
- valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
- if valid_signs >= signs_required:
- break # Break if we has enough signs
- if valid_signs < signs_required:
- raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required))
- else:
- return True
- else: # Old style signing
- if CryptBitcoin.verify(sign_content, self.site.address, sign):
- return True
- else:
- raise VerifyError("Invalid old-style sign")
- except Exception, err:
- self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err)))
- raise err
- else: # Check using sha512 hash
- file_info = self.getFileInfo(inner_path)
- if file_info:
- if CryptHash.sha512sum(file) != file_info.get("sha512", ""):
- raise VerifyError("Invalid hash")
- if file_info.get("size", 0) != file.tell():
- raise VerifyError(
- "File size does not match %s <> %s" %
- (inner_path, file.tell(), file_info.get("size", 0))
- )
- return True
- else: # File not in content.json
- raise VerifyError("File not in content.json")
- def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):
- if size is None:
- size = self.site.storage.getSize(inner_path)
- done = self.hashfield.appendHashId(hash_id)
- self.site.settings["optional_downloaded"] += size
- return done
- def optionalRemoved(self, inner_path, hash_id, size=None):
- if size is None:
- size = self.site.storage.getSize(inner_path)
- done = self.hashfield.removeHashId(hash_id)
- self.site.settings["optional_downloaded"] -= size
- return done
|