sydent/sydent/sydent.py

# Copyright 2014 OpenMarket Ltd
# Copyright 2018 New Vector Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import configparser
import copy
import gc
import logging
import logging.handlers
import os
from typing import Set

import twisted.internet.reactor
from jinja2 import Environment, FileSystemLoader
from twisted.internet import address, task
from twisted.python import log

from sydent.config import SydentConfig
from sydent.db.hashing_metadata import HashingMetadataStore
from sydent.db.sqlitedb import SqliteDatabase
from sydent.db.valsession import ThreePidValSessionStore
from sydent.hs_federation.verifier import Verifier
from sydent.http.httpcommon import SslComponents
from sydent.http.httpsclient import ReplicationHttpsClient
from sydent.http.httpserver import (
    ClientApiHttpServer,
    InternalApiHttpServer,
    ReplicationHttpsServer,
)
from sydent.http.servlets.accountservlet import AccountServlet
from sydent.http.servlets.blindlysignstuffservlet import BlindlySignStuffServlet
from sydent.http.servlets.bulklookupservlet import BulkLookupServlet
from sydent.http.servlets.emailservlet import (
    EmailRequestCodeServlet,
    EmailValidateCodeServlet,
)
from sydent.http.servlets.getvalidated3pidservlet import GetValidated3pidServlet
from sydent.http.servlets.hashdetailsservlet import HashDetailsServlet
from sydent.http.servlets.logoutservlet import LogoutServlet
from sydent.http.servlets.lookupservlet import LookupServlet
from sydent.http.servlets.lookupv2servlet import LookupV2Servlet
from sydent.http.servlets.msisdnservlet import (
    MsisdnRequestCodeServlet,
    MsisdnValidateCodeServlet,
)
from sydent.http.servlets.pubkeyservlets import (
    Ed25519Servlet,
    EphemeralPubkeyIsValidServlet,
    PubkeyIsValidServlet,
)
from sydent.http.servlets.registerservlet import RegisterServlet
from sydent.http.servlets.replication import ReplicationPushServlet
from sydent.http.servlets.store_invite_servlet import StoreInviteServlet
from sydent.http.servlets.termsservlet import TermsServlet
from sydent.http.servlets.threepidbindservlet import ThreePidBindServlet
from sydent.http.servlets.threepidunbindservlet import ThreePidUnbindServlet
from sydent.http.servlets.v1_servlet import V1Servlet
from sydent.http.servlets.v2_servlet import V2Servlet
from sydent.replication.pusher import Pusher
from sydent.threepid.bind import ThreepidBinder
from sydent.util.hash import sha256_and_url_safe_base64
from sydent.util.ip_range import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
from sydent.util.tokenutils import generateAlphanumericTokenOfLength
from sydent.validators.emailvalidator import EmailValidator
from sydent.validators.msisdnvalidator import MsisdnValidator

logger = logging.getLogger(__name__)

CONFIG_DEFAULTS = {
    "general": {
        "server.name": os.environ.get("SYDENT_SERVER_NAME", ""),
        "log.path": "",
        "log.level": "INFO",
        "pidfile.path": os.environ.get("SYDENT_PID_FILE", "sydent.pid"),
        "terms.path": "",
        "address_lookup_limit": "10000",  # Maximum amount of addresses in a single /lookup request
        # The root path to use for load templates. This should contain branded
        # directories. Each directory should contain the following templates:
        #
        # * invite_template.eml
        # * verification_template.eml
        # * verify_response_template.html
        "templates.path": "res",
        # The brand directory to use if no brand hint (or an invalid brand hint)
        # is provided by the request.
        "brand.default": "matrix-org",
        # The following can be added to your local config file to enable prometheus
        # support.
        # 'prometheus_port': '8080',  # The port to serve metrics on
        # 'prometheus_addr': '',  # The address to bind to. Empty string means bind to all.
        # The following can be added to your local config file to enable sentry support.
        # 'sentry_dsn': 'https://...'  # The DSN has configured in the sentry instance project.
        # Whether clients and homeservers can register an association using v1 endpoints.
        "enable_v1_associations": "true",
        "delete_tokens_on_bind": "true",
        # Prevent outgoing requests from being sent to the following blacklisted
        # IP address CIDR ranges. If this option is not specified or empty then
        # it defaults to private IP address ranges.
        #
        # The blacklist applies to all outbound requests except replication
        # requests.
        #
        # (0.0.0.0 and :: are always blacklisted, whether or not they are
        # explicitly listed here, since they correspond to unroutable
        # addresses.)
        "ip.blacklist": "",
        # List of IP address CIDR ranges that should be allowed for outbound
        # requests. This is useful for specifying exceptions to wide-ranging
        # blacklisted target IP ranges.
        #
        # This whitelist overrides `ip.blacklist` and defaults to an empty
        # list.
        "ip.whitelist": "",
    },
    "db": {
        "db.file": os.environ.get("SYDENT_DB_PATH", "sydent.db"),
    },
    "http": {
        "clientapi.http.bind_address": "::",
        "clientapi.http.port": "8090",
        "internalapi.http.bind_address": "::1",
        "internalapi.http.port": "",
        "replication.https.certfile": "",
        "replication.https.cacert": "",  # This should only be used for testing
        "replication.https.bind_address": "::",
        "replication.https.port": "4434",
        "obey_x_forwarded_for": "False",
        "federation.verifycerts": "True",
        # verify_response_template is deprecated, but still used if defined Define
        # templates.path and brand.default under general instead.
        #
        # 'verify_response_template': 'res/verify_response_page_template',
        "client_http_base": "",
    },
    "email": {
        # email.template and email.invite_template are deprecated, but still used
        # if defined. Define templates.path and brand.default under general instead.
        #
        # 'email.template': 'res/verification_template.eml',
        # 'email.invite_template': 'res/invite_template.eml',
        "email.from": "Sydent Validation <noreply@{hostname}>",
        "email.subject": "Your Validation Token",
        "email.invite.subject": "%(sender_display_name)s has invited you to chat",
        "email.invite.subject_space": "%(sender_display_name)s has invited you to a space",
        "email.smtphost": "localhost",
        "email.smtpport": "25",
        "email.smtpusername": "",
        "email.smtppassword": "",
        "email.hostname": "",
        "email.tlsmode": "0",
        # The web client location which will be used if it is not provided by
        # the homeserver.
        #
        # This should be the scheme and hostname only, see res/invite_template.eml
        # for the full URL that gets generated.
        "email.default_web_client_location": "https://app.element.io",
        # When a user is invited to a room via their email address, that invite is
        # displayed in the room list using an obfuscated version of the user's email
        # address. These config options determine how much of the email address to
        # obfuscate. Note that the '@' sign is always included.
        #
        # If the string is longer than a configured limit below, it is truncated to that limit
        # with '...' added. Otherwise:
        #
        # * If the string is longer than 5 characters, it is truncated to 3 characters + '...'
        # * If the string is longer than 1 character, it is truncated to 1 character + '...'
        # * If the string is 1 character long, it is converted to '...'
        #
        # This ensures that a full email address is never shown, even if it is extremely
        # short.
        #
        # The number of characters from the beginning to reveal of the email's username
        # portion (left of the '@' sign)
        "email.third_party_invite_username_obfuscate_characters": "3",
        # The number of characters from the beginning to reveal of the email's domain
        # portion (right of the '@' sign)
        "email.third_party_invite_domain_obfuscate_characters": "3",
    },
    "sms": {
        "bodyTemplate": "Your code is {token}",
        "username": "",
        "password": "",
    },
    "crypto": {
        "ed25519.signingkey": "",
    },
}


class Sydent:
    def __init__(
        self,
        cfg,
        sydent_config: SydentConfig,
        reactor=twisted.internet.reactor,
        use_tls_for_federation=True,
    ):
        self.cfg = cfg
        self.config = sydent_config

        self.reactor = reactor
        self.config_file = get_config_file_path()
        self.use_tls_for_federation = use_tls_for_federation

        logger.info("Starting Sydent server")

        self.pidfile = self.cfg.get("general", "pidfile.path")

        self.db = SqliteDatabase(self).db

        if self.cfg.has_option("general", "sentry_dsn"):
            # Only import and start sentry SDK if configured.
            import sentry_sdk

            sentry_sdk.init(
                dsn=self.cfg.get("general", "sentry_dsn"),
            )
            with sentry_sdk.configure_scope() as scope:
                scope.set_tag("sydent_server_name", self.config.general.server_name)

        if self.cfg.has_option("general", "prometheus_port"):
            import prometheus_client

            prometheus_client.start_http_server(
                port=self.cfg.getint("general", "prometheus_port"),
                addr=self.cfg.get("general", "prometheus_addr"),
            )

        if self.cfg.has_option("general", "templates.path"):
            # Get the possible brands by looking at directories under the
            # templates.path directory.
            root_template_path = self.cfg.get("general", "templates.path")
            if os.path.exists(root_template_path):
                self.valid_brands = {
                    p
                    for p in os.listdir(root_template_path)
                    if os.path.isdir(os.path.join(root_template_path, p))
                }
            else:
                # This is a legacy code-path and assumes that verify_response_template,
                # email.template, and email.invite_template are defined.
                self.valid_brands = set()

        self.enable_v1_associations = parse_cfg_bool(
            self.cfg.get("general", "enable_v1_associations")
        )

        self.delete_tokens_on_bind = parse_cfg_bool(
            self.cfg.get("general", "delete_tokens_on_bind")
        )

        ip_blacklist = set_from_comma_sep_string(
            self.cfg.get("general", "ip.blacklist")
        )
        if not ip_blacklist:
            ip_blacklist = DEFAULT_IP_RANGE_BLACKLIST

        ip_whitelist = set_from_comma_sep_string(
            self.cfg.get("general", "ip.whitelist")
        )

        self.ip_blacklist = generate_ip_set(ip_blacklist)
        self.ip_whitelist = generate_ip_set(ip_whitelist)

        self.template_environment = Environment(
            loader=FileSystemLoader(self.cfg.get("general", "templates.path")),
            autoescape=True,
        )

        # See if a pepper already exists in the database
        # Note: This MUST be run before we start serving requests, otherwise lookups for
        # 3PID hashes may come in before we've completed generating them
        hashing_metadata_store = HashingMetadataStore(self)
        lookup_pepper = hashing_metadata_store.get_lookup_pepper()
        if not lookup_pepper:
            # No pepper defined in the database, generate one
            lookup_pepper = generateAlphanumericTokenOfLength(5)

            # Store it in the database and rehash 3PIDs
            hashing_metadata_store.store_lookup_pepper(
                sha256_and_url_safe_base64, lookup_pepper
            )

        self.validators = Validators()
        self.validators.email = EmailValidator(self)
        self.validators.msisdn = MsisdnValidator(self)

        self.keyring = Keyring()
        self.keyring.ed25519 = self.config.crypto.signing_key
        self.keyring.ed25519.alg = "ed25519"

        self.sig_verifier = Verifier(self)

        self.servlets = Servlets()
        self.servlets.v1 = V1Servlet(self)
        self.servlets.v2 = V2Servlet(self)
        self.servlets.emailRequestCode = EmailRequestCodeServlet(self)
        self.servlets.emailRequestCodeV2 = EmailRequestCodeServlet(
            self, require_auth=True
        )
        self.servlets.emailValidate = EmailValidateCodeServlet(self)
        self.servlets.emailValidateV2 = EmailValidateCodeServlet(
            self, require_auth=True
        )
        self.servlets.msisdnRequestCode = MsisdnRequestCodeServlet(self)
        self.servlets.msisdnRequestCodeV2 = MsisdnRequestCodeServlet(
            self, require_auth=True
        )
        self.servlets.msisdnValidate = MsisdnValidateCodeServlet(self)
        self.servlets.msisdnValidateV2 = MsisdnValidateCodeServlet(
            self, require_auth=True
        )
        self.servlets.lookup = LookupServlet(self)
        self.servlets.bulk_lookup = BulkLookupServlet(self)
        self.servlets.hash_details = HashDetailsServlet(self, lookup_pepper)
        self.servlets.lookup_v2 = LookupV2Servlet(self, lookup_pepper)
        self.servlets.pubkey_ed25519 = Ed25519Servlet(self)
        self.servlets.pubkeyIsValid = PubkeyIsValidServlet(self)
        self.servlets.ephemeralPubkeyIsValid = EphemeralPubkeyIsValidServlet(self)
        self.servlets.threepidBind = ThreePidBindServlet(self)
        self.servlets.threepidBindV2 = ThreePidBindServlet(self, require_auth=True)
        self.servlets.threepidUnbind = ThreePidUnbindServlet(self)
        self.servlets.replicationPush = ReplicationPushServlet(self)
        self.servlets.getValidated3pid = GetValidated3pidServlet(self)
        self.servlets.getValidated3pidV2 = GetValidated3pidServlet(
            self, require_auth=True
        )
        self.servlets.storeInviteServlet = StoreInviteServlet(self)
        self.servlets.storeInviteServletV2 = StoreInviteServlet(self, require_auth=True)
        self.servlets.blindlySignStuffServlet = BlindlySignStuffServlet(self)
        self.servlets.blindlySignStuffServletV2 = BlindlySignStuffServlet(
            self, require_auth=True
        )
        self.servlets.termsServlet = TermsServlet(self)
        self.servlets.accountServlet = AccountServlet(self)
        self.servlets.registerServlet = RegisterServlet(self)
        self.servlets.logoutServlet = LogoutServlet(self)

        self.threepidBinder = ThreepidBinder(self)

        self.sslComponents = SslComponents(self)

        self.clientApiHttpServer = ClientApiHttpServer(self)
        self.replicationHttpsServer = ReplicationHttpsServer(self)
        self.replicationHttpsClient = ReplicationHttpsClient(self)

        self.pusher = Pusher(self)

        # A dedicated validation session store just to clean up old sessions every N minutes
        self.cleanupValSession = ThreePidValSessionStore(self)
        cb = task.LoopingCall(self.cleanupValSession.deleteOldSessions)
        cb.clock = self.reactor
        cb.start(10 * 60.0)

        # workaround for https://github.com/getsentry/sentry-python/issues/803: we
        # disable automatic GC and run it periodically instead.
        gc.disable()
        cb = task.LoopingCall(run_gc)
        cb.clock = self.reactor
        cb.start(1.0)

    def save_config(self):
        fp = open(self.config_file, "w")
        self.cfg.write(fp)
        fp.close()

    def run(self):
        self.clientApiHttpServer.setup()
        self.replicationHttpsServer.setup()
        self.pusher.setup()

        if self.config.http.internal_api_enabled:
            internalport = self.config.http.internal_port
            interface = self.config.http.internal_bind_address

            self.internalApiHttpServer = InternalApiHttpServer(self)
            self.internalApiHttpServer.setup(interface, internalport)

        if self.pidfile:
            with open(self.pidfile, "w") as pidfile:
                pidfile.write(str(os.getpid()) + "\n")

        self.reactor.run()

    def ip_from_request(self, request):
        if self.config.http.obey_x_forwarded_for and request.requestHeaders.hasHeader(
            "X-Forwarded-For"
        ):
            return request.requestHeaders.getRawHeaders("X-Forwarded-For")[0]
        client = request.getClientAddress()
        if isinstance(client, (address.IPv4Address, address.IPv6Address)):
            return client.host
        else:
            return None

    def brand_from_request(self, request):
        """
        If the brand GET parameter is passed, returns that as a string, otherwise returns None.

        :param request: The incoming request.
        :type request: twisted.web.http.Request

        :return: The brand to use or None if no hint is found.
        :rtype: str or None
        """
        if b"brand" in request.args:
            return request.args[b"brand"][0].decode("utf-8")
        return None

    def get_branded_template(self, brand, template_name, deprecated_template_name):
        """
        Calculate a (maybe) branded template filename to use.

        If the deprecated email.template setting is defined, always use it.
        Otherwise, attempt to use the hinted brand from the request if the brand
        is valid. Otherwise, fallback to the default brand.

        :param brand: The hint of which brand to use.
        :type brand: str or None
        :param template_name: The name of the template file to load.
        :type template_name: str
        :param deprecated_template_name: The deprecated setting to use, if provided.
        :type deprecated_template_name: Tuple[str]

        :return: The template filename to use.
        :rtype: str
        """

        # If the deprecated setting is defined, return it.
        try:
            return self.cfg.get(*deprecated_template_name)
        except configparser.NoOptionError:
            pass

        # If a brand hint is provided, attempt to use it if it is valid.
        if brand:
            if brand not in self.valid_brands:
                brand = None

        # If the brand hint is not valid, or not provided, fallback to the default brand.
        if not brand:
            brand = self.cfg.get("general", "brand.default")

        root_template_path = self.cfg.get("general", "templates.path")

        # Grab jinja template if it exists
        if os.path.exists(
            os.path.join(root_template_path, brand, template_name + ".j2")
        ):
            return os.path.join(brand, template_name + ".j2")
        else:
            return os.path.join(root_template_path, brand, template_name)


class Validators:
    pass


class Servlets:
    pass


class Keyring:
    pass


def parse_config_dict(config_dict):
    """Parse the given config from a dictionary, populating missing items and sections

    Args:
        config_dict (dict): the configuration dictionary to be parsed
    """
    # Build a config dictionary from the defaults merged with the given dictionary
    config = copy.deepcopy(CONFIG_DEFAULTS)
    for section, section_dict in config_dict.items():
        if section not in config:
            config[section] = {}
        for option in section_dict.keys():
            config[section][option] = config_dict[section][option]

    # Build a ConfigParser from the merged dictionary
    cfg = configparser.ConfigParser()
    for section, section_dict in config.items():
        cfg.add_section(section)
        for option, value in section_dict.items():
            cfg.set(section, option, value)

    return cfg


def parse_config_file(config_file):
    """Parse the given config from a filepath, populating missing items and
    sections
    Args:
        config_file (str): the file to be parsed
    """
    # if the config file doesn't exist, prepopulate the config object
    # with the defaults, in the right section.
    #
    # otherwise, we have to put the defaults in the DEFAULT section,
    # to ensure that they don't override anyone's settings which are
    # in their config file in the default section (which is likely,
    # because sydent used to be braindead).
    use_defaults = not os.path.exists(config_file)
    cfg = configparser.ConfigParser()
    for sect, entries in CONFIG_DEFAULTS.items():
        cfg.add_section(sect)
        for k, v in entries.items():
            cfg.set(configparser.DEFAULTSECT if use_defaults else sect, k, v)

    cfg.read(config_file)

    return cfg


def setup_logging(cfg):
    log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s"
    formatter = logging.Formatter(log_format)

    logPath = cfg.get("general", "log.path")
    if logPath != "":
        handler = logging.handlers.TimedRotatingFileHandler(
            logPath, when="midnight", backupCount=365
        )
        handler.setFormatter(formatter)

        def sighup(signum, stack):
            logger.info("Closing log file due to SIGHUP")
            handler.doRollover()
            logger.info("Opened new log file due to SIGHUP")

    else:
        handler = logging.StreamHandler()

    handler.setFormatter(formatter)
    rootLogger = logging.getLogger("")
    rootLogger.setLevel(cfg.get("general", "log.level"))
    rootLogger.addHandler(handler)

    observer = log.PythonLoggingObserver()
    observer.start()


def get_config_file_path():
    return os.environ.get("SYDENT_CONF", "sydent.conf")


def parse_cfg_bool(value):
    return value.lower() == "true"


def set_from_comma_sep_string(rawstr: str) -> Set[str]:
    if rawstr == "":
        return set()
    return {x.strip() for x in rawstr.split(",")}


def run_gc():
    threshold = gc.get_threshold()
    counts = gc.get_count()
    for i in reversed(range(len(threshold))):
        if threshold[i] < counts[i]:
            gc.collect(i)


if __name__ == "__main__":
    cfg = parse_config_file(get_config_file_path())
    setup_logging(cfg)

    sydent_config = SydentConfig()
    cfg_needs_saving = sydent_config.parse_from_config_parser(cfg)

    syd = Sydent(cfg, sydent_config=sydent_config)

    if cfg_needs_saving:
        syd.save_config()

    syd.run()