Merge branch 'master' into dinsic

.gitignore

@@ -4,6 +4,9 @@
 .vscode/
 *.iml
 _trial_temp
+*.egg
+*.egg-info
+.python-version
 
 # Runtime files
 sydent.conf

CHANGELOG.md

@@ -1,3 +1,38 @@
+Sydent 2.2.0 (2020-09-11)
+=========================
+
+Bugfixes
+--------
+
+- Fix intermittent deadlock in Sentry integration. ([\#312](https://github.com/matrix-org/sydent/issues/312))
+
+
+Sydent 2.1.0 (2020-09-10)
+=========================
+
+Features
+--------
+
+- Add a Dockerfile and allow environment variables `SYDENT_SERVER_NAME`, `SYDENT_PID_FILE` and `SYDENT_DB_PATH` to modify default configuration values. ([\#290](https://github.com/matrix-org/sydent/issues/290))
+- Add config options for controlling how email addresses are obfuscated in third party invites. ([\#311](https://github.com/matrix-org/sydent/issues/311))
+
+
+Bugfixes
+--------
+
+- Fix a bug in the error handling of 3PID session validation, if the token submitted is incorrect. ([\#296](https://github.com/matrix-org/sydent/issues/296))
+- Stop sending the unspecified `success` parameter in responses to `/requestToken` requests. ([\#302](https://github.com/matrix-org/sydent/issues/302))
+- Fix a bug causing Sydent to ignore `nextLink` parameters. ([\#303](https://github.com/matrix-org/sydent/issues/303))
+- Fix the HTTP status code returned during some error responses. ([\#305](https://github.com/matrix-org/sydent/issues/305))
+- Sydent now correctly enforces the valid characters in the `client_secret` parameter used in various endpoints. ([\#309](https://github.com/matrix-org/sydent/issues/309))
+
+
+Internal Changes
+----------------
+
+- Replace instances of Riot with Element. ([\#308](https://github.com/matrix-org/sydent/issues/308))
+
+
 Sydent 2.0.1 (2020-05-20)
 =========================
 

Dockerfile

@@ -1,17 +1,15 @@
 #
 # Step 1: Build sydent and install dependencies
 #
-FROM docker.io/python:3.8-alpine as builder
+FROM docker.io/python:3.8-slim as builder
 
 # Install dev packages
-RUN apk add --no-cache \
-        build-base \
-        libressl-dev \
-        libffi-dev
+RUN apt-get update && apt-get install -y \
+    build-essential
 
 # Add user sydent
-RUN addgroup -S -g 993 sydent \
-    && adduser -D --home /sydent -S -u 993 -G sydent -s /bin/ash sydent \
+RUN addgroup --system --gid 993 sydent \
+    && adduser --disabled-password --home /sydent --system --uid 993 --gecos sydent sydent \
     && echo "sydent:$(dd if=/dev/random bs=32 count=1 | base64)" | chpasswd
 
 # Copy resources
@@ -21,9 +19,10 @@ COPY --chown=sydent:sydent ["sydent", "/sydent/sydent"]
 COPY --chown=sydent:sydent ["README.rst", "setup.cfg", "setup.py", "/sydent/"]
 
 # Install dependencies
-RUN cd /sydent \
-    && su sydent -c 'pip install --user --upgrade pip setuptools sentry-sdk' \
-    && su sydent -c 'pip install --user -e .' \
+USER sydent
+WORKDIR /sydent
+RUN pip install --user --upgrade pip setuptools sentry-sdk prometheus_client \
+    && pip install --user . \
     && rm -rf /sydent/.cache \
     && find /sydent -name '*.pyc' -delete
 
@@ -31,16 +30,11 @@ RUN cd /sydent \
 # Step 2: Reduce image size and layers
 #
 
-FROM docker.io/python:3.8-alpine
-
-# Install packages
-RUN apk add --no-cache \
-        libressl \
-        libffi
+FROM docker.io/python:3.8-slim
 
 # Add user sydent and create /data directory
-RUN addgroup -S -g 993 sydent \
-    && adduser -D --home /sydent -S -u 993 -G sydent -s /bin/ash sydent \
+RUN addgroup --system --gid 993 sydent \
+    && adduser --disabled-password --home /sydent --system --uid 993 --gecos sydent sydent \
     && echo "sydent:$(dd if=/dev/random bs=32 count=1 | base64)" | chpasswd \
     && mkdir /data \
     && chown sydent:sydent /data

changelog.d/290.feature

@@ -1 +0,0 @@
-Add a Dockerfile and allow environment variables `SYDENT_SERVER_NAME`, `SYDENT_PID_FILE` and `SYDENT_DB_PATH` to modify default configuration values.

changelog.d/296.bugfix

@@ -1 +0,0 @@
-Fix a bug in the error handling of 3PID session validation, if the token submitted is incorrect.

changelog.d/302.bugfix

@@ -1 +0,0 @@
-Stop sending the unspecified `success` parameter in responses to `/requestToken` requests.

changelog.d/303.bugfix

@@ -1 +0,0 @@
-Fix a bug causing Sydent to ignore `nextLink` parameters.

changelog.d/305.bugfix

@@ -1 +0,0 @@
-Fix the HTTP status code returned during some error responses.

changelog.d/308.misc

@@ -1 +0,0 @@
-Replace instances of Riot with Element.

changelog.d/309.bugfix

@@ -1 +0,0 @@
-Sydent now correctly enforces the valid characters in the `client_secret` parameter used in various endpoints.

changelog.d/311.feature

@@ -1 +0,0 @@
-Add config options for controlling how email addresses are obfuscated in third party invites.

changelog.d/317.bugfix

@@ -0,0 +1 @@
+Fix a regression in v2.2.0 where the wrong characters would be obfuscated in a 3pid invite.

changelog.d/318.misc

@@ -0,0 +1 @@
+Fix test logging to allow braces in log output.

changelog.d/325.misc

@@ -0,0 +1 @@
+Install prometheus_client in the Docker image.

changelog.d/326.feature

@@ -0,0 +1 @@
+Accept an optional `web_client_location` argument to the invite endpoint which allows customisation of the email template.

changelog.d/328.feature

@@ -0,0 +1 @@
+Move templates to a per-brand subdirectory of /res. Add templates.path and brand.default config options

changelog.d/334.misc

@@ -0,0 +1 @@
+Bump the version of signedjson to 1.1.1.

changelog.d/335.docker

@@ -0,0 +1 @@
+Base docker image on Debian rather than Alpine Linux.

changelog.d/337.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where invalid JSON would be accepted over the HTTP interfaces.

matrix_is_test/launcher.py

@@ -25,7 +25,6 @@ CFG_TEMPLATE = """
 clientapi.http.bind_address = localhost
 clientapi.http.port = {port}
 client_http_base = http://localhost:{port}
-verify_response_template = {testsubject_path}/res/verify_response_template
 federation.verifycerts = False
 
 [db]
@@ -35,16 +34,16 @@ db.file = :memory:
 server.name = test.local
 terms.path = {terms_path}
 info_path = {info_path}
+templates.path = {testsubject_path}/res
+brand.default = is-test
 
 [email]
 email.tlsmode = 0
-email.template = {testsubject_path}/res/verification_template.eml
 email.invite.subject = %(sender_display_name)s has invited you to chat
 email.smtphost = localhost
 email.from = Sydent Validation <noreply@localhost>
 email.smtpport = 9925
 email.subject = Your Validation Token
-email.invite_template = {testsubject_path}/res/invite_template.eml
 """
 
 class MatrixIsTestLauncher(object):

pyproject.toml

@@ -14,6 +14,11 @@
         name = "Bugfixes"
         showcontent = true
 
+    [[tool.towncrier.type]]
+        directory = "docker"
+        name = "Updates to the Docker image"
+        showcontent = true
+
     [[tool.towncrier.type]]
         directory = "doc"
         name = "Improved Documentation"

res/invite_template.eml → res/matrix-org/invite_template.eml

@@ -18,7 +18,7 @@ Matrix. To join the conversation, either pick a Matrix client from
 https://matrix.org/docs/projects/try-matrix-now.html or use the single-click
 link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android)
 
-https://app.element.io/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s
+%(web_client_location)/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s
 
 
 About Matrix:

res/invite_template_vector.eml → res/vector-im/invite_template.eml

@@ -16,7 +16,7 @@ Hi,
 %(sender_display_name)s has invited you into a room %(bracketed_room_name)s on
 Element. To join the conversation please follow the link below.
 
-https://app.element.io/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s
+%(web_client_location)/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s
 
 Element is an open source collaboration app built on the Matrix.org
 open standard for interoperable communication: supporting group chat,

setup.py

@@ -38,7 +38,7 @@ setup(
     packages=find_packages(),
     description="Reference Matrix Identity Verification and Lookup Server",
     install_requires=[
-        "signedjson==1.0.0",
+        "signedjson==1.1.1",
         "unpaddedbase64==1.1.0",
 
         # We require defer.Deferred.addTimeout, which was introduced in 16.5

sydent/__init__.py

@@ -1 +1 @@
-__version__ = '2.0.1'
+__version__ = '2.2.0'

sydent/http/httpclient.py

@@ -25,6 +25,7 @@ from twisted.web.http_headers import Headers
 from sydent.http.matrixfederationagent import MatrixFederationAgent
 
 from sydent.http.federation_tls_options import ClientTLSOptionsFactory
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -52,7 +53,7 @@ class HTTPClient(object):
         body = yield readBody(response)
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            json_body = json.loads(body.decode("UTF-8"))
+            json_body = json_decoder.decode(body.decode("UTF-8"))
         except Exception as e:
             logger.exception("Error parsing JSON from %s", uri)
             raise

sydent/http/matrixfederationagent.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 from __future__ import absolute_import
 
-import json
 import logging
 import random
 import time
@@ -32,6 +31,7 @@ from twisted.web.http_headers import Headers
 from twisted.web.iweb import IAgent
 
 from sydent.http.srvresolver import SrvResolver, pick_server_from_list
+from sydent.util import json_decoder
 from sydent.util.ttlcache import TTLCache
 
 # period to cache .well-known results for by default
@@ -320,7 +320,7 @@ class MatrixFederationAgent(object):
             if response.code != 200:
                 raise Exception("Non-200 response %s" % (response.code, ))
 
-            parsed_body = json.loads(body.decode('utf-8'))
+            parsed_body = json_decoder.decode(body.decode('utf-8'))
             logger.info("Response from .well-known: %s", parsed_body)
             if not isinstance(parsed_body, dict):
                 raise Exception("not a dict")
@@ -439,4 +439,4 @@ class _RoutingResult(object):
     The port we should route the TCP connection to (the target of the SRV record, or
     the port from the URL/.well-known, or 8448)
     :type: int
-    """
+    """

sydent/http/servlets/__init__.py

@@ -22,6 +22,7 @@ import functools
 from twisted.internet import defer
 from twisted.web import server
 
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +77,7 @@ def get_args(request, args, required=True):
     ):
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            request_args = json.loads(request.content.read().decode("UTF-8"))
+            request_args = json_decoder.decode(request.content.read().decode("UTF-8"))
         except ValueError:
             raise MatrixRestError(400, 'M_BAD_JSON', 'Malformed JSON')
 

sydent/http/servlets/emailservlet.py

@@ -62,6 +62,7 @@ class EmailRequestCodeServlet(Resource):
             }
 
         ipaddress = self.sydent.ip_from_request(request)
+        brand = self.sydent.brand_from_request(request)
 
         nextLink = None
         if 'next_link' in args and not args['next_link'].startswith("file:///"):
@@ -78,7 +79,7 @@ class EmailRequestCodeServlet(Resource):
 
         try:
             sid = self.sydent.validators.email.requestToken(
-                email, clientSecret, sendAttempt, nextLink, ipaddress=ipaddress
+                email, clientSecret, sendAttempt, nextLink, ipaddress=ipaddress, brand=brand,
             )
             resp = {'sid': str(sid)}
         except EmailAddressException:
@@ -121,7 +122,12 @@ class EmailValidateCodeServlet(Resource):
             #msg = "Verification failed: you may need to request another verification email"
             msg = u"La vérification a échoué: essayez de recommencer la procédure."
 
-        templateFile = self.sydent.cfg.get('http', 'verify_response_template')
+        brand = self.sydent.brand_from_request(request)
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "verify_response_template.html",
+            ('http', 'verify_response_template'),
+        )
 
         request.setHeader("Content-Type", "text/html")
         res = open(templateFile).read() % {'message': msg}

sydent/http/servlets/lookupservlet.py

@@ -20,12 +20,11 @@ from twisted.web.resource import Resource
 from sydent.db.threepid_associations import GlobalAssociationStore
 
 import logging
-import json
 import signedjson.sign
 
 from sydent.http.servlets import get_args, jsonwrap, send_cors, MatrixRestError
 from sydent.http.auth import authIfV2
-
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +41,7 @@ class LookupServlet(Resource):
         Look up an individual threepid.
 
         ** DEPRECATED **
-        
+
         Params: 'medium': the medium of the threepid
                 'address': the address of the threepid
         Returns: A signed association if the threepid has a corresponding mxid, otherwise the empty object.
@@ -63,7 +62,7 @@ class LookupServlet(Resource):
         if not sgassoc:
             return {}
 
-        sgassoc = json.loads(sgassoc)
+        sgassoc = json_decoder.decode(sgassoc)
         if not self.sydent.server_name in sgassoc['signatures']:
             # We have not yet worked out what the proper trust model should be.
             #

sydent/http/servlets/msisdnservlet.py

@@ -87,9 +87,10 @@ class MsisdnRequestCodeServlet(Resource):
             phone_number_object, phonenumbers.PhoneNumberFormat.INTERNATIONAL
         )
 
+        brand = self.sydent.brand_from_request(request)
         try:
             sid = self.sydent.validators.msisdn.requestToken(
-                phone_number_object, clientSecret, sendAttempt
+                phone_number_object, clientSecret, sendAttempt, brand
             )
             resp = {
                 'success': True, 'sid': str(sid),
@@ -135,7 +136,12 @@ class MsisdnValidateCodeServlet(Resource):
                 request.setResponseCode(400)
                 msg = "Verification failed: you may need to request another verification text"
 
-        templateFile = self.sydent.cfg.get('http', 'verify_response_template')
+        brand = self.sydent.brand_from_request(request)
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "verify_response_template.html",
+            ('http', 'verify_response_template'),
+        )
 
         request.setHeader("Content-Type", "text/html")
         return open(templateFile).read().decode('utf8') % {'message': msg}

sydent/http/servlets/replication.py

@@ -21,6 +21,7 @@ from twisted.web import server
 from twisted.internet import defer
 from sydent.http.servlets import jsonwrap, MatrixRestError
 from sydent.threepid import threePidAssocFromDict
+from sydent.util import json_decoder
 
 from sydent.util.hash import sha256_and_url_safe_base64
 
@@ -87,7 +88,7 @@ class ReplicationPushServlet(Resource):
 
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            inJson = json.loads(request.content.read().decode("UTF-8"))
+            inJson = json_decoder.decode(request.content.read().decode("UTF-8"))
         except ValueError:
             logger.warn("Peer %s made push connection with malformed JSON", peer.servername)
             raise MatrixRestError(400, 'M_BAD_JSON', 'Malformed JSON')

sydent/http/servlets/store_invite_servlet.py

@@ -19,7 +19,6 @@ from __future__ import absolute_import
 import nacl.signing
 import random
 import string
-import time
 from email.header import Header
 
 from six import string_types
@@ -81,33 +80,48 @@ class StoreInviteServlet(Resource):
         tokenStore.storeEphemeralPublicKey(ephemeralPublicKeyBase64)
         tokenStore.storeToken(medium, address, roomId, sender, token)
 
+        # Variables to substitute in the template.
         substitutions = {}
+        # Include all arguments sent via the request.
         for k, v in args.items():
             if isinstance(v, string_types):
                 substitutions[k] = v
         substitutions["token"] = token
 
-        required = [
+        # Substitutions that the template requires, but are optional to provide
+        # to the API.
+        extra_substitutions = [
             'sender_display_name',
             'token',
             'room_name',
             'bracketed_room_name',
             'room_avatar_url',
-            'sender_display_name',
+            'sender_avatar_url',
             'guest_user_id',
             'guest_access_token',
         ]
-        for k in required:
+        for k in extra_substitutions:
             substitutions.setdefault(k, '')
 
         substitutions["ephemeral_private_key"] = ephemeralPrivateKeyBase64
         if substitutions["room_name"] != '':
             substitutions["bracketed_room_name"] = "(%s)" % substitutions["room_name"]
 
+        substitutions["web_client_location"] = self.sydent.default_web_client_location
+        if 'org.matrix.web_client_location' in substitutions:
+            substitutions["web_client_location"] = substitutions.pop("org.matrix.web_client_location")
+
         subject_header = Header(self.sydent.cfg.get('email', 'email.invite.subject', raw=True) % substitutions, 'utf8')
         substitutions["subject_header_value"] = subject_header.encode()
 
-        sendEmail(self.sydent, "email.invite_template", address, substitutions)
+        brand = self.sydent.brand_from_request(request)
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "invite_template.eml",
+            ('email', 'email.invite_template'),
+        )
+
+        sendEmail(self.sydent, templateFile, address, substitutions)
 
         pubKey = self.sydent.keyring.ed25519.verify_key
         pubKeyBase64 = encode_base64(pubKey.encode())

sydent/http/servlets/threepidunbindservlet.py

@@ -24,6 +24,7 @@ from signedjson.sign import SignatureVerifyException
 
 from sydent.http.servlets import dict_to_json_bytes
 from sydent.db.valsession import ThreePidValSessionStore
+from sydent.util import json_decoder
 from sydent.util.stringutils import is_valid_client_secret
 from sydent.validators import (
     IncorrectClientSecretException,
@@ -51,7 +52,7 @@ class ThreePidUnbindServlet(Resource):
         try:
             try:
                 # json.loads doesn't allow bytes in Python 3.5
-                body = json.loads(request.content.read().decode("UTF-8"))
+                body = json_decoder.decode(request.content.read().decode("UTF-8"))
             except ValueError:
                 request.setResponseCode(400)
                 request.write(dict_to_json_bytes({'errcode': 'M_BAD_JSON', 'error': 'Malformed JSON'}))
@@ -81,7 +82,7 @@ class ThreePidUnbindServlet(Resource):
             # and "client_secret" fields, they are trying to prove that they
             # were the original author of the bind. We then check that what
             # they supply matches and if it does, allow the unbind.
-            # 
+            #
             # However if these fields are not supplied, we instead check
             # whether the request originated from a homeserver, and if so the
             # same homeserver that originally created the bind. We do this by
@@ -121,7 +122,7 @@ class ThreePidUnbindServlet(Resource):
                         'error': "This validation session has not yet been completed"
                     }))
                     return
-                
+
                 if s.medium != threepid['medium'] or s.address != threepid['address']:
                     request.setResponseCode(403)
                     request.write(dict_to_json_bytes({

sydent/replication/peer.py

@@ -22,6 +22,7 @@ from sydent.db.threepid_associations import GlobalAssociationStore
 from sydent.db.hashing_metadata import HashingMetadataStore
 from sydent.threepid import threePidAssocFromDict
 from sydent.config import ConfigError
+from sydent.util import json_decoder
 from sydent.util.hash import sha256_and_url_safe_base64
 from unpaddedbase64 import decode_base64
 
@@ -269,7 +270,7 @@ class RemotePeer(Peer):
         :param updateDeferred: The deferred to call the error callback of.
         :type updateDeferred: twisted.internet.defer.Deferred
         """
-        errObj = json.loads(body.decode("utf8"))
+        errObj = json_decoder.decode(body.decode("utf8"))
         e = RemotePeerError()
         e.errorDict = errObj
         updateDeferred.errback(e)

sydent/sydent.py

@@ -17,6 +17,8 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+import gc
+
 from six.moves import configparser
 import copy
 import logging
@@ -112,6 +114,17 @@ CONFIG_DEFAULTS = {
         # If empty, no whitelist is applied
         'next_link.domain_whitelist': '',
 
+        # The root path to use for load templates. This should contain branded
+        # directories. Each directory should contain the following templates:
+        #
+        # * invite_template.eml
+        # * verification_template.eml
+        # * verify_response_template.html
+        'templates.path': 'res',
+        # The brand directory to use if no brand hint (or an invalid brand hint)
+        # is provided by the request.
+        'brand.default': 'matrix-org',
+
         # The following can be added to your local config file to enable prometheus
         # support.
         # 'prometheus_port': '8080',  # The port to serve metrics on
@@ -138,12 +151,18 @@ CONFIG_DEFAULTS = {
         'replication.https.port': '4434',
         'obey_x_forwarded_for': 'False',
         'federation.verifycerts': 'True',
-        'verify_response_template': '',
+        # verify_response_template is deprecated, but still used if defined Define
+        # templates.path and brand.default under general instead.
+        #
+        # 'verify_response_template': 'res/verify_response_page_template',
         'client_http_base': '',
     },
     'email': {
-        'email.template': 'res/email.template',
-        'email.invite_template': 'res/invite.template',
+        # email.template and email.invite_template are deprecated, but still used
+        # if defined. Define templates.path and brand.default under general instead.
+        #
+        # 'email.template': 'res/verification_template.eml',
+        # 'email.invite_template': 'res/invite_template.eml',
         'email.from': 'Sydent Validation <noreply@{hostname}>',
         'email.subject': 'Your Validation Token',
         'email.invite.subject': '%(sender_display_name)s has invited you to chat',
@@ -153,6 +172,12 @@ CONFIG_DEFAULTS = {
         'email.smtppassword': '',
         'email.hostname': '',
         'email.tlsmode': '0',
+        # The web client location which will be used if it is not provided by
+        # the homeserver.
+        #
+        # This should be the scheme and hostname only, see res/invite_template.eml
+        # for the full URL that gets generated.
+        'email.default_web_client_location': 'https://app.element.io',
 
         # When a user is invited to a room via their email address, that invite is
         # displayed in the room list using an obfuscated version of the user's email
@@ -282,6 +307,19 @@ class Sydent:
                 addr=self.cfg.get("general", "prometheus_addr"),
             )
 
+        if self.cfg.has_option("general", "templates.path"):
+            # Get the possible brands by looking at directories under the
+            # templates.path directory.
+            root_template_path = self.cfg.get("general", "templates.path")
+            if os.path.exists(root_template_path):
+                self.valid_brands = {
+                    p for p in os.listdir(root_template_path) if os.path.isdir(os.path.join(root_template_path, p))
+                }
+            else:
+                # This is a legacy code-path and assumes that verify_response_template,
+                # email.template, and email.invite_template are defined.
+                self.valid_brands = set()
+
         self.enable_v1_associations = parse_cfg_bool(
             self.cfg.get("general", "enable_v1_associations")
         )
@@ -328,6 +366,13 @@ class Sydent:
             self.cfg.get("email", "email.always_obfuscate")
         )
 
+        self.default_web_client_location = self.cfg.get(
+            "email", "email.default_web_client_location"
+        )
+        self.username_obfuscate_characters = int(self.cfg.get(
+            "email", "email.third_party_invite_username_obfuscate_characters"
+        ))
+
         # See if a pepper already exists in the database
         # Note: This MUST be run before we start serving requests, otherwise lookups for
         # 3PID hashes may come in before we've completed generating them
@@ -398,6 +443,13 @@ class Sydent:
         cb.clock = self.reactor
         cb.start(10 * 60.0)
 
+        # workaround for https://github.com/getsentry/sentry-python/issues/803: we
+        # disable automatic GC and run it periodically instead.
+        gc.disable()
+        cb = task.LoopingCall(run_gc)
+        cb.clock = self.reactor
+        cb.start(1.0)
+
     def save_config(self):
         fp = open(self.config_file, 'w')
         self.cfg.write(fp)
@@ -429,6 +481,57 @@ class Sydent:
             return request.requestHeaders.getRawHeaders("X-Forwarded-For")[0]
         return request.getClientIP()
 
+    def brand_from_request(self, request):
+        """
+        If the brand GET parameter is passed, returns that as a string, otherwise returns None.
+
+        :param request: The incoming request.
+        :type request: twisted.web.http.Request
+
+        :return: The brand to use or None if no hint is found.
+        :rtype: str or None
+        """
+        if b"brand" in request.args:
+            return request.args[b"brand"][0].decode("utf-8")
+        return None
+
+    def get_branded_template(self, brand, template_name, deprecated_template_name):
+        """
+        Calculate a (maybe) branded template filename to use.
+
+        If the deprecated email.template setting is defined, always use it.
+        Otherwise, attempt to use the hinted brand from the request if the brand
+        is valid. Otherwise, fallback to the default brand.
+
+        :param brand: The hint of which brand to use.
+        :type brand: str or None
+        :param template_name: The name of the template file to load.
+        :type template_name: str
+        :param deprecated_template_name: The deprecated setting to use, if provided.
+        :type deprecated_template_name: Tuple[str]
+
+        :return: The template filename to use.
+        :rtype: str
+        """
+
+        # If the deprecated setting is defined, return it.
+        try:
+            return self.cfg.get(*deprecated_template_name)
+        except configparser.NoOptionError:
+            pass
+
+        # If a brand hint is provided, attempt to use it if it is valid.
+        if brand:
+            if brand not in self.valid_brands:
+                brand = None
+
+        # If the brand hint is not valid, or not provided, fallback to the default brand.
+        if not brand:
+            brand = self.cfg.get("general", "brand.default")
+
+        root_template_path = self.cfg.get("general", "templates.path")
+        return os.path.join(root_template_path, brand, template_name)
+
 
 class Validators:
     pass
@@ -554,6 +657,14 @@ def parse_cfg_bool(value):
     return value.lower() == "true"
 
 
+def run_gc():
+    threshold = gc.get_threshold()
+    counts = gc.get_count()
+    for i in reversed(range(len(threshold))):
+        if threshold[i] < counts[i]:
+            gc.collect(i)
+
+
 if __name__ == '__main__':
     cfg = parse_config_file(get_config_file_path())
     setup_logging(cfg)

sydent/util/__init__.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import time
 
-
 def time_msec():
     """
     Get the current time in milliseconds.
@@ -25,3 +25,12 @@ def time_msec():
     :rtype: int
     """
     return int(time.time() * 1000)
+
+
+def _reject_invalid_json(val):
+    """Do not allow Infinity, -Infinity, or NaN values in JSON."""
+    raise ValueError("Invalid JSON value: '%s'" % val)
+
+
+# a custom JSON decoder which will reject Python extensions to JSON.
+json_decoder = json.JSONDecoder(parse_constant=_reject_invalid_json)

sydent/util/emailutils.py

@@ -39,23 +39,22 @@ from sydent.util import time_msec
 logger = logging.getLogger(__name__)
 
 
-def sendEmail(sydent, templateName, mailTo, substitutions):
+def sendEmail(sydent, templateFile, mailTo, substitutions):
     """
     Sends an email with the given parameters.
 
     :param sydent: The Sydent instance to use when building the configuration to send the
         email with.
     :type sydent: sydent.sydent.Sydent
-    :param templateName: The name of the template to use when building the body of the
+    :param templateFile: The filename of the template to use when building the body of the
         email.
-    :type templateName: str
+    :type templateFile: str
     :param mailTo: The email address to send the email to.
     :type mailTo: unicode
     :param substitutions: The substitutions to use with the template.
     :type substitutions: dict[str, str]
     """
     mailFrom = sydent.cfg.get('email', 'email.from')
-    mailTemplateFile = sydent.cfg.get('email', templateName)
 
     myHostname = sydent.cfg.get('email', 'email.hostname')
     if myHostname == '':
@@ -74,17 +73,9 @@ def sendEmail(sydent, templateName, mailTo, substitutions):
     for k, v in substitutions.items():
         allSubstitutions[k] = v
         allSubstitutions[k+"_forhtml"] = escape(v)
-
-        if six.PY2 and isinstance(v, unicode):
-            # urllib.parse.quote doesn't support unicode in Python 2, because at that
-            # time unicode in URLs weren't a thing. So convert the value to ascii and
-            # ignore error so we don't return an error if a parameter (e.g. the room's
-            # name contains ascii).
-            v = v.encode("utf-8", errors="ignore")
-
         allSubstitutions[k+"_forurl"] = urllib.parse.quote(v)
 
-    mailString = open(mailTemplateFile, encoding="utf-8").read() % allSubstitutions
+    mailString = open(templateFile, encoding="utf-8").read() % allSubstitutions
     parsedFrom = email.utils.parseaddr(mailFrom)[1]
     parsedTo = email.utils.parseaddr(mailTo)[1]
     if parsedFrom == '' or parsedTo == '':

sydent/validators/emailvalidator.py

@@ -31,7 +31,7 @@ class EmailValidator:
     def __init__(self, sydent):
         self.sydent = sydent
 
-    def requestToken(self, emailAddress, clientSecret, sendAttempt, nextLink, ipaddress=None):
+    def requestToken(self, emailAddress, clientSecret, sendAttempt, nextLink, ipaddress=None, brand=None):
         """
         Creates or retrieves a validation session and sends an email to the corresponding
         email address with a token to use to verify the association.
@@ -47,6 +47,8 @@ class EmailValidator:
         :type nextLink: unicode
         :param ipaddress: The requester's IP address.
         :type ipaddress: str or None
+        :param brand: A hint at a brand from the request.
+        :type brand: str or None
 
         :return: The ID of the session created (or of the existing one if any)
         :rtype: int
@@ -58,6 +60,12 @@ class EmailValidator:
 
         valSessionStore.setMtime(valSession.id, time_msec())
 
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "verification_template.eml",
+            ('email', 'email.template'),
+        )
+
         if int(valSession.sendAttemptNumber) >= int(sendAttempt):
             logger.info("Not mailing code because current send attempt (%d) is not less than given send attempt (%s)", int(sendAttempt), int(valSession.sendAttemptNumber))
             return valSession.id
@@ -73,7 +81,7 @@ class EmailValidator:
             "Attempting to mail code %s (nextLink: %s) to %s",
             valSession.token, nextLink, emailAddress,
         )
-        sendEmail(self.sydent, 'email.template', emailAddress, substitutions)
+        sendEmail(self.sydent, templateFile, emailAddress, substitutions)
 
         valSessionStore.setSendAttemptNumber(valSession.id, sendAttempt)
 

sydent/validators/msisdnvalidator.py

@@ -64,7 +64,7 @@ class MsisdnValidator:
 
                 self.smsRules[country] = action
 
-    def requestToken(self, phoneNumber, clientSecret, sendAttempt):
+    def requestToken(self, phoneNumber, clientSecret, sendAttempt, brand=None):
         """
         Creates or retrieves a validation session and sends an text message to the
         corresponding phone number address with a token to use to verify the association.
@@ -75,6 +75,8 @@ class MsisdnValidator:
         :type clientSecret: unicode
         :param sendAttempt: The current send attempt.
         :type sendAttempt: int
+        :param brand: A hint at a brand from the request.
+        :type brand: str or None
 
         :return: The ID of the session created (or of the existing one if any)
         :rtype: int

tests/test_email.py

@@ -0,0 +1,82 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os.path
+from mock import Mock, patch
+
+from twisted.web.client import Response
+from twisted.trial import unittest
+
+from sydent.db.invite_tokens import JoinTokenStore
+from sydent.http.httpclient import FederationHttpClient
+from sydent.http.servlets.store_invite_servlet import StoreInviteServlet
+from tests.utils import make_request, make_sydent
+
+
+class TestRequestCode(unittest.TestCase):
+    def setUp(self):
+        # Create a new sydent
+        config = {
+            "general": {
+                "templates.path": os.path.join(os.path.dirname(os.path.dirname(__file__)), "res"),
+            },
+        }
+        self.sydent = make_sydent(test_config=config)
+
+    def _render_request(self, request):
+        # Patch out the email sending so we can investigate the resulting email.
+        with patch("sydent.util.emailutils.smtplib") as smtplib:
+            request.render(self.sydent.servlets.emailRequestCode)
+
+        # Fish out the SMTP object and return it.
+        smtp = smtplib.SMTP.return_value
+        smtp.sendmail.assert_called_once()
+
+        return smtp
+
+    def test_request_code(self):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor, "POST", "/_matrix/identity/v1/validate/email/requestToken",
+            {
+                "email": "test@test",
+                "client_secret": "oursecret",
+                "send_attempt": 0,
+            }
+        )
+        smtp = self._render_request(request)
+        self.assertEqual(channel.code, 200)
+
+        # Ensure the email is as expected.
+        email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
+        self.assertIn("Confirm your email address for Matrix", email_contents)
+
+    def test_branded_request_code(self):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor, "POST", "/_matrix/identity/v1/validate/email/requestToken?brand=vector-im",
+            {
+                "email": "test@test",
+                "client_secret": "oursecret",
+                "send_attempt": 0,
+            }
+        )
+        smtp = self._render_request(request)
+        self.assertEqual(channel.code, 200)
+
+        # Ensure the email is as expected.
+        email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
+        self.assertIn("Confirm your email address for Element", email_contents)

tests/utils.py

@@ -201,6 +201,8 @@ def make_request(
     if not path.startswith(b"/"):
         path = b"/" + path
 
+    if isinstance(content, dict):
+        content = json.dumps(content)
     if isinstance(content, text_type):
         content = content.encode("utf8")
 
@@ -240,8 +242,7 @@ class ToTwistedHandler(logging.Handler):
         log_entry = self.format(record)
         log_level = record.levelname.lower().replace("warning", "warn")
         self.tx_log.emit(
-            twisted.logger.LogLevel.levelWithName(log_level),
-            log_entry.replace("{", r"(").replace("}", r")"),
+            twisted.logger.LogLevel.levelWithName(log_level), "{entry}", entry=log_entry
         )