Support branded templates. (#328)

This allows a request to specify a brand hint which sydent attempts
to use to render different email templates.

changelog.d/328.feature

@@ -0,0 +1 @@
+Move templates to a per-brand subdirectory of /res. Add templates.path and brand.default config options

matrix_is_test/launcher.py

@@ -25,7 +25,6 @@ CFG_TEMPLATE = """
 clientapi.http.bind_address = localhost
 clientapi.http.port = {port}
 client_http_base = http://localhost:{port}
-verify_response_template = {testsubject_path}/res/verify_response_template
 federation.verifycerts = False
 
 [db]
@@ -34,16 +33,16 @@ db.file = :memory:
 [general]
 server.name = test.local
 terms.path = {terms_path}
+templates.path = {testsubject_path}/res
+brand.default = is-test
 
 [email]
 email.tlsmode = 0
-email.template = {testsubject_path}/res/verification_template.eml
 email.invite.subject = %(sender_display_name)s has invited you to chat
 email.smtphost = localhost
 email.from = Sydent Validation <noreply@localhost>
 email.smtpport = 9925
 email.subject = Your Validation Token
-email.invite_template = {testsubject_path}/res/invite_template.eml
 """
 
 class MatrixIsTestLauncher(object):

sydent/http/servlets/emailservlet.py

@@ -57,6 +57,7 @@ class EmailRequestCodeServlet(Resource):
             }
 
         ipaddress = self.sydent.ip_from_request(request)
+        brand = self.sydent.brand_from_request(request)
 
         nextLink = None
         if 'next_link' in args and not args['next_link'].startswith("file:///"):
@@ -64,7 +65,7 @@ class EmailRequestCodeServlet(Resource):
 
         try:
             sid = self.sydent.validators.email.requestToken(
-                email, clientSecret, sendAttempt, nextLink, ipaddress=ipaddress
+                email, clientSecret, sendAttempt, nextLink, ipaddress=ipaddress, brand=brand,
             )
             resp = {'sid': str(sid)}
         except EmailAddressException:
@@ -105,7 +106,12 @@ class EmailValidateCodeServlet(Resource):
         else:
             msg = "Verification failed: you may need to request another verification email"
 
-        templateFile = self.sydent.cfg.get('http', 'verify_response_template')
+        brand = self.sydent.brand_from_request(request)
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "verify_response_template.html",
+            ('http', 'verify_response_template'),
+        )
 
         request.setHeader("Content-Type", "text/html")
         res = open(templateFile).read() % {'message': msg}

sydent/http/servlets/msisdnservlet.py

@@ -79,9 +79,10 @@ class MsisdnRequestCodeServlet(Resource):
             phone_number_object, phonenumbers.PhoneNumberFormat.INTERNATIONAL
         )
 
+        brand = self.sydent.brand_from_request(request)
         try:
             sid = self.sydent.validators.msisdn.requestToken(
-                phone_number_object, clientSecret, sendAttempt
+                phone_number_object, clientSecret, sendAttempt, brand
             )
             resp = {
                 'success': True, 'sid': str(sid),
@@ -127,7 +128,12 @@ class MsisdnValidateCodeServlet(Resource):
                 request.setResponseCode(400)
                 msg = "Verification failed: you may need to request another verification text"
 
-        templateFile = self.sydent.cfg.get('http', 'verify_response_template')
+        brand = self.sydent.brand_from_request(request)
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "verify_response_template.html",
+            ('http', 'verify_response_template'),
+        )
 
         request.setHeader("Content-Type", "text/html")
         return open(templateFile).read() % {'message': msg}

sydent/http/servlets/store_invite_servlet.py

@@ -113,7 +113,14 @@ class StoreInviteServlet(Resource):
         subject_header = Header(self.sydent.cfg.get('email', 'email.invite.subject', raw=True) % substitutions, 'utf8')
         substitutions["subject_header_value"] = subject_header.encode()
 
-        sendEmail(self.sydent, "email.invite_template", address, substitutions)
+        brand = self.sydent.brand_from_request(request)
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "invite_template.eml",
+            ('email', 'email.invite_template'),
+        )
+
+        sendEmail(self.sydent, templateFile, address, substitutions)
 
         pubKey = self.sydent.keyring.ed25519.verify_key
         pubKeyBase64 = encode_base64(pubKey.encode())

sydent/sydent.py

@@ -85,6 +85,17 @@ CONFIG_DEFAULTS = {
         'terms.path': '',
         'address_lookup_limit': '10000',  # Maximum amount of addresses in a single /lookup request
 
+        # The root path to use for load templates. This should contain branded
+        # directories. Each directory should contain the following templates:
+        #
+        # * invite_template.eml
+        # * verification_template.eml
+        # * verify_response_template.html
+        'templates.path': 'res',
+        # The brand directory to use if no brand hint (or an invalid brand hint)
+        # is provided by the request.
+        'brand.default': 'matrix-org',
+
         # The following can be added to your local config file to enable prometheus
         # support.
         # 'prometheus_port': '8080',  # The port to serve metrics on
@@ -111,12 +122,18 @@ CONFIG_DEFAULTS = {
         'replication.https.port': '4434',
         'obey_x_forwarded_for': 'False',
         'federation.verifycerts': 'True',
-        'verify_response_template': '',
+        # verify_response_template is deprecated, but still used if defined Define
+        # templates.path and brand.default under general instead.
+        #
+        # 'verify_response_template': 'res/verify_response_page_template',
         'client_http_base': '',
     },
     'email': {
-        'email.template': 'res/email.template',
-        'email.invite_template': 'res/invite.template',
+        # email.template and email.invite_template are deprecated, but still used
+        # if defined. Define templates.path and brand.default under general instead.
+        #
+        # 'email.template': 'res/verification_template.eml',
+        # 'email.invite_template': 'res/invite_template.eml',
         'email.from': 'Sydent Validation <noreply@{hostname}>',
         'email.subject': 'Your Validation Token',
         'email.invite.subject': '%(sender_display_name)s has invited you to chat',
@@ -203,6 +220,19 @@ class Sydent:
                 addr=self.cfg.get("general", "prometheus_addr"),
             )
 
+        if self.cfg.has_option("general", "templates.path"):
+            # Get the possible brands by looking at directories under the
+            # templates.path directory.
+            root_template_path = self.cfg.get("general", "templates.path")
+            if os.path.exists(root_template_path):
+                self.valid_brands = {
+                    p for p in os.listdir(root_template_path) if os.path.isdir(os.path.join(root_template_path, p))
+                }
+            else:
+                # This is a legacy code-path and assumes that verify_response_template,
+                # email.template, and email.invite_template are defined.
+                self.valid_brands = set()
+
         self.enable_v1_associations = parse_cfg_bool(
             self.cfg.get("general", "enable_v1_associations")
         )
@@ -323,6 +353,57 @@ class Sydent:
             return request.requestHeaders.getRawHeaders("X-Forwarded-For")[0]
         return request.getClientIP()
 
+    def brand_from_request(self, request):
+        """
+        If the brand GET parameter is passed, returns that as a string, otherwise returns None.
+
+        :param request: The incoming request.
+        :type request: twisted.web.http.Request
+
+        :return: The brand to use or None if no hint is found.
+        :rtype: str or None
+        """
+        if b"brand" in request.args:
+            return request.args[b"brand"][0].decode("utf-8")
+        return None
+
+    def get_branded_template(self, brand, template_name, deprecated_template_name):
+        """
+        Calculate a (maybe) branded template filename to use.
+
+        If the deprecated email.template setting is defined, always use it.
+        Otherwise, attempt to use the hinted brand from the request if the brand
+        is valid. Otherwise, fallback to the default brand.
+
+        :param brand: The hint of which brand to use.
+        :type brand: str or None
+        :param template_name: The name of the template file to load.
+        :type template_name: str
+        :param deprecated_template_name: The deprecated setting to use, if provided.
+        :type deprecated_template_name: Tuple[str]
+
+        :return: The template filename to use.
+        :rtype: str
+        """
+
+        # If the deprecated setting is defined, return it.
+        try:
+            return self.cfg.get(*deprecated_template_name)
+        except configparser.NoOptionError:
+            pass
+
+        # If a brand hint is provided, attempt to use it if it is valid.
+        if brand:
+            if brand not in self.valid_brands:
+                brand = None
+
+        # If the brand hint is not valid, or not provided, fallback to the default brand.
+        if not brand:
+            brand = self.cfg.get("general", "brand.default")
+
+        root_template_path = self.cfg.get("general", "templates.path")
+        return os.path.join(root_template_path, brand, template_name)
+
 
 class Validators:
     pass

sydent/util/emailutils.py

@@ -38,23 +38,22 @@ from sydent.util import time_msec
 logger = logging.getLogger(__name__)
 
 
-def sendEmail(sydent, templateName, mailTo, substitutions):
+def sendEmail(sydent, templateFile, mailTo, substitutions):
     """
     Sends an email with the given parameters.
 
     :param sydent: The Sydent instance to use when building the configuration to send the
         email with.
     :type sydent: sydent.sydent.Sydent
-    :param templateName: The name of the template to use when building the body of the
+    :param templateFile: The filename of the template to use when building the body of the
         email.
-    :type templateName: str
+    :type templateFile: str
     :param mailTo: The email address to send the email to.
     :type mailTo: unicode
     :param substitutions: The substitutions to use with the template.
     :type substitutions: dict[str, str]
     """
     mailFrom = sydent.cfg.get('email', 'email.from')
-    mailTemplateFile = sydent.cfg.get('email', templateName)
 
     myHostname = sydent.cfg.get('email', 'email.hostname')
     if myHostname == '':
@@ -75,7 +74,7 @@ def sendEmail(sydent, templateName, mailTo, substitutions):
         allSubstitutions[k+"_forhtml"] = escape(v)
         allSubstitutions[k+"_forurl"] = urllib.parse.quote(v)
 
-    mailString = open(mailTemplateFile).read() % allSubstitutions
+    mailString = open(templateFile).read() % allSubstitutions
     parsedFrom = email.utils.parseaddr(mailFrom)[1]
     parsedTo = email.utils.parseaddr(mailTo)[1]
     if parsedFrom == '' or parsedTo == '':

sydent/validators/emailvalidator.py

@@ -31,7 +31,7 @@ class EmailValidator:
     def __init__(self, sydent):
         self.sydent = sydent
 
-    def requestToken(self, emailAddress, clientSecret, sendAttempt, nextLink, ipaddress=None):
+    def requestToken(self, emailAddress, clientSecret, sendAttempt, nextLink, ipaddress=None, brand=None):
         """
         Creates or retrieves a validation session and sends an email to the corresponding
         email address with a token to use to verify the association.
@@ -47,6 +47,8 @@ class EmailValidator:
         :type nextLink: unicode
         :param ipaddress: The requester's IP address.
         :type ipaddress: str or None
+        :param brand: A hint at a brand from the request.
+        :type brand: str or None
 
         :return: The ID of the session created (or of the existing one if any)
         :rtype: int
@@ -58,6 +60,12 @@ class EmailValidator:
 
         valSessionStore.setMtime(valSession.id, time_msec())
 
+        templateFile = self.sydent.get_branded_template(
+            brand,
+            "verification_template.eml",
+            ('email', 'email.template'),
+        )
+
         if int(valSession.sendAttemptNumber) >= int(sendAttempt):
             logger.info("Not mailing code because current send attempt (%d) is not less than given send attempt (%s)", int(sendAttempt), int(valSession.sendAttemptNumber))
             return valSession.id
@@ -73,7 +81,7 @@ class EmailValidator:
             "Attempting to mail code %s (nextLink: %s) to %s",
             valSession.token, nextLink, emailAddress,
         )
-        sendEmail(self.sydent, 'email.template', emailAddress, substitutions)
+        sendEmail(self.sydent, templateFile, emailAddress, substitutions)
 
         valSessionStore.setSendAttemptNumber(valSession.id, sendAttempt)
 

sydent/validators/msisdnvalidator.py

@@ -64,7 +64,7 @@ class MsisdnValidator:
 
                 self.smsRules[country] = action
 
-    def requestToken(self, phoneNumber, clientSecret, sendAttempt):
+    def requestToken(self, phoneNumber, clientSecret, sendAttempt, brand=None):
         """
         Creates or retrieves a validation session and sends an text message to the
         corresponding phone number address with a token to use to verify the association.
@@ -75,6 +75,8 @@ class MsisdnValidator:
         :type clientSecret: unicode
         :param sendAttempt: The current send attempt.
         :type sendAttempt: int
+        :param brand: A hint at a brand from the request.
+        :type brand: str or None
 
         :return: The ID of the session created (or of the existing one if any)
         :rtype: int

tests/test_email.py

@@ -0,0 +1,82 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os.path
+from mock import Mock, patch
+
+from twisted.web.client import Response
+from twisted.trial import unittest
+
+from sydent.db.invite_tokens import JoinTokenStore
+from sydent.http.httpclient import FederationHttpClient
+from sydent.http.servlets.store_invite_servlet import StoreInviteServlet
+from tests.utils import make_request, make_sydent
+
+
+class TestRequestCode(unittest.TestCase):
+    def setUp(self):
+        # Create a new sydent
+        config = {
+            "general": {
+                "templates.path": os.path.join(os.path.dirname(os.path.dirname(__file__)), "res"),
+            },
+        }
+        self.sydent = make_sydent(test_config=config)
+
+    def _render_request(self, request):
+        # Patch out the email sending so we can investigate the resulting email.
+        with patch("sydent.util.emailutils.smtplib") as smtplib:
+            request.render(self.sydent.servlets.emailRequestCode)
+
+        # Fish out the SMTP object and return it.
+        smtp = smtplib.SMTP.return_value
+        smtp.sendmail.assert_called_once()
+
+        return smtp
+
+    def test_request_code(self):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor, "POST", "/_matrix/identity/v1/validate/email/requestToken",
+            {
+                "email": "test@test",
+                "client_secret": "oursecret",
+                "send_attempt": 0,
+            }
+        )
+        smtp = self._render_request(request)
+        self.assertEqual(channel.code, 200)
+
+        # Ensure the email is as expected.
+        email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
+        self.assertIn("Confirm your email address for Matrix", email_contents)
+
+    def test_branded_request_code(self):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor, "POST", "/_matrix/identity/v1/validate/email/requestToken?brand=vector-im",
+            {
+                "email": "test@test",
+                "client_secret": "oursecret",
+                "send_attempt": 0,
+            }
+        )
+        smtp = self._render_request(request)
+        self.assertEqual(channel.code, 200)
+
+        # Ensure the email is as expected.
+        email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
+        self.assertIn("Confirm your email address for Element", email_contents)

tests/utils.py

@@ -201,6 +201,8 @@ def make_request(
     if not path.startswith(b"/"):
         path = b"/" + path
 
+    if isinstance(content, dict):
+        content = json.dumps(content)
     if isinstance(content, text_type):
         content = content.encode("utf8")