Merge remote-tracking branch 'origin/master'

changelog.d/337.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where invalid JSON would be accepted over the HTTP interfaces.

sydent/http/httpclient.py

@@ -27,6 +27,7 @@ from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper
 from sydent.http.matrixfederationagent import MatrixFederationAgent
 from sydent.http.federation_tls_options import ClientTLSOptionsFactory
 from sydent.http.httpcommon import BodyExceededMaxSize, read_body_with_max_size
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +58,7 @@ class HTTPClient(object):
         body = yield read_body_with_max_size(response, max_size)
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            json_body = json.loads(body.decode("UTF-8"))
+            json_body = json_decoder.decode(body.decode("UTF-8"))
         except Exception as e:
             logger.exception("Error parsing JSON from %s", uri)
             raise

sydent/http/matrixfederationagent.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 from __future__ import absolute_import
 
-import json
 import logging
 import random
 import time
@@ -33,6 +32,7 @@ from twisted.web.iweb import IAgent
 
 from sydent.http.httpcommon import BodyExceededMaxSize, read_body_with_max_size
 from sydent.http.srvresolver import SrvResolver, pick_server_from_list
+from sydent.util import json_decoder
 from sydent.util.ttlcache import TTLCache
 
 # period to cache .well-known results for by default
@@ -324,7 +324,7 @@ class MatrixFederationAgent(object):
             if response.code != 200:
                 raise Exception("Non-200 response %s" % (response.code, ))
 
-            parsed_body = json.loads(body.decode('utf-8'))
+            parsed_body = json_decoder.decode(body.decode('utf-8'))
             logger.info("Response from .well-known: %s", parsed_body)
             if not isinstance(parsed_body, dict):
                 raise Exception("not a dict")
@@ -444,4 +444,4 @@ class _RoutingResult(object):
     The port we should route the TCP connection to (the target of the SRV record, or
     the port from the URL/.well-known, or 8448)
     :type: int
-    """
+    """

sydent/http/servlets/__init__.py

@@ -22,6 +22,7 @@ import functools
 from twisted.internet import defer
 from twisted.web import server
 
+from sydent.util import json_decoder
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +77,7 @@ def get_args(request, args, required=True):
     ):
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            request_args = json.loads(request.content.read().decode("UTF-8"))
+            request_args = json_decoder.decode(request.content.read().decode("UTF-8"))
         except ValueError:
             raise MatrixRestError(400, 'M_BAD_JSON', 'Malformed JSON')
 

sydent/http/servlets/lookupservlet.py

@@ -20,10 +20,10 @@ from twisted.web.resource import Resource
 from sydent.db.threepid_associations import GlobalAssociationStore
 
 import logging
-import json
 import signedjson.sign
 
 from sydent.http.servlets import get_args, jsonwrap, send_cors, MatrixRestError
+from sydent.util import json_decoder
 
 
 logger = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ class LookupServlet(Resource):
         Look up an individual threepid.
 
         ** DEPRECATED **
-        
+
         Params: 'medium': the medium of the threepid
                 'address': the address of the threepid
         Returns: A signed association if the threepid has a corresponding mxid, otherwise the empty object.
@@ -60,7 +60,7 @@ class LookupServlet(Resource):
         if not sgassoc:
             return {}
 
-        sgassoc = json.loads(sgassoc)
+        sgassoc = json_decoder.decode(sgassoc)
         if not self.sydent.server_name in sgassoc['signatures']:
             # We have not yet worked out what the proper trust model should be.
             #

sydent/http/servlets/replication.py

@@ -19,6 +19,7 @@ import twisted.python.log
 from twisted.web.resource import Resource
 from sydent.http.servlets import jsonwrap, MatrixRestError
 from sydent.threepid import threePidAssocFromDict
+from sydent.util import json_decoder
 
 from sydent.util.hash import sha256_and_url_safe_base64
 
@@ -60,7 +61,7 @@ class ReplicationPushServlet(Resource):
 
         try:
             # json.loads doesn't allow bytes in Python 3.5
-            inJson = json.loads(request.content.read().decode("UTF-8"))
+            inJson = json_decoder.decode(request.content.read().decode("UTF-8"))
         except ValueError:
             logger.warn("Peer %s made push connection with malformed JSON", peer.servername)
             raise MatrixRestError(400, 'M_BAD_JSON', 'Malformed JSON')

sydent/http/servlets/threepidunbindservlet.py

@@ -24,6 +24,7 @@ from signedjson.sign import SignatureVerifyException
 
 from sydent.http.servlets import dict_to_json_bytes
 from sydent.db.valsession import ThreePidValSessionStore
+from sydent.util import json_decoder
 from sydent.util.stringutils import is_valid_client_secret
 from sydent.validators import (
     IncorrectClientSecretException,
@@ -51,7 +52,7 @@ class ThreePidUnbindServlet(Resource):
         try:
             try:
                 # json.loads doesn't allow bytes in Python 3.5
-                body = json.loads(request.content.read().decode("UTF-8"))
+                body = json_decoder.decode(request.content.read().decode("UTF-8"))
             except ValueError:
                 request.setResponseCode(400)
                 request.write(dict_to_json_bytes({'errcode': 'M_BAD_JSON', 'error': 'Malformed JSON'}))

sydent/replication/peer.py

@@ -22,6 +22,7 @@ from sydent.db.threepid_associations import GlobalAssociationStore
 from sydent.db.hashing_metadata import HashingMetadataStore
 from sydent.threepid import threePidAssocFromDict
 from sydent.config import ConfigError
+from sydent.util import json_decoder
 from sydent.util.hash import sha256_and_url_safe_base64
 from unpaddedbase64 import decode_base64
 
@@ -241,7 +242,7 @@ class RemotePeer(Peer):
         :param updateDeferred: The deferred to call the error callback of.
         :type updateDeferred: twisted.internet.defer.Deferred
         """
-        errObj = json.loads(body.decode("utf8"))
+        errObj = json_decoder.decode(body.decode("utf8"))
         e = RemotePeerError()
         e.errorDict = errObj
         updateDeferred.errback(e)

sydent/util/__init__.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import time
 
-
 def time_msec():
     """
     Get the current time in milliseconds.
@@ -25,3 +25,12 @@ def time_msec():
     :rtype: int
     """
     return int(time.time() * 1000)
+
+
+def _reject_invalid_json(val):
+    """Do not allow Infinity, -Infinity, or NaN values in JSON."""
+    raise ValueError("Invalid JSON value: '%s'" % val)
+
+
+# a custom JSON decoder which will reject Python extensions to JSON.
+json_decoder = json.JSONDecoder(parse_constant=_reject_invalid_json)