|
@@ -192,6 +192,41 @@ class SQLBaseStore(object):
|
|
|
|
|
|
self.database_engine = hs.database_engine
|
|
|
|
|
|
+ # A set of tables that are not safe to use native upserts in.
|
|
|
+ self._unsafe_to_upsert_tables = {"user_ips"}
|
|
|
+
|
|
|
+ if self.database_engine.can_native_upsert:
|
|
|
+ # Check ASAP (and then later, every 1s) to see if we have finished
|
|
|
+ # background updates of tables that aren't safe to update.
|
|
|
+ self._clock.call_later(0.0, self._check_safe_to_upsert)
|
|
|
+
|
|
|
+ @defer.inlineCallbacks
|
|
|
+ def _check_safe_to_upsert(self):
|
|
|
+ """
|
|
|
+ Is it safe to use native UPSERT?
|
|
|
+
|
|
|
+ If there are background updates, we will need to wait, as they may be
|
|
|
+ the addition of indexes that set the UNIQUE constraint that we require.
|
|
|
+
|
|
|
+ If the background updates have not completed, wait a second and check again.
|
|
|
+ """
|
|
|
+ updates = yield self._simple_select_list(
|
|
|
+ "background_updates",
|
|
|
+ keyvalues=None,
|
|
|
+ retcols=["update_name"],
|
|
|
+ desc="check_background_updates",
|
|
|
+ )
|
|
|
+ updates = [x["update_name"] for x in updates]
|
|
|
+
|
|
|
+ # The User IPs table in schema #53 was missing a unique index, which we
|
|
|
+ # run as a background update.
|
|
|
+ if "user_ips_device_unique_index" not in updates:
|
|
|
+ self._unsafe_to_upsert_tables.discard("user_id")
|
|
|
+
|
|
|
+ # If there's any tables left to check, reschedule to run.
|
|
|
+ if self._unsafe_to_upsert_tables:
|
|
|
+ self._clock.call_later(1.0, self._check_safe_to_upsert)
|
|
|
+
|
|
|
def start_profiling(self):
|
|
|
self._previous_loop_ts = self._clock.time_msec()
|
|
|
|
|
@@ -494,8 +529,15 @@ class SQLBaseStore(object):
|
|
|
txn.executemany(sql, vals)
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
- def _simple_upsert(self, table, keyvalues, values,
|
|
|
- insertion_values={}, desc="_simple_upsert", lock=True):
|
|
|
+ def _simple_upsert(
|
|
|
+ self,
|
|
|
+ table,
|
|
|
+ keyvalues,
|
|
|
+ values,
|
|
|
+ insertion_values={},
|
|
|
+ desc="_simple_upsert",
|
|
|
+ lock=True
|
|
|
+ ):
|
|
|
"""
|
|
|
|
|
|
`lock` should generally be set to True (the default), but can be set
|
|
@@ -516,16 +558,21 @@ class SQLBaseStore(object):
|
|
|
inserting
|
|
|
lock (bool): True to lock the table when doing the upsert.
|
|
|
Returns:
|
|
|
- Deferred(bool): True if a new entry was created, False if an
|
|
|
- existing one was updated.
|
|
|
+ Deferred(None or bool): Native upserts always return None. Emulated
|
|
|
+ upserts return True if a new entry was created, False if an existing
|
|
|
+ one was updated.
|
|
|
"""
|
|
|
attempts = 0
|
|
|
while True:
|
|
|
try:
|
|
|
result = yield self.runInteraction(
|
|
|
desc,
|
|
|
- self._simple_upsert_txn, table, keyvalues, values, insertion_values,
|
|
|
- lock=lock
|
|
|
+ self._simple_upsert_txn,
|
|
|
+ table,
|
|
|
+ keyvalues,
|
|
|
+ values,
|
|
|
+ insertion_values,
|
|
|
+ lock=lock,
|
|
|
)
|
|
|
defer.returnValue(result)
|
|
|
except self.database_engine.module.IntegrityError as e:
|
|
@@ -537,12 +584,59 @@ class SQLBaseStore(object):
|
|
|
|
|
|
# presumably we raced with another transaction: let's retry.
|
|
|
logger.warn(
|
|
|
- "IntegrityError when upserting into %s; retrying: %s",
|
|
|
- table, e
|
|
|
+ "%s when upserting into %s; retrying: %s", e.__name__, table, e
|
|
|
)
|
|
|
|
|
|
- def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={},
|
|
|
- lock=True):
|
|
|
+ def _simple_upsert_txn(
|
|
|
+ self,
|
|
|
+ txn,
|
|
|
+ table,
|
|
|
+ keyvalues,
|
|
|
+ values,
|
|
|
+ insertion_values={},
|
|
|
+ lock=True,
|
|
|
+ ):
|
|
|
+ """
|
|
|
+ Pick the UPSERT method which works best on the platform. Either the
|
|
|
+ native one (Pg9.5+, recent SQLites), or fall back to an emulated method.
|
|
|
+
|
|
|
+ Args:
|
|
|
+ txn: The transaction to use.
|
|
|
+ table (str): The table to upsert into
|
|
|
+ keyvalues (dict): The unique key tables and their new values
|
|
|
+ values (dict): The nonunique columns and their new values
|
|
|
+ insertion_values (dict): additional key/values to use only when
|
|
|
+ inserting
|
|
|
+ lock (bool): True to lock the table when doing the upsert.
|
|
|
+ Returns:
|
|
|
+ Deferred(None or bool): Native upserts always return None. Emulated
|
|
|
+ upserts return True if a new entry was created, False if an existing
|
|
|
+ one was updated.
|
|
|
+ """
|
|
|
+ if (
|
|
|
+ self.database_engine.can_native_upsert
|
|
|
+ and table not in self._unsafe_to_upsert_tables
|
|
|
+ ):
|
|
|
+ return self._simple_upsert_txn_native_upsert(
|
|
|
+ txn,
|
|
|
+ table,
|
|
|
+ keyvalues,
|
|
|
+ values,
|
|
|
+ insertion_values=insertion_values,
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ return self._simple_upsert_txn_emulated(
|
|
|
+ txn,
|
|
|
+ table,
|
|
|
+ keyvalues,
|
|
|
+ values,
|
|
|
+ insertion_values=insertion_values,
|
|
|
+ lock=lock,
|
|
|
+ )
|
|
|
+
|
|
|
+ def _simple_upsert_txn_emulated(
|
|
|
+ self, txn, table, keyvalues, values, insertion_values={}, lock=True
|
|
|
+ ):
|
|
|
# We need to lock the table :(, unless we're *really* careful
|
|
|
if lock:
|
|
|
self.database_engine.lock_table(txn, table)
|
|
@@ -577,12 +671,44 @@ class SQLBaseStore(object):
|
|
|
sql = "INSERT INTO %s (%s) VALUES (%s)" % (
|
|
|
table,
|
|
|
", ".join(k for k in allvalues),
|
|
|
- ", ".join("?" for _ in allvalues)
|
|
|
+ ", ".join("?" for _ in allvalues),
|
|
|
)
|
|
|
txn.execute(sql, list(allvalues.values()))
|
|
|
# successfully inserted
|
|
|
return True
|
|
|
|
|
|
+ def _simple_upsert_txn_native_upsert(
|
|
|
+ self, txn, table, keyvalues, values, insertion_values={}
|
|
|
+ ):
|
|
|
+ """
|
|
|
+ Use the native UPSERT functionality in recent PostgreSQL versions.
|
|
|
+
|
|
|
+ Args:
|
|
|
+ table (str): The table to upsert into
|
|
|
+ keyvalues (dict): The unique key tables and their new values
|
|
|
+ values (dict): The nonunique columns and their new values
|
|
|
+ insertion_values (dict): additional key/values to use only when
|
|
|
+ inserting
|
|
|
+ Returns:
|
|
|
+ None
|
|
|
+ """
|
|
|
+ allvalues = {}
|
|
|
+ allvalues.update(keyvalues)
|
|
|
+ allvalues.update(values)
|
|
|
+ allvalues.update(insertion_values)
|
|
|
+
|
|
|
+ sql = (
|
|
|
+ "INSERT INTO %s (%s) VALUES (%s) "
|
|
|
+ "ON CONFLICT (%s) DO UPDATE SET %s"
|
|
|
+ ) % (
|
|
|
+ table,
|
|
|
+ ", ".join(k for k in allvalues),
|
|
|
+ ", ".join("?" for _ in allvalues),
|
|
|
+ ", ".join(k for k in keyvalues),
|
|
|
+ ", ".join(k + "=EXCLUDED." + k for k in values),
|
|
|
+ )
|
|
|
+ txn.execute(sql, list(allvalues.values()))
|
|
|
+
|
|
|
def _simple_select_one(self, table, keyvalues, retcols,
|
|
|
allow_none=False, desc="_simple_select_one"):
|
|
|
"""Executes a SELECT query on the named table, which is expected to
|