postgres.html 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413
  1. <!DOCTYPE HTML>
  2. <html lang="en" class="sidebar-visible no-js light">
  3. <head>
  4. <!-- Book generated using mdBook -->
  5. <meta charset="UTF-8">
  6. <title>Using Postgres - Synapse</title>
  7. <!-- Custom HTML head -->
  8. <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
  9. <meta name="description" content="">
  10. <meta name="viewport" content="width=device-width, initial-scale=1">
  11. <meta name="theme-color" content="#ffffff" />
  12. <link rel="icon" href="favicon.svg">
  13. <link rel="shortcut icon" href="favicon.png">
  14. <link rel="stylesheet" href="css/variables.css">
  15. <link rel="stylesheet" href="css/general.css">
  16. <link rel="stylesheet" href="css/chrome.css">
  17. <link rel="stylesheet" href="css/print.css" media="print">
  18. <!-- Fonts -->
  19. <link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
  20. <link rel="stylesheet" href="fonts/fonts.css">
  21. <!-- Highlight.js Stylesheets -->
  22. <link rel="stylesheet" href="highlight.css">
  23. <link rel="stylesheet" href="tomorrow-night.css">
  24. <link rel="stylesheet" href="ayu-highlight.css">
  25. <!-- Custom theme stylesheets -->
  26. <link rel="stylesheet" href="docs/website_files/table-of-contents.css">
  27. <link rel="stylesheet" href="docs/website_files/remove-nav-buttons.css">
  28. <link rel="stylesheet" href="docs/website_files/indent-section-headers.css">
  29. <link rel="stylesheet" href="docs/website_files/version-picker.css">
  30. </head>
  31. <body>
  32. <!-- Provide site root to javascript -->
  33. <script type="text/javascript">
  34. var path_to_root = "";
  35. var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
  36. </script>
  37. <!-- Work around some values being stored in localStorage wrapped in quotes -->
  38. <script type="text/javascript">
  39. try {
  40. var theme = localStorage.getItem('mdbook-theme');
  41. var sidebar = localStorage.getItem('mdbook-sidebar');
  42. if (theme.startsWith('"') && theme.endsWith('"')) {
  43. localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
  44. }
  45. if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
  46. localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
  47. }
  48. } catch (e) { }
  49. </script>
  50. <!-- Set the theme before any content is loaded, prevents flash -->
  51. <script type="text/javascript">
  52. var theme;
  53. try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
  54. if (theme === null || theme === undefined) { theme = default_theme; }
  55. var html = document.querySelector('html');
  56. html.classList.remove('no-js')
  57. html.classList.remove('light')
  58. html.classList.add(theme);
  59. html.classList.add('js');
  60. </script>
  61. <!-- Hide / unhide sidebar before it is displayed -->
  62. <script type="text/javascript">
  63. var html = document.querySelector('html');
  64. var sidebar = 'hidden';
  65. if (document.body.clientWidth >= 1080) {
  66. try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
  67. sidebar = sidebar || 'visible';
  68. }
  69. html.classList.remove('sidebar-visible');
  70. html.classList.add("sidebar-" + sidebar);
  71. </script>
  72. <nav id="sidebar" class="sidebar" aria-label="Table of contents">
  73. <div class="sidebar-scrollbox">
  74. <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="postgres.html" class="active">Using Postgres</a></li><li class="chapter-item expanded "><a href="reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="turn-howto.html">Configuring a Turn Server</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="setup/turn/coturn.html">coturn TURN server</a></li><li class="chapter-item expanded "><a href="setup/turn/eturnal.html">eturnal TURN server</a></li></ol></li><li class="chapter-item expanded "><a href="delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="federate.html">Federation</a></li><li class="chapter-item expanded "><a href="usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/config_documentation.html">Configuration Manual</a></li><li class="chapter-item expanded "><a href="usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="templates.html">Templates</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="jwt.html">JSON Web Tokens</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/refresh_tokens.html">Refresh Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_data_callbacks.html">Account data callbacks</a></li><li class="chapter-item expanded "><a href="modules/add_extra_fields_to_client_events_unsigned.html">Add extra fields to client events unsigned section callbacks</a></li><li class="chapter-item expanded "><a href="modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="admin_api/experimental_features.html">Experimental Features</a></li><li class="chapter-item expanded "><a href="admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="metrics-howto.html">Monitoring</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/administration/monitoring/reporting_homeserver_usage_statistics.html">Reporting Homeserver Usage Statistics</a></li></ol></li><li class="chapter-item expanded "><a href="usage/administration/monthly_active_users.html">Monthly Active Users</a></li><li class="chapter-item expanded "><a href="usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="development/reviews.html">Reviewing Code</a></li><li class="chapter-item expanded "><a href="development/releases.html">Release Cycle</a></li><li class="chapter-item expanded "><a href="development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/demo.html">Demo scripts</a></li></ol></li><li class="chapter-item expanded "><a href="opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><a href="development/dependencies.html">Dependency management</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/synapse_architecture/cancellation.html">Cancellation</a></li><li class="chapter-item expanded "><a href="log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="replication.html">Replication</a></li><li class="chapter-item expanded "><a href="development/synapse_architecture/streams.html">Streams</a></li><li class="chapter-item expanded "><a href="tcp_replication.html">TCP Replication</a></li><li class="chapter-item expanded "><a href="development/synapse_architecture/faster_joins.html">Faster remote joins</a></li></ol></li><li class="chapter-item expanded "><a href="development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol>
  75. </div>
  76. <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
  77. </nav>
  78. <div id="page-wrapper" class="page-wrapper">
  79. <div class="page">
  80. <div id="menu-bar-hover-placeholder"></div>
  81. <div id="menu-bar" class="menu-bar sticky bordered">
  82. <div class="left-buttons">
  83. <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
  84. <i class="fa fa-bars"></i>
  85. </button>
  86. <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
  87. <i class="fa fa-paint-brush"></i>
  88. </button>
  89. <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
  90. <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
  91. <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
  92. <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
  93. <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
  94. <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
  95. </ul>
  96. <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
  97. <i class="fa fa-search"></i>
  98. </button>
  99. <div class="version-picker">
  100. <div class="dropdown">
  101. <div class="select">
  102. <span></span>
  103. <i class="fa fa-chevron-down"></i>
  104. </div>
  105. <input type="hidden" name="version">
  106. <ul class="dropdown-menu">
  107. <!-- Versions will be added dynamically in version-picker.js -->
  108. </ul>
  109. </div>
  110. </div>
  111. </div>
  112. <h1 class="menu-title">Synapse</h1>
  113. <div class="right-buttons">
  114. <a href="print.html" title="Print this book" aria-label="Print this book">
  115. <i id="print-button" class="fa fa-print"></i>
  116. </a>
  117. <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository">
  118. <i id="git-repository-button" class="fa fa-github"></i>
  119. </a>
  120. <a href="https://github.com/matrix-org/synapse/edit/develop/docs/postgres.md" title="Suggest an edit" aria-label="Suggest an edit">
  121. <i id="git-edit-button" class="fa fa-edit"></i>
  122. </a>
  123. </div>
  124. </div>
  125. <div id="search-wrapper" class="hidden">
  126. <form id="searchbar-outer" class="searchbar-outer">
  127. <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
  128. </form>
  129. <div id="searchresults-outer" class="searchresults-outer hidden">
  130. <div id="searchresults-header" class="searchresults-header"></div>
  131. <ul id="searchresults">
  132. </ul>
  133. </div>
  134. </div>
  135. <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
  136. <script type="text/javascript">
  137. document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
  138. document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
  139. Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
  140. link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
  141. });
  142. </script>
  143. <div id="content" class="content">
  144. <main>
  145. <!-- Page table of contents -->
  146. <div class="sidetoc">
  147. <nav class="pagetoc"></nav>
  148. </div>
  149. <h1 id="using-postgres"><a class="header" href="#using-postgres">Using Postgres</a></h1>
  150. <p>The minimum supported version of PostgreSQL is determined by the <a href="deprecation_policy.html">Dependency
  151. Deprecation Policy</a>.</p>
  152. <h2 id="install-postgres-client-libraries"><a class="header" href="#install-postgres-client-libraries">Install postgres client libraries</a></h2>
  153. <p>Synapse will require the python postgres client library in order to
  154. connect to a postgres database.</p>
  155. <ul>
  156. <li>
  157. <p>If you are using the <a href="setup/installation.html#matrixorg-packages">matrix.org debian/ubuntu
  158. packages</a>, the necessary python
  159. library will already be installed, but you will need to ensure the
  160. low-level postgres library is installed, which you can do with
  161. <code>apt install libpq5</code>.</p>
  162. </li>
  163. <li>
  164. <p>For other pre-built packages, please consult the documentation from
  165. the relevant package.</p>
  166. </li>
  167. <li>
  168. <p>If you installed synapse <a href="setup/installation.html#installing-as-a-python-module-from-pypi">in a
  169. virtualenv</a>, you can install
  170. the library with:</p>
  171. <pre><code>~/synapse/env/bin/pip install &quot;matrix-synapse[postgres]&quot;
  172. </code></pre>
  173. <p>(substituting the path to your virtualenv for <code>~/synapse/env</code>, if
  174. you used a different path). You will require the postgres
  175. development files. These are in the <code>libpq-dev</code> package on
  176. Debian-derived distributions.</p>
  177. </li>
  178. </ul>
  179. <h2 id="set-up-database"><a class="header" href="#set-up-database">Set up database</a></h2>
  180. <p>Assuming your PostgreSQL database user is called <code>postgres</code>, first authenticate as the database user with:</p>
  181. <pre><code class="language-sh">su - postgres
  182. # Or, if your system uses sudo to get administrative rights
  183. sudo -u postgres bash
  184. </code></pre>
  185. <p>Then, create a postgres user and a database with:</p>
  186. <pre><code class="language-sh"># this will prompt for a password for the new user
  187. createuser --pwprompt synapse_user
  188. createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
  189. </code></pre>
  190. <p>The above will create a user called <code>synapse_user</code>, and a database called
  191. <code>synapse</code>.</p>
  192. <p>Note that the PostgreSQL database <em>must</em> have the correct encoding set
  193. (as shown above), otherwise it will not be able to store UTF8 strings.</p>
  194. <p>You may need to enable password authentication so <code>synapse_user</code> can
  195. connect to the database. See
  196. <a href="https://www.postgresql.org/docs/current/auth-pg-hba-conf.html">https://www.postgresql.org/docs/current/auth-pg-hba-conf.html</a>.</p>
  197. <h2 id="synapse-config"><a class="header" href="#synapse-config">Synapse config</a></h2>
  198. <p>When you are ready to start using PostgreSQL, edit the <code>database</code>
  199. section in your config file to match the following lines:</p>
  200. <pre><code class="language-yaml">database:
  201. name: psycopg2
  202. args:
  203. user: &lt;user&gt;
  204. password: &lt;pass&gt;
  205. dbname: &lt;db&gt;
  206. host: &lt;host&gt;
  207. cp_min: 5
  208. cp_max: 10
  209. </code></pre>
  210. <p>All key, values in <code>args</code> are passed to the <code>psycopg2.connect(..)</code>
  211. function, except keys beginning with <code>cp_</code>, which are consumed by the
  212. twisted adbapi connection pool. See the <a href="https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS">libpq
  213. documentation</a>
  214. for a list of options which can be passed.</p>
  215. <p>You should consider tuning the <code>args.keepalives_*</code> options if there is any danger of
  216. the connection between your homeserver and database dropping, otherwise Synapse
  217. may block for an extended period while it waits for a response from the
  218. database server. Example values might be:</p>
  219. <pre><code class="language-yaml">database:
  220. args:
  221. # ... as above
  222. # seconds of inactivity after which TCP should send a keepalive message to the server
  223. keepalives_idle: 10
  224. # the number of seconds after which a TCP keepalive message that is not
  225. # acknowledged by the server should be retransmitted
  226. keepalives_interval: 10
  227. # the number of TCP keepalives that can be lost before the client's connection
  228. # to the server is considered dead
  229. keepalives_count: 3
  230. </code></pre>
  231. <h2 id="tuning-postgres"><a class="header" href="#tuning-postgres">Tuning Postgres</a></h2>
  232. <p>The default settings should be fine for most deployments. For larger
  233. scale deployments tuning some of the settings is recommended, details of
  234. which can be found at
  235. <a href="https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server">https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server</a>.</p>
  236. <p>In particular, we've found tuning the following values helpful for
  237. performance:</p>
  238. <ul>
  239. <li><code>shared_buffers</code></li>
  240. <li><code>effective_cache_size</code></li>
  241. <li><code>work_mem</code></li>
  242. <li><code>maintenance_work_mem</code></li>
  243. <li><code>autovacuum_work_mem</code></li>
  244. </ul>
  245. <p>Note that the appropriate values for those fields depend on the amount
  246. of free memory the database host has available.</p>
  247. <p>Additionally, admins of large deployments might want to consider using huge pages
  248. to help manage memory, especially when using large values of <code>shared_buffers</code>. You
  249. can read more about that <a href="https://www.postgresql.org/docs/10/kernel-resources.html#LINUX-HUGE-PAGES">here</a>.</p>
  250. <h2 id="porting-from-sqlite"><a class="header" href="#porting-from-sqlite">Porting from SQLite</a></h2>
  251. <h3 id="overview"><a class="header" href="#overview">Overview</a></h3>
  252. <p>The script <code>synapse_port_db</code> allows porting an existing synapse server
  253. backed by SQLite to using PostgreSQL. This is done in as a two phase
  254. process:</p>
  255. <ol>
  256. <li>Copy the existing SQLite database to a separate location and run
  257. the port script against that offline database.</li>
  258. <li>Shut down the server. Rerun the port script to port any data that
  259. has come in since taking the first snapshot. Restart server against
  260. the PostgreSQL database.</li>
  261. </ol>
  262. <p>The port script is designed to be run repeatedly against newer snapshots
  263. of the SQLite database file. This makes it safe to repeat step 1 if
  264. there was a delay between taking the previous snapshot and being ready
  265. to do step 2.</p>
  266. <p>It is safe to at any time kill the port script and restart it.</p>
  267. <p>However, under no circumstances should the SQLite database be <code>VACUUM</code>ed between
  268. multiple runs of the script. Doing so can lead to an inconsistent copy of your database
  269. into Postgres.
  270. To avoid accidental error, the script will check that SQLite's <code>auto_vacuum</code> mechanism
  271. is disabled, but the script is not able to protect against a manual <code>VACUUM</code> operation
  272. performed either by the administrator or by any automated task that the administrator
  273. may have configured.</p>
  274. <p>Note that the database may take up significantly more (25% - 100% more)
  275. space on disk after porting to Postgres.</p>
  276. <h3 id="using-the-port-script"><a class="header" href="#using-the-port-script">Using the port script</a></h3>
  277. <p>Firstly, shut down the currently running synapse server and copy its
  278. database file (typically <code>homeserver.db</code>) to another location. Once the
  279. copy is complete, restart synapse. For instance:</p>
  280. <pre><code class="language-sh">synctl stop
  281. cp homeserver.db homeserver.db.snapshot
  282. synctl start
  283. </code></pre>
  284. <p>Copy the old config file into a new config file:</p>
  285. <pre><code class="language-sh">cp homeserver.yaml homeserver-postgres.yaml
  286. </code></pre>
  287. <p>Edit the database section as described in the section <em>Synapse config</em>
  288. above and with the SQLite snapshot located at <code>homeserver.db.snapshot</code>
  289. simply run:</p>
  290. <pre><code class="language-sh">synapse_port_db --sqlite-database homeserver.db.snapshot \
  291. --postgres-config homeserver-postgres.yaml
  292. </code></pre>
  293. <p>The flag <code>--curses</code> displays a coloured curses progress UI.</p>
  294. <p>If the script took a long time to complete, or time has otherwise passed
  295. since the original snapshot was taken, repeat the previous steps with a
  296. newer snapshot.</p>
  297. <p>To complete the conversion shut down the synapse server and run the port
  298. script one last time, e.g. if the SQLite database is at <code>homeserver.db</code>
  299. run:</p>
  300. <pre><code class="language-sh">synapse_port_db --sqlite-database homeserver.db \
  301. --postgres-config homeserver-postgres.yaml
  302. </code></pre>
  303. <p>Once that has completed, change the synapse config to point at the
  304. PostgreSQL database configuration file <code>homeserver-postgres.yaml</code>:</p>
  305. <pre><code class="language-sh">synctl stop
  306. mv homeserver.yaml homeserver-old-sqlite.yaml
  307. mv homeserver-postgres.yaml homeserver.yaml
  308. synctl start
  309. </code></pre>
  310. <p>Synapse should now be running against PostgreSQL.</p>
  311. <h2 id="troubleshooting"><a class="header" href="#troubleshooting">Troubleshooting</a></h2>
  312. <h3 id="alternative-auth-methods"><a class="header" href="#alternative-auth-methods">Alternative auth methods</a></h3>
  313. <p>If you get an error along the lines of <code>FATAL: Ident authentication failed for user &quot;synapse_user&quot;</code>, you may need to use an authentication method other than
  314. <code>ident</code>:</p>
  315. <ul>
  316. <li>
  317. <p>If the <code>synapse_user</code> user has a password, add the password to the <code>database:</code>
  318. section of <code>homeserver.yaml</code>. Then add the following to <code>pg_hba.conf</code>:</p>
  319. <pre><code>host synapse synapse_user ::1/128 md5 # or `scram-sha-256` instead of `md5` if you use that
  320. </code></pre>
  321. </li>
  322. <li>
  323. <p>If the <code>synapse_user</code> user does not have a password, then a password doesn't
  324. have to be added to <code>homeserver.yaml</code>. But the following does need to be added
  325. to <code>pg_hba.conf</code>:</p>
  326. <pre><code>host synapse synapse_user ::1/128 trust
  327. </code></pre>
  328. </li>
  329. </ul>
  330. <p>Note that line order matters in <code>pg_hba.conf</code>, so make sure that if you do add a
  331. new line, it is inserted before:</p>
  332. <pre><code>host all all ::1/128 ident
  333. </code></pre>
  334. <h3 id="fixing-incorrect-collate-or-ctype"><a class="header" href="#fixing-incorrect-collate-or-ctype">Fixing incorrect <code>COLLATE</code> or <code>CTYPE</code></a></h3>
  335. <p>Synapse will refuse to set up a new database if it has the wrong values of
  336. <code>COLLATE</code> and <code>CTYPE</code> set. Synapse will also refuse to start an existing database with incorrect values
  337. of <code>COLLATE</code> and <code>CTYPE</code> unless the config flag <code>allow_unsafe_locale</code>, found in the
  338. <code>database</code> section of the config, is set to true. Using different locales can cause issues if the locale library is updated from
  339. underneath the database, or if a different version of the locale is used on any
  340. replicas.</p>
  341. <p>If you have a database with an unsafe locale, the safest way to fix the issue is to dump the database and recreate it with
  342. the correct locale parameter (as shown above). It is also possible to change the
  343. parameters on a live database and run a <code>REINDEX</code> on the entire database,
  344. however extreme care must be taken to avoid database corruption.</p>
  345. <p>Note that the above may fail with an error about duplicate rows if corruption
  346. has already occurred, and such duplicate rows will need to be manually removed.</p>
  347. <h3 id="fixing-inconsistent-sequences-error"><a class="header" href="#fixing-inconsistent-sequences-error">Fixing inconsistent sequences error</a></h3>
  348. <p>Synapse uses Postgres sequences to generate IDs for various tables. A sequence
  349. and associated table can get out of sync if, for example, Synapse has been
  350. downgraded and then upgraded again.</p>
  351. <p>To fix the issue shut down Synapse (including any and all workers) and run the
  352. SQL command included in the error message. Once done Synapse should start
  353. successfully.</p>
  354. </main>
  355. <nav class="nav-wrapper" aria-label="Page navigation">
  356. <!-- Mobile navigation buttons -->
  357. <a rel="prev" href="setup/installation.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
  358. <i class="fa fa-angle-left"></i>
  359. </a>
  360. <a rel="next" href="reverse_proxy.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
  361. <i class="fa fa-angle-right"></i>
  362. </a>
  363. <div style="clear: both"></div>
  364. </nav>
  365. </div>
  366. </div>
  367. <nav class="nav-wide-wrapper" aria-label="Page navigation">
  368. <a rel="prev" href="setup/installation.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
  369. <i class="fa fa-angle-left"></i>
  370. </a>
  371. <a rel="next" href="reverse_proxy.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
  372. <i class="fa fa-angle-right"></i>
  373. </a>
  374. </nav>
  375. </div>
  376. <script type="text/javascript">
  377. window.playground_copyable = true;
  378. </script>
  379. <script src="elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
  380. <script src="mark.min.js" type="text/javascript" charset="utf-8"></script>
  381. <script src="searcher.js" type="text/javascript" charset="utf-8"></script>
  382. <script src="clipboard.min.js" type="text/javascript" charset="utf-8"></script>
  383. <script src="highlight.js" type="text/javascript" charset="utf-8"></script>
  384. <script src="book.js" type="text/javascript" charset="utf-8"></script>
  385. <!-- Custom JS scripts -->
  386. <script type="text/javascript" src="docs/website_files/table-of-contents.js"></script>
  387. <script type="text/javascript" src="docs/website_files/version-picker.js"></script>
  388. <script type="text/javascript" src="docs/website_files/version.js"></script>
  389. </body>
  390. </html>