metrics-howto.html 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. <!DOCTYPE HTML>
  2. <html lang="en" class="sidebar-visible no-js light">
  3. <head>
  4. <!-- Book generated using mdBook -->
  5. <meta charset="UTF-8">
  6. <title>Monitoring - Synapse</title>
  7. <!-- Custom HTML head -->
  8. <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
  9. <meta name="description" content="">
  10. <meta name="viewport" content="width=device-width, initial-scale=1">
  11. <meta name="theme-color" content="#ffffff" />
  12. <link rel="icon" href="favicon.svg">
  13. <link rel="shortcut icon" href="favicon.png">
  14. <link rel="stylesheet" href="css/variables.css">
  15. <link rel="stylesheet" href="css/general.css">
  16. <link rel="stylesheet" href="css/chrome.css">
  17. <link rel="stylesheet" href="css/print.css" media="print">
  18. <!-- Fonts -->
  19. <link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
  20. <link rel="stylesheet" href="fonts/fonts.css">
  21. <!-- Highlight.js Stylesheets -->
  22. <link rel="stylesheet" href="highlight.css">
  23. <link rel="stylesheet" href="tomorrow-night.css">
  24. <link rel="stylesheet" href="ayu-highlight.css">
  25. <!-- Custom theme stylesheets -->
  26. <link rel="stylesheet" href="docs/website_files/table-of-contents.css">
  27. <link rel="stylesheet" href="docs/website_files/remove-nav-buttons.css">
  28. <link rel="stylesheet" href="docs/website_files/indent-section-headers.css">
  29. <link rel="stylesheet" href="docs/website_files/version-picker.css">
  30. </head>
  31. <body>
  32. <!-- Provide site root to javascript -->
  33. <script type="text/javascript">
  34. var path_to_root = "";
  35. var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
  36. </script>
  37. <!-- Work around some values being stored in localStorage wrapped in quotes -->
  38. <script type="text/javascript">
  39. try {
  40. var theme = localStorage.getItem('mdbook-theme');
  41. var sidebar = localStorage.getItem('mdbook-sidebar');
  42. if (theme.startsWith('"') && theme.endsWith('"')) {
  43. localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
  44. }
  45. if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
  46. localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
  47. }
  48. } catch (e) { }
  49. </script>
  50. <!-- Set the theme before any content is loaded, prevents flash -->
  51. <script type="text/javascript">
  52. var theme;
  53. try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
  54. if (theme === null || theme === undefined) { theme = default_theme; }
  55. var html = document.querySelector('html');
  56. html.classList.remove('no-js')
  57. html.classList.remove('light')
  58. html.classList.add(theme);
  59. html.classList.add('js');
  60. </script>
  61. <!-- Hide / unhide sidebar before it is displayed -->
  62. <script type="text/javascript">
  63. var html = document.querySelector('html');
  64. var sidebar = 'hidden';
  65. if (document.body.clientWidth >= 1080) {
  66. try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
  67. sidebar = sidebar || 'visible';
  68. }
  69. html.classList.remove('sidebar-visible');
  70. html.classList.add("sidebar-" + sidebar);
  71. </script>
  72. <nav id="sidebar" class="sidebar" aria-label="Table of contents">
  73. <div class="sidebar-scrollbox">
  74. <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="turn-howto.html">Configuring a Turn Server</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="setup/turn/coturn.html">coturn TURN server</a></li><li class="chapter-item expanded "><a href="setup/turn/eturnal.html">eturnal TURN server</a></li></ol></li><li class="chapter-item expanded "><a href="delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="federate.html">Federation</a></li><li class="chapter-item expanded "><a href="usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/config_documentation.html">Configuration Manual</a></li><li class="chapter-item expanded "><a href="usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="templates.html">Templates</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="jwt.html">JSON Web Tokens</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/refresh_tokens.html">Refresh Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_data_callbacks.html">Account data callbacks</a></li><li class="chapter-item expanded "><a href="modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="metrics-howto.html" class="active">Monitoring</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/administration/monitoring/reporting_homeserver_usage_statistics.html">Reporting Homeserver Usage Statistics</a></li></ol></li><li class="chapter-item expanded "><a href="usage/administration/monthly_active_users.html">Monthly Active Users</a></li><li class="chapter-item expanded "><a href="usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="development/reviews.html">Reviewing Code</a></li><li class="chapter-item expanded "><a href="development/releases.html">Release Cycle</a></li><li class="chapter-item expanded "><a href="development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/demo.html">Demo scripts</a></li></ol></li><li class="chapter-item expanded "><a href="opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><a href="development/dependencies.html">Dependency management</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/synapse_architecture/cancellation.html">Cancellation</a></li><li class="chapter-item expanded "><a href="log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="replication.html">Replication</a></li><li class="chapter-item expanded "><a href="tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol>
  75. </div>
  76. <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
  77. </nav>
  78. <div id="page-wrapper" class="page-wrapper">
  79. <div class="page">
  80. <div id="menu-bar-hover-placeholder"></div>
  81. <div id="menu-bar" class="menu-bar sticky bordered">
  82. <div class="left-buttons">
  83. <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
  84. <i class="fa fa-bars"></i>
  85. </button>
  86. <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
  87. <i class="fa fa-paint-brush"></i>
  88. </button>
  89. <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
  90. <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
  91. <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
  92. <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
  93. <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
  94. <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
  95. </ul>
  96. <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
  97. <i class="fa fa-search"></i>
  98. </button>
  99. <div class="version-picker">
  100. <div class="dropdown">
  101. <div class="select">
  102. <span></span>
  103. <i class="fa fa-chevron-down"></i>
  104. </div>
  105. <input type="hidden" name="version">
  106. <ul class="dropdown-menu">
  107. <!-- Versions will be added dynamically in version-picker.js -->
  108. </ul>
  109. </div>
  110. </div>
  111. </div>
  112. <h1 class="menu-title">Synapse</h1>
  113. <div class="right-buttons">
  114. <a href="print.html" title="Print this book" aria-label="Print this book">
  115. <i id="print-button" class="fa fa-print"></i>
  116. </a>
  117. <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository">
  118. <i id="git-repository-button" class="fa fa-github"></i>
  119. </a>
  120. <a href="https://github.com/matrix-org/synapse/edit/develop/docs/metrics-howto.md" title="Suggest an edit" aria-label="Suggest an edit">
  121. <i id="git-edit-button" class="fa fa-edit"></i>
  122. </a>
  123. </div>
  124. </div>
  125. <div id="search-wrapper" class="hidden">
  126. <form id="searchbar-outer" class="searchbar-outer">
  127. <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
  128. </form>
  129. <div id="searchresults-outer" class="searchresults-outer hidden">
  130. <div id="searchresults-header" class="searchresults-header"></div>
  131. <ul id="searchresults">
  132. </ul>
  133. </div>
  134. </div>
  135. <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
  136. <script type="text/javascript">
  137. document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
  138. document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
  139. Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
  140. link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
  141. });
  142. </script>
  143. <div id="content" class="content">
  144. <main>
  145. <!-- Page table of contents -->
  146. <div class="sidetoc">
  147. <nav class="pagetoc"></nav>
  148. </div>
  149. <h1 id="how-to-monitor-synapse-metrics-using-prometheus"><a class="header" href="#how-to-monitor-synapse-metrics-using-prometheus">How to monitor Synapse metrics using Prometheus</a></h1>
  150. <ol>
  151. <li>
  152. <p>Install Prometheus:</p>
  153. <p>Follow instructions at
  154. <a href="http://prometheus.io/docs/introduction/install/">http://prometheus.io/docs/introduction/install/</a></p>
  155. </li>
  156. <li>
  157. <p>Enable Synapse metrics:</p>
  158. <p>In <code>homeserver.yaml</code>, make sure <code>enable_metrics</code> is
  159. set to <code>True</code>.</p>
  160. </li>
  161. <li>
  162. <p>Enable the <code>/_synapse/metrics</code> Synapse endpoint that Prometheus uses to
  163. collect data:</p>
  164. <p>There are two methods of enabling the metrics endpoint in Synapse.</p>
  165. <p>The first serves the metrics as a part of the usual web server and
  166. can be enabled by adding the <code>metrics</code> resource to the existing
  167. listener as such as in this example:</p>
  168. <pre><code class="language-yaml">listeners:
  169. - port: 8008
  170. tls: false
  171. type: http
  172. x_forwarded: true
  173. bind_addresses: ['::1', '127.0.0.1']
  174. resources:
  175. # added &quot;metrics&quot; in this line
  176. - names: [client, federation, metrics]
  177. compress: false
  178. </code></pre>
  179. <p>This provides a simple way of adding metrics to your Synapse
  180. installation, and serves under <code>/_synapse/metrics</code>. If you do not
  181. wish your metrics be publicly exposed, you will need to either
  182. filter it out at your load balancer, or use the second method.</p>
  183. <p>The second method runs the metrics server on a different port, in a
  184. different thread to Synapse. This can make it more resilient to
  185. heavy load meaning metrics cannot be retrieved, and can be exposed
  186. to just internal networks easier. The served metrics are available
  187. over HTTP only, and will be available at <code>/_synapse/metrics</code>.</p>
  188. <p>Add a new listener to homeserver.yaml as in this example:</p>
  189. <pre><code class="language-yaml">listeners:
  190. - port: 8008
  191. tls: false
  192. type: http
  193. x_forwarded: true
  194. bind_addresses: ['::1', '127.0.0.1']
  195. resources:
  196. - names: [client, federation]
  197. compress: false
  198. # beginning of the new metrics listener
  199. - port: 9000
  200. type: metrics
  201. bind_addresses: ['::1', '127.0.0.1']
  202. </code></pre>
  203. </li>
  204. <li>
  205. <p>Restart Synapse.</p>
  206. </li>
  207. <li>
  208. <p>Add a Prometheus target for Synapse.</p>
  209. <p>It needs to set the <code>metrics_path</code> to a non-default value (under
  210. <code>scrape_configs</code>):</p>
  211. <pre><code class="language-yaml"> - job_name: &quot;synapse&quot;
  212. scrape_interval: 15s
  213. metrics_path: &quot;/_synapse/metrics&quot;
  214. static_configs:
  215. - targets: [&quot;my.server.here:port&quot;]
  216. </code></pre>
  217. <p>where <code>my.server.here</code> is the IP address of Synapse, and <code>port</code> is
  218. the listener port configured with the <code>metrics</code> resource.</p>
  219. <p>If your prometheus is older than 1.5.2, you will need to replace
  220. <code>static_configs</code> in the above with <code>target_groups</code>.</p>
  221. </li>
  222. <li>
  223. <p>Restart Prometheus.</p>
  224. </li>
  225. <li>
  226. <p>Consider using the <a href="https://github.com/matrix-org/synapse/tree/master/contrib/grafana/">grafana dashboard</a>
  227. and required <a href="https://github.com/matrix-org/synapse/tree/master/contrib/prometheus/">recording rules</a> </p>
  228. </li>
  229. </ol>
  230. <h2 id="monitoring-workers"><a class="header" href="#monitoring-workers">Monitoring workers</a></h2>
  231. <p>To monitor a Synapse installation using <a href="workers.html">workers</a>,
  232. every worker needs to be monitored independently, in addition to
  233. the main homeserver process. This is because workers don't send
  234. their metrics to the main homeserver process, but expose them
  235. directly (if they are configured to do so).</p>
  236. <p>To allow collecting metrics from a worker, you need to add a
  237. <code>metrics</code> listener to its configuration, by adding the following
  238. under <code>worker_listeners</code>:</p>
  239. <pre><code class="language-yaml"> - type: metrics
  240. bind_address: ''
  241. port: 9101
  242. </code></pre>
  243. <p>The <code>bind_address</code> and <code>port</code> parameters should be set so that
  244. the resulting listener can be reached by prometheus, and they
  245. don't clash with an existing worker.
  246. With this example, the worker's metrics would then be available
  247. on <code>http://127.0.0.1:9101</code>.</p>
  248. <p>Example Prometheus target for Synapse with workers:</p>
  249. <pre><code class="language-yaml"> - job_name: &quot;synapse&quot;
  250. scrape_interval: 15s
  251. metrics_path: &quot;/_synapse/metrics&quot;
  252. static_configs:
  253. - targets: [&quot;my.server.here:port&quot;]
  254. labels:
  255. instance: &quot;my.server&quot;
  256. job: &quot;master&quot;
  257. index: 1
  258. - targets: [&quot;my.workerserver.here:port&quot;]
  259. labels:
  260. instance: &quot;my.server&quot;
  261. job: &quot;generic_worker&quot;
  262. index: 1
  263. - targets: [&quot;my.workerserver.here:port&quot;]
  264. labels:
  265. instance: &quot;my.server&quot;
  266. job: &quot;generic_worker&quot;
  267. index: 2
  268. - targets: [&quot;my.workerserver.here:port&quot;]
  269. labels:
  270. instance: &quot;my.server&quot;
  271. job: &quot;media_repository&quot;
  272. index: 1
  273. </code></pre>
  274. <p>Labels (<code>instance</code>, <code>job</code>, <code>index</code>) can be defined as anything.
  275. The labels are used to group graphs in grafana.</p>
  276. <h2 id="renaming-of-metrics--deprecation-of-old-names-in-12"><a class="header" href="#renaming-of-metrics--deprecation-of-old-names-in-12">Renaming of metrics &amp; deprecation of old names in 1.2</a></h2>
  277. <p>Synapse 1.2 updates the Prometheus metrics to match the naming
  278. convention of the upstream <code>prometheus_client</code>. The old names are
  279. considered deprecated and will be removed in a future version of
  280. Synapse.
  281. <strong>The old names will be disabled by default in Synapse v1.71.0 and removed
  282. altogether in Synapse v1.73.0.</strong></p>
  283. <table><thead><tr><th>New Name</th><th>Old Name</th></tr></thead><tbody>
  284. <tr><td>python_gc_objects_collected_total</td><td>python_gc_objects_collected</td></tr>
  285. <tr><td>python_gc_objects_uncollectable_total</td><td>python_gc_objects_uncollectable</td></tr>
  286. <tr><td>python_gc_collections_total</td><td>python_gc_collections</td></tr>
  287. <tr><td>process_cpu_seconds_total</td><td>process_cpu_seconds</td></tr>
  288. <tr><td>synapse_federation_client_sent_transactions_total</td><td>synapse_federation_client_sent_transactions</td></tr>
  289. <tr><td>synapse_federation_client_events_processed_total</td><td>synapse_federation_client_events_processed</td></tr>
  290. <tr><td>synapse_event_processing_loop_count_total</td><td>synapse_event_processing_loop_count</td></tr>
  291. <tr><td>synapse_event_processing_loop_room_count_total</td><td>synapse_event_processing_loop_room_count</td></tr>
  292. <tr><td>synapse_util_caches_cache_hits</td><td>synapse_util_caches_cache:hits</td></tr>
  293. <tr><td>synapse_util_caches_cache_size</td><td>synapse_util_caches_cache:size</td></tr>
  294. <tr><td>synapse_util_caches_cache_evicted_size</td><td>synapse_util_caches_cache:evicted_size</td></tr>
  295. <tr><td>synapse_util_caches_cache</td><td>synapse_util_caches_cache:total</td></tr>
  296. <tr><td>synapse_util_caches_response_cache_size</td><td>synapse_util_caches_response_cache:size</td></tr>
  297. <tr><td>synapse_util_caches_response_cache_hits</td><td>synapse_util_caches_response_cache:hits</td></tr>
  298. <tr><td>synapse_util_caches_response_cache_evicted_size</td><td>synapse_util_caches_response_cache:evicted_size</td></tr>
  299. <tr><td>synapse_util_metrics_block_count_total</td><td>synapse_util_metrics_block_count</td></tr>
  300. <tr><td>synapse_util_metrics_block_time_seconds_total</td><td>synapse_util_metrics_block_time_seconds</td></tr>
  301. <tr><td>synapse_util_metrics_block_ru_utime_seconds_total</td><td>synapse_util_metrics_block_ru_utime_seconds</td></tr>
  302. <tr><td>synapse_util_metrics_block_ru_stime_seconds_total</td><td>synapse_util_metrics_block_ru_stime_seconds</td></tr>
  303. <tr><td>synapse_util_metrics_block_db_txn_count_total</td><td>synapse_util_metrics_block_db_txn_count</td></tr>
  304. <tr><td>synapse_util_metrics_block_db_txn_duration_seconds_total</td><td>synapse_util_metrics_block_db_txn_duration_seconds</td></tr>
  305. <tr><td>synapse_util_metrics_block_db_sched_duration_seconds_total</td><td>synapse_util_metrics_block_db_sched_duration_seconds</td></tr>
  306. <tr><td>synapse_background_process_start_count_total</td><td>synapse_background_process_start_count</td></tr>
  307. <tr><td>synapse_background_process_ru_utime_seconds_total</td><td>synapse_background_process_ru_utime_seconds</td></tr>
  308. <tr><td>synapse_background_process_ru_stime_seconds_total</td><td>synapse_background_process_ru_stime_seconds</td></tr>
  309. <tr><td>synapse_background_process_db_txn_count_total</td><td>synapse_background_process_db_txn_count</td></tr>
  310. <tr><td>synapse_background_process_db_txn_duration_seconds_total</td><td>synapse_background_process_db_txn_duration_seconds</td></tr>
  311. <tr><td>synapse_background_process_db_sched_duration_seconds_total</td><td>synapse_background_process_db_sched_duration_seconds</td></tr>
  312. <tr><td>synapse_storage_events_persisted_events_total</td><td>synapse_storage_events_persisted_events</td></tr>
  313. <tr><td>synapse_storage_events_persisted_events_sep_total</td><td>synapse_storage_events_persisted_events_sep</td></tr>
  314. <tr><td>synapse_storage_events_state_delta_total</td><td>synapse_storage_events_state_delta</td></tr>
  315. <tr><td>synapse_storage_events_state_delta_single_event_total</td><td>synapse_storage_events_state_delta_single_event</td></tr>
  316. <tr><td>synapse_storage_events_state_delta_reuse_delta_total</td><td>synapse_storage_events_state_delta_reuse_delta</td></tr>
  317. <tr><td>synapse_federation_server_received_pdus_total</td><td>synapse_federation_server_received_pdus</td></tr>
  318. <tr><td>synapse_federation_server_received_edus_total</td><td>synapse_federation_server_received_edus</td></tr>
  319. <tr><td>synapse_handler_presence_notified_presence_total</td><td>synapse_handler_presence_notified_presence</td></tr>
  320. <tr><td>synapse_handler_presence_federation_presence_out_total</td><td>synapse_handler_presence_federation_presence_out</td></tr>
  321. <tr><td>synapse_handler_presence_presence_updates_total</td><td>synapse_handler_presence_presence_updates</td></tr>
  322. <tr><td>synapse_handler_presence_timers_fired_total</td><td>synapse_handler_presence_timers_fired</td></tr>
  323. <tr><td>synapse_handler_presence_federation_presence_total</td><td>synapse_handler_presence_federation_presence</td></tr>
  324. <tr><td>synapse_handler_presence_bump_active_time_total</td><td>synapse_handler_presence_bump_active_time</td></tr>
  325. <tr><td>synapse_federation_client_sent_edus_total</td><td>synapse_federation_client_sent_edus</td></tr>
  326. <tr><td>synapse_federation_client_sent_pdu_destinations_count_total</td><td>synapse_federation_client_sent_pdu_destinations:count</td></tr>
  327. <tr><td>synapse_federation_client_sent_pdu_destinations_total</td><td>synapse_federation_client_sent_pdu_destinations:total</td></tr>
  328. <tr><td>synapse_handlers_appservice_events_processed_total</td><td>synapse_handlers_appservice_events_processed</td></tr>
  329. <tr><td>synapse_notifier_notified_events_total</td><td>synapse_notifier_notified_events</td></tr>
  330. <tr><td>synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total</td><td>synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter</td></tr>
  331. <tr><td>synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total</td><td>synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter</td></tr>
  332. <tr><td>synapse_http_httppusher_http_pushes_processed_total</td><td>synapse_http_httppusher_http_pushes_processed</td></tr>
  333. <tr><td>synapse_http_httppusher_http_pushes_failed_total</td><td>synapse_http_httppusher_http_pushes_failed</td></tr>
  334. <tr><td>synapse_http_httppusher_badge_updates_processed_total</td><td>synapse_http_httppusher_badge_updates_processed</td></tr>
  335. <tr><td>synapse_http_httppusher_badge_updates_failed_total</td><td>synapse_http_httppusher_badge_updates_failed</td></tr>
  336. <tr><td>synapse_admin_mau_current</td><td>synapse_admin_mau:current</td></tr>
  337. <tr><td>synapse_admin_mau_max</td><td>synapse_admin_mau:max</td></tr>
  338. <tr><td>synapse_admin_mau_registered_reserved_users</td><td>synapse_admin_mau:registered_reserved_users</td></tr>
  339. </tbody></table>
  340. <h2 id="removal-of-deprecated-metrics--time-based-counters-becoming-histograms-in-0310"><a class="header" href="#removal-of-deprecated-metrics--time-based-counters-becoming-histograms-in-0310">Removal of deprecated metrics &amp; time based counters becoming histograms in 0.31.0</a></h2>
  341. <p>The duplicated metrics deprecated in Synapse 0.27.0 have been removed.</p>
  342. <p>All time duration-based metrics have been changed to be seconds. This
  343. affects:</p>
  344. <table><thead><tr><th>msec -&gt; sec metrics</th></tr></thead><tbody>
  345. <tr><td>python_gc_time</td></tr>
  346. <tr><td>python_twisted_reactor_tick_time</td></tr>
  347. <tr><td>synapse_storage_query_time</td></tr>
  348. <tr><td>synapse_storage_schedule_time</td></tr>
  349. <tr><td>synapse_storage_transaction_time</td></tr>
  350. </tbody></table>
  351. <p>Several metrics have been changed to be histograms, which sort entries
  352. into buckets and allow better analysis. The following metrics are now
  353. histograms:</p>
  354. <table><thead><tr><th>Altered metrics</th></tr></thead><tbody>
  355. <tr><td>python_gc_time</td></tr>
  356. <tr><td>python_twisted_reactor_pending_calls</td></tr>
  357. <tr><td>python_twisted_reactor_tick_time</td></tr>
  358. <tr><td>synapse_http_server_response_time_seconds</td></tr>
  359. <tr><td>synapse_storage_query_time</td></tr>
  360. <tr><td>synapse_storage_schedule_time</td></tr>
  361. <tr><td>synapse_storage_transaction_time</td></tr>
  362. </tbody></table>
  363. <h2 id="block-and-response-metrics-renamed-for-0270"><a class="header" href="#block-and-response-metrics-renamed-for-0270">Block and response metrics renamed for 0.27.0</a></h2>
  364. <p>Synapse 0.27.0 begins the process of rationalising the duplicate
  365. <code>*:count</code> metrics reported for the resource tracking for code blocks and
  366. HTTP requests.</p>
  367. <p>At the same time, the corresponding <code>*:total</code> metrics are being renamed,
  368. as the <code>:total</code> suffix no longer makes sense in the absence of a
  369. corresponding <code>:count</code> metric.</p>
  370. <p>To enable a graceful migration path, this release just adds new names
  371. for the metrics being renamed. A future release will remove the old
  372. ones.</p>
  373. <p>The following table shows the new metrics, and the old metrics which
  374. they are replacing.</p>
  375. <table><thead><tr><th>New name</th><th>Old name</th></tr></thead><tbody>
  376. <tr><td>synapse_util_metrics_block_count</td><td>synapse_util_metrics_block_timer:count</td></tr>
  377. <tr><td>synapse_util_metrics_block_count</td><td>synapse_util_metrics_block_ru_utime:count</td></tr>
  378. <tr><td>synapse_util_metrics_block_count</td><td>synapse_util_metrics_block_ru_stime:count</td></tr>
  379. <tr><td>synapse_util_metrics_block_count</td><td>synapse_util_metrics_block_db_txn_count:count</td></tr>
  380. <tr><td>synapse_util_metrics_block_count</td><td>synapse_util_metrics_block_db_txn_duration:count</td></tr>
  381. <tr><td>synapse_util_metrics_block_time_seconds</td><td>synapse_util_metrics_block_timer:total</td></tr>
  382. <tr><td>synapse_util_metrics_block_ru_utime_seconds</td><td>synapse_util_metrics_block_ru_utime:total</td></tr>
  383. <tr><td>synapse_util_metrics_block_ru_stime_seconds</td><td>synapse_util_metrics_block_ru_stime:total</td></tr>
  384. <tr><td>synapse_util_metrics_block_db_txn_count</td><td>synapse_util_metrics_block_db_txn_count:total</td></tr>
  385. <tr><td>synapse_util_metrics_block_db_txn_duration_seconds</td><td>synapse_util_metrics_block_db_txn_duration:total</td></tr>
  386. <tr><td>synapse_http_server_response_count</td><td>synapse_http_server_requests</td></tr>
  387. <tr><td>synapse_http_server_response_count</td><td>synapse_http_server_response_time:count</td></tr>
  388. <tr><td>synapse_http_server_response_count</td><td>synapse_http_server_response_ru_utime:count</td></tr>
  389. <tr><td>synapse_http_server_response_count</td><td>synapse_http_server_response_ru_stime:count</td></tr>
  390. <tr><td>synapse_http_server_response_count</td><td>synapse_http_server_response_db_txn_count:count</td></tr>
  391. <tr><td>synapse_http_server_response_count</td><td>synapse_http_server_response_db_txn_duration:count</td></tr>
  392. <tr><td>synapse_http_server_response_time_seconds</td><td>synapse_http_server_response_time:total</td></tr>
  393. <tr><td>synapse_http_server_response_ru_utime_seconds</td><td>synapse_http_server_response_ru_utime:total</td></tr>
  394. <tr><td>synapse_http_server_response_ru_stime_seconds</td><td>synapse_http_server_response_ru_stime:total</td></tr>
  395. <tr><td>synapse_http_server_response_db_txn_count</td><td>synapse_http_server_response_db_txn_count:total</td></tr>
  396. <tr><td>synapse_http_server_response_db_txn_duration_seconds</td><td>synapse_http_server_response_db_txn_duration:total</td></tr>
  397. </tbody></table>
  398. <h2 id="standard-metric-names"><a class="header" href="#standard-metric-names">Standard Metric Names</a></h2>
  399. <p>As of synapse version 0.18.2, the format of the process-wide metrics has
  400. been changed to fit prometheus standard naming conventions. Additionally
  401. the units have been changed to seconds, from milliseconds.</p>
  402. <table><thead><tr><th>New name</th><th>Old name</th></tr></thead><tbody>
  403. <tr><td>process_cpu_user_seconds_total</td><td>process_resource_utime / 1000</td></tr>
  404. <tr><td>process_cpu_system_seconds_total</td><td>process_resource_stime / 1000</td></tr>
  405. <tr><td>process_open_fds (no 'type' label)</td><td>process_fds</td></tr>
  406. </tbody></table>
  407. <p>The python-specific counts of garbage collector performance have been
  408. renamed.</p>
  409. <table><thead><tr><th>New name</th><th>Old name</th></tr></thead><tbody>
  410. <tr><td>python_gc_time</td><td>reactor_gc_time</td></tr>
  411. <tr><td>python_gc_unreachable_total</td><td>reactor_gc_unreachable</td></tr>
  412. <tr><td>python_gc_counts</td><td>reactor_gc_counts</td></tr>
  413. </tbody></table>
  414. <p>The twisted-specific reactor metrics have been renamed.</p>
  415. <table><thead><tr><th>New name</th><th>Old name</th></tr></thead><tbody>
  416. <tr><td>python_twisted_reactor_pending_calls</td><td>reactor_pending_calls</td></tr>
  417. <tr><td>python_twisted_reactor_tick_time</td><td>reactor_tick_time</td></tr>
  418. </tbody></table>
  419. </main>
  420. <nav class="nav-wrapper" aria-label="Page navigation">
  421. <!-- Mobile navigation buttons -->
  422. <a rel="prev" href="manhole.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
  423. <i class="fa fa-angle-left"></i>
  424. </a>
  425. <a rel="next" href="usage/administration/monitoring/reporting_homeserver_usage_statistics.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
  426. <i class="fa fa-angle-right"></i>
  427. </a>
  428. <div style="clear: both"></div>
  429. </nav>
  430. </div>
  431. </div>
  432. <nav class="nav-wide-wrapper" aria-label="Page navigation">
  433. <a rel="prev" href="manhole.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
  434. <i class="fa fa-angle-left"></i>
  435. </a>
  436. <a rel="next" href="usage/administration/monitoring/reporting_homeserver_usage_statistics.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
  437. <i class="fa fa-angle-right"></i>
  438. </a>
  439. </nav>
  440. </div>
  441. <script type="text/javascript">
  442. window.playground_copyable = true;
  443. </script>
  444. <script src="elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
  445. <script src="mark.min.js" type="text/javascript" charset="utf-8"></script>
  446. <script src="searcher.js" type="text/javascript" charset="utf-8"></script>
  447. <script src="clipboard.min.js" type="text/javascript" charset="utf-8"></script>
  448. <script src="highlight.js" type="text/javascript" charset="utf-8"></script>
  449. <script src="book.js" type="text/javascript" charset="utf-8"></script>
  450. <!-- Custom JS scripts -->
  451. <script type="text/javascript" src="docs/website_files/table-of-contents.js"></script>
  452. <script type="text/javascript" src="docs/website_files/version-picker.js"></script>
  453. <script type="text/javascript" src="docs/website_files/version.js"></script>
  454. </body>
  455. </html>