From 0e0bdd2973a7febd3545ea7b604ca24701bb80f9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 9 Sep 2026 00:07:10 +0100 Subject: [PATCH 01/41] Speed up by recursive relations by 10-100x by joining events inside the CTE (#20182) This solves the root cause of a user opening the thread panel in Element Web DoSing synapse with recursive relation requests, starving out delayed events and causing MatrixRTC calls to drop - see https://github.com/matrix-org/matrix-js-sdk/pull/5519 for papering over it clientside. Fixes https://github.com/element-hq/synapse/issues/18788 Claude rationale: Postgres cannot estimate the size of a recursive CTE. When it guesses large it stops probing events by event_id and instead hashes every event in the room, so a single recursive `/relations` request in a busy room takes seconds and a Threads-panel fan-out of 30 of them can pin a client-reader's DB pool for a minute. Joining events per recursion step keeps the lookups as index probes regardless of the estimate. The recursion also moves from `UNION` to `UNION ALL`. An event carries a single `m.relates_to` and is stored as exactly one `event_relations` row (unique index on `event_id`), so the relation graph is a tree: every node is reached along one path and `UNION` never had duplicates to remove. `UNION ALL` drops the sort-and-dedupe pass over the working table on every iteration, which matters more now that each row also carries the joined events columns. A cycle from bogus events is still terminated by the depth bound, as before, and `UNION` gave no protection there anyway since such rows differ in depth. Measured on Postgres 16 against a synthetic corpus modelled on a large homeserver: 4 rooms x 300k events; one 40-reply thread rooted 280k events back in the timeline, with 2 reactions per reply; every 5th event elsewhere a reaction, plus 200 popular roots with 2000 reactions each so that the `relates_to_id statistics` are skewed the way they are in production. Default limit (6 rows), warm cache, JIT off: ``` before after single request 77 ms 1.4 ms 20 concurrent requests 1.21 s 0.14 s ``` Before: Hash Join with a Hash over all 300k events of the room (4 batches). After: Nested Loop with an Index Scan on `events_event_id_key` per row. (found/solved by fable) --------- Co-authored-by: Eric Eastwood --- changelog.d/20182.bugfix | 1 + synapse/storage/databases/main/relations.py | 25 ++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20182.bugfix diff --git a/changelog.d/20182.bugfix b/changelog.d/20182.bugfix new file mode 100644 index 0000000000..637c3efb96 --- /dev/null +++ b/changelog.d/20182.bugfix @@ -0,0 +1 @@ +Fix slow recursive `/relations` requests in large rooms by joining events inside the recursive query. diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 9d9c37e2a4..58f9321628 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -237,19 +237,28 @@ async def get_relations_for_event( # If no recursion is needed then the event_relations table is queried # for direct children of the requested event. if recurse: + # The events table is joined inside the recursion rather than + # after it: Postgres cannot accurately estimate the size of a recursive CTE, + # and when the guess is large it joins the CTE against a scan of + # *every* event in the room, which takes seconds in busy rooms. + # Joining per step keeps every events lookup an index probe. sql = """ WITH RECURSIVE related_events AS ( - SELECT event_id, relation_type, relates_to_id, 0 AS depth - FROM event_relations - WHERE relates_to_id = ? - UNION SELECT e.event_id, e.relation_type, e.relates_to_id, depth + 1 - FROM event_relations e - INNER JOIN related_events r ON r.event_id = e.relates_to_id - WHERE depth <= 3 + SELECT er.event_id, er.relation_type, ev.room_id, ev.sender, + ev.topological_ordering, ev.stream_ordering, ev.type, 0 AS depth + FROM event_relations er + INNER JOIN events ev ON ev.event_id = er.event_id + WHERE er.relates_to_id = ? + UNION ALL + SELECT er.event_id, er.relation_type, ev.room_id, ev.sender, + ev.topological_ordering, ev.stream_ordering, ev.type, r.depth + 1 + FROM related_events r + INNER JOIN event_relations er ON er.relates_to_id = r.event_id + INNER JOIN events ev ON ev.event_id = er.event_id + WHERE r.depth <= 3 ) SELECT event_id, relation_type, sender, topological_ordering, stream_ordering FROM related_events - INNER JOIN events USING (event_id) WHERE %s ORDER BY topological_ordering %s, stream_ordering %s LIMIT ?; From 3a2fa70157973193f7f7f5950443fa7ccbe2d179 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 9 Sep 2026 14:04:35 +0100 Subject: [PATCH 02/41] Improve the rendering of the set inequality errors produced by `assertEqual` in the tests. (#20193) This also changes the rendering of our custom helper `assertIncludes`. Spawns from https://github.com/element-hq/synapse/pull/20019#discussion_r3676539092 This PR hijacks `assertEqual` in order to substitute in our own error rendering logic for set inequality. (The motivation to do this is that we should just render sets in our preferred style by default, without having to think about `assertIncludes` with the `exact` flag or risk forgetting it.) The error rendering for `assertIncludes` is adapted to make it reusable in our `assertEqual` and to make it clearer _to me_ (I found it a bit jarring that `+` was used more like a tick, when in other test frameworks I expect to see that as a diff marker). I have tried to make it as clear as I could without being cryptic. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20193.misc | 1 + tests/unittest.py | 186 ++++++++++++++++++++++++++++++++++------- 2 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 changelog.d/20193.misc diff --git a/changelog.d/20193.misc b/changelog.d/20193.misc new file mode 100644 index 0000000000..03ab2b0f7e --- /dev/null +++ b/changelog.d/20193.misc @@ -0,0 +1 @@ +Improve the rendering of the set inequality errors produced by `assertEqual` in the tests. \ No newline at end of file diff --git a/tests/unittest.py b/tests/unittest.py index 152fb341cb..4881a6650a 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -25,10 +25,12 @@ import hmac import json import logging +import os import secrets +import sys import time +from collections.abc import Set from typing import ( - AbstractSet, Any, Awaitable, Callable, @@ -46,7 +48,7 @@ import canonicaljson import signedjson.key import unpaddedbase64 -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import Concatenate, Never, ParamSpec, override from twisted.internet import defer from twisted.internet.defer import Deferred, ensureDeferred @@ -105,6 +107,19 @@ S = TypeVar("S") +def _use_colour() -> bool: + """ + Whether assertion failures should be coloured with ANSI escapes. + + Follow the `NO_COLOR`/`FORCE_COLOR` conventions (https://no-color.org, https://force-color.org/). + """ + if os.environ.get("NO_COLOR"): + return False + if os.environ.get("FORCE_COLOR"): + return True + return sys.stdout.isatty() + + class _TypedFailure(Generic[_ExcType], Protocol): """Extension to twisted.Failure, where the 'value' has a certain type.""" @@ -269,10 +284,145 @@ def assert_dict(self, required: Mapping, actual: Mapping) -> None: required[key], actual[key], msg="%s mismatch. %s" % (key, actual) ) + def _fail_with_set_inequality( + self, + actual_items: Set[TV], + expected_items: Set[TV], + message: str | None, + exact: bool, + ) -> Never: + """ + Fail the current test, printing a rich message showing + the set inequality. + + If `exact` is True, we expect no extra items in `actual_items`. + If `exact` is False, we clarify that extra items are permissible. + """ + if _use_colour(): + BOLD = "\033[1m" + DIM_GREY = "\033[2;37m" + RED = "\033[31m" + BRIGHT_YELLOW = "\033[93m" + DIM_BLUE = "\033[94m" + RESET = "\033[0m" + else: + BOLD = DIM_GREY = RED = BRIGHT_YELLOW = DIM_BLUE = RESET = "" + + # If it's correct, use dim grey: we don't want to draw your attention to it + # Grey out the whole item + CORRECT_MARKER = f"{DIM_GREY} ok" + + # If it's wrong, use red: that's the most important information here + MISSING_MARKER = f" {RED}missing{RESET}{BRIGHT_YELLOW}" + UNEXPECTED_MARKER = f"{RED}unwanted{RESET}{BRIGHT_YELLOW}" + + # If it's a harmless extra, give it a slight bit more emphasis than something correct + # (as it could be a sign of something unexpected), but not enough to make it seem wrong. + EXTRA_MARKER = f"{DIM_BLUE} extra{RESET}" + + used_markers = set() + expected_lines: list[str] = [] + + # Sort the items in the sets for ease of reading. Motivation: + # - keeps `(A, B)` and `(A, C)` next to each other in sets of tuples + # - makes it easier to cross-compare the two sets visually if they are in the same order + # + # sorted() only accepts objects that support at least `<` or `>`. + # Virtually every immutable data type in Python supports these, so virtually + # everything inside a set supports these. + # Ignore the type error arising from not proving that `TV` supports comparison. + for expected_item in sorted(expected_items): # type: ignore[type-var] + is_missing = expected_item not in actual_items + + marker = MISSING_MARKER if is_missing else CORRECT_MARKER + + used_markers.add(marker) + expected_lines.append(f"{marker} {expected_item!r}{RESET}") + + actual_lines: list[str] = [] + # See note above about type ignore. + for actual_item in sorted(actual_items): # type: ignore[type-var] + is_expected = actual_item in expected_items + + if is_expected: + marker = CORRECT_MARKER + elif exact: + # We want an exact match, so this 'extra' is actively 'unwanted' + marker = UNEXPECTED_MARKER + else: + # Harmless 'extra' + marker = EXTRA_MARKER + + used_markers.add(marker) + actual_lines.append(f"{marker} {actual_item!r}{RESET}") + + newline = "\n" + expected_string = f"{BOLD}Expected items:{RESET}\n {{\n{newline.join(expected_lines)}\n }}" + actual_string = f"{BOLD}Actually received items:{RESET}\n {{\n{newline.join(actual_lines)}\n }}" + first_message = ( + "Items must match exactly (sets are not equal)" + if exact + else "Some expected items are missing." + ) + + legend = f"{BOLD}Legend:{RESET}\n" + if CORRECT_MARKER in used_markers: + legend += f" {CORRECT_MARKER}{RESET}: item is correct as it was both expected and received\n" + if MISSING_MARKER in used_markers: + legend += ( + f" {MISSING_MARKER}{RESET}: item is expected but was not received\n" + ) + if UNEXPECTED_MARKER in used_markers: + legend += ( + f" {UNEXPECTED_MARKER}{RESET}: item was received but is not expected\n" + ) + if EXTRA_MARKER in used_markers: + legend += f" {EXTRA_MARKER}{RESET}: item was received and allowed, though not explicitly expected\n" + + diff_message = ( + f"{first_message}\n{legend}\n{expected_string}\n\n{actual_string}" + ) + + extra_message = "" + if message is not None: + extra_message = "f\n{message}" + + self.fail(f"{diff_message}{extra_message}") + + @override + def assertEqual(self, first: TV, second: TV, msg: object | None = None) -> None: + """ + Override of `assertEqual` to make it print better errors. + + Note that `first` is treated as 'actual' and `second` as 'expected`. + (We can't rename them in our override because that is not a compatible change, + Mypy forbids it.) + + Specifically: + - better errors for set inequality + """ + if first == second: + return + + if isinstance(first, Set) and isinstance(second, Set): + self._fail_with_set_inequality( + first, + second, + # Any `str()`-able object is valid as a message. We frequently pass in raw JSON responses, for instance. + str(msg) if msg is not None else msg, + exact=True, + ) + + # Fall back to the base implementation for other types + # Since we know `first == second` does not hold, + # we expect this must diverge by raising an exception. + super().assertEqual(first=first, second=second, msg=msg) + raise AssertionError("unreachable") + def assertIncludes( self, - actual_items: AbstractSet[TV], - expected_items: AbstractSet[TV], + actual_items: Set[TV], + expected_items: Set[TV], exact: bool = False, message: str | None = None, ) -> None: @@ -295,33 +445,7 @@ def assertIncludes( elif not exact and actual_items >= expected_items: return - expected_lines: list[str] = [] - for expected_item in expected_items: - is_expected_in_actual = expected_item in actual_items - expected_lines.append( - "{} {}".format(" " if is_expected_in_actual else "?", expected_item) - ) - - actual_lines: list[str] = [] - for actual_item in actual_items: - is_actual_in_expected = actual_item in expected_items - actual_lines.append( - "{} {}".format("+" if is_actual_in_expected else " ", actual_item) - ) - - newline = "\n" - expected_string = f"Expected items to be in actual ('?' = missing expected items):\n {{\n{newline.join(expected_lines)}\n }}" - actual_string = f"Actual ('+' = found expected items):\n {{\n{newline.join(actual_lines)}\n }}" - first_message = ( - "Items must match exactly" if exact else "Some expected items are missing." - ) - diff_message = f"{first_message}\n{expected_string}\n{actual_string}" - - extra_message = "" - if message is not None: - extra_message = "f\n{message}" - - self.fail(f"{diff_message}{extra_message}") + self._fail_with_set_inequality(actual_items, expected_items, message, exact) def DEBUG(target: TV) -> TV: From 4a15ec41b028ec4162229bb1491f06e550e95b33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:41:23 +0000 Subject: [PATCH 03/41] Bump log from 0.4.33 to 0.4.34 in the patches group (#20195) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6062122c15..13228a5542 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,9 +750,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" From ca02ad7cca32d02809fce03d5c940c07c45c1379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:09:54 +0000 Subject: [PATCH 04/41] Bump gitpython from 3.1.58 to 3.1.59 (#20199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.1.58 to 3.1.59.
Release notes

Sourced from gitpython's releases.

3.1.59 - Security

What's Changed

Full Changelog: https://github.com/gitpython-developers/GitPython/compare/3.1.58...3.1.59

Commits
  • 66340d7 prepare changelog prior to release
  • a5e047d Merge pull request #2211 from gitpython-developers/config-sanitize-more
  • ef7568e fix: ignore includes in submodule configuration
  • 4b4e47f fix: preserve multiline config values when writing
  • b473abb Merge pull request #2210 from gitpython-developers/fix-clone-unsafe-option
  • 5ff52cc Merge pull request #2209 from caroescm/fix-index-add-chmod
  • b68afff Block separate git directories during clone
  • 93677a0 fix: index.add() now supports filters (#2021)
  • 9729ed3 Merge pull request #2208 from gitpython-developers/security-fixes
  • ce9d8e8 prepare next release
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=gitpython&package-manager=pip&previous-version=3.1.58&new-version=3.1.59)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 508379acd5..572cd5d195 100644 --- a/poetry.lock +++ b/poetry.lock @@ -583,14 +583,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.58" +version = "3.1.62" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f"}, - {file = "gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22"}, + {file = "gitpython-3.1.62-py3-none-any.whl", hash = "sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e"}, + {file = "gitpython-3.1.62.tar.gz", hash = "sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af"}, ] [package.dependencies] From c69d7eeb592dec24816a96365508eaeb16204c0b Mon Sep 17 00:00:00 2001 From: Ben Banfield-Zanin Date: Thu, 10 Sep 2026 13:50:42 +0100 Subject: [PATCH 05/41] Document MAS admin APIs as being workerisable (#20209) ### Pull Request Checklist Replacement for https://github.com/element-hq/synapse/pull/19752 as that has bit-rotted with https://github.com/element-hq/synapse/pull/19895 being merged. `/_synapse/mas` is mounted on a worker on matrix.org and can be seen to be workerisable via * https://github.com/element-hq/synapse/blob/v1.160.0/synapse/app/generic_worker.py#L202 * https://github.com/element-hq/synapse/blob/v1.160.0/synapse/rest/synapse/client/__init__.py#L71 * https://github.com/element-hq/synapse/blob/v1.160.0/synapse/rest/synapse/mas/__init__.py#L46-L71 * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20209.doc | 1 + docs/workers.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20209.doc diff --git a/changelog.d/20209.doc b/changelog.d/20209.doc new file mode 100644 index 0000000000..cfee70c67c --- /dev/null +++ b/changelog.d/20209.doc @@ -0,0 +1 @@ +Document the paths that can be handled on workers with stabilised delegated authentication. diff --git a/docs/workers.md b/docs/workers.md index 8959becea6..5914b9d1cb 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -291,6 +291,9 @@ information. # Unstable MSC4140 support ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/.*/restart)?$ + # Stabilised Delegated Authentication support (`matrix_authentication_service.enabled: true`) + ^/_synapse/mas/ + Additionally, the following REST endpoints can be handled for GET requests: # Push rules requests @@ -315,7 +318,7 @@ for the room are in flight: Additionally, the following endpoints should be included if Synapse is configured to use SSO (you only need to include the ones for whichever SSO provider you're -using): +using) and delegated authentication isn't enabled: # for all SSO providers ^/_matrix/client/(api/v1|r0|v3|unstable)/login/sso/redirect From f7cd91f6b0dc5e771ef01caf1680c4ab3a572fcc Mon Sep 17 00:00:00 2001 From: Ben Banfield-Zanin Date: Thu, 10 Sep 2026 17:25:24 +0100 Subject: [PATCH 06/41] Document that the new get single delayed event endpoint is workerisable (#20210) ### Pull Request Checklist Missed off of https://github.com/element-hq/synapse/pull/19926/changes IMO. My reading of https://github.com/element-hq/synapse/blob/v1.161.0rc1/synapse/rest/client/delayed_events.py is that the new endpoint (pattern `r"/org\.matrix\.msc4140/delayed_events/(?P[^/]+)$"` is workerisable) given how `register_servlets` flows. However given `UpdateDelayedEventServlet` covers the same pattern but for `POST` requests, this should go in the `GET` only part of the documentation. * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20210.doc | 1 + docs/workers.md | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/20210.doc diff --git a/changelog.d/20210.doc b/changelog.d/20210.doc new file mode 100644 index 0000000000..3b0f828036 --- /dev/null +++ b/changelog.d/20210.doc @@ -0,0 +1 @@ +Document the endpoint for getting a single delayed event is workerisable. diff --git a/docs/workers.md b/docs/workers.md index 5914b9d1cb..5587ba9c6d 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -309,6 +309,9 @@ Additionally, the following REST endpoints can be handled for GET requests: # Admin API requests ^/_synapse/admin/v2/users/[^/]+$ + # Unstable MSC4140 support + ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events/[^/]+$ + Pagination requests can also be handled, but all requests for a given room must be routed to the same instance. Additionally, care must be taken to ensure that the purge history admin API is not used while pagination requests From 54c0a78f76cbd54ff1e72f9aaee55e219ecf9b77 Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:44:40 +0100 Subject: [PATCH 07/41] MSC4242: State DAGs (serving) (#20133) Adds the serving functions needed for MSC4242: State DAGs. This PR adds MSC4242 support to /make_join, /send_join and /get_missing_events, as well as calculates the destinations for /send events correctly using `prev_state_events`. Built on top of https://github.com/element-hq/synapse/pull/19718 for the storage functions it makes. Split out from https://github.com/element-hq/synapse/pull/19425 Part of a series of 5x PRs to land the federation part of [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) ([storage](https://github.com/element-hq/synapse/pull/19718), [fedclient](https://github.com/element-hq/synapse/pull/20127), serving (this PR), inbound-joins, inbound-pulls). Whilst this is mostly a port of the code in #19425 there are a few changes: - `/get_missing_events` accepts message events when walking the state DAG, in which case it resolves the first hop to be that event's `prev_state_events`. The original PR made the client `/event` the message event and then set `latest=[prev_state_events]` on its own. This is not very efficient (extra round trip to fetch the event) and there's no reason why the server can't do the message->prev_state_events lookup, so we do so. This matches the MSC examples. - We cap the amount of events fetched via `/get_missing_events`. The MSC allows it, so it's a good safety check. - We sort the returned state DAG in `/send_join` by depth then event ID so it's "mostly" sorted. This is more a formality than anything else, the MSC does not mandate this, but it makes `/send_join` responses deterministic. - `notify_on_event_delivered_over_federation` is a new thing since #19425, so we include state DAG events in it like we do with state/auth_chain. This PR does remove the forced `m.federate: false` setting for MSC4242 rooms, so it makes it possible for federated MSC4242 rooms to be made. This is mostly so we can test via the endpoints. Given you must opt-in to MSC4242 via the experimental features config option, it seems reasonable to loosen this setting. The forced no-federation flag existed prior to review saying that the MSC4242 room version could itself be gated behind an experimental feature. Reviewable commit-by-commit. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood --- changelog.d/20133.misc | 1 + synapse/api/constants.py | 10 + synapse/federation/federation_server.py | 60 +++- synapse/federation/sender/__init__.py | 6 +- .../federation/transport/server/federation.py | 4 +- synapse/handlers/federation.py | 92 ++++- synapse/handlers/room.py | 4 - .../databases/main/event_federation.py | 4 - tests/federation/test_federation_server.py | 340 ++++++++++++++++-- 9 files changed, 458 insertions(+), 63 deletions(-) create mode 100644 changelog.d/20133.misc diff --git a/changelog.d/20133.misc b/changelog.d/20133.misc new file mode 100644 index 0000000000..9590db6934 --- /dev/null +++ b/changelog.d/20133.misc @@ -0,0 +1 @@ +Add HTTP serving functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 8e91b2e203..840930312f 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -500,3 +500,13 @@ class StickyEvent: This is the default specified in the MSC. Chosen arbitrarily. """ + + +class StateDag: + GET_MISSING_EVENTS_FIELD: Final = "org.matrix.msc4242.state_dag" + MAX_MISSING_EVENTS: Final = 1000 + """ + Max number of events to return in a single /get_missing_events response. + Arbitrary, servers usually never get this high so it's mostly to bound the max size of + a single /get_missing_events response. + """ diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2379b2ed2c..b90bb67ba2 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -30,6 +30,7 @@ Callable, Collection, Mapping, + Sequence, ) from prometheus_client import Counter, Gauge, Histogram @@ -56,6 +57,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import EventPersistencePair from synapse.federation.federation_base import ( FederationBase, @@ -864,6 +866,11 @@ async def on_send_join_request( event, context = await self._on_send_membership_event( origin, content, Membership.JOIN, room_id ) + + if supports_msc4242_state_dag(event): + # FIXME: We don't yet support faster room joins in Synapse with MSC4242 + caller_supports_partial_state = False + # Use the join event's own stream ordering as the upper bound when fetching # forward extremities (below), so we only consider extremities that existed at # or before the join rather than those introduced by concurrent writes that @@ -890,28 +897,43 @@ async def on_send_join_request( state_event_ids = prev_state_ids.values() servers_in_room = None - auth_chain_event_ids = await self.store.get_auth_chain_ids( - room_id, state_event_ids - ) + state_dag: Sequence[EventBase] = () + state_events: Sequence[EventBase] = () + auth_chain_events: Sequence[EventBase] = () - # if the caller has opted in, we can omit any auth_chain events which are - # already in state_event_ids - if caller_supports_partial_state: - auth_chain_event_ids.difference_update(state_event_ids) + if supports_msc4242_state_dag(event): + state_dag_map = await self.store.get_state_dag( + room_id, set(event.prev_state_events) + ) + state_dag = list(state_dag_map.values()) + else: + auth_chain_event_ids = await self.store.get_auth_chain_ids( + room_id, state_event_ids + ) - auth_chain_events = await self.store.get_events_as_list(auth_chain_event_ids) - state_events = await self.store.get_events_as_list(state_event_ids) + # if the caller has opted in, we can omit any auth_chain events which are + # already in state_event_ids + if caller_supports_partial_state: + auth_chain_event_ids.difference_update(state_event_ids) + + auth_chain_events = await self.store.get_events_as_list( + auth_chain_event_ids + ) + state_events = await self.store.get_events_as_list(state_event_ids) # we try to do all the async stuff before this point, so that time_now is as # accurate as possible. time_now = self._clock.time_msec() event_json = event.get_pdu_json(time_now) - resp = { + resp: JsonDict = { "event": event_json, - "state": serialize_and_filter_pdus(state_events, time_now), - "auth_chain": serialize_and_filter_pdus(auth_chain_events, time_now), "members_omitted": caller_supports_partial_state, } + if supports_msc4242_state_dag(event): + resp["state_dag"] = serialize_and_filter_pdus(state_dag, time_now) + else: + resp["state"] = serialize_and_filter_pdus(state_events, time_now) + resp["auth_chain"] = serialize_and_filter_pdus(auth_chain_events, time_now) # Check the forward extremities for the room here. If there is more than one, it # is likely that another event was created in the room during the @@ -942,7 +964,7 @@ async def on_send_join_request( await self._federation_callbacks.notify_on_event_delivered_over_federation( origin, - [event, *state_events, *auth_chain_events], + [event, *state_dag, *state_events, *auth_chain_events], FederatedEventDeliveryMethod.SEND_JOIN, ) @@ -1246,6 +1268,7 @@ async def on_get_missing_events( earliest_events: list[str], latest_events: list[str], limit: int, + walk_state_dag: bool = False, ) -> dict[str, list]: async with self._server_linearizer.queue((origin, room_id)): origin_host, _ = parse_server_name(origin) @@ -1259,9 +1282,14 @@ async def on_get_missing_events( limit, ) - missing_events = await self.handler.on_get_missing_events( - origin, room_id, earliest_events, latest_events, limit - ) + if walk_state_dag: + missing_events = await self.handler.on_get_missing_events_state_dag( + origin, room_id, earliest_events, latest_events, limit + ) + else: + missing_events = await self.handler.on_get_missing_events( + origin, room_id, earliest_events, latest_events, limit + ) if len(missing_events) < 5: logger.debug( diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index f7240c2f7f..9b48bf33fb 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -149,6 +149,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.presence import UserPresenceState from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.federation.sender.per_destination_queue import ( CATCHUP_RETRY_INTERVAL, PerDestinationQueue, @@ -660,7 +661,10 @@ async def handle_event(event: EventBase) -> None: # banned then it won't receive the event because it won't # be in the room after the ban. destinations = await self.state.get_hosts_in_room_at_events( - event.room_id, event_ids=event.prev_event_ids() + event.room_id, + event_ids=event.prev_state_events + if supports_msc4242_state_dag(event) + else event.prev_event_ids(), ) except Exception: logger.exception( diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index 8a8914bf4f..af6c733029 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -28,7 +28,7 @@ Sequence, ) -from synapse.api.constants import Direction, EduTypes +from synapse.api.constants import Direction, EduTypes, StateDag from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersions from synapse.api.urls import FEDERATION_UNSTABLE_PREFIX, FEDERATION_V2_PREFIX @@ -639,6 +639,7 @@ async def on_POST( limit = int(content.get("limit", 10)) earliest_events = content.get("earliest_events", []) latest_events = content.get("latest_events", []) + walk_state_dag = bool(content.get(StateDag.GET_MISSING_EVENTS_FIELD, False)) result = await self.handler.on_get_missing_events( origin, @@ -646,6 +647,7 @@ async def on_POST( earliest_events=earliest_events, latest_events=latest_events, limit=limit, + walk_state_dag=walk_state_dag, ) return 200, result diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c67974c575..655e8d244a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -40,7 +40,13 @@ from unpaddedbase64 import decode_base64 from synapse import event_auth -from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership +from synapse.api.constants import ( + MAX_DEPTH, + EventContentFields, + EventTypes, + Membership, + StateDag, +) from synapse.api.errors import ( AuthError, CodeMessageException, @@ -49,6 +55,7 @@ FederationError, FederationPullAttemptBackoffError, HttpResponseException, + InvalidAPICallError, NotFoundError, PartialStateConflictError, RequestSendFailed, @@ -58,7 +65,8 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase +from synapse.events import EventBase, event_exists_in_state_dag +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import EventContext, UnpersistedEventContextBase from synapse.events.validator import EventValidator from synapse.federation.federation_client import InvalidResponseError @@ -1042,6 +1050,11 @@ async def on_make_join_request( # Note that this requires the /send_join request to come back to the # same server. prev_event_ids = None + prev_state_events = None + if room_version.msc4242_state_dags: + prev_state_events = list( + await self.store.get_state_dag_extremities(room_id) + ) if room_version.restricted_join_rule: # Note that the room's state can change out from under us and render our # nice join rules-conformant event non-conformant by the time we build the @@ -1098,6 +1111,7 @@ async def on_make_join_request( ) = await self.event_creation_handler.create_new_client_event( builder=builder, prev_event_ids=prev_event_ids, + prev_state_events=prev_state_events, ) except SynapseError as e: logger.warning("Failed to create join to %s because %s", room_id, e) @@ -1517,6 +1531,80 @@ async def on_get_missing_events( return missing_events + async def on_get_missing_events_state_dag( + self, + origin: str, + room_id: str, + earliest_events: list[str], + latest_events: list[str], + limit: int, + ) -> list[EventBase]: + """Processes a /get_missing_events request for the state DAG. + + This is similar to processing the normal DAG with a few notable exceptions: + * The max 20 limit does not apply. As the entire state DAG needs to be filled + in, we cannot arbitrarily set a low limit. If the state DAG delta is 1000s of + events, we rely on the sender to set sensible limits depending on the + bandwidth/round trip tradeoff, capped at `StateDag.MAX_MISSING_EVENTS` so a + single request cannot ask for an unbounded response. + * We do not filter any events in the state DAG. History visibility does not + filter out delivery of auth chain events, so neither should this. All of the + returned events will be treated as outliers and as such will not be delivered + to clients. + * `latest_events` may name events which are not themselves in the state DAG, + because the caller seeds this request with whatever it received over /send. + Only state events have `msc4242_state_dag_edges` rows, so we walk from such an + event's `prev_state_events` instead, and return those as the first hop. + """ + # guard against extreme cases + if len(earliest_events) > 100 or len(latest_events) > 100: + raise InvalidAPICallError( + "'earliest_events' and/or 'latest_events' too large, size must be less than 100" + ) + + await self._event_auth_handler.assert_host_in_room(room_id, origin, True) + limit = min(limit, StateDag.MAX_MISSING_EVENTS) + earliest_event_set = set(earliest_events) + + seed_events = await self.store.get_events(latest_events) + + seed_event_ids: list[str] = [] + first_hop_event_ids: set[str] = set() + for event_id, event in seed_events.items(): + if event_exists_in_state_dag(event): + seed_event_ids.append(event_id) + continue + # event is a message, so its prev_state_events are the first returned hop + assert supports_msc4242_state_dag( + event + ) # type-assert to access .prev_state_events + first_hop_event_ids.update( + prev_state_event_id + for prev_state_event_id in event.prev_state_events + if prev_state_event_id not in earliest_event_set + ) + # `latest_events` can mix state and non-state events. If an event's + # prev_state_event is also one of the state-event seeds, it will be walked from + # below, so returning it as a first hop as well would just duplicate it. + first_hop_event_ids.difference_update(seed_event_ids) + first_hop_events: list[EventBase] = [] + if first_hop_event_ids: + first_hop_events = await self.store.get_events_as_list( + sorted(first_hop_event_ids) + ) + # the sort exists to make [:limit] truncation deterministic + first_hop_events.sort(key=lambda ev: ev.event_id) + first_hop_events = first_hop_events[:limit] + seed_event_ids.extend(ev.event_id for ev in first_hop_events) + + missing_events = await self.store.get_missing_events_state_dag( + room_id=room_id, + earliest_event_ids=earliest_events, + latest_event_ids=seed_event_ids, + limit=limit - len(first_hop_events), + ) + return first_hop_events + missing_events + async def exchange_third_party_invite( self, sender_user_id: str, target_user_id: str, room_id: str, signed: JsonDict ) -> None: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 26ebd42574..92d82aeda1 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1244,10 +1244,6 @@ async def create_room( creation_content = config.get("creation_content", {}) # override any attempt to set room versions via the creation_content creation_content["room_version"] = room_version.identifier - # We do not currently support federating state DAG rooms. - # See related restriction in /send_join requests in federation_client.py. - if room_version.msc4242_state_dags: - creation_content[EventContentFields.FEDERATE] = False # trusted private chats have the invited users marked as additional creators if ( diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 1908b26f68..87748c4072 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1195,8 +1195,6 @@ def _get_auth_chain_difference_txn( # Return all events where not all sets can reach them. return {eid for eid, n in event_to_missing_sets.items() if n} - # FIXME(2026-04-22): Remove comment when used. Unused currently, but will be used in - # future MSC4242 PRs. async def get_state_dag( self, room_id: str, forward_extrems: set[str] ) -> dict[str, MSC4242Event]: @@ -1260,8 +1258,6 @@ def _get_state_events_txn(txn: LoggingTransaction, room_id: str) -> list[str]: return result - # FIXME(2026-04-22): Remove comment when used. Unused currently, but will be used in - # future MSC4242 PRs. async def get_missing_events_state_dag( self, *, diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 6056448f20..61dcb7446b 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -20,18 +20,20 @@ # import logging from http import HTTPStatus +from unittest import skip as skip_test from unittest.mock import Mock from parameterized import parameterized from twisted.internet.testing import MemoryReactor -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, StateDag from synapse.api.errors import Codes, FederationError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.http.types import QueryParams from synapse.logging.context import LoggingContext from synapse.rest import admin @@ -393,6 +395,180 @@ def _create_acl_event(content: JsonDict) -> EventBase: ) +class GetMissingEventsStateDagTests(unittest.FederatingHomeserverTestCase): + """ + Tests for walking the MSC4242 state DAG via /get_missing_events. + + In future, these can be ported to Complement to benefit other servers, + but as of writing MSC4242 is not fully implemented so can't be in Complement yet. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = {"msc4242_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.local_user_id = self.register_user("alice", "pass") + self.local_user_token = self.login("alice", "pass") + self.room_id = self.helper.create_room_as( + room_creator=self.local_user_id, + tok=self.local_user_token, + room_version=RoomVersions.MSC4242v12.identifier, + ) + self.inject_room_member( + self.room_id, f"@remote:{self.OTHER_SERVER_NAME}", "join" + ) + + # a message is interleaved between the two topics, so it is an ancestor of + # `second_topic_id` in the timeline DAG but absent from the state DAG + self.first_topic_id = self.helper.send_state( + self.room_id, "m.room.topic", {"topic": "one"}, tok=self.local_user_token + )["event_id"] + self.message_id = self.helper.send( + self.room_id, body="a message", tok=self.local_user_token + )["event_id"] + self.second_topic_id = self.helper.send_state( + self.room_id, "m.room.topic", {"topic": "two"}, tok=self.local_user_token + )["event_id"] + + def _get_missing_events( + self, + earliest_events: list[str], + latest_events: list[str], + walk_state_dag: bool, + limit: int = 10, + ) -> list[JsonDict]: + content: JsonDict = { + "earliest_events": earliest_events, + "latest_events": latest_events, + "limit": limit, + } + if walk_state_dag: + content[StateDag.GET_MISSING_EVENTS_FIELD] = True + + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_id}", + content=content, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.json_body) + return channel.json_body["events"] + + def test_walks_the_state_dag(self) -> None: + events = self._get_missing_events( + earliest_events=[], + latest_events=[self.second_topic_id], + limit=20, + walk_state_dag=True, + ) + + returned = [(ev["type"], ev.get("state_key")) for ev in events] + self.assertIncludes( + set(returned), + { + ("m.room.create", ""), + ("m.room.join_rules", ""), + ("m.room.history_visibility", ""), + ("m.room.power_levels", ""), + ("m.room.topic", ""), + ("m.room.member", self.local_user_id), + ("m.room.member", f"@remote:{self.OTHER_SERVER_NAME}"), + }, + exact=True, + ) + + # the seed itself is not returned, so only the first topic is + self.assertEqual(len([ev for ev in returned if ev[0] == "m.room.topic"]), 1) + + # the state DAG contains no message events + self.assertEqual([ev for ev in returned if ev[0] == "m.room.message"], []) + + def test_stops_at_earliest_events(self) -> None: + events = self._get_missing_events( + earliest_events=[self.first_topic_id], + latest_events=[self.second_topic_id], + limit=20, + walk_state_dag=True, + ) + self.assertEqual(events, []) + + def test_honours_limit(self) -> None: + events = self._get_missing_events( + earliest_events=[], + latest_events=[self.second_topic_id], + limit=2, + walk_state_dag=True, + ) + self.assertEqual(len(events), 2) + + def test_is_opt_in(self) -> None: + state_dag_events = self._get_missing_events( + earliest_events=[], + latest_events=[self.second_topic_id], + limit=20, + walk_state_dag=True, + ) + timeline_events = self._get_missing_events( + earliest_events=[], + latest_events=[self.second_topic_id], + limit=20, + walk_state_dag=False, + ) + + self.assertEqual( + [ev for ev in state_dag_events if ev["type"] == "m.room.message"], [] + ) + self.assertNotEqual( + [ev for ev in timeline_events if ev["type"] == "m.room.message"], [] + ) + + def test_walks_from_a_non_state_event(self) -> None: + store = self.hs.get_datastores().main + message = self.get_success(store.get_event(self.message_id)) + assert supports_msc4242_state_dag(message) + state_dag = self.get_success( + store.get_state_dag(self.room_id, set(message.prev_state_events)) + ) + + events = self._get_missing_events( + earliest_events=[], + latest_events=[self.message_id], + walk_state_dag=True, + limit=20, + ) + + # We get all the same events + self.assertIncludes( + {(ev["type"], ev.get("state_key")) for ev in events}, + {(ev.type, ev.state_key) for ev in state_dag.values()}, + exact=True, + ) + # and no messages (we aren't walking up prev_events) + self.assertEqual([ev for ev in events if ev["type"] == "m.room.message"], []) + + def test_non_state_seed_stops_at_earliest_events(self) -> None: + message = self.get_success( + self.hs.get_datastores().main.get_event(self.message_id) + ) + assert supports_msc4242_state_dag(message) + events = self._get_missing_events( + earliest_events=list(message.prev_state_events), + latest_events=[self.message_id], + walk_state_dag=True, + limit=20, + ) + self.assertEqual(events, []) + + class MessageAcceptTests(unittest.FederatingHomeserverTestCase): """ Tests to make sure that we don't accept flawed events from federation (incoming). @@ -1002,36 +1178,45 @@ def _test_send_join_common(self, room_version: str) -> None: ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - # we should get complete room state back - returned_state = [ - (ev["type"], ev["state_key"]) for ev in channel.json_body["state"] + expected_state = [ + ("m.room.create", ""), + ("m.room.power_levels", ""), + ("m.room.join_rules", ""), + ("m.room.history_visibility", ""), + ("m.room.member", f"@kermit_v{room_version}:test"), + ("m.room.member", f"@fozzie_v{room_version}:test"), + # nb: *not* the joining user ] - self.assertCountEqual( - returned_state, - [ - ("m.room.create", ""), - ("m.room.power_levels", ""), - ("m.room.join_rules", ""), - ("m.room.history_visibility", ""), - ("m.room.member", f"@kermit_v{room_version}:test"), - ("m.room.member", f"@fozzie_v{room_version}:test"), - # nb: *not* the joining user - ], - ) - # also check the auth chain - returned_auth_chain_events = [ - (ev["type"], ev["state_key"]) for ev in channel.json_body["auth_chain"] - ] - self.assertCountEqual( - returned_auth_chain_events, - [ - ("m.room.create", ""), - ("m.room.member", f"@kermit_v{room_version}:test"), - ("m.room.power_levels", ""), - ("m.room.join_rules", ""), - ], - ) + if KNOWN_ROOM_VERSIONS[room_version].msc4242_state_dags: + returned_state_dag = [ + (ev["type"], ev["state_key"]) for ev in channel.json_body["state_dag"] + ] + self.assertIncludes( + set(returned_state_dag), set(expected_state), exact=True + ) + self.assertNotIn("state", channel.json_body) + self.assertNotIn("auth_chain", channel.json_body) + else: + # we should get complete room state back + returned_state = [ + (ev["type"], ev["state_key"]) for ev in channel.json_body["state"] + ] + self.assertCountEqual(returned_state, expected_state) + + # also check the auth chain + returned_auth_chain_events = [ + (ev["type"], ev["state_key"]) for ev in channel.json_body["auth_chain"] + ] + self.assertCountEqual( + returned_auth_chain_events, + [ + ("m.room.create", ""), + ("m.room.member", f"@kermit_v{room_version}:test"), + ("m.room.power_levels", ""), + ("m.room.join_rules", ""), + ], + ) # the room should show that the new user is a member r = self.get_success(self._storage_controllers.state.get_current_state(room_id)) @@ -1041,20 +1226,105 @@ def _test_send_join_common(self, room_version: str) -> None: @override_config({"use_frozen_dicts": True}) def test_send_join_with_frozen_dicts(self, room_version: str) -> None: """Test send_join with USE_FROZEN_DICTS=True""" - if room_version == RoomVersions.MSC4242v12.identifier: - # TODO: This room version doesn't work over federation in this PR. - return self._test_send_join_common(room_version) @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) @override_config({"use_frozen_dicts": False}) def test_send_join_without_frozen_dicts(self, room_version: str) -> None: """Test send_join with USE_FROZEN_DICTS=False""" - if room_version == RoomVersions.MSC4242v12.identifier: - # TODO: This room version doesn't work over federation in this PR. - return self._test_send_join_common(room_version) + @skip_test("requires MSC4242 inbound event auth") + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_send_join_state_dag(self) -> None: + """ + KNOWN_ROOM_VERSIONS lacks MSC4242v12 rooms because it is behind an experimental features flag + so set the flag and do the same test as above. + + FIXME: When MSC4242 rooms are not gated behind a + config flag this test can be deleted. + """ + self._test_send_join_common(RoomVersions.MSC4242v12.identifier) + + @skip_test("requires MSC4242 inbound event auth") + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_send_join_state_dag_ignores_partial_state(self) -> None: + # FIXME: when we support partial joins this test can be deleted + room_version = RoomVersions.MSC4242v12.identifier + creator_user_id = self.register_user("user1_msc4242", "test") + tok = self.login(creator_user_id, "test") + room_id = self.helper.create_room_as( + room_creator=creator_user_id, tok=tok, room_version=room_version + ) + + joining_user = "@misspiggy:" + self.OTHER_SERVER_NAME + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{room_id}/{joining_user}?ver={room_version}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + join_event_dict = channel.json_body["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, + KNOWN_ROOM_VERSIONS[room_version], + ) + # Ask to join as a partial state (omit_members=true) which should be ignored + # because we don't support partial joins in state DAG rooms just yet + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{room_id}/x?omit_members=true", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["members_omitted"], False) + self.assertNotIn("servers_in_room", channel.json_body) + self.assertIn("state_dag", channel.json_body) + + # the full state DAG is returned, including the member events that a + # partial state join would have omitted to prove we are in fact ignoring the + # partial state flag. + self.assertIncludes( + {(ev["type"], ev["state_key"]) for ev in channel.json_body["state_dag"]}, + { + ("m.room.create", ""), + ("m.room.power_levels", ""), + ("m.room.join_rules", ""), + ("m.room.history_visibility", ""), + ("m.room.member", creator_user_id), + }, + exact=True, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_make_join_state_dag(self) -> None: + room_version = RoomVersions.MSC4242v12.identifier + creator_user_id = self.register_user("user1_msc4242", "test") + tok = self.login("user1_msc4242", "test") + room_id = self.helper.create_room_as( + room_creator=creator_user_id, tok=tok, room_version=room_version + ) + + joining_user = "@misspiggy:" + self.OTHER_SERVER_NAME + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{room_id}/{joining_user}?ver={room_version}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + event = channel.json_body["event"] + # MSC4242 events don't have an auth_events field. + self.assertNotIn("auth_events", event) + + extremities = self.get_success( + self.hs.get_datastores().main.get_state_dag_extremities(room_id) + ) + self.assertGreater(len(extremities), 0) + # When creating events, prev_state_events should be set to the state DAG fwd extremities + self.assertIncludes( + set(event["prev_state_events"]), set(extremities), exact=True + ) + def test_send_join_partial_state(self) -> None: """/send_join should return partial state, if requested""" joining_user = "@misspiggy:" + self.OTHER_SERVER_NAME From 1e69b49255891100892b5e1d0a3e8f30e427e6fb Mon Sep 17 00:00:00 2001 From: Mohammed Sufiyan Ahmed Date: Fri, 11 Sep 2026 17:25:13 +0530 Subject: [PATCH 08/41] Fix `Schema Diff` CI failing to post comment on PRs from forks (#20207) Fixes: #20167 The `Schema Diff` workflow posts a PR comment showing the effective schema diff. For PRs from forks, `GITHUB_TOKEN` is downgraded to read-only, so the comment-posting step was silently failing. ### Changes - `schema_diff.yml`: only post the comment directly when the PR is from the same repository. For forked PRs, upload the diff (and PR number) as a short-lived artifact instead of trying to comment. - `schema_diff_comment.yml` (new): triggered by `workflow_run` after `Schema Diff` completes, with `pull-requests: write` permission (granted because this workflow always runs in the context of the base repository). It downloads the artifact, if present, and posts the comment on behalf of the forked PR. This avoids `pull_request_target`, per the security concerns raised in the issue (zizmor flags it as dangerous). The new workflow only ever treats the downloaded artifact as inert comment text -- it is never executed. --------- Co-authored-by: Olivier 'reivilibre --- .github/workflows/schema_diff.yml | 29 ++++++++- .github/workflows/schema_diff_comment.yml | 71 +++++++++++++++++++++++ changelog.d/20207.misc | 1 + 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/schema_diff_comment.yml create mode 100644 changelog.d/20207.misc diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index f0d148b071..eaa25b53fa 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -1,4 +1,4 @@ -name: Schema Diff +name: Schema Diff # If this is changed, need to update `schema_diff_comment.yml` as well on: pull_request: @@ -95,9 +95,34 @@ jobs: echo "⚠️ Schema diff generation failed. See job logs for details." \ > "${{ runner.temp }}/schema_diff.md" + # Post a comment. + # + # For same-repo PRs, we can do this directly and we are done. - name: Post sticky PR comment uses: marocchino/sticky-pull-request-comment@3d7b8546315c63df45a03981d50a43ec19237f80 # v3 - if: always() + # Only run for same-repo PRs + if: always() && github.event.pull_request.head.repo.full_name == github.repository with: header: schema-diff path: ${{ runner.temp }}/schema_diff.md + + # For PRs from forks, we only have a read-only `GITHUB_TOKEN` here and need to work around + # by triggering a `workflow_run` onto the main repo. + # + # We upload an artifact and the `schema_diff_comment.yml` workflow will be triggered + # on the main repo by this workflow exiting. + - name: Save PR number for forked PRs + # Only run for fork PRs + if: always() && github.event.pull_request.head.repo.full_name != github.repository + run: echo "${{ github.event.pull_request.number }}" > "${{ runner.temp }}/pr_number" + + - name: Upload schema diff artifact for forked PRs + # Only run for fork PRs + if: always() && github.event.pull_request.head.repo.full_name != github.repository + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: schema-diff + path: | + ${{ runner.temp }}/schema_diff.md + ${{ runner.temp }}/pr_number + retention-days: 1 diff --git a/.github/workflows/schema_diff_comment.yml b/.github/workflows/schema_diff_comment.yml new file mode 100644 index 0000000000..4aefc3a123 --- /dev/null +++ b/.github/workflows/schema_diff_comment.yml @@ -0,0 +1,71 @@ +name: Post schema diff comment for forked PRs + +# The "Schema Diff" workflow can't post a PR comment for PRs from forks, +# since GITHUB_TOKEN is downgraded to read-only in that context. Instead it +# uploads the diff (and PR number) as an artifact, which we pick up here and +# post as a comment. +# +# This workflow runs in the context of the main repository, so its +# `GITHUB_TOKEN` can have write access. +# The downloaded artifact comes from an untrusted fork though, so it must only +# ever be treated as inert comment text, never executed. +# The PR number it carries is likewise untrusted: we cross-check it against the trusted +# `workflow_run` event before posting, so a fork PR can't point the comment +# at some other PR or issue. +on: + workflow_run: + workflows: ["Schema Diff"] + types: [completed] + +# DANGER: Due to untrusted input and untrusted triggers, +# think very carefully before expanding the permissions +permissions: + # To read the artifacts from the `schema_diff.yml` run + actions: read + # To post a comment to the PR + pull-requests: write + +jobs: + comment: + name: Post schema diff comment + runs-on: ubuntu-latest + # Only run when triggered by a pull request and only on a PR from a fork + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name != github.repository && github.event.workflow_run.conclusion != 'cancelled' + steps: + - name: Download schema diff artifact + id: download + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: schema-diff + path: schema-diff + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + # The artifact (including the PR number) comes from an untrusted fork. + # Cross-check the number against the trusted workflow_run event before + # posting, so a fork PR cannot target another PR or issue. + - name: Verify PR matches the run that produced the artifact + id: verify + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # No checkout in this job, so tell gh which repo to talk to. + GH_REPO: ${{ github.repository }} + EXPECTED_BRANCH: ${{ github.event.workflow_run.head_branch }} + EXPECTED_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + run: | + pr_number="$(cat schema-diff/pr_number)" + read -r pr_branch pr_head_repo < <( + gh api "repos/{owner}/{repo}/pulls/$pr_number" --jq '.head.ref + " " + .head.repo.full_name' + ) + if [ "$pr_branch" != "$EXPECTED_BRANCH" ] || [ "$pr_head_repo" != "$EXPECTED_HEAD_REPO" ]; then + echo "::error::Artifact PR $pr_number does not match the triggering run; refusing to post a comment" + exit 1 + fi + echo "number=$pr_number" >> "$GITHUB_OUTPUT" + + - name: Post sticky PR comment + uses: marocchino/sticky-pull-request-comment@3d7b8546315c63df45a03981d50a43ec19237f80 # v3 + with: + header: schema-diff + number_force: ${{ steps.verify.outputs.number }} + path: schema-diff/schema_diff.md diff --git a/changelog.d/20207.misc b/changelog.d/20207.misc new file mode 100644 index 0000000000..3709ad46f1 --- /dev/null +++ b/changelog.d/20207.misc @@ -0,0 +1 @@ +Fix the `Schema Diff` CI check failing to post a comment on pull requests from forks. From fa9dbbd3ecc261a4c77b2e522fef244b4a62f273 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Fri, 11 Sep 2026 14:59:15 +0200 Subject: [PATCH 09/41] Stabilize the `M_UNKNOWN_DEVICE` error code (#20181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of: #19415 TLDR: return the `M_UNKNOWN_DEVICE` error code instead of the unstable `ORG.MATRIX.MSC4326.M_UNKNOWN_DEVICE` identifier. > **[Added in `v1.17`]** Application services MAY similarly masquerade as a specific device ID belonging the user ID through use of the `device_id` query string parameter on the request. If the given device ID is not known to belong to the user, the server will return a 400 `M_UNKNOWN_DEVICE` error. > > — [Matrix v1.19, Application Service API — Identity assertion](https://spec.matrix.org/v1.19/application-service-api/#identity-assertion) Synapse returns the correct 400, but with the unstable identifier. MSC4326 was stabilized in Matrix 1.17 and its experimental flag was already removed in #19033; only the error code identifier was left behind. Before: ``` GET /_matrix/client/v3/sync?user_id=@alice:test&device_id=NOT_A_REAL_DEVICE_ID # appservice token 400 {"errcode": "ORG.MATRIX.MSC4326.M_UNKNOWN_DEVICE"} ``` After: ``` GET /_matrix/client/v3/sync?user_id=@alice:test&device_id=NOT_A_REAL_DEVICE_ID # appservice token 400 {"errcode": "M_UNKNOWN_DEVICE"} ``` I verified that no implementation was currently handling the prefixed error code. --- changelog.d/20181.bugfix | 1 + synapse/api/errors.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20181.bugfix diff --git a/changelog.d/20181.bugfix b/changelog.d/20181.bugfix new file mode 100644 index 0000000000..477902b97f --- /dev/null +++ b/changelog.d/20181.bugfix @@ -0,0 +1 @@ +Return the stable `M_UNKNOWN_DEVICE` error code, added in Matrix 1.17, instead of its unstable [MSC4326](https://github.com/matrix-org/matrix-spec-proposals/pull/4326)-prefixed identifier. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 038578dbed..75cf8add27 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -157,7 +157,7 @@ class Codes(str, Enum): MSC4306_NOT_IN_THREAD = "IO.ELEMENT.MSC4306.M_NOT_IN_THREAD" # Part of MSC4326 - UNKNOWN_DEVICE = "ORG.MATRIX.MSC4326.M_UNKNOWN_DEVICE" + UNKNOWN_DEVICE = "M_UNKNOWN_DEVICE" USER_LIMIT_EXCEEDED = "M_USER_LIMIT_EXCEEDED" From 46f3eb082effb9e96db4f61164a071e37280b4a5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:15:39 +0100 Subject: [PATCH 10/41] Fix warnings generated by `mdbook build` (#20217) --- changelog.d/20217.doc | 1 + docs/development/synapse_architecture/cancellation.md | 1 + docs/openid.md | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20217.doc diff --git a/changelog.d/20217.doc b/changelog.d/20217.doc new file mode 100644 index 0000000000..a7d4590cfa --- /dev/null +++ b/changelog.d/20217.doc @@ -0,0 +1 @@ +Fix small warnings in documentation building tooling output. \ No newline at end of file diff --git a/docs/development/synapse_architecture/cancellation.md b/docs/development/synapse_architecture/cancellation.md index a12f119fb5..91d8c8ca4d 100644 --- a/docs/development/synapse_architecture/cancellation.md +++ b/docs/development/synapse_architecture/cancellation.md @@ -284,6 +284,7 @@ await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") ``` + ### Uncancelled processing diff --git a/docs/openid.md b/docs/openid.md index e91d375c41..3115401961 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -140,7 +140,7 @@ The synapse config will look like this: 3. Add a rule with any name to add the `preferred_username` claim. (See https://auth0.com/docs/customize/rules/create-rules for more information on how to create rules.) -
+
Code sample ```js From 3f17bd706ae9043e31db3e9d24a259ca42c9a312 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Fri, 11 Sep 2026 17:54:31 +0200 Subject: [PATCH 11/41] Return `200 {}` again for an unset `displayname`/`avatar_url` (#20200) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of: #19414 When [MSC4133](https://github.com/matrix-org/matrix-spec-proposals/pull/4133) (custom profile fields) was implemented, the returned value for an unset display name changed from `200 {}` to `200 { displayname: null }`. This happened first on the unstable `uk.tcpip.msc4133` path in #17488 (1.123.0), then on the stable path when #18635 (1.135.0) unified the `displayname`, `avatar_url` and custom field servlets. Neither PR discussed the change in review, so it looks like an unintended side effect of the refactor rather than a deliberate decision. The v1.16 spec mandated to change from returning `200 {}` to `404` but change was not identified as breaking and was eventually not implemented in other clients and server. This PR has a sister MSC that proposes to return to the pre-1.16 error codes: [MSC4537](https://github.com/matrix-org/matrix-spec-proposals/pull/4537). Before: ``` GET /_matrix/client/v3/profile/@alice:test/displayname 200 {"displayname": null} ``` After: ``` GET /_matrix/client/v3/profile/@alice:test/displayname 200 {} ``` ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20200.bugfix | 1 + synapse/rest/client/profile.py | 25 ++++++++++++++++++------- tests/rest/client/test_profile.py | 21 +++++++++++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 changelog.d/20200.bugfix diff --git a/changelog.d/20200.bugfix b/changelog.d/20200.bugfix new file mode 100644 index 0000000000..41eadbcf6e --- /dev/null +++ b/changelog.d/20200.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.135.0 where fetching an unset `displayname` or `avatar_url` via `GET /_matrix/client/v3/profile/{userId}/{field}` returned the field with a `null` value instead of an empty object. diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 4431aa2b2c..58a6c7c4e2 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -36,7 +36,7 @@ ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns -from synapse.types import JsonDict, JsonValue, UserID +from synapse.types import JsonDict, UserID from synapse.util.stringutils import is_namedspaced_grammar if TYPE_CHECKING: @@ -145,16 +145,27 @@ async def on_GET( user = UserID.from_string(user_id) await self.profile_handler.check_profile_query_allowed(user, requester_user) + ret: JsonDict = {} if field_name == ProfileFields.DISPLAYNAME: - field_value: ( - JsonValue | dict[str, JsonValue] - ) = await self.profile_handler.get_displayname(user) + displayname = await self.profile_handler.get_displayname(user) + if displayname is not None: + ret[field_name] = displayname elif field_name == ProfileFields.AVATAR_URL: - field_value = await self.profile_handler.get_avatar_url(user) + avatar_url = await self.profile_handler.get_avatar_url(user) + if avatar_url is not None: + ret[field_name] = avatar_url else: - field_value = await self.profile_handler.get_profile_field(user, field_name) + # Custom fields deliberately behave differently from `displayname` and + # `avatar_url`: an unset custom field raises a 404 rather than returning + # `200 {}`. The spec allows both, see MSC4537: + # https://github.com/matrix-org/matrix-spec-proposals/pull/4537 + # This is likely to change once the spec settles on either 404 or + # `200 {}` only, which would be a breaking change. + ret[field_name] = await self.profile_handler.get_profile_field( + user, field_name + ) - return 200, {field_name: field_value} + return 200, ret async def on_PUT( self, request: SynapseRequest, user_id: str, field_name: str diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 524045c751..30f58f1cc3 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -563,6 +563,27 @@ def test_msc4069_inhibit_propagation_like_default(self) -> None: # The client requested ?propagate=true, so it should have happened. self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + def test_get_unset_field_omits_key(self) -> None: + """ + Fetching a standard profile field which the user has not set should + return a 200 with an empty body, rather than the key with a null value. + """ + for field in ("displayname", "avatar_url"): + channel = self.make_request( + "DELETE", + f"/_matrix/client/v3/profile/{self.owner}/{field}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + for path in ( + f"/profile/{self.owner}/{field}", + f"/_matrix/client/v3/profile/{self.owner}/{field}", + ): + channel = self.make_request("GET", path) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {}, path) + def test_get_missing_custom_field(self) -> None: channel = self.make_request( "GET", From 6825c98eb22185d020aa1dd94fa3c1d0abd7b334 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 14 Sep 2026 11:41:21 +0100 Subject: [PATCH 12/41] Add support for un-soft-failing MSC4354 Sticky Events when room state changes, making federation support more reliable. (#20204) Part of: MSC4354 Experimental feature tracking issue: https://github.com/element-hq/synapse/issues/19409 Related Complement tests currently in https://github.com/matrix-org/complement/pull/806/files#diff-6c9d6d169485d0848c6b20dd9b43f6fe669a8a710e42f953d08fa25a99cc8f4cR509 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20204.bugfix | 1 + synapse/storage/databases/main/events.py | 30 ++ .../storage/databases/main/events_worker.py | 16 +- .../storage/databases/main/sticky_events.py | 341 +++++++++++++++++- tests/storage/test_sticky_events.py | 142 +++++++- 5 files changed, 526 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20204.bugfix diff --git a/changelog.d/20204.bugfix b/changelog.d/20204.bugfix new file mode 100644 index 0000000000..783bedee14 --- /dev/null +++ b/changelog.d/20204.bugfix @@ -0,0 +1 @@ +Add support for un-soft-failing [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) when room state changes, making federation support more reliable. \ No newline at end of file diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 35f387576f..7550a03a43 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -23,6 +23,7 @@ import itertools import logging from collections import OrderedDict +from collections.abc import Set from typing import ( TYPE_CHECKING, Any, @@ -379,6 +380,24 @@ async def _persist_events_and_state_updates( ) ) + sticky_events_to_un_soft_fail: set[str] = set() + if self._msc4354_enabled and state_delta_for_room is not None: + # When we change the room's current state with `state_delta_for_room`, + # that might cause some previously soft-failed sticky events to now pass + # the state-dependent auth checks. + # In other words, the sticky events could have been valid if they had + # waited for these state changes. + # For that reason, we give sticky events a second chance. + # We compute them here and then un-soft-fail them atomically with the + # persistence of the events. + sticky_events_to_un_soft_fail = ( + await self.store.compute_sticky_events_to_un_soft_fail( + room_id, + events_and_contexts, + state_delta_for_room, + ) + ) + await self.db_pool.runInteraction( "persist_events", self._persist_events_txn, @@ -390,6 +409,7 @@ async def _persist_events_and_state_updates( new_event_links=new_event_links, sliding_sync_table_changes=sliding_sync_table_changes, new_state_dag_forward_extremities=new_state_dag_forward_extremities, + sticky_events_to_un_soft_fail=sticky_events_to_un_soft_fail, ) persist_event_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( len(events_and_contexts) @@ -1055,6 +1075,7 @@ def _persist_events_txn( new_event_links: dict[str, NewEventChainLinks], sliding_sync_table_changes: SlidingSyncTableChanges | None, new_state_dag_forward_extremities: set[str] | None = None, + sticky_events_to_un_soft_fail: Set[str] = frozenset(), ) -> None: """Insert some number of room events into the necessary database tables. @@ -1083,6 +1104,8 @@ def _persist_events_txn( `sliding_sync_membership_snapshots` and `sliding_sync_joined_rooms` tables derived from the given `delta_state` (see `_calculate_sliding_sync_table_changes(...)`) + sticky_events_to_un_soft_fail: + Sticky events which will be un-soft-failed when persisting the events. Raises: PartialStateConflictError: if attempting to persist a partial state event in @@ -1213,6 +1236,13 @@ def _persist_events_txn( txn, [ev for ev, _ in events_and_contexts] ) + # Un-soft-fail any sticky events that the state delta applied just above + # has made valid. + if sticky_events_to_un_soft_fail: + self.store.un_soft_fail_sticky_events_txn( + txn, sticky_events_to_un_soft_fail + ) + # We only update the sliding sync tables for non-backfilled events. self._update_sliding_sync_tables_with_new_persisted_events_txn( txn, room_id, events_and_contexts diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 27dab290b3..215c8e44e3 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -68,7 +68,12 @@ from synapse.metrics.background_process_metrics import ( wrap_as_background_process, ) -from synapse.replication.tcp.streams import BackfillStream, UnPartialStatedEventStream +from synapse.replication.tcp.streams import ( + BackfillStream, + StickyEventsStream, + UnPartialStatedEventStream, +) +from synapse.replication.tcp.streams._base import StickyEventsStreamRow from synapse.replication.tcp.streams.events import EventsStream from synapse.replication.tcp.streams.partial_state import UnPartialStatedEventStreamRow from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause @@ -470,6 +475,15 @@ def process_replication_rows( # If the partial-stated event became rejected or unrejected # when it wasn't before, we need to invalidate this cache. self._invalidate_local_get_event_cache(row.event_id) + elif stream_name == StickyEventsStream.NAME: + for row in rows: + assert isinstance(row, StickyEventsStreamRow) + + # A sticky event only gets a new row on this stream when it is first + # persisted (in which case there's nothing cached to invalidate) or when + # its soft-failure status changed, which is stored in the event's + # internal metadata, so invalidate the cached event. + self._invalidate_local_get_event_cache(row.event_id) super().process_replication_rows(stream_name, instance_name, token, rows) diff --git a/synapse/storage/databases/main/sticky_events.py b/synapse/storage/databases/main/sticky_events.py index eee6b92415..bc2c2e67e4 100644 --- a/synapse/storage/databases/main/sticky_events.py +++ b/synapse/storage/databases/main/sticky_events.py @@ -12,12 +12,18 @@ # . import logging import random +from collections.abc import Set from dataclasses import dataclass +from itertools import chain from typing import TYPE_CHECKING, Collection, cast from twisted.internet.defer import Deferred +from synapse import event_auth +from synapse.api.constants import EventTypes +from synapse.api.errors import AuthError from synapse.events import EventBase +from synapse.events.snapshot import EventPersistencePair from synapse.replication.tcp.streams._base import StickyEventsStream from synapse.storage.database import ( DatabasePool, @@ -26,10 +32,14 @@ make_in_list_sql_clause, ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore +from synapse.storage.databases.main.events import DeltaState from synapse.storage.databases.main.state import StateGroupWorkerStore from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.types import StateKey +from synapse.types.state import StateFilter from synapse.util.duration import Duration +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer @@ -127,7 +137,7 @@ def process_replication_position( super().process_replication_position(stream_name, instance_name, token) def get_max_sticky_events_stream_id(self) -> int: - """Get the current maximum stream_id for thread subscriptions. + """Get the current maximum stream_id for sticky events. Returns: The maximum stream_id @@ -391,6 +401,335 @@ def insert_sticky_events_txn( ], ) + async def compute_sticky_events_to_un_soft_fail( + self, + room_id: str, + events_and_contexts: list[EventPersistencePair], + state_delta_for_room: DeltaState, + ) -> set[str]: + """ + Determine which soft-failed sticky events in the given room will become + un-soft-failed once `state_delta_for_room` has been applied to the current state. + + As per MSC4354: + > **Re-evaluate soft-failure** of soft-failed unexpired sticky events when the membership state of the sender changes.[^softfail] + > + > [^softfail]: Not all servers will agree on soft-failure status due to the check considering the “current state” of the room. + > To ensure all servers agree on which events are sticky, we need to re-evaluate soft-failed status when the current room state changes. + > This becomes particularly important when room state is rolled back. For example, if Charlie sends some sticky event E and + > then Bob kicks Charlie, but concurrently Alice kicks Bob then whether or not a receiving server would accept E would depend + > on whether they saw “Alice kicks Bob” or “Bob kicks Charlie”. If they saw “Alice kicks Bob” then E would be accepted. If they + > saw “Bob kicks Charlie” then E would be rejected, and would need to be rolled back when they see “Alice kicks Bob”. + > + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4ad14b0cd3b09205dcba59e45cbf1cab1e75edf7/proposals/4354-sticky-events.md#L95 + + Must be called from within the per-room event persistence critical section (see + `_EventPeristenceQueue`) and immediately before the persist transaction, so that + nothing else can change the room's current state in the meantime. + + Args: + room_id: The room that all of the events belong to + events_and_contexts: The events about to be persisted. These are not eligible + for re-evaluation. + state_delta_for_room: The changes about to be made to the current state, used + to detect if we need to re-evaluate soft-failed sticky events. + + Returns: + The event IDs of sticky events which are currently recorded as soft-failed + but which pass auth against the new current state. + """ + assert self._can_write_to_sticky_events + + # Fetch the soft-failed sticky events to recheck + event_ids_to_check = await self._get_soft_failed_sticky_events_to_recheck( + room_id, state_delta_for_room + ) + # Defensively filter out soft-failed events in events_and_contexts: they haven't been + # inserted into `sticky_events` yet, but be defensive in case we are asked to + # re-persist an event which is already there (e.g. de-outliering), as their + # soft failure status won't have changed for them. + persisting_event_ids = {ev.event_id for ev, _ in events_and_contexts} + event_ids_to_check = [ + event_id + for event_id in event_ids_to_check + if event_id not in persisting_event_ids + ] + if not event_ids_to_check: + return set() + + events_to_check = await self.get_events( + event_ids_to_check, allow_rejected=False + ) + + # Calculate what (state event type, state key) tuples are needed as auth events for the + # soft-failed events we are reconsidering? + # e.g. [('m.room.member', '@user:example.org'), ('m.room.power_levels', ''), ...] + needed_state_tuples_for_auth: set[StateKey] = set() + for soft_failed_event in events_to_check.values(): + needed_state_tuples_for_auth.update( + event_auth.auth_types_for_event( + soft_failed_event.room_version, soft_failed_event + ) + ) + + # Load the needed auth state from the current state + # (type, state key) -> event_id + current_auth_state_ids_map = dict( + await self.get_partial_filtered_current_state_ids( + room_id, StateFilter.from_types(needed_state_tuples_for_auth) + ) + ) + # `state_delta_for_room` hasn't yet been applied to the room's persisted current state, + # so we need to apply it here to the auth state we are using for the re-evaluation + for deleted_key in state_delta_for_room.to_delete: + current_auth_state_ids_map.pop(deleted_key, None) + for inserted_key, inserted_event_id in state_delta_for_room.to_insert.items(): + if inserted_key in needed_state_tuples_for_auth: + current_auth_state_ids_map[inserted_key] = inserted_event_id + + # Now load in the auth events + persisting_events_by_id = {ev.event_id: ev for ev, _ in events_and_contexts} + current_auth_events: list[EventBase] = [] + current_auth_state_event_ids_to_fetch: list[str] = [] + for event_id in current_auth_state_ids_map.values(): + persisting_event = persisting_events_by_id.get(event_id) + if persisting_event is not None: + # This event is one we are about to persist, so just use it + current_auth_events.append(persisting_event) + else: + # This event needs to be loaded from the database + current_auth_state_event_ids_to_fetch.append(event_id) + current_auth_events.extend( + await self.get_events_as_list(current_auth_state_event_ids_to_fetch) + ) + + passing_event_ids: set[str] = set() + for soft_failed_event in events_to_check.values(): + try: + # We don't need to check_state_independent_auth_rules as that doesn't depend on room state, + # so if it passed once it'll pass again. + event_auth.check_state_dependent_auth_rules( + soft_failed_event, current_auth_events + ) + + # Ready to be un-soft-failed + passing_event_ids.add(soft_failed_event.event_id) + except AuthError: + # state-dependent auth rules still unsatisfied: remain soft-failed + pass + + if passing_event_ids: + logger.info( + "%s soft-failed events now pass current state checks in room %s : %s", + len(passing_event_ids), + room_id, + shortstr(passing_event_ids), + ) + + return passing_event_ids + + async def _get_soft_failed_sticky_events_to_recheck( + self, + room_id: str, + state_delta_for_room: DeltaState, + ) -> list[str]: + """ + Fetch soft-failed sticky events which should be rechecked against the current state. + + Returns: + A list of event IDs to recheck + """ + + if state_delta_for_room.no_longer_in_room: + # We're leaving the room, so the current state is about to be wiped and + # nothing can pass auth against it. + return [] + + # Only a change to critical auth state may change soft failure status. + # This means any changes to join rules, power levels or member events. + # If the state has changed but these types are unchanged, we don't need to recheck. + CRITICAL_AUTH_TYPES = ( + EventTypes.JoinRules, + EventTypes.PowerLevels, + EventTypes.Member, + ) + + critical_auth_types_changed = { + typ + for typ, _ in chain( + state_delta_for_room.to_insert, state_delta_for_room.to_delete + ) + if typ in CRITICAL_AUTH_TYPES + } + if len(critical_auth_types_changed) == 0: + # No change to critical auth events. + # No way soft failure status could be different. + return [] + + if critical_auth_types_changed == {EventTypes.Member}: + # If the only critical auth state that changed is user memberships, + # then we can restrict our re-evaluation to only reconsider soft-failed sticky events sent + # by the users who have their membership changed. + # Events sent by any other user can not be affected, + # with the pedantic yet possible exception of sticky invite/kick/ban `m.room.member` + # state events (where state key ≠ sender). + # That said: we don't expect to use those and it is not possible to create one with + # the Client-Server API. + changed_members = { + membership_user_id + for event_type, membership_user_id in chain( + state_delta_for_room.to_insert, state_delta_for_room.to_delete + ) + if event_type == EventTypes.Member + } + + return await self.db_pool.runInteraction( + "_get_soft_failed_sticky_events_to_recheck_members", + self._get_soft_failed_sticky_events_txn, + room_id, + # Only reconsider events from changed members + senders=changed_members, + ) + + # If we reach here, then it must be the case that there have been changes in + # power level or join rules. + # In both of these cases we want to re-evaluate soft failure status of all the + # soft-failed events in the room. + # + # NB: event auth checks are NOT recursive. We don't need to specifically handle the case where + # an admin user's membership changes which causes a PL event to be allowed, as when the PL event + # gets allowed we will re-evaluate anyway. E.g: + # + # PL(send_event=0, sender=Admin) #1 + # ^ ^_____________________ + # | | + # . PL(send_event=50, sender=Mod) #2 sticky event (sender=User) #3 + # + # In this scenario, the sticky event is soft-failed due to the Mod updating the PL event to + # set send_event=50, which User does not have. If we learn of an event which makes Mod's PL + # event invalid (say, Mod was banned by Admin concurrently to Mod setting the PL event), then + # the act of seeing the ban event will cause the old PL event to be in the state delta, meaning + # we will re-evaluate the sticky event due to the PL changing. We don't need to specially handle + # this case. + return await self.db_pool.runInteraction( + "_get_soft_failed_sticky_events_to_recheck", + self._get_soft_failed_sticky_events_txn, + room_id, + # Consider everyone + senders=None, + ) + + def _get_soft_failed_sticky_events_txn( + self, + txn: LoggingTransaction, + room_id: str, + *, + senders: Collection[str] | None, + ) -> list[str]: + """ + Fetch the event IDs of (unexpired) soft-failed sticky events in a room. + + Args: + room_id: the room to look in. + senders: + If present, only return sticky events sent by one of these users. + If None, do not restrict by sender. + """ + sender_clause = "" + sender_args: Collection[str] = () + if senders is not None: + if not senders: + return [] + sender_sql, sender_args = make_in_list_sql_clause( + txn.database_engine, "se.sender", senders + ) + sender_clause = f"AND {sender_sql}" + + if isinstance(self.database_engine, PostgresEngine): + expr_soft_failed = "COALESCE(((ej.internal_metadata::jsonb)->>'soft_failed')::boolean, FALSE)" + else: + expr_soft_failed = "COALESCE(ej.internal_metadata->>'soft_failed', FALSE)" + + # Note that we are relying on the 1h stickiness limit to make this + # tractable, as we can't realistically apply any LIMIT here. + txn.execute( + f""" + SELECT se.event_id + FROM sticky_events se + INNER JOIN event_json ej USING (event_id) + WHERE + se.room_id = ? + AND ? < se.expires_at + AND {expr_soft_failed} + {sender_clause} + """, + (room_id, self.clock.time_msec(), *sender_args), + ) + return [event_id for (event_id,) in txn] + + def un_soft_fail_sticky_events_txn( + self, txn: LoggingTransaction, sticky_event_ids: Set[str] + ) -> None: + """ + For the given soft-failed sticky events: + + - removes their soft-failed status + - moves them to the end of the `sticky_events` stream so that clients get told about them + """ + if not sticky_event_ids: + return + + # Update the internal metadata on the event itself. + event_id_in_list_clause, event_id_in_list_args = make_in_list_sql_clause( + txn.database_engine, + "event_id", + sticky_event_ids, + ) + if isinstance(txn.database_engine, PostgresEngine): + # It's a bit sad that internal_metadata is TEXT and not JSONB... + txn.execute( + f""" + UPDATE event_json + SET internal_metadata = ( + jsonb_set(internal_metadata::jsonb, '{{soft_failed}}', 'false'::jsonb) + )::text + WHERE {event_id_in_list_clause} + """, + event_id_in_list_args, + ) + else: + assert isinstance(txn.database_engine, Sqlite3Engine) + txn.execute( + f""" + UPDATE event_json + SET internal_metadata = json_set(internal_metadata, '$.soft_failed', json('false')) + WHERE {event_id_in_list_clause} + """, + event_id_in_list_args, + ) + + # Invalidate caches as a result + for event_id in sticky_event_ids: + self.invalidate_get_event_cache_after_txn(txn, event_id) + + # Move the events to the end of the sticky events stream + new_stream_ids = self._sticky_events_id_gen.get_next_mult_txn( + txn, len(sticky_event_ids) + ) + self.db_pool.simple_update_many_txn( + txn, + table="sticky_events", + key_names=("event_id",), + key_values=[(event_id,) for event_id in sticky_event_ids], + value_names=( + "stream_id", + "instance_name", + ), + value_values=[ + (stream_id, self._instance_name) for stream_id in new_stream_ids + ], + ) + async def _delete_expired_sticky_events(self) -> None: await self.db_pool.runInteraction( "_delete_expired_sticky_events", diff --git a/tests/storage/test_sticky_events.py b/tests/storage/test_sticky_events.py index e77b362f52..6335553f0a 100644 --- a/tests/storage/test_sticky_events.py +++ b/tests/storage/test_sticky_events.py @@ -11,6 +11,7 @@ # See the GNU Affero General Public License for more details: # . import sqlite3 +from http import HTTPStatus from twisted.internet.testing import MemoryReactor @@ -23,7 +24,7 @@ ) from synapse.api.room_versions import RoomVersions from synapse.rest import admin -from synapse.rest.client import login, register, room +from synapse.rest.client import login, register, room, sync from synapse.server import HomeServer from synapse.types import JsonDict, create_requester from synapse.util.clock import Clock @@ -45,6 +46,7 @@ class StickyEventsTestCase(unittest.HomeserverTestCase): servlets = [ room.register_servlets, + sync.register_servlets, login.register_servlets, register.register_servlets, admin.register_servlets, @@ -52,7 +54,10 @@ class StickyEventsTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = {"msc4354_enabled": True} + config["experimental_features"] = { + "msc3575_enabled": True, + "msc4354_enabled": True, + } return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: @@ -428,3 +433,136 @@ def test_spam_checker_spammy_events_are_not_tracked(self) -> None: self.assertEqual(len(updates), 1) self.assertEqual(updates[0].event_id, valid_sticky_event.event_id) + + def _get_visible_sticky_event_ids(self) -> set[str]: + """ + Returns the IDs of the sticky events visible to clients in sync. + """ + sync_body: JsonDict = { + "lists": { + "main": { + "ranges": [[0, 0]], + "required_state": [], + # We don't want any timeline events, just sticky events + "timeline_limit": 0, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + channel = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync", + sync_body, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + sticky_events = channel.json_body["extensions"].get( + "org.matrix.msc4354.sticky_events" + ) + if sticky_events is None: + return set() + events_in_room = ( + sticky_events.get("rooms", {}).get(self.room_id, {}).get("events", []) + ) + return {event["event_id"] for event in events_in_room} + + def test_soft_failure_cleared_when_state_changes(self) -> None: + """ + Tests that a soft-failed sticky event stops being soft-failed once a change to + the room's current state means that it passes auth after all. + """ + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + self.helper.join(self.room_id, user2_id, tok=user2_tok) + + # Devoice user2, so that their sticky event will (realistically) fail auth + # against the room's current state. + self.helper.send_state( + self.room_id, + EventTypes.PowerLevels, + body={"users": {self.user_id: 100, user2_id: -1}, "events_default": 0}, + tok=self.token, + ) + + # Inject a soft-failed sticky event. This is cheating a bit for brevity. + # In the real world, we'd need to craft a soft-failed sticky event to arrive over federation. + # The Complement test will do this: https://github.com/matrix-org/complement/pull/806/files#diff-6c9d6d169485d0848c6b20dd9b43f6fe669a8a710e42f953d08fa25a99cc8f4cR509 + event_id = self.get_success( + inject_event( + self.hs, + room_id=self.room_id, + sender=user2_id, + type=EventTypes.Message, + content={"body": "sticky", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ).event_id + + # Whilst it is soft-failed, the event isn't shown to clients. + self.assertEqual(self._get_visible_sticky_event_ids(), set()) + + # Change the room's power levels to voice user2 back. + # This triggers the soft-fail re-evaluation and also allows the soft-failed sticky + # event to pass state-dependent auth checks against the current state, becoming + # un-soft-failed + self.helper.send_state( + self.room_id, + EventTypes.PowerLevels, + body={"users": {self.user_id: 100, user2_id: 0}, "events_default": 0}, + tok=self.token, + ) + + # The event has been re-evaluated and is now shown to clients... + self.assertEqual(self._get_visible_sticky_event_ids(), {event_id}) + # ...and the soft-failure flag has been cleared. + event = self.get_success(self.store.get_event(event_id)) + self.assertFalse(event.internal_metadata.is_soft_failed()) + + def test_soft_failure_cleared_when_sender_membership_changes(self) -> None: + """ + Tests that soft-failure status of a sticky event is reconsidered when + the sender's membership changes. + """ + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Inject a soft-failed sticky event from user2 + event_id = self.get_success( + inject_event( + self.hs, + room_id=self.room_id, + sender=user2_id, + type=EventTypes.Message, + content={"body": "sticky", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ).event_id + + # Whilst it is soft-failed, the event isn't shown to clients. + self.assertEqual(self._get_visible_sticky_event_ids(), set()) + + # Check that an irrelevant user's membership changing doesn't affect the event + user3_id = self.register_user("user3", "pass") + user3_tok = self.login(user3_id, "pass") + self.helper.join(self.room_id, user3_id, tok=user3_tok) + self.assertEqual(self._get_visible_sticky_event_ids(), set()) + + # The sender joins, so the event now passes auth and is un-soft-failed. + self.helper.join(self.room_id, user2_id, tok=user2_tok) + self.assertEqual(self._get_visible_sticky_event_ids(), {event_id}) + # ...and the soft-failure has been cleared from the event itself. + event = self.get_success(self.store.get_event(event_id)) + self.assertFalse(event.internal_metadata.is_soft_failed()) From 6115b472623935d792bd89cb7b6fe1e1c4fa24ba Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 14 Sep 2026 13:05:15 +0100 Subject: [PATCH 13/41] Run the in-repo Complement test suite in CI, even when the standard Complement suite fails. (#20161) Supersedes: https://github.com/element-hq/synapse-private/pull/155 It would be useful to have the in-repo Complement suite give a status, even when the normal suite fails (e.g. flakes). --------- Signed-off-by: Olivier 'reivilibre Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- .github/workflows/complement_tests.yml | 4 ++++ changelog.d/20161.misc | 1 + 2 files changed, 5 insertions(+) create mode 100644 changelog.d/20161.misc diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 220ea57e1a..a98ba23f43 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -160,6 +160,10 @@ jobs: run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" - name: Run in-repo Complement Tests + # Always run as long as we attempted to run the original Complement tests. + # This lets us get a status report on the in-repo tests, even if the + # standard Complement test suite fails. + if: always() && steps.run_complement_tests.outcome != 'skipped' id: run_in_repo_complement_tests # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes # are underpowered and don't like running tons of Synapse instances at once. diff --git a/changelog.d/20161.misc b/changelog.d/20161.misc new file mode 100644 index 0000000000..4351f779c2 --- /dev/null +++ b/changelog.d/20161.misc @@ -0,0 +1 @@ +Run the in-repo Complement test suite in CI, even when the standard Complement suite fails. \ No newline at end of file From 8c8138d57247258f5e1ae90e52d01b90c7d140a0 Mon Sep 17 00:00:00 2001 From: Skye Elliot Date: Mon, 14 Sep 2026 16:05:23 +0100 Subject: [PATCH 14/41] Reject OTK uploads if clients exceed 500 server-side keys (#20162) Co-authored-by: Andrew Morgan Signed-off-by: Skye Elliot --- changelog.d/20162.feature | 1 + synapse/handlers/e2e_keys.py | 36 ++++++++++++++++++++++- tests/handlers/test_e2e_keys.py | 51 +++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20162.feature diff --git a/changelog.d/20162.feature b/changelog.d/20162.feature new file mode 100644 index 0000000000..a197d13664 --- /dev/null +++ b/changelog.d/20162.feature @@ -0,0 +1 @@ +Limit the number of end-to-end encryption one-time keys stored per device to 500 per algorithm, rejecting uploads which would exceed the limit with a `400 Bad Request`. diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 6284f75319..1594112602 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -20,6 +20,7 @@ # # import logging +from collections import Counter from typing import TYPE_CHECKING, Iterable, Mapping import attr @@ -60,6 +61,18 @@ ONE_TIME_KEY_UPLOAD = "one_time_key_upload_lock" +# The maximum number of one-time keys, per algorithm, to keep for a device. Uploads +# which would exceed it are rejected, which protects against clients that keep +# uploading keys they will never be able to use. +# +# The spec allows clients to discard their oldest private one-time keys once they hold +# too many, and both libolm (at 100 keys) and vodozemac (at 5000) do, so keys beyond +# that bound could never be used anyway. The limit sits comfortably above the 50 keys +# clients built on the matrix-rust-sdk aim to keep on the server, and well below +# vodozemac's private key bound, so we never reject an upload from a well-behaved +# client nor hold a key the client has already discarded. +MAX_ONE_TIME_KEYS_PER_ALGORITHM_PER_DEVICE = 500 + class E2eKeysHandler: def __init__(self, hs: "HomeServer"): @@ -121,7 +134,6 @@ def __init__(self, hs: "HomeServer"): self._query_appservices_for_keys = ( hs.config.experimental.msc3984_appservice_key_query ) - self._task_scheduler.register_action( self._delete_old_one_time_keys_task, "delete_old_otks" ) @@ -989,6 +1001,28 @@ async def _upload_one_time_keys_for_user( (algorithm, key_id, encode_canonical_json(key).decode("ascii")) ) + # Reject uploads which would take the device over the limit, rather than + # quietly discarding keys, so that a client which keeps uploading keys + # regardless of how many the server holds gets told about it. + counts = await self.store.count_e2e_one_time_keys(user_id, device_id) + for algorithm, new_count in Counter( + algorithm for algorithm, _, _ in new_keys + ).items(): + total = counts.get(algorithm, 0) + new_count + if total > MAX_ONE_TIME_KEYS_PER_ALGORITHM_PER_DEVICE: + raise SynapseError( + 400, + "Uploading %i more %s one-time keys would leave the device " + "holding %i, over the limit of %i" + % ( + new_count, + algorithm, + total, + MAX_ONE_TIME_KEYS_PER_ALGORITHM_PER_DEVICE, + ), + Codes.TOO_LARGE, + ) + log_kv({"message": "Inserting new one_time_keys.", "keys": new_keys}) await self.store.add_e2e_one_time_keys( user_id, device_id, time_now, new_keys diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 4790c339d9..1bb4a05bb1 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -405,6 +405,57 @@ def test_claim_one_time_key_bulk_ordering(self) -> None: for key_id in claimed_keys.keys(): self.assertIn(key_id, ["alg1:k20", "alg1:k21", "alg1:k22"]) + @mock.patch( + "synapse.handlers.e2e_keys.MAX_ONE_TIME_KEYS_PER_ALGORITHM_PER_DEVICE", 5 + ) + def test_upload_one_time_keys_over_limit_is_rejected(self) -> None: + """Uploading one-time keys which would take a device over the limit fails""" + local_user = "@boris:" + self.hs.hostname + device_id = "xyz" + + # Uploading exactly up to the limit is fine. + res = self.get_success( + self.handler.upload_keys_for_user( + local_user, + device_id, + {"one_time_keys": {f"alg1:k{i}": f"key{i}" for i in range(1, 6)}}, + ) + ) + self.assertEqual(res["one_time_key_counts"]["alg1"], 5) + + # Uploading any more is rejected, with a 400. + error = self.get_failure( + self.handler.upload_keys_for_user( + local_user, + device_id, + {"one_time_keys": {"alg1:k6": "key6"}}, + ), + SynapseError, + ).value + self.assertEqual(error.code, 400) + self.assertEqual(error.errcode, Codes.TOO_LARGE) + + # The limit is per algorithm, so keys of another algorithm are still accepted. + res = self.get_success( + self.handler.upload_keys_for_user( + local_user, + device_id, + {"one_time_keys": {"alg2:k1": "key1"}}, + ) + ) + self.assertEqual(res["one_time_key_counts"]["alg1"], 5) + self.assertEqual(res["one_time_key_counts"]["alg2"], 1) + + # Nothing from the rejected upload was stored. + keys = self.get_success( + self.store.get_e2e_one_time_keys( + local_user, device_id, [f"k{i}" for i in range(1, 7)] + ) + ) + self.assertEqual( + set(keys), {("alg1", f"k{i}") for i in range(1, 6)} | {("alg2", "k1")} + ) + def test_fallback_key(self) -> None: local_user = "@boris:" + self.hs.hostname device_id = "xyz" From af01910b2a410df39e4f39e35e7f1ed1ff0d09d2 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 15 Sep 2026 11:40:59 +0100 Subject: [PATCH 15/41] Refactor the federation transmission code to delineate transaction preparation and completion. (#20166) A key refactoring for, and split out of, https://github.com/element-hq/synapse/pull/20165 Would be easier to land first to isolate the diff. Should be a standalone change with no behavioural change. Motivation is that #20165 will round-robin between 'main queue' transactions and 'sticky event' transactions. To keep the data flow clear, I wanted to insert a typed struct (well, `attrs` dataclass) as an interface between the 'preparation' of a transaction and its 'completion'. Doing this whilst keeping the asynchronous context manager style did not lead to a readable result in my opinion. (I would also say the async context manager is a touch 'magic' / obscures control flow, but I suspect this is largely down to opinion.) Replace _TransactionQueueManager with prepare/complete transaction methods --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20166.misc | 1 + .../sender/per_destination_queue.py | 298 +++++++++++------- 2 files changed, 186 insertions(+), 113 deletions(-) create mode 100644 changelog.d/20166.misc diff --git a/changelog.d/20166.misc b/changelog.d/20166.misc new file mode 100644 index 0000000000..716e910f8f --- /dev/null +++ b/changelog.d/20166.misc @@ -0,0 +1 @@ +Refactor the federation transmission code to delineate transaction preparation and completion. \ No newline at end of file diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 32f8630c9d..4c0918e0ac 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -22,7 +22,6 @@ import datetime import logging from collections import OrderedDict -from types import TracebackType from typing import TYPE_CHECKING, Hashable, Iterable import attr @@ -79,6 +78,52 @@ MAX_PRESENCE_STATES_PER_EDU = 50 +@attr.s(slots=True, auto_attribs=True, frozen=True) +class _PreparedTransaction: + """ + A transaction that has been prepared for sending: what to send, along with the + information that is useful for marking the transaction as complete once it has + been successfully sent. + + Produced by `PerDestinationQueue._prepare_transaction` and consumed by + `PerDestinationQueue._complete_transaction`. + """ + + pdus: list[EventBase] + """ + The PDUs to send in this transaction. + """ + + edus: list[Edu] + """ + The EDUs to send in this transaction. + """ + + to_device_message_stream_id: int | None + """ + This is the stream ID of the latest to-device message (`device_federation_outbox`) to be + sent (or None if none sent). + + When the transaction completes, to-device messages up to this point will be deleted from + the outbox. + """ + + device_list_stream_id: int | None + """ + This is the stream ID of the latest device list to be sent (or None if none sent). + + When the transaction completes, we will mark device lists up to this point as having been + sent. + """ + + last_stream_ordering: int | None + """ + This is the stream ordering of the last PDU that was sent (or None if none sent). + + When the transaction completes, this should be stored as our position in the events stream. + """ + + class PerDestinationQueue: """ Manages the per-destination transmission queues. @@ -343,7 +388,7 @@ def attempt_new_transaction(self) -> None: ) async def _transaction_transmission_loop(self) -> None: - pending_pdus: list[EventBase] = [] + transaction: _PreparedTransaction | None = None try: self.transmission_loop_running = True # This will throw if we wouldn't retry. We do this here so we fail @@ -366,46 +411,48 @@ async def _transaction_transmission_loop(self) -> None: while self._transmission_loop_enabled: self._new_data_to_send = False + # Clear `transaction` (as it's used in the exception handler + # for error reporting) + transaction = None + transaction = await self._prepare_transaction() + + if transaction is None: + logger.debug("TX [%s] Nothing to send", self._destination) + + # If we've gotten told about new things to send during + # checking for things to send, we try looking again. + # Otherwise new PDUs or EDUs might arrive in the meantime, + # but not get sent because we currently have an + # `_active_transmission_loop` running. + if self._new_data_to_send: + continue + else: + return - async with _TransactionQueueManager(self) as ( - pending_pdus, # noqa: F811 - pending_edus, - ): - if not pending_pdus and not pending_edus: - logger.debug("TX [%s] Nothing to send", self._destination) - - # If we've gotten told about new things to send during - # checking for things to send, we try looking again. - # Otherwise new PDUs or EDUs might arrive in the meantime, - # but not get sent because we currently have an - # `_active_transmission_loop` running. - if self._new_data_to_send: - continue - else: - return - - if pending_pdus: - logger.debug( - "TX [%s] len(pending_pdus_by_dest[dest]) = %d", - self._destination, - len(pending_pdus), - ) - - await self._transaction_manager.send_new_transaction( - self._destination, pending_pdus, pending_edus + if transaction.pdus: + logger.debug( + "TX [%s] len(pending_pdus_by_dest[dest]) = %d", + self._destination, + len(transaction.pdus), ) - sent_transactions_counter.labels( - **{SERVER_NAME_LABEL: self.server_name} + await self._transaction_manager.send_new_transaction( + self._destination, transaction.pdus, transaction.edus + ) + + sent_transactions_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc() + sent_edus_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( + len(transaction.edus) + ) + for edu in transaction.edus: + sent_edus_by_type.labels( + type=edu.edu_type, + **{SERVER_NAME_LABEL: self.server_name}, ).inc() - sent_edus_counter.labels( - **{SERVER_NAME_LABEL: self.server_name} - ).inc(len(pending_edus)) - for edu in pending_edus: - sent_edus_by_type.labels( - type=edu.edu_type, - **{SERVER_NAME_LABEL: self.server_name}, - ).inc() + + await self._complete_transaction(transaction) except NotRetryingDestination as e: logger.debug( @@ -455,16 +502,19 @@ async def _transaction_transmission_loop(self) -> None: "TX [%s] Failed to send transaction: %s", self._destination, e ) - for p in pending_pdus: - logger.info( - "Failed to send event %s to %s", p.event_id, self._destination - ) + if transaction is not None: + for p in transaction.pdus: + logger.info( + "Failed to send event %s to %s", p.event_id, self._destination + ) except Exception: logger.exception("TX [%s] Failed to send transaction", self._destination) - for p in pending_pdus: - logger.info( - "Failed to send event %s to %s", p.event_id, self._destination - ) + + if transaction is not None: + for p in transaction.pdus: + logger.info( + "Failed to send event %s to %s", p.event_id, self._destination + ) finally: # We want to be *very* sure we clear this after we stop processing self.active_transmission_loop = None @@ -736,21 +786,31 @@ def _start_catching_up(self) -> None: self._catching_up = True self._pending_pdus = [] + async def _prepare_transaction(self) -> _PreparedTransaction | None: + """ + Work out what should go in the next transaction to this destination, by + calculating what we want to send and preparing the information that is + useful once we have completed the transaction. + + Side effects: + - Dequeues pending EDUs + - `_pending_presence` + - `_pending_receipt_edus` + - `_pending_edus` (currently unused in practice) + - `_pending_edus_keyed` + - Advances our devices stream positions (only if there is nothing to send + in that range, so we don't lose any updates) + + PDUs are not dequeued until acknowledged by `_complete_transaction`. + + Returns: + - the prepared transaction; or + - None if there is nothing to send and no progress to record + + Once the prepared transaction has been sent successfully, + `_complete_transaction` must be called with it. + """ -@attr.s(slots=True, auto_attribs=True) -class _TransactionQueueManager: - """A helper async context manager for pulling stuff off the queues and - tracking what was last successfully sent, etc. - """ - - queue: PerDestinationQueue - - _device_stream_id: int | None = None - _device_list_id: int | None = None - _last_stream_ordering: int | None = None - _pdus: list[EventBase] = attr.Factory(list) - - async def __aenter__(self) -> tuple[list[EventBase], list[Edu]]: # First we calculate the EDUs we want to send, if any. # There's a maximum number of EDUs that can be sent with a transaction, @@ -767,30 +827,30 @@ async def __aenter__(self) -> tuple[list[EventBase], list[Edu]]: pending_edus = [] # Add presence EDU. - if self.queue._pending_presence: + if self._pending_presence: # Only send max 50 presence entries in the EDU, to bound the amount # of data we're sending. presence_to_add: list[JsonDict] = [] while ( - self.queue._pending_presence + self._pending_presence and len(presence_to_add) < MAX_PRESENCE_STATES_PER_EDU ): - _, presence = self.queue._pending_presence.popitem(last=False) + _, presence = self._pending_presence.popitem(last=False) presence_to_add.append( - format_user_presence_state(presence, self.queue._clock.time_msec()) + format_user_presence_state(presence, self._clock.time_msec()) ) pending_edus.append( Edu( - origin=self.queue.server_name, - destination=self.queue._destination, + origin=self.server_name, + destination=self._destination, edu_type=EduTypes.PRESENCE, content={"push": presence_to_add}, ) ) # Add read receipt EDUs. - pending_edus.extend(self.queue._get_receipt_edus(limit=5)) + pending_edus.extend(self._get_receipt_edus(limit=5)) edu_limit = MAX_EDUS_PER_TRANSACTION - len(pending_edus) # Next, prioritize to-device messages so that existing encryption channels @@ -799,91 +859,103 @@ async def __aenter__(self) -> tuple[list[EventBase], list[Edu]]: ( to_device_edus, device_stream_id, - ) = await self.queue._get_to_device_message_edus( + ) = await self._get_to_device_message_edus( edu_limit - NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION ) + device_stream_id_upon_completion: int | None = None if to_device_edus: - self._device_stream_id = device_stream_id + # We can advance our position in the device stream after the transaction completes. + device_stream_id_upon_completion = device_stream_id else: - self.queue._last_device_stream_id = device_stream_id + # We can advance our position in the device stream immediately, as there's nothing to send. + self._last_device_stream_id = device_stream_id pending_edus.extend(to_device_edus) edu_limit -= len(to_device_edus) # Add device list update EDUs. - device_update_edus, dev_list_id = await self.queue._get_device_update_edus( - edu_limit - ) + device_update_edus, dev_list_id = await self._get_device_update_edus(edu_limit) + device_list_id_upon_completion: int | None = None if device_update_edus: - self._device_list_id = dev_list_id + # We can advance our position in the device list stream after the transaction completes. + device_list_id_upon_completion = dev_list_id else: - self.queue._last_device_list_stream_id = dev_list_id + # We can advance our position in the device list stream immediately, as there's nothing to send. + self._last_device_list_stream_id = dev_list_id pending_edus.extend(device_update_edus) edu_limit -= len(device_update_edus) # Finally add any other types of EDUs if there is room. - other_edus = self.queue._pop_pending_edus(edu_limit) + other_edus = self._pop_pending_edus(edu_limit) pending_edus.extend(other_edus) edu_limit -= len(other_edus) - while edu_limit > 0 and self.queue._pending_edus_keyed: - _, val = self.queue._pending_edus_keyed.popitem() + while edu_limit > 0 and self._pending_edus_keyed: + _, val = self._pending_edus_keyed.popitem() pending_edus.append(val) edu_limit -= 1 # Now we look for any PDUs to send, by getting up to 50 PDUs from the # queue - self._pdus = self.queue._pending_pdus[:50] - - if not self._pdus and not pending_edus: - return [], [] - - if self._pdus: - self._last_stream_ordering = self._pdus[ - -1 - ].internal_metadata.stream_ordering - assert self._last_stream_ordering - - return self._pdus, pending_edus + pdus = self._pending_pdus[:50] + + if not pdus and not pending_edus: + # There is nothing to send. There's also nothing to record upon + # completion: the only progress we could have made without sending + # anything is advancing our positions in the device streams, and that + # has already been done above. + return None + + last_stream_ordering: int | None = None + if pdus: + last_stream_ordering = pdus[-1].internal_metadata.stream_ordering + assert last_stream_ordering + + return _PreparedTransaction( + pdus=pdus, + edus=pending_edus, + to_device_message_stream_id=device_stream_id_upon_completion, + device_list_stream_id=device_list_id_upon_completion, + last_stream_ordering=last_stream_ordering, + ) - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc: BaseException | None, - tb: TracebackType | None, - ) -> None: - if exc_type is not None: - # Failed to send transaction, so we bail out. - return + async def _complete_transaction(self, transaction: _PreparedTransaction) -> None: + """ + Handle the fact that a transaction has been successfully completed. + Must not be called if sending the transaction failed, as it records how far + through the various streams we have now got. + """ # Successfully sent transactions, so we remove pending PDUs from the queue - if self._pdus: - self.queue._pending_pdus = self.queue._pending_pdus[len(self._pdus) :] + if transaction.pdus: + self._pending_pdus = self._pending_pdus[len(transaction.pdus) :] # Succeeded to send the transaction so we record where we have sent up # to in the various streams - if self._device_stream_id: - await self.queue._store.delete_device_msgs_for_remote( - self.queue._destination, self._device_stream_id + if transaction.to_device_message_stream_id: + await self._store.delete_device_msgs_for_remote( + self._destination, transaction.to_device_message_stream_id ) - self.queue._last_device_stream_id = self._device_stream_id + self._last_device_stream_id = transaction.to_device_message_stream_id # also mark the device updates as sent - if self._device_list_id: + if transaction.device_list_stream_id: logger.info( - "Marking as sent %r %r", self.queue._destination, self._device_list_id + "Marking as sent %r %r", + self._destination, + transaction.device_list_stream_id, ) - await self.queue._store.mark_as_sent_devices_by_remote( - self.queue._destination, self._device_list_id + await self._store.mark_as_sent_devices_by_remote( + self._destination, transaction.device_list_stream_id ) - self.queue._last_device_list_stream_id = self._device_list_id + self._last_device_list_stream_id = transaction.device_list_stream_id - if self._last_stream_ordering: + if transaction.last_stream_ordering: # we sent some PDUs and it was successful, so update our # last_successful_stream_ordering in the destinations table. - await self.queue._store.set_destination_last_successful_stream_ordering( - self.queue._destination, self._last_stream_ordering + await self._store.set_destination_last_successful_stream_ordering( + self._destination, transaction.last_stream_ordering ) From 177f8b5b763830be6dbc6184b27d63379fd6e503 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:58:08 +0100 Subject: [PATCH 16/41] Fix htmltest CI; put docs dependencies in `pyproject.toml` (#20216) --- .github/workflows/docs-pr.yaml | 27 +++++++++++---- .github/workflows/docs.yaml | 14 ++++---- changelog.d/20216.misc | 1 + docs/README.md | 15 ++++++-- poetry.lock | 62 +++++++++++++++++----------------- pyproject.toml | 6 +++- 6 files changed, 77 insertions(+), 48 deletions(-) create mode 100644 changelog.d/20216.misc diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 6e5428dee0..2b094f7219 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -7,6 +7,8 @@ on: - book.toml - .github/workflows/docs-pr.yaml - scripts-dev/schema_versions.py + # If the docs' python dependencies change, rebuild the book and test links. + - pyproject.toml jobs: pages: @@ -23,19 +25,21 @@ jobs: with: mdbook-version: '0.5.2' - - name: Setup python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + # Building the book requires running the `scripts-dev/schema_versions.py` + # script first, which has some python dependencies. + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - python-version: "3.x" - - - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + install-project: "false" + groups: "docs" + poetry-version: "2.4.1" - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page # as the default. Let's opt for the welcome page instead. run: | - mdbook build + poetry run mdbook build cp book/welcome_and_overview.html book/index.html - name: Upload Artifact @@ -57,6 +61,15 @@ jobs: with: mdbook-version: '0.5.2' + # Building the book requires running the `scripts-dev/schema_versions.py` + # script first, which has some python dependencies. + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + install-project: "false" + groups: "docs" + poetry-version: "2.4.1" + - name: Setup htmltest run: | wget https://github.com/wjdp/htmltest/releases/download/v0.17.0/htmltest_0.17.0_linux_amd64.tar.gz @@ -66,7 +79,7 @@ jobs: - name: Test links with htmltest run: | # Build the book with `./` as the site URL (to make checks on 404.html possible) - MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build + MDBOOK_OUTPUT__HTML__SITE_URL="./" poetry run mdbook build # Delete the contents of the print.html file, as it can raise false # positives during link checking. diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 0d1138f988..9a51d726cb 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -63,19 +63,21 @@ jobs: - name: Set version of docs run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - - name: Setup python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + # Building the book requires running the `scripts-dev/schema_versions.py` + # script first, which has some python dependencies. + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - python-version: "3.x" - - - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + install-project: "false" + groups: "docs" + poetry-version: "2.4.1" - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page # as the default. Let's opt for the welcome page instead. run: | - mdbook build + poetry run mdbook build cp book/welcome_and_overview.html book/index.html - name: Prepare and publish schema files diff --git a/changelog.d/20216.misc b/changelog.d/20216.misc new file mode 100644 index 0000000000..130a5b1339 --- /dev/null +++ b/changelog.d/20216.misc @@ -0,0 +1 @@ +Move documentation building python dependencies into `pyproject.toml` under a new `docs` dependency group. \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 7802d3c3ce..2462384714 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,11 +43,20 @@ line linking to the new documentation file: The documentation is built with [mdbook](https://rust-lang.github.io/mdBook/), and the outline of the documentation is determined by the structure of [SUMMARY.md](SUMMARY.md). -First, [get mdbook](https://github.com/rust-lang/mdBook#installation). Then, **from the root of the repository**, -build the documentation with: +Building the documentation requires running some python. Ensure you have set up `poetry`, then run: + +``` +poetry install --only docs --no-root +``` + +to install just the python dependencies needed to build the documentation. + +Then, [install mdbook](https://github.com/rust-lang/mdBook#installation), which will convert the source markdown to HTML pages. + +Finally, **from the root of the repository**, build the documentation with: ```sh -mdbook build +poetry run mdbook build ``` The rendered contents will be outputted to a new `book/` directory at the root of the repository. Please note that diff --git a/poetry.lock b/poetry.lock index 572cd5d195..00a0fddd12 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -532,7 +532,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -557,7 +557,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -572,7 +572,7 @@ version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -587,7 +587,7 @@ version = "3.1.62" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "gitpython-3.1.62-py3-none-any.whl", hash = "sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e"}, {file = "gitpython-3.1.62.tar.gz", hash = "sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af"}, @@ -607,7 +607,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -890,7 +890,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -931,7 +931,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1123,7 +1123,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1240,7 +1240,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1548,7 +1548,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1833,7 +1833,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1847,7 +1847,7 @@ version = "26.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, @@ -2028,7 +2028,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2046,7 +2046,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2062,7 +2062,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2108,7 +2108,7 @@ files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] -markers = {main = "implementation_name != \"PyPy\"", dev = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\""} +markers = {main = "implementation_name != \"PyPy\"", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} [[package]] name = "pydantic" @@ -2342,7 +2342,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2474,7 +2474,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2499,7 +2499,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2527,7 +2527,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2931,7 +2931,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3089,7 +3089,7 @@ version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, @@ -3131,7 +3131,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3147,7 +3147,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.24.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efe85c4508adaf6c9e6f7fac2b1c3c9beb4b39b19c375519966335a70b28e64a"}, {file = "thrift-0.24.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2baaec0c5cd7ba3eace54b26d81fdf0f5a85010468687cf4a131871d65abfed"}, @@ -3257,7 +3257,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163"}, {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100"}, @@ -3389,7 +3389,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3650,7 +3650,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3671,7 +3671,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -3784,4 +3784,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "53ecef32887866656603697f9e11043dae7b2917b1a2e7727f5133f604acf9de" +content-hash = "4628b733106afbf7e6306058bcac72c1b273533941eb7cc8621c74b91e947b63" diff --git a/pyproject.toml b/pyproject.toml index f3bf9682d8..fad346d442 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -290,7 +290,6 @@ dev = [ # The following are used by the release script "click>=8.1.3", - # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. "GitPython>=3.1.47", "markdown-it-py>=3.0.0", "pygithub>=1.59", @@ -305,6 +304,11 @@ dev = [ # Used for checking the schema delta files "sqlglot>=28.0.0", ] +# Dependencies for building Synapse's documentation. +docs = [ + "packaging>=20.0", + "GitPython>=3.1.20", +] [tool.towncrier] package = "synapse" From ac771446a61b2fbf1944d747e8dc944f8421b451 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 15 Sep 2026 06:15:23 -0500 Subject: [PATCH 17/41] Allow 3rd party rules `check_event_allowed` callback to function for room version "12" creation events (#19768) --- changelog.d/19768.bugfix | 1 + synapse/handlers/message.py | 35 ++++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 changelog.d/19768.bugfix diff --git a/changelog.d/19768.bugfix b/changelog.d/19768.bugfix new file mode 100644 index 0000000000..8314491ced --- /dev/null +++ b/changelog.d/19768.bugfix @@ -0,0 +1 @@ +Allow the Third Party Rules `check_event_allowed()` callback to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c61fccb5f2..5230772e49 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2449,11 +2449,15 @@ async def _rebuild_event_after_third_party_rules( original_event.room_version, third_party_result ) self.validator.validate_builder(builder) - assert builder.room_id is not None + except SynapseError as e: - raise Exception( - "Third party rules module created an invalid event: " + e.msg, - ) + # Prepend the error message with some context. This will be raised as a + # `400` since assumption of the validator is that it came directly from a + # client. Change this to a `500`, as the module will have changed something + # and that is a fault of the server + e.msg = "Third party rules module created an invalid event: " + e.msg + e.code = HTTPStatus.INTERNAL_SERVER_ERROR + raise immutable_fields = [ # changing the room is going to break things: we've already checked that the @@ -2484,12 +2488,25 @@ async def _rebuild_event_after_third_party_rules( for k, v in original_event.internal_metadata.get_dict().items(): setattr(builder.internal_metadata, k, v) - # modules can send new state events, so we re-calculate the auth events just in - # case. - prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) + # Creation events using msc4242 and msc4291 rooms will not have a room_id, and + # will also not have prev_events nor prev_state_events(below). + prev_event_ids = [] + if builder.room_id is not None and not ( + builder.type == EventTypes.Create + and original_event.room_version.msc4291_room_ids_as_hashes + ): + prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) - prev_state_events = None - if original_event.room_version.msc4242_state_dags: + prev_state_events: list[str] | None = ( + [] if original_event.room_version.msc4242_state_dags else None + ) + if ( + original_event.room_version.msc4242_state_dags + and builder.type != EventTypes.Create + and builder.room_id is not None + ): + # modules can send new state events, so we re-calculate the auth events just + # in case. prev_state_events = list( await self.store.get_state_dag_extremities(builder.room_id) ) From cd2c84b5a50115ad045ed1d8988c3a5740247dba Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 15 Sep 2026 15:49:51 +0100 Subject: [PATCH 18/41] Stop treating unset display names and avatar URLs as profile fields with a `null` value. (#20145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead, treat them as absent fields as they feel like they should be. The database implementation detail that these fields have a dedicated column with `NULL` when unset is kept to the storage layer. The goal here is to reduce the amount of special casing needed for these two original profile fields and treat them a little bit more like regular profile fields. Follows: #20003 Follows: #20147 (needed as a bugfix to continue sending them down oldschool sync when they get deleted. Without #20147, this PR would break that — which matches how custom profile fields were broken too.) --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20145.bugfix | 1 + synapse/handlers/sliding_sync/extensions.py | 20 +------- synapse/storage/databases/main/profile.py | 32 +++++++----- tests/handlers/test_sync.py | 50 ++++++++++++------- .../sliding_sync/test_extension_profiles.py | 4 -- 5 files changed, 56 insertions(+), 51 deletions(-) create mode 100644 changelog.d/20145.bugfix diff --git a/changelog.d/20145.bugfix b/changelog.d/20145.bugfix new file mode 100644 index 0000000000..e2ce9d52db --- /dev/null +++ b/changelog.d/20145.bugfix @@ -0,0 +1 @@ +Stop treating unset display names and avatar URLs as profile fields with a `null` value. \ No newline at end of file diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 7ee26079ed..0e193cae36 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -29,7 +29,6 @@ AccountDataTypes, EduTypes, EventTypes, - ProfileFields, ProfileUpdateAction, StickyEvent, ) @@ -1421,28 +1420,13 @@ async def get_profiles_extension_response( per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} per_user_removals: set[str] = set() for field_name in user_fields: - # For custom fields the lack of a field means it will be `Absent`, - # for displayname/avatar_url it will be `None`, due to way we store - # things differently. - # FIXME: I intend to simplify this by pushing the special-case logic - # for these 'original' profile fields into the storage layer instead. - absent_type = ( - Absent - if field_name - not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) - else None - ) field_value: JsonValue | dict[str, JsonValue] | AbsentType = ( - profile_data.get(field_name, absent_type) + profile_data.get(field_name, Absent) ) if ( # If the field isn't found on the profile and it is present in # `updated_fields`, that means an existing field has been removed. - # We need the check against `updated_fields` as some profile fields - # are `None` by default, for example each and every user created - # by Synapse will have `avatar_url: None`, and we don't want to - # constantly send that to the clients. - field_value is absent_type and field_name in updated_fields + field_value is Absent and field_name in updated_fields ): per_user_removals.add(field_name) else: diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 1a2dea7460..7f860258ee 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -20,7 +20,7 @@ # import json from collections.abc import Set -from typing import TYPE_CHECKING, Collection, cast +from typing import TYPE_CHECKING, Collection, Iterable, cast import attr from canonicaljson import encode_canonical_json @@ -644,12 +644,19 @@ async def get_profile_data_for_users( user_ids: List of user IDs to filter against. Returns: - Dictionary of displayname/avatar_url/custom fields for a list of users. + Dictionary from user_id -> field name -> field value + for the requested users. + + This includes `displayname`, `avatar_url` and all custom fields. + For `displayname` and `avatar_url`, when they are stored as NULL + in the database column, the dictionary entry will be omitted. """ if not user_ids: return {} - rows = await self.db_pool.simple_select_many_batch( + rows: Iterable[ + tuple[str, str | None, str | None, str | JsonDict | None] + ] = await self.db_pool.simple_select_many_batch( table="profiles", column="full_user_id", iterable=user_ids, @@ -659,15 +666,16 @@ async def get_profile_data_for_users( results: dict[str, dict[str, JsonValue | dict[str, JsonValue]]] = {} for full_user_id, displayname, avatar_url, fields in rows: - user_fields = fields or {} - # The SQLite driver doesn't have a JSON datatype. - if isinstance(self.database_engine, Sqlite3Engine) and fields: - user_fields = json.loads(fields) - base_fields = { - ProfileFields.DISPLAYNAME: displayname, - ProfileFields.AVATAR_URL: avatar_url, - } - user_fields.update(base_fields) + user_fields = db_to_json(fields or {}) + + # When the displayname and avatar URL aren't set, + # they are stored as NULL in the database. + # To make them behave the same as custom fields, + # when they are NULL, we treat them as not being set at all. + if displayname is not None: + user_fields[ProfileFields.DISPLAYNAME] = displayname + if avatar_url is not None: + user_fields[ProfileFields.AVATAR_URL] = avatar_url results[full_user_id] = user_fields diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index e100943413..a74bd68df1 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1175,6 +1175,13 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.other_user = self.register_user("other_user", "password") self.other_tok = self.login("other_user", "password") self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.get_success( + self.store.set_profile_field( + UserID.from_string(self.user), + ProfileFields.AVATAR_URL, + "mxc://example.invalid/abcdef", + ) + ) self.get_success( self.store.set_profile_field( user_id=UserID.from_string(self.user), @@ -1978,8 +1985,11 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_ ) assert incremental_result.profile_updates["@other_user:test"] is not None self.assertEqual( - set(incremental_result.profile_updates["@other_user:test"].keys()), - {"avatar_url", "displayname"}, + incremental_result.profile_updates["@other_user:test"], + { + "displayname": "other_user", + # avatar_url unset (user doesn't have one) + }, ) # If we have more events from the other_user, and do another lazy sync, @@ -2259,7 +2269,7 @@ def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joi user=third_user, tok=third_tok, ) - # Set a status field we don't except to see in sync + # Set a status field we don't expect to see in sync self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(third_user), @@ -2292,14 +2302,12 @@ def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joi [third_user], ) self.assertEqual( - incremental_result.profile_updates["@third_user:test"]["displayname"], - "third_user", - ) - self.assertIsNone( - incremental_result.profile_updates["@third_user:test"]["avatar_url"], - ) - self.assertFalse( - "m.status" in incremental_result.profile_updates["@third_user:test"].keys(), + incremental_result.profile_updates["@third_user:test"], + { + "displayname": "third_user", + # avatar_url unset (user doesn't have one) + # m.status unset (not requested in sync) + }, ) @parameterized.expand( @@ -2335,6 +2343,14 @@ def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> N request_key=generate_request_key(), ) ) + # Sanity-check that initial sync includes the fields + self.assertEqual( + initial_result.profile_updates["@user:test"], + { + "m.status": {"text": "Swimming in the Great Lakes!", "emoji": "🏊"}, + "avatar_url": "mxc://example.invalid/abcdef", + }, + ) self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.user), @@ -2359,12 +2375,12 @@ def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> N ) assert incremental_result.profile_updates["@user:test"] is not None self.assertEqual( - incremental_result.profile_updates["@user:test"]["m.status"], - {"text": "On holiday", "emoji": "🏖"}, - ) - # We didn't ask for displayname - self.assertFalse( - "displayname" in incremental_result.profile_updates["@user:test"].keys(), + incremental_result.profile_updates["@user:test"], + { + "m.status": {"text": "On holiday", "emoji": "🏖"}, + # avatar_url not included (it didn't change during this sync window) + # displayname not included (we didn't request it in sync) + }, ) @parameterized.expand([[True, False], [True, True], [False, False], [False, True]]) diff --git a/tests/rest/client/sliding_sync/test_extension_profiles.py b/tests/rest/client/sliding_sync/test_extension_profiles.py index 40d426fb87..5c2d17547c 100644 --- a/tests/rest/client/sliding_sync/test_extension_profiles.py +++ b/tests/rest/client/sliding_sync/test_extension_profiles.py @@ -480,7 +480,6 @@ def test_all_fields_returned_if_no_fields_specified(self, is_initial: bool) -> N ], { "updated": { - "avatar_url": None, "displayname": "other_user", "field": "value", } @@ -582,8 +581,6 @@ def test_profile_returned_if_user_left_then_rejoined(self) -> None: { "updated": { "displayname": "other_user", - # FIXME: This shouldn't be returned, but currently is - "avatar_url": None, } }, ) @@ -625,7 +622,6 @@ def test_all_fields_returned_in_incremental_non_lazy_sync_if_someone_joined( expectation = { "updated": { - "avatar_url": None, "displayname": "third_user", } } From f7c16d0148ab1e1329eab70ead51a581a8bfdf41 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 15 Sep 2026 17:16:47 +0000 Subject: [PATCH 19/41] Support configuring a username for Redis connections (#20187) Adds a `redis.username` config option. Details: A `username` without a `password` (or `password_path`) is refused at startup. Redis has no wire form for a username without a password, and txredisapi only sends `AUTH` when a password is set, so the username would otherwise be silently ignored. An explicitly empty password is accepted, since that is how a `nopass` ACL user is configured. This relies on txredisapi 1.4.12, the first release to accept a `username` kwarg. That upstream support was contributed by @karolyi specifically to unblock this. Fixes #19238. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Claude Sonnet 5 --- changelog.d/20187.feature | 1 + .../configuration/config_documentation.md | 5 ++ poetry.lock | 10 +-- pyproject.toml | 6 +- schema/synapse-config.schema.yaml | 11 +++ stubs/txredisapi.pyi | 3 + synapse/config/redis.py | 11 +++ synapse/replication/tcp/redis.py | 7 ++ synapse/server.py | 2 + tests/config/test_redis.py | 85 +++++++++++++++++++ tests/replication/_base.py | 9 ++ tests/replication/tcp/test_redis_auth.py | 60 +++++++++++++ 12 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 changelog.d/20187.feature create mode 100644 tests/config/test_redis.py create mode 100644 tests/replication/tcp/test_redis_auth.py diff --git a/changelog.d/20187.feature b/changelog.d/20187.feature new file mode 100644 index 0000000000..d5c53f1510 --- /dev/null +++ b/changelog.d/20187.feature @@ -0,0 +1 @@ +Add support for configuring a `username` for Redis connections, for Redis 6+ ACL authentication. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 4882569b7c..74ac85256f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4670,6 +4670,8 @@ _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_ _Changed in Synapse 1.116.0: Added password\_path_ +_Changed in Synapse 1.162.0: Added username_ + This setting has the following sub-options: * `enabled` (boolean): Whether to use Redis support. Defaults to `false`. @@ -4680,6 +4682,8 @@ This setting has the following sub-options: * `path` (string): The full path to a local Unix socket file. **If this is used, `host` and `port` are ignored.** Defaults to `"/tmp/redis.sock"`. +* `username` (string|null): Optional username if configured on the Redis instance (Redis 6+ ACL authentication). Requires `password` (or `password_path`) to also be set. Defaults to `null`. + * `password` (string|null): Optional password if configured on the Redis instance. Defaults to `null`. * `password_path` (string|null): Alternative to `password`, reading the password from an external file. The file should be a plain text file, containing only the password. Synapse reads the password from the given file once at startup. Defaults to `null`. @@ -4702,6 +4706,7 @@ redis: enabled: true host: localhost port: 6379 + username: password_path: dbid: ``` diff --git a/poetry.lock b/poetry.lock index 00a0fddd12..38c00e1af0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -3384,15 +3384,15 @@ windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)" [[package]] name = "txredisapi" -version = "1.4.11" +version = "1.4.12" description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] markers = "extra == \"redis\" or extra == \"all\"" files = [ - {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, - {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, + {file = "txredisapi-1.4.12-py3-none-any.whl", hash = "sha256:c698fff24a0b82e8932ef18c5623496b0a8b5195e4e94bf8524800a00ca6b7c1"}, + {file = "txredisapi-1.4.12.tar.gz", hash = "sha256:98e2440ff2e297c9048c5f9d516e7b1302d447cd9cd4568c986c35a8523de3c7"}, ] [package.dependencies] @@ -3784,4 +3784,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "4628b733106afbf7e6306058bcac72c1b273533941eb7cc8621c74b91e947b63" +content-hash = "3e9a70d3ec7ee5edd32ea5e86a0b56e7633e9ed1b59589890a4df8df25a135d5" diff --git a/pyproject.toml b/pyproject.toml index 3606443fcf..e1965b71fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,7 +153,9 @@ opentracing = [ jwt = ["authlib"] # hiredis is not a *strict* dependency, but it makes things much faster. # (if it is not installed, we fall back to slow code.) -redis = ["txredisapi>=1.4.7", "hiredis>=0.3"] +# txredisapi 1.4.12 is the first release to accept a `username` kwarg for +# Redis ACL auth: https://github.com/IlyaSkriblovsky/txredisapi/releases/tag/1.4.12 +redis = ["txredisapi>=1.4.12", "hiredis>=0.3"] # Required to use experimental `caches.track_memory_usage` config option. cache-memory = ["pympler>=1.0"] # If this is updated, don't forget to update the equivalent lines in @@ -186,7 +188,7 @@ all = [ # opentracing "jaeger-client>=4.2.0", "opentracing>=2.2.0", # redis - "txredisapi>=1.4.7", "hiredis>=0.3", + "txredisapi>=1.4.12", "hiredis>=0.3", # cache-memory # 1.0 added support for python 3.10, our current minimum supported python version "pympler>=1.0", diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 4a4b9eed3f..b34bc374f3 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5823,6 +5823,9 @@ properties: _Changed in Synapse 1.116.0: Added password\_path_ + + + _Changed in Synapse 1.162.0: Added username_ properties: enabled: type: boolean @@ -5842,6 +5845,13 @@ properties: The full path to a local Unix socket file. **If this is used, `host` and `port` are ignored.** default: /tmp/redis.sock + username: + type: ["string", "null"] + description: >- + Optional username if configured on the Redis instance (Redis 6+ + ACL authentication). Requires `password` (or `password_path`) to + also be set. + default: null password: type: ["string", "null"] description: Optional password if configured on the Redis instance. @@ -5885,6 +5895,7 @@ properties: - enabled: true host: localhost port: 6379 + username: "" password_path: "" dbid: "" worker_app: diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi index 50ab54037a..7454ad5d14 100644 --- a/stubs/txredisapi.pyi +++ b/stubs/txredisapi.pyi @@ -38,6 +38,7 @@ class RedisProtocol(protocol.Protocol): class SubscriberProtocol(RedisProtocol): def __init__(self, *args: object, **kwargs: object): ... + username: str | None password: str | None def subscribe(self, channels: str | list[str]) -> "Deferred[None]": ... def connectionMade(self) -> None: ... @@ -56,6 +57,7 @@ def lazyConnection( connectTimeout: int | None = ..., replyTimeout: int | None = ..., convertNumbers: bool = ..., + username: str | None = ..., ) -> RedisProtocol: ... # ConnectionHandler doesn't actually inherit from RedisProtocol, but it proxies @@ -82,6 +84,7 @@ class RedisFactory(protocol.ReconnectingClientFactory): password: str | None = None, replyTimeout: int | None = None, convertNumbers: int | None = True, + username: str | None = None, ): ... def buildProtocol(self, addr: IAddress) -> RedisProtocol: ... diff --git a/synapse/config/redis.py b/synapse/config/redis.py index 948c95eef7..cf5707c2c3 100644 --- a/synapse/config/redis.py +++ b/synapse/config/redis.py @@ -49,6 +49,7 @@ def read_config( self.redis_port = redis_config.get("port", 6379) self.redis_path = redis_config.get("path", None) self.redis_dbid = redis_config.get("dbid", None) + self.redis_username = redis_config.get("username", None) self.redis_password = redis_config.get("password") if self.redis_password and not allow_secrets_in_config: raise ConfigError( @@ -67,6 +68,16 @@ def read_config( ), ).strip() + # An empty password is not the same as an unset one: Redis has no wire form + # for a username without a password, but a `nopass` ACL user accepts any. + if self.redis_username and self.redis_password is None: + raise ConfigError( + "`redis.username` was set without `redis.password` (or " + "`redis.password_path`). Redis username authentication requires " + "a password to also be configured.", + ("redis", "username"), + ) + self.redis_use_tls = redis_config.get("use_tls", False) self.redis_certificate = redis_config.get("certificate_file", None) self.redis_private_key = redis_config.get("private_key_file", None) diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 93ba48b406..705ea4172c 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -299,6 +299,7 @@ def __init__( isLazy: bool = False, handler: type = ConnectionHandler, charset: str = "utf-8", + username: str | None = None, password: str | None = None, replyTimeout: int = 30, convertNumbers: int | None = True, @@ -310,6 +311,7 @@ def __init__( isLazy=isLazy, handler=handler, charset=charset, + username=username, password=password, replyTimeout=replyTimeout, convertNumbers=convertNumbers, @@ -390,6 +392,7 @@ def __init__( dbid=None, poolsize=1, replyTimeout=30, + username=hs.config.redis.redis_username, password=hs.config.redis.redis_password, ) @@ -425,6 +428,7 @@ def lazyConnection( port: int = 6379, dbid: int | None = None, reconnect: bool = True, + username: str | None = None, password: str | None = None, replyTimeout: int = 30, ) -> ConnectionHandler: @@ -440,6 +444,7 @@ def lazyConnection( poolsize=1, isLazy=True, handler=ConnectionHandler, + username=username, password=password, replyTimeout=replyTimeout, ) @@ -474,6 +479,7 @@ def lazyUnixConnection( path: str = "/tmp/redis.sock", dbid: int | None = None, reconnect: bool = True, + username: str | None = None, password: str | None = None, replyTimeout: int = 30, ) -> ConnectionHandler: @@ -493,6 +499,7 @@ def lazyUnixConnection( poolsize=1, isLazy=True, handler=UnixConnectionHandler, + username=username, password=password, replyTimeout=replyTimeout, ) diff --git a/synapse/server.py b/synapse/server.py index b756223e54..9422882167 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1216,6 +1216,7 @@ def get_outbound_redis_connection(self) -> "ConnectionHandler": host=self.config.redis.redis_host, port=self.config.redis.redis_port, dbid=self.config.redis.redis_dbid, + username=self.config.redis.redis_username, password=self.config.redis.redis_password, reconnect=True, ) @@ -1229,6 +1230,7 @@ def get_outbound_redis_connection(self) -> "ConnectionHandler": hs=self, path=self.config.redis.redis_path, dbid=self.config.redis.redis_dbid, + username=self.config.redis.redis_username, password=self.config.redis.redis_password, reconnect=True, ) diff --git a/tests/config/test_redis.py b/tests/config/test_redis.py new file mode 100644 index 0000000000..3a1f810c33 --- /dev/null +++ b/tests/config/test_redis.py @@ -0,0 +1,85 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import tempfile + +from synapse.config._base import ConfigError +from synapse.config.homeserver import HomeServerConfig + +from tests.unittest import TestCase +from tests.utils import default_config + +try: + import hiredis +except ImportError: + hiredis = None # type: ignore + + +class RedisConfigTestCase(TestCase): + # hiredis is part of the `redis` extra, which `RedisConfig` requires to + # parse an enabled redis config. + if not hiredis: + skip = "Requires hiredis" + + def _make_config(self, redis_config: dict) -> HomeServerConfig: + config_dict = default_config(server_name="test") + config_dict["redis"] = redis_config + config = HomeServerConfig() + config.parse_config_dict(config_dict, "", "") + return config + + def test_username_defaults_to_none(self) -> None: + """`redis.username` is `None` when not configured.""" + config = self._make_config({"enabled": True, "password": "hunter2"}) + self.assertIsNone(config.redis.redis_username) + + def test_username_is_parsed(self) -> None: + """`redis.username` is parsed through to `redis_username` when set + alongside `password`.""" + config = self._make_config( + {"enabled": True, "username": "alice", "password": "hunter2"} + ) + self.assertEqual(config.redis.redis_username, "alice") + + def test_username_is_parsed_with_password_path(self) -> None: + """`redis.username` is also accepted alongside `password_path`, the + documented alternative to an inline `password`.""" + with tempfile.NamedTemporaryFile(buffering=0) as password_file: + password_file.write(b"hunter2") + + config = self._make_config( + { + "enabled": True, + "username": "alice", + "password_path": password_file.name, + } + ) + self.assertEqual(config.redis.redis_username, "alice") + self.assertEqual(config.redis.redis_password, "hunter2") + + def test_username_with_empty_password_is_accepted(self) -> None: + """`redis.username` with an explicitly empty `password` is allowed: that + is how a Redis ACL user declared `nopass` is configured.""" + config = self._make_config( + {"enabled": True, "username": "alice", "password": ""} + ) + self.assertEqual(config.redis.redis_username, "alice") + self.assertEqual(config.redis.redis_password, "") + + def test_username_without_password_is_rejected(self) -> None: + """`redis.username` without any of `password`/`password_path` is + rejected: Redis ACL authentication requires both.""" + with self.assertRaises(ConfigError): + self._make_config({"enabled": True, "username": "alice"}) diff --git a/tests/replication/_base.py b/tests/replication/_base.py index b23696668f..791a6beab5 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -509,6 +509,10 @@ def __init__(self) -> None: defaultdict(set) ) + # The arguments of every `AUTH` command received, in order: `(password,)` + # when only a password is configured, `(username, password)` with both. + self.auth_attempts: list[tuple[bytes, ...]] = [] + def add_subscriber(self, conn: "FakeRedisPubSubProtocol", channel: bytes) -> None: """A connection has called SUBSCRIBE""" self._subscribers_by_channel[channel].add(conn) @@ -581,6 +585,11 @@ def handle_command(self, command: bytes, *args: bytes) -> None: elif command == b"PING": self.send("PONG") + # We don't check the credentials, just record that they were sent. + elif command == b"AUTH": + self._server.auth_attempts.append(args) + self.send("OK") + else: raise Exception(f"Unknown command: {command!r}") diff --git a/tests/replication/tcp/test_redis_auth.py b/tests/replication/tcp/test_redis_auth.py new file mode 100644 index 0000000000..ad025b987e --- /dev/null +++ b/tests/replication/tcp/test_redis_auth.py @@ -0,0 +1,60 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from typing import Any + +from tests.replication._base import BaseMultiWorkerStreamTestCase + + +class RedisUsernameAuthTestCase(BaseMultiWorkerStreamTestCase): + """Tests that Synapse authenticates to Redis with a username *and* password + when both are configured, rather than with a bare password. + """ + + USERNAME = b"synapse-user" + PASSWORD = b"correct-horse-battery-staple" + + def default_config(self) -> dict[str, Any]: + config = super().default_config() + config["redis"]["username"] = self.USERNAME.decode("utf-8") + config["redis"]["password"] = self.PASSWORD.decode("utf-8") + return config + + def test_auth_sent_with_username(self) -> None: + """Both Redis connections the main process opens (one outbound, one + subscriber) send `AUTH `. + """ + # Let the AUTH replies flow back; nothing here needs virtual time to pass. + self.reactor.advance(0) + + self.assertEqual( + self._redis_server.auth_attempts, + [(self.USERNAME, self.PASSWORD)] * 2, + ) + + def test_workers_authenticate_with_username_too(self) -> None: + """A worker authenticates the same way as the main process, and both end + up subscribed to the replication stream over those connections. + """ + self.make_worker_hs("synapse.app.generic_worker") + + # Let the AUTH and SUBSCRIBE replies flow back. + self.reactor.advance(0) + + self.assertEqual( + self._redis_server.auth_attempts, + [(self.USERNAME, self.PASSWORD)] * 4, + ) + self.assertEqual(len(self._redis_server._subscribers_by_channel[b"test"]), 2) From 99f20bef0533407c8d0536aa15102708af75a3fc Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 15 Sep 2026 15:36:36 -0500 Subject: [PATCH 20/41] Get `join_rules` from state directly instead of stats tables when populating the room summary (#20205) Fix #19905 Directly retrieve the `m.room.join_rules` event so the room summary data is correct and matches `allowed_room_ids`. Otherwise there could be a mis-match delay or the `join_rule` could be missing altogether. --- changelog.d/20205.misc | 1 + synapse/api/constants.py | 3 +++ synapse/handlers/room_summary.py | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20205.misc diff --git a/changelog.d/20205.misc b/changelog.d/20205.misc new file mode 100644 index 0000000000..5a5d03b44a --- /dev/null +++ b/changelog.d/20205.misc @@ -0,0 +1 @@ +Fix `/room_summary` returning stale `join_rules`. Contributed by @famedly. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 840930312f..17e6a4cfdb 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -316,6 +316,9 @@ class EventContentFields: M_TOPIC: Final = "m.topic" M_TEXT: Final = "m.text" + # `m.room.join_rules` `join_rule` field + JOIN_RULE: Final = "join_rule" + class EventUnsignedContentFields: """Fields found inside the 'unsigned' data on events""" diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 4afd449292..c79a94e930 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -27,6 +27,7 @@ import attr from synapse.api.constants import ( + EventContentFields, EventTypes, HistoryVisibility, JoinRules, @@ -796,7 +797,6 @@ async def _build_room_entry(self, room_id: str) -> JsonDict: "canonical_alias": stats.canonical_alias, "num_joined_members": stats.joined_members, "avatar_url": stats.avatar, - "join_rule": stats.join_rules, "world_readable": ( stats.history_visibility == HistoryVisibility.WORLD_READABLE ), @@ -815,6 +815,23 @@ async def _build_room_entry(self, room_id: str) -> JsonDict: state_filter=StateFilter.from_types([(EventTypes.JoinRules, "")]), ) ) + if join_event_id := join_rules_state_ids.get((EventTypes.JoinRules, ""), None): + # To get the freshest data available, pull the state for the join_rules + # directly. In the unlikely case it is None, it will still be filtered out + # below. + # + # XXX: The current `/room_summary` spec (as of 2026-09-15) says that the + # room is assumed to be `public` when `join_rule` isn't present but this + # directly contradicts the scenarios where `join_rule` doesn't exist. For + # example, if there is no `m.room.join_rules` event in the room, there is + # no default and the the auth rules effectively make it so no one can join + # except the room creator. The other scenario is if `join_rule` isn't a + # string (not a valid `m.room.join_rules` event). See + # https://github.com/matrix-org/matrix-spec/issues/2444 + join_event = await self._store.get_event(join_event_id) + join_rule_content = join_event.content.get(EventContentFields.JOIN_RULE) + if isinstance(join_rule_content, str): + entry["join_rule"] = join_rule_content try: room_version = await self._store.get_room_version(room_id) From 8c11b13f63c8cf27f8f1f65ff4f4826e2191563c Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Wed, 16 Sep 2026 11:07:25 +0200 Subject: [PATCH 21/41] Fix state events missing from MSC4222 `state_after` when the `since` token falls inside a persist batch (#20171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes the issue described as comment here: https://github.com/element-hq/synapse/issues/18793#issuecomment-3502202379 In Element Call, this shows up as ghost participants: someone who left the call keeps being displayed until a later state change refreshes the room. The bug is not specific to Element Call: any state event can be affected, RTC membership just changes often enough to make it visible. ## What happens Alice has a client syncing against a homeserver where events are persisted by one worker (the event persister) and `/sync` is served by another (the sync worker). Her client is parked in a long-poll: `GET /sync?since=s99&timeout=30000`. Bob joins a call at the same moment Carol sends a message. Carol's message reaches the persister first; Bob's `m.call.member` arrives while that write is still in flight, so the per-room persist queue groups them into one transaction: ``` events (each gets its own stream ordering): stream_ordering 100: m.room.message Carol stream_ordering 101: m.call.member Bob (state) current_state_delta_stream (how state_after finds state changes): stream_id 100 ────► (m.call.member, @bob) -> $bob_join_call ▲ └─ stamped with the batch MINIMUM (100), not the event's own 101 (see `_update_current_state_txn`) ``` The transaction commits: both events and the delta row are now in the database, atomically. The persister then announces the new events over replication, one RDATA token per stream ordering — rows are only merged into one token when they share a position, and 100 and 101 don't. So the sync worker's events-stream position steps 99 → 100 → 101, and on reaching 100 it pokes the notifier. Alice's long-poll wakes at exactly that moment. Her response is built at the worker's *current* position — `end = 100` — with RDATA 101 still in the queue: ``` Sync A (since=99, end=100): timeline: events 99 < ordering ≤ 100 → [Carol's message] state_after: deltas 99 < stream_id ≤ 100 → [$bob_join_call] ← delivered EARLY next_batch: s100 ← mid-batch token ``` No race on the client's side is needed: the server *hands out* the mid-batch token as `next_batch`. Alice's client re-polls with it, as every sync client does. The worker has meanwhile processed RDATA 101: ``` Sync B (since=100, end=101): timeline: events 100 < ordering ≤ 101 → [Bob's m.call.member @101] ✓ state_after: deltas 100 < stream_id ≤ 101 → [] row is stamped 100 ✗ ``` A state event in the timeline with an empty `state_after`. An MSC4222 client trusts `state_after` over timeline state events, so Alice's copy of Bob's call membership never updates from this response. On a single process this cannot happen: the batch's stream IDs are released as a whole, so the position visible to `/sync` jumps 99 → 101 and `s100` is never handed out. Only a process that learns its position from replication — any sync worker — ticks through the middle of a batch. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20171.bugfix | 1 + synapse/handlers/sync.py | 14 +- synapse/storage/databases/main/events.py | 5 + .../storage/databases/main/state_deltas.py | 264 +++++++++++++- synapse/storage/databases/main/stream.py | 2 +- tests/replication/tcp/streams/test_events.py | 71 +++- tests/rest/client/test_sync.py | 296 ++++++++++++++- tests/storage/test_state_deltas.py | 337 ++++++++++++++++++ tests/test_utils/event_injection.py | 47 +++ 9 files changed, 1024 insertions(+), 13 deletions(-) create mode 100644 changelog.d/20171.bugfix create mode 100644 tests/storage/test_state_deltas.py diff --git a/changelog.d/20171.bugfix b/changelog.d/20171.bugfix new file mode 100644 index 0000000000..ab54bf8df4 --- /dev/null +++ b/changelog.d/20171.bugfix @@ -0,0 +1 @@ +Fix state events being omitted from the [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) `state_after` sync response when the client's `since` token falls inside an event persistence batch, as could happen on worker deployments. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 049726a97e..c66034cd0e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1398,8 +1398,8 @@ async def _compute_state_delta_for_full_sync( # Now roll back the state by looking at the state deltas between # end_token and now. - deltas = await self.store.get_current_state_deltas_for_room( - room_id, + deltas = await self.store.get_current_state_deltas_for_room_by_event_position( + room_id=room_id, from_token=end_token.room_key, to_token=self.store.get_room_max_token(), ) @@ -1554,10 +1554,12 @@ async def _compute_state_delta_for_incremental_sync( # # i.e. we return all state deltas, including membership changes that # we'd normally exclude due to LL. - deltas = await self.store.get_current_state_deltas_for_room( - room_id=room_id, - from_token=since_token.room_key, - to_token=end_token.room_key, + deltas = ( + await self.store.get_current_state_deltas_for_room_by_event_position( + room_id=room_id, + from_token=since_token.room_key, + to_token=end_token.room_key, + ) ) for delta in deltas: if delta.event_id is None: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 7550a03a43..26a2b92f8d 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1829,6 +1829,11 @@ def _update_current_state_txn( stream_id: This is expected to be the minimum `stream_ordering` for the batch of events that we are persisting; which means we do not end up in a situation where workers see events before the `current_state_delta` updates. + Note that this stamps a row *before* its own event; readers that pair + deltas with the events in the same window bound each delta on its + event's position instead, see + `get_current_state_deltas_for_room_by_event_position(...)`, which stays + correct if this stamp is ever changed. FIXME: However, this function also gets called with next upcoming `stream_ordering` when we re-sync the state of a partial stated room (see `update_current_state(...)`) which may be "correct" but it would be good to diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py index a5d5407327..1e71c9ed7b 100644 --- a/synapse/storage/databases/main/state_deltas.py +++ b/synapse/storage/databases/main/state_deltas.py @@ -20,7 +20,7 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import attr @@ -59,8 +59,9 @@ class StateDelta: class StateDeltasStore(SQLBaseStore): # This class must be mixed in with a child class which provides the following - # attribute. TODO: can we get static analysis to enforce this? + # attributes. TODO: can we get static analysis to enforce this? _curr_state_delta_stream_cache: StreamChangeCache + _events_stream_cache: StreamChangeCache def __init__( self, @@ -226,7 +227,9 @@ def get_current_state_deltas_for_room_txn( to_token: RoomStreamToken | None, ) -> list[StateDelta]: """ - Get the state deltas between two tokens. + Get the state deltas between two tokens, bounded on the delta rows' + own `stream_id`. See `get_current_state_deltas_for_room` for when that + is the wrong bound. (> `from_token` and <= `to_token`) """ @@ -272,9 +275,18 @@ async def get_current_state_deltas_for_room( to_token: RoomStreamToken | None, ) -> list[StateDelta]: """ - Get the state deltas between two tokens. + Get the state deltas between two tokens, bounded on the delta rows' + own `stream_id`. (> `from_token` and <= `to_token`) + + Rows carry the minimum stream ordering of their persist batch as + `stream_id` (see `_update_current_state_txn`), so a token that falls inside + a batch -- which a worker reading the events stream from replication + routinely observes -- misses that batch's deltas here. Callers that + pair the deltas with the events in the same window (a timeline) + should use `get_current_state_deltas_for_room_by_event_position` + instead, which bounds each delta on its event's position. """ # We can bail early if the `from_token` is after the `to_token` if ( @@ -300,6 +312,250 @@ async def get_current_state_deltas_for_room( to_token=to_token, ) + def get_current_state_deltas_for_room_by_event_position_txn( + self, + txn: LoggingTransaction, + room_id: str, + *, + from_token: RoomStreamToken | None, + to_token: RoomStreamToken | None, + ) -> list[StateDelta]: + """ + Get the state deltas of a room between two tokens. + + A delta is included if its position is > `from_token` and <= `to_token` + (a `None` token is unbounded). The position of a delta is the later of + the delta row's own `stream_id` and the stream ordering of its event: + + * a delta whose event was persisted later in the stream than the row's + `stream_id` is positioned at the event, so it is reported in the same + window as the event itself; + * a delta with no event (state that was removed, e.g. when the last + local user left the room) is positioned at the row's `stream_id`; + * a delta whose row was written later than its event (existing state + re-announced by a partial-state resync) is positioned at the row's + `stream_id`. + + Returns: + The deltas in `stream_id` order, which for any two deltas of + different persist batches is also position order. The order of + deltas within one batch is unspecified; a batch writes at most one + delta per state key. + """ + # `current_state_delta_stream.stream_id` is not the position of the + # delta's event: a persist batch writes all its rows with the *first* + # stream ordering of the batch (see `_update_current_state_txn`), so a + # state event persisted mid-batch has `stream_id` < `stream_ordering`. + # A worker reading the events stream from replication advances one + # event at a time, so its tokens routinely fall inside a batch: + # + # stream_ordering event delta row `stream_id` + # from-token batch 10 message + # from_token = 11 -> 11 message + # 12 state event X X: 10 + # ... any number of batches + # to-token batch 90 state event Y Y: 90 + # to_token = 91 -> 91 message + # 92 state event Z Z: 90 + # + # On `stream_id` alone, the window (11, 91] misses X's delta (10 <= 11, + # though event 12 is in the window) and includes Z's (90 <= 91, though + # event 92 is past it). Y's delta is right either way. + # + # Batches of a room follow each other in the stream, so only two of + # them can be cut by a token: the one that contains `from_token` (the + # from-token batch) and the one that contains `to_token` (the to-token + # batch). Every other batch is entirely inside the window or entirely + # outside it, and so are its delta rows, whether judged on `stream_id` + # or on `stream_ordering`. + # + # The from-token batch is found without looking at `events`: its rows in + # `current_state_delta_stream` carry the latest `stream_id` <= + # `from_token` in the room, since every later batch writes its rows + # after `from_token`. So instead of fetching the delta rows with + # `from_token` < `stream_id`, fetch from that batch's `stream_id` + # inclusive (`from_batch_stream_id` below): the same delta rows plus + # those of the from-token batch. (If it changed no state, this picks the + # previous batch that did; its events are all before `from_token`, so + # its delta rows are dropped below.) + # + # Among the fetched delta rows, the from-token batch's are those at + # `from_batch_stream_id`, and the to-token batch's are found the same + # way, at the latest `stream_id` <= `to_token`. Only these need their + # event's `stream_ordering`, looked up in `events` by `event_id`; such a + # delta row is kept when + # `from_token` < max(`stream_id`, `stream_ordering`) <= `to_token`. + # Every other delta row is kept when + # `from_token` < `stream_id` <= `to_token`. + args: list[str | int] = [room_id] + + lower_clause = "" + if from_token is not None: + # The from-token batch's `stream_id`: the latest `stream_id` <= + # `from_token`. + # (`from_token.stream` is the minimum over writers; the delta rows + # of a writer that is further ahead are in the range anyway and are + # judged against that writer's position below.) + txn.execute( + """ + SELECT MAX(stream_id) FROM current_state_delta_stream + WHERE room_id = ? AND stream_id <= ? + """, + (room_id, from_token.stream), + ) + row = txn.fetchone() + from_batch_stream_id: int | None = row[0] if row is not None else None + if from_batch_stream_id is not None: + # No delta row has `from_batch_stream_id` < `stream_id` <= + # `from_token.stream`, so this is the from-token batch plus + # everything after `from_token`. + lower_clause = "AND ? <= stream_id" + args.append(from_batch_stream_id) + else: + lower_clause = "AND ? < stream_id" + args.append(from_token.stream) + + upper_clause = "" + if to_token is not None: + upper_clause = "AND stream_id <= ?" + args.append(to_token.get_max_stream_pos()) + + sql = f""" + SELECT instance_name, stream_id, type, state_key, event_id, prev_event_id + FROM current_state_delta_stream + WHERE room_id = ? {lower_clause} {upper_clause} + ORDER BY stream_id ASC + """ + txn.execute(sql, args) + rows = cast( + list[tuple[str | None, int, str, str, str | None, str | None]], + txn.fetchall(), + ) + + # The `stream_id` of the from-token batch and of the to-token batch, + # per writer: the latest `stream_id` <= the writer's position in + # `from_token` and in `to_token`. Historic delta rows have no instance + # name and count as "master", as in `_filter_results_by_stream`. + token_batch_stream_ids: set[tuple[str, int]] = set() + for token in (from_token, to_token): + if token is None: + continue + latest_by_instance: dict[str, int] = {} + for instance_name, stream_id, _, _, _, _ in rows: + instance_name = instance_name or "master" + if stream_id <= token.get_stream_pos_for_instance(instance_name): + latest_by_instance[instance_name] = max( + latest_by_instance.get(instance_name, stream_id), stream_id + ) + token_batch_stream_ids.update(latest_by_instance.items()) + + # Only the delta rows of the two token batches need their event's + # position. + token_batch_event_ids = [ + event_id + for instance_name, stream_id, _, _, event_id, _ in rows + if event_id is not None + and (instance_name or "master", stream_id) in token_batch_stream_ids + ] + event_positions: dict[str, tuple[str | None, int]] = {} + for chunk in batch_iter(token_batch_event_ids, 1000): + clause, clause_args = make_in_list_sql_clause( + self.database_engine, "event_id", chunk + ) + txn.execute( + f""" + SELECT event_id, instance_name, stream_ordering + FROM events + WHERE {clause} + """, + clause_args, + ) + for event_id, event_instance, event_stream in txn: + if event_stream is not None: + event_positions[event_id] = (event_instance, event_stream) + + deltas = [] + for ( + row_instance, + row_stream, + event_type, + state_key, + event_id, + prev_event_id, + ) in rows: + # The delta's position: its `stream_id`, unless it belongs to + # a token batch and its event sits later in the stream. + effective_instance, effective_stream = row_instance, row_stream + if ( + event_id is not None + and (row_instance or "master", row_stream) in token_batch_stream_ids + ): + position = event_positions.get(event_id) + if position is not None and position[1] > row_stream: + effective_instance, effective_stream = position + + if _filter_results_by_stream( + from_token, to_token, effective_instance, effective_stream + ): + deltas.append( + StateDelta( + stream_id=row_stream, + room_id=room_id, + event_type=event_type, + state_key=state_key, + event_id=event_id, + prev_event_id=prev_event_id, + ) + ) + + return deltas + + @trace + async def get_current_state_deltas_for_room_by_event_position( + self, + room_id: str, + *, + from_token: RoomStreamToken | None, + to_token: RoomStreamToken | None, + ) -> list[StateDelta]: + """ + Get the state deltas between two tokens, bounding each delta on the + position of its state event rather than on the delta row's `stream_id`. + See `get_current_state_deltas_for_room_by_event_position_txn`. + + (> `from_token` and <= `to_token`) + """ + # We can bail early if the `from_token` is after the `to_token` + if ( + to_token is not None + and from_token is not None + and to_token.is_before_or_eq(from_token) + ): + return [] + + # A delta's effective position is beyond `from_token` only if the row's + # `stream_id` is (the delta stream cache) or its event's stream + # ordering is (the events stream cache); if neither cache has seen the + # room change there is nothing to return. + if ( + from_token is not None + and not self._curr_state_delta_stream_cache.has_entity_changed( + room_id, from_token.stream + ) + and not self._events_stream_cache.has_entity_changed( + room_id, from_token.stream + ) + ): + return [] + + return await self.db_pool.runInteraction( + "get_current_state_deltas_for_room_by_event_position", + self.get_current_state_deltas_for_room_by_event_position_txn, + room_id, + from_token=from_token, + to_token=to_token, + ) + @trace async def get_current_state_deltas_for_rooms( self, diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 7d14f9f4d8..1d08601071 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -608,7 +608,7 @@ def __init__( stream_column="stream_ordering", max_value=events_max, ) - self._events_stream_cache = StreamChangeCache( + self._events_stream_cache: StreamChangeCache = StreamChangeCache( name="EventsRoomStreamChangeCache", server_name=self.server_name, current_stream_pos=min_event_val, diff --git a/tests/replication/tcp/streams/test_events.py b/tests/replication/tcp/streams/test_events.py index 484fd6b6db..486d3af517 100644 --- a/tests/replication/tcp/streams/test_events.py +++ b/tests/replication/tcp/streams/test_events.py @@ -27,6 +27,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase from synapse.replication.tcp.commands import RdataCommand +from synapse.replication.tcp.resource import _batch_updates from synapse.replication.tcp.streams._base import _STREAM_UPDATE_TARGET_ROW_COUNT from synapse.replication.tcp.streams.events import ( _MAX_STATE_UPDATES_PER_ROOM, @@ -41,8 +42,13 @@ from synapse.server import HomeServer from synapse.util.clock import Clock +from tests import unittest from tests.replication._base import BaseStreamTestCase -from tests.test_utils.event_injection import inject_event, inject_member_event +from tests.test_utils.event_injection import ( + inject_event, + inject_member_event, + persist_message_and_state_event_in_one_batch, +) class EventsStreamTestCase(BaseStreamTestCase): @@ -563,3 +569,66 @@ def _inject_state_event( content={"body": body}, ) ) + + +class EventsStreamBatchTokensTestCase(unittest.HomeserverTestCase): + """Tests for the tokens `EventsStream` hands to readers, which need no + worker (so they run under SQLite too, unlike `EventsStreamTestCase`).""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("u1", "pass") + self.user_tok = self.login("u1", "pass") + self.room_id = self.helper.create_room_as(tok=self.user_tok) + + def test_update_function_splits_persist_batches(self) -> None: + """The events stream emits a distinct token for every event stream + ordering (`_batch_updates` only collapses rows that share a token), and + `_process_rdata` calls `on_rdata` -- and hence + `process_replication_position` -- once per token. So a reader worker + (e.g. a sync worker) advances its events-stream position *through* the + middle of a persist batch, one event at a time, while reading + `current_state_delta_stream` -- whose rows are stamped with the batch + minimum -- straight from the database. + + This is what makes the hand-built `since` token in + `tests/rest/client/test_sync.py::SyncStateAfterTimelineStateTestCase` + one a real deployment hands out. + """ + store = self.hs.get_datastores().main + before = store.get_room_max_token().stream + message, state_event = self.get_success( + persist_message_and_state_event_in_one_batch( + self.hs, self.room_id, self.user_id + ) + ) + after = store.get_room_max_token().stream + + message_pos = message.internal_metadata.stream_ordering + state_pos = state_event.internal_metadata.stream_ordering + assert message_pos is not None and state_pos is not None + self.assertLess(message_pos, state_pos) + + stream = EventsStream(self.hs) + updates, _upto, _limited = self.get_success( + # A limit of 100 is comfortably more than the handful of rows this + # test persists, so the update batch is never truncated. + stream._update_function("master", before, after, 100) + ) + + # The tokens a reader will actually advance to, in order. + delivered = [token for token, _row in _batch_updates(updates) if token] + + self.assertIn( + message_pos, + delivered, + "a reader worker advances to the batch minimum before it sees the " + "state event", + ) + self.assertIn(state_pos, delivered) + self.assertLess(delivered.index(message_pos), delivered.index(state_pos)) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 039aea4d78..2da5348492 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -33,10 +33,11 @@ ReceiptTypes, RelationTypes, ) +from synapse.events import EventBase from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import devices, knock, login, read_marker, receipts, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, RoomStreamToken, StreamKeyType, StreamToken from synapse.util.clock import Clock from tests import unittest @@ -45,6 +46,10 @@ ) from tests.rest.client.test_rooms import make_request_with_cancellation_test from tests.server import TimedOutException +from tests.test_utils.event_injection import ( + inject_event, + persist_message_and_state_event_in_one_batch, +) logger = logging.getLogger(__name__) @@ -1275,6 +1280,295 @@ def test_incremental_sync(self) -> None: self.assertEqual(200, channel.code, msg=channel.result["body"]) +class SyncStateAfterTimelineStateTestCase(unittest.HomeserverTestCase): + """Tests for the MSC4222 invariant that any state event served in a sync + response's *timeline* is reflected in `state_after` (unless it lost state + resolution): `state at since` + `state_after` must equal the state at the + end of the timeline. See + https://github.com/matrix-org/matrix-spec-proposals/pull/4222. + + This breaks when the `since` token falls inside a persist batch: the + `current_state_delta_stream` row for a state event persisted in a batch is + stamped with the *minimum* stream ordering of the batch, so such a token + selects the event for the timeline but not its delta. A worker reading the + events stream from replication routinely observes such a token, which is + why this is seen on worker deployments and not on a single process. + + `test_state_after_with_token_inside_persist_batch` reproduces this end to + end with a hand-built `since` token. Its preconditions -- the batch-minimum + stamp, and that the events replication stream really hands out a token + inside a batch -- are pinned in `tests/storage/test_state_deltas.py` and + `tests/replication/tcp/streams/test_events.py`. + + The remaining tests guard that whatever repairs this reports the *resolved* + state at the end of the timeline rather than replaying the timeline. + """ + + servlets = [ + synapse.rest.admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + self.alice = self.register_user("alice", "password") + self.alice_tok = self.login("alice", "password") + self.bob = self.register_user("bob", "password") + self.bob_tok = self.login("bob", "password") + + self.get_success( + self.store.set_features_for_user( + self.alice, {ExperimentalFeature.MSC4222: True} + ) + ) + + # Named room, so that hero calculation doesn't inject current + # membership state into the response and confuse the assertions. + self.room_id = self.helper.create_room_as( + self.alice, tok=self.alice_tok, extra_content={"name": "test room"} + ) + self.helper.join(self.room_id, self.bob, tok=self.bob_tok) + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def _sync_url(self, lazy_load_members: bool, timeline_limit: int = 10) -> str: + # The default `timeline_limit` of 10 is arbitrary: comfortably more + # events than any test here produces between syncs, so the timeline is + # never truncated unless a test passes a smaller limit on purpose. + sync_filter: JsonDict = {"room": {"timeline": {"limit": timeline_limit}}} + if lazy_load_members: + sync_filter["room"]["state"] = {"lazy_load_members": True} + return f"/sync?filter={json.dumps(sync_filter)}&org.matrix.msc4222.use_state_after=true" + + def _sync(self, sync_url: str, since: str | None = None) -> JsonDict: + url = sync_url if since is None else f"{sync_url}&since={since}" + channel = self.make_request("GET", url, access_token=self.alice_tok) + self.assertEqual(channel.code, 200, channel.result) + return channel.json_body + + def _joined_room(self, response: JsonDict) -> JsonDict: + rooms = response["rooms"].get("join", {}) + self.assertIn(self.room_id, rooms, f"room missing from sync: {response}") + return rooms[self.room_id] + + def _timeline_ids(self, room: JsonDict) -> list[str]: + return [e["event_id"] for e in room["timeline"]["events"]] + + def _state_after_ids(self, room: JsonDict) -> list[str]: + return [e["event_id"] for e in room["org.matrix.msc4222.state_after"]["events"]] + + def _persist_batch(self) -> tuple[EventBase, EventBase]: + return self.get_success( + persist_message_and_state_event_in_one_batch( + self.hs, self.room_id, self.alice + ) + ) + + def _assert_state_after_is_current_state(self, room: JsonDict) -> None: + """For a joined room `end_token` is the global `now_token`, so every + entry in `state_after` must be the room's current state for that key. + + This is the guard against "fix" shapes that seed `state_after` straight + from the timeline: a state event can be in the timeline and *not* be the + state at the end of it. + """ + current_state = self.get_success( + self.store.get_partial_current_state_ids(self.room_id) + ) + current_ids = set(current_state.values()) + for event in room["org.matrix.msc4222.state_after"]["events"]: + self.assertIn( + event["event_id"], + current_ids, + f"state_after reports {event['type']}/{event['state_key']} = " + f"{event['event_id']}, which is not the current state " + f"({current_state.get((event['type'], event['state_key']))})", + ) + + # ------------------------------------------------------------------ + # The persist-batch window + # ------------------------------------------------------------------ + + def test_state_after_with_token_inside_persist_batch(self) -> None: + """The end-to-end symptom of the batch-minimum stamp: if a client's + `since` token lands strictly inside a persist batch (which a *reader* + worker can + observe, because it advances its events-stream position from replication + RDATA batches that may split a persist batch), the state event is in the + timeline of the next sync and must also be in `state_after`. + """ + sync_url = self._sync_url(lazy_load_members=False) + base = self._sync(sync_url)["next_batch"] + base_token = self.get_success(StreamToken.from_string(self.store, base)) + + message, state_event = self._persist_batch() + message_pos = message.internal_metadata.stream_ordering + assert message_pos is not None + + # A token positioned just after the message but before the state event + # -- i.e. in the middle of the persist batch. + split_token = base_token.copy_and_replace( + StreamKeyType.ROOM, RoomStreamToken(stream=message_pos) + ) + + deltas_up_to_split = self.get_success( + self.store.get_current_state_deltas_for_room( + self.room_id, + from_token=base_token.room_key, + to_token=RoomStreamToken(stream=message_pos), + ) + ) + self.assertIn( + state_event.event_id, + [d.event_id for d in deltas_up_to_split], + "expected the delta to be visible *before* its event", + ) + + split_since = self.get_success(split_token.to_string(self.store)) + room = self._joined_room(self._sync(sync_url, split_since)) + self.assertIn(state_event.event_id, self._timeline_ids(room)) + self.assertIn( + state_event.event_id, + self._state_after_ids(room), + f"state event in timeline but missing from state_after when the " + f"since token splits a persist batch: {room}", + ) + + # ------------------------------------------------------------------ + # Guards: `state_after` must be the *resolved* state, not a replay of + # the timeline + # ------------------------------------------------------------------ + + def test_state_res_loser_alone_in_timeline(self) -> None: + """A state event that appears in the timeline but *loses* state + resolution must not be reported in `state_after` -- `state_after` is + the resolved state at the end of the timeline, not a replay of the + timeline. + + Here the client has already synced past the first of two conflicting + events, so the second one arrives alone in the timeline with no delta of + its own. + """ + sync_url = self._sync_url(lazy_load_members=False) + + fork_point = self.get_success(self.store.get_prev_events_for_room(self.room_id)) + + # Pin the state resolution outcome: conflicted events at the same + # mainline position are applied in `(origin_server_ts, event_id)` + # order with the last application winning, so giving the first event + # the later timestamp makes it deterministically beat the second one. + now = self.clock.time_msec() + + first = self.get_success( + inject_event( + self.hs, + room_id=self.room_id, + type="m.call.member", + state_key=self.alice, + sender=self.alice, + content={"memberships": [{"device_id": "FIRST"}]}, + prev_event_ids=fork_point, + origin_server_ts=now + 1000, + ) + ) + + # The client syncs past the first event. + since = self._sync(sync_url)["next_batch"] + + # A conflicting event forked off the same point arrives afterwards. + second = self.get_success( + inject_event( + self.hs, + room_id=self.room_id, + type="m.call.member", + state_key=self.alice, + sender=self.alice, + content={"memberships": [{"device_id": "SECOND"}]}, + prev_event_ids=fork_point, + origin_server_ts=now, + ) + ) + + current_state = self.get_success( + self.store.get_partial_current_state_ids(self.room_id) + ) + self.assertEqual( + current_state[("m.call.member", self.alice)], + first.event_id, + "test setup: expected the first event to win state resolution", + ) + + room = self._joined_room(self._sync(sync_url, since)) + self.assertIn(second.event_id, self._timeline_ids(room)) + self._assert_state_after_is_current_state(room) + self.assertNotIn( + second.event_id, + self._state_after_ids(room), + "a state-res loser leaked into state_after", + ) + + def test_state_after_is_current_state_with_split_token(self) -> None: + """Whatever mechanism repairs the persist-batch window must not report + anything other than the state at the end of the timeline.""" + sync_url = self._sync_url(lazy_load_members=False) + base = self._sync(sync_url)["next_batch"] + base_token = self.get_success(StreamToken.from_string(self.store, base)) + + message, state_event = self._persist_batch() + message_pos = message.internal_metadata.stream_ordering + assert message_pos is not None + + split_token = base_token.copy_and_replace( + StreamKeyType.ROOM, RoomStreamToken(stream=message_pos) + ) + since = self.get_success(split_token.to_string(self.store)) + + room = self._joined_room(self._sync(sync_url, since)) + self._assert_state_after_is_current_state(room) + self.assertIn(state_event.event_id, self._state_after_ids(room)) + + def test_gappy_timeline_with_split_token(self) -> None: + """The gappy variant of the persist-batch window: the `since` token + splits a persist batch *and* the timeline is truncated so that the + state event falls outside the window. The state event is then in + neither the timeline nor the deltas, but it is a state change since + `since`, so `state_after` must still report it -- with `limited: true` + the client relies entirely on `state_after` to bridge the gap. + """ + sync_url = self._sync_url(lazy_load_members=False, timeline_limit=3) + base = self._sync(sync_url)["next_batch"] + base_token = self.get_success(StreamToken.from_string(self.store, base)) + + message, state_event = self._persist_batch() + message_pos = message.internal_metadata.stream_ordering + assert message_pos is not None + + # Push the state event out of the timeline window: more fillers than + # the timeline limit of 3, so the window holds fillers only. + for i in range(10): + self.helper.send(self.room_id, body=f"filler {i}", tok=self.bob_tok) + + split_token = base_token.copy_and_replace( + StreamKeyType.ROOM, RoomStreamToken(stream=message_pos) + ) + since = self.get_success(split_token.to_string(self.store)) + + room = self._joined_room(self._sync(sync_url, since)) + self.assertTrue(room["timeline"].get("limited"), room) + self.assertNotIn(state_event.event_id, self._timeline_ids(room)) + self.assertIn( + state_event.event_id, + self._state_after_ids(room), + f"state event outside a truncated timeline is missing from " + f"state_after when the since token splits a persist batch: {room}", + ) + + class SyncStateAfterArchivedRoomTestCase(unittest.HomeserverTestCase): """Tests MSC4222 `state_after` behaviour for rooms the syncing user has left (i.e. rooms in the `leave` section of the sync response).""" diff --git a/tests/storage/test_state_deltas.py b/tests/storage/test_state_deltas.py new file mode 100644 index 0000000000..831096b6c0 --- /dev/null +++ b/tests/storage/test_state_deltas.py @@ -0,0 +1,337 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import random +from typing import Any + +from immutabledict import immutabledict + +from twisted.test.proto_helpers import MemoryReactor + +import synapse.rest.admin +from synapse.api.constants import EventTypes +from synapse.events import EventBase +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import RoomStreamToken +from synapse.util.clock import Clock + +from tests import unittest +from tests.test_utils.event_injection import ( + create_event, + persist_message_and_state_event_in_one_batch, +) + + +class StateDeltasByEventPositionTestCase(unittest.HomeserverTestCase): + """Tests for `get_current_state_deltas_for_room_by_event_position`, which + bounds each delta on the position of its state event rather than on the + delta row's own `stream_id` (rows are stamped with the minimum stream + ordering of their persist batch, so the two can differ).""" + + servlets = [ + synapse.rest.admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + self.alice = self.register_user("alice", "password") + self.alice_tok = self.login("alice", "password") + self.room_id = self.helper.create_room_as(self.alice, tok=self.alice_tok) + + def _persist_batch(self) -> tuple[EventBase, EventBase]: + return self.get_success( + persist_message_and_state_event_in_one_batch( + self.hs, self.room_id, self.alice + ) + ) + + def _batch_positions(self) -> tuple[str, int, int]: + """Persist a batch and return (state event id, batch minimum position, + state event position), sanity-checking the batch shape.""" + message, state_event = self._persist_batch() + message_pos = message.internal_metadata.stream_ordering + state_pos = state_event.internal_metadata.stream_ordering + assert message_pos is not None and state_pos is not None + self.assertLess(message_pos, state_pos) + return state_event.event_id, message_pos, state_pos + + def test_delta_row_is_stamped_at_batch_minimum(self) -> None: + """Documents the precondition: the `current_state_delta_stream` row of + a state event persisted in a batch is stamped with the batch's + *minimum* stream ordering (see `_update_current_state_txn`), not the + state event's own, so a window bounded on the stamp and a window + bounded on the event disagree about which side of a mid-batch token + the delta falls on.""" + state_event_id, message_pos, _state_pos = self._batch_positions() + + rows = self.get_success( + self.store.db_pool.simple_select_list( + table="current_state_delta_stream", + keyvalues={"room_id": self.room_id, "type": "m.call.member"}, + retcols=("stream_id", "event_id"), + desc="test_delta_row_is_stamped_at_batch_minimum", + ) + ) + self.assertEqual(rows, [(message_pos, state_event_id)]) + + def test_mid_batch_delta_is_in_window(self) -> None: + """A window whose lower bound splits a persist batch contains the + batch's state event delta, which the stamp-bounded query misses.""" + state_event_id, message_pos, state_pos = self._batch_positions() + + from_token = RoomStreamToken(stream=message_pos) + to_token = RoomStreamToken(stream=state_pos) + + # The stamp-bounded query misses the delta: the row is stamped at the + # batch minimum, below the window. + deltas = self.get_success( + self.store.get_current_state_deltas_for_room( + self.room_id, from_token=from_token, to_token=to_token + ) + ) + self.assertEqual([d.event_id for d in deltas], []) + + # The by-event-position query recovers it. + deltas = self.get_success( + self.store.get_current_state_deltas_for_room_by_event_position( + self.room_id, from_token=from_token, to_token=to_token + ) + ) + self.assertEqual([d.event_id for d in deltas], [state_event_id]) + + def test_delta_is_not_reported_before_its_event(self) -> None: + """A window ending at the batch minimum must not contain the state + event's delta: its effective position is the event's own, beyond the + window. (The stamp-bounded query reports it here, one window early.)""" + state_event_id, message_pos, _state_pos = self._batch_positions() + + to_token = RoomStreamToken(stream=message_pos) + + deltas = self.get_success( + self.store.get_current_state_deltas_for_room( + self.room_id, from_token=None, to_token=to_token + ) + ) + self.assertIn(state_event_id, [d.event_id for d in deltas]) + + deltas = self.get_success( + self.store.get_current_state_deltas_for_room_by_event_position( + self.room_id, from_token=None, to_token=to_token + ) + ) + self.assertNotIn(state_event_id, [d.event_id for d in deltas]) + + def test_no_lower_bound(self) -> None: + """With no lower bound the event-driven query is unnecessary and the + query returns everything up to the upper bound, batch rows included.""" + state_event_id, _message_pos, state_pos = self._batch_positions() + + deltas = self.get_success( + self.store.get_current_state_deltas_for_room_by_event_position( + self.room_id, + from_token=None, + to_token=RoomStreamToken(stream=state_pos), + ) + ) + # The room's creation state plus the batched state event. + self.assertIn(state_event_id, [d.event_id for d in deltas]) + + def test_no_upper_bound(self) -> None: + """With no upper bound a mid-batch lower bound still recovers the + batch's state event delta.""" + state_event_id, message_pos, _state_pos = self._batch_positions() + + deltas = self.get_success( + self.store.get_current_state_deltas_for_room_by_event_position( + self.room_id, + from_token=RoomStreamToken(stream=message_pos), + to_token=None, + ) + ) + self.assertEqual([d.event_id for d in deltas], [state_event_id]) + + def test_rows_without_an_event_keep_their_stamp(self) -> None: + """Rows with no event -- the clearance of the room's state when the + last local user leaves -- have no event position to bound on and are + returned at their own stamp, exactly as the stamp-bounded query + returns them.""" + before = self.store.get_room_max_token() + self.helper.leave(self.room_id, self.alice, tok=self.alice_tok) + after = self.store.get_room_max_token() + + deltas = self.get_success( + self.store.get_current_state_deltas_for_room_by_event_position( + self.room_id, from_token=before, to_token=after + ) + ) + + self.assertTrue(deltas) + self.assertEqual({d.event_id for d in deltas}, {None}) + self.assertIn( + (EventTypes.Create, ""), {(d.event_type, d.state_key) for d in deltas} + ) + self.assertCountEqual( + deltas, + self.get_success( + self.store.get_current_state_deltas_for_room( + self.room_id, from_token=before, to_token=after + ) + ), + ) + + def _persist_mixed_batch(self, kinds: list[bool], rng: random.Random) -> None: + """Persist one batch of events, a state event for each True in + `kinds` (over a handful of reused state keys) and a message for each + False, all forked off the same forward extremities.""" + persistence = self.hs.get_storage_controllers().persistence + assert persistence is not None + prev_event_ids = self.get_success( + self.store.get_prev_events_for_room(self.room_id) + ) + events = [] + for i, is_state in enumerate(kinds): + kwargs: dict[str, Any] + if is_state: + kwargs = { + "type": "m.call.member", + "state_key": f"k{rng.randrange(4)}", + "content": {"memberships": [{"device_id": f"d{rng.random()}"}]}, + } + else: + kwargs = { + "type": "m.room.message", + "content": {"msgtype": "m.text", "body": f"msg {i}"}, + } + events.append( + self.get_success( + create_event( + self.hs, + room_id=self.room_id, + sender=self.alice, + prev_event_ids=prev_event_ids, + **kwargs, + ) + ) + ) + self.get_success(persistence.persist_events(events)) + + def test_matches_brute_force_over_every_window(self) -> None: + """Cross-checks the query against a brute-force oracle. + + Persists a mix of batches (lone messages, lone state events, batches of + two to four events with state events at varying positions, state keys + reused across batches), then for every pair of positions in the room's + stream -- and for tokens whose position for the room's writer is ahead + of their minimum, which is what a multi-writer token looks like -- + checks that the deltas returned are exactly the rows whose effective + position (the maximum of the row's stamp and its event's stream + ordering) lies in the window, in stamp order. + """ + rng = random.Random(4222) + for _ in range(20): + size = rng.choice([1, 1, 2, 3, 4]) + kinds = [rng.random() < 0.6 for _ in range(size)] + if size == 1: + kinds = [rng.random() < 0.5] + self._persist_mixed_batch(kinds, rng) + + # The oracle: every row's effective position, straight from the tables. + rows = self.get_success( + self.store.db_pool.simple_select_list( + table="current_state_delta_stream", + keyvalues={"room_id": self.room_id}, + retcols=("stream_id", "event_id"), + desc="oracle_rows", + ) + ) + event_positions = dict( + self.get_success( + self.store.db_pool.simple_select_list( + table="events", + keyvalues={"room_id": self.room_id}, + retcols=("event_id", "stream_ordering"), + desc="oracle_events", + ) + ) + ) + effective = { + (stream_id, event_id): max( + stream_id, event_positions.get(event_id, stream_id) + ) + for stream_id, event_id in rows + } + self.assertGreater(len(rows), 10) + self.assertTrue( + any(effective[key] > key[0] for key in effective), + "test setup: expected some rows stamped before their event", + ) + + lowest = min(event_positions.values()) - 1 + highest = max(event_positions.values()) + positions: list[int | None] = [None, *range(lowest, highest + 1)] + + def check( + from_token: RoomStreamToken | None, + to_token: RoomStreamToken | None, + from_pos: int | None, + to_pos: int | None, + ) -> None: + expected = { + key + for key, pos in effective.items() + if (from_pos is None or from_pos < pos) + and (to_pos is None or pos <= to_pos) + } + deltas = self.get_success( + self.store.get_current_state_deltas_for_room_by_event_position( + self.room_id, from_token=from_token, to_token=to_token + ) + ) + self.assertEqual( + {(d.stream_id, d.event_id) for d in deltas}, + expected, + f"window ({from_token}, {to_token}]", + ) + stamps = [d.stream_id for d in deltas] + self.assertEqual(stamps, sorted(stamps), "deltas not in stamp order") + + for from_pos in positions: + for to_pos in positions: + if from_pos is not None and to_pos is not None and to_pos <= from_pos: + continue + check( + RoomStreamToken(stream=from_pos) if from_pos is not None else None, + RoomStreamToken(stream=to_pos) if to_pos is not None else None, + from_pos, + to_pos, + ) + # The same window as a multi-writer token: the room's writer + # ("master") sits at the position, the token's minimum behind. + if from_pos is not None and to_pos is not None and from_pos % 3 == 0: + check( + RoomStreamToken( + stream=from_pos - 2, + instance_map=immutabledict({"master": from_pos}), + ), + RoomStreamToken( + stream=to_pos - 2, + instance_map=immutabledict({"master": to_pos}), + ), + from_pos, + to_pos, + ) diff --git a/tests/test_utils/event_injection.py b/tests/test_utils/event_injection.py index 36f6baa123..c775b0c6d7 100644 --- a/tests/test_utils/event_injection.py +++ b/tests/test_utils/event_injection.py @@ -127,6 +127,53 @@ async def create_event( return event, context +async def persist_message_and_state_event_in_one_batch( + hs: synapse.server.HomeServer, + room_id: str, + sender: str, +) -> tuple[EventBase, EventBase]: + """Persist a message and a state event in a *single* persist batch (one + `_persist_events_and_state_updates` call), with the message first. + + The message's stream ordering is then the batch minimum, so the state + event's `current_state_delta_stream` row (which is stamped with the batch + minimum, see `_update_current_state_txn`) sits *before* the state event's + own position. That is the shape behind + https://github.com/element-hq/synapse/issues/18793. + + Both events are created off the same forward extremities, i.e. as + siblings, which is how two events sent concurrently end up in one batch. + + Returns: + The persisted (message, state event) pair. + """ + store = hs.get_datastores().main + persistence = hs.get_storage_controllers().persistence + assert persistence is not None + prev_event_ids = await store.get_prev_events_for_room(room_id) + + message, message_ctx = await create_event( + hs, + room_id=room_id, + type=EventTypes.Message, + sender=sender, + content={"msgtype": "m.text", "body": "batched message"}, + prev_event_ids=prev_event_ids, + ) + state_event, state_ctx = await create_event( + hs, + room_id=room_id, + type="m.call.member", + state_key=sender, + sender=sender, + content={"memberships": [{"device_id": "BATCHED"}]}, + prev_event_ids=prev_event_ids, + ) + + await persistence.persist_events([(message, message_ctx), (state_event, state_ctx)]) + return message, state_event + + async def mark_event_as_partial_state( hs: synapse.server.HomeServer, event_id: str, From 84ac9ffe37a1acaeb95a7379d746752549f15219 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Wed, 16 Sep 2026 11:08:36 +0200 Subject: [PATCH 22/41] Drop federation device list updates from non-compliant user IDs (#20115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This partially fixes the bug https://github.com/element-hq/synapse/issues/20116 Device list update EDUs from non-compliant (grandfathered historical) user IDs are currently accepted over federation, stored, and surfaced to clients in `/sync`'s `device_lists.changed` array. > For current room versions, servers must still accept events using such user IDs over federation; however they SHOULD NOT forward such user IDs to clients when referenced outside the context of an event. For example, device list updates from non-compliant user IDs would be dropped by the receiving server. > > -- [Matrix spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids), clarified in Matrix v1.14 by [matrix-spec#1506](https://github.com/matrix-org/matrix-spec/issues/1506) ### Problem Example A remote server sends an `m.device_list_update` EDU for `@héllo:remote.example` (non-ASCII localpart, outside the compliant U+0021–U+007E range). Synapse: - accepts and processes the update (resyncing the user's device list if needed) - stores it in the remote device list cache - forwards `@héllo:remote.example` to local clients via `device_lists.changed` in `/sync` (**the leak** — a non-compliant user ID referenced outside event context) --- ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20115.bugfix | 1 + synapse/handlers/device.py | 23 +++++++++ tests/federation/test_federation_devices.py | 55 +++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 changelog.d/20115.bugfix diff --git a/changelog.d/20115.bugfix b/changelog.d/20115.bugfix new file mode 100644 index 0000000000..4cb368a1b3 --- /dev/null +++ b/changelog.d/20115.bugfix @@ -0,0 +1 @@ +Drop incoming federation device list updates from non-compliant (grandfathered historical) user IDs, so that such user IDs are not forwarded to clients outside the context of an event, as per [the spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids). diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index ae61bc24bf..6826c36745 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -70,6 +70,7 @@ UserID, get_domain_from_id, get_verify_key_from_cross_signing_key, + is_compliant_user_id_localpart, ) from synapse.util import stringutils from synapse.util.async_helpers import Linearizer @@ -1538,6 +1539,28 @@ async def incoming_device_list_update( ) return + if not is_compliant_user_id_localpart(UserID.from_string(user_id).localpart): + # We SHOULD NOT forward non-compliant (grandfathered historical) + # user IDs to clients outside the context of an event, and the spec + # gives dropping their device list updates as the example. See + # https://spec.matrix.org/v1.14/appendices/#historical-user-ids + logger.warning( + "Dropping device list update edu for non-compliant user ID %r from %r", + user_id, + origin, + ) + + set_tag("error", True) + log_kv( + { + "message": "Got a device list update edu from a " + "non-compliant user ID, dropping it.", + "user_id": user_id, + "device_id": device_id, + } + ) + return + # Check if we are partially joining any rooms. If so we need to store # all device list updates so that we can handle them correctly once we # know who is in the room. diff --git a/tests/federation/test_federation_devices.py b/tests/federation/test_federation_devices.py index c935669ce2..1ea5e7d3cb 100644 --- a/tests/federation/test_federation_devices.py +++ b/tests/federation/test_federation_devices.py @@ -100,6 +100,61 @@ def query_user_devices( self.reactor.advance(30) self.assertEqual(self.resync_attempts, 2) + def test_non_compliant_user_id_update_dropped(self) -> None: + """Tests that device list updates from non-compliant (grandfathered + historical) user IDs are dropped, per + https://spec.matrix.org/v1.14/appendices/#historical-user-ids + """ + remote_user_id = "@héllo:test_remote" + remote_origin = "test_remote" + + # Track the number of attempts to resync the user's device list. + self.resync_attempts = 0 + + # When this function is called, increment the number of resync attempts (only if + # we're querying devices for the right user ID), then raise a + # NotRetryingDestination error to fail the resync gracefully. + def query_user_devices( + destination: str, user_id: str, timeout: int = 30000 + ) -> JsonDict: + if user_id == remote_user_id: + self.resync_attempts += 1 + + raise NotRetryingDestination(0, 0, destination) + + # Register the mock on the federation client. + federation_client = self.hs.get_federation_client() + federation_client.query_user_devices = Mock(side_effect=query_user_devices) # type: ignore[method-assign] + + # Share room with user. + self.store.get_rooms_for_user = AsyncMock(return_value=["!someroom:test"]) + + # Manually inject a fake device list update from the non-compliant user ID. + device_list_updater = self.hs.get_device_handler().device_list_updater + assert isinstance(device_list_updater, DeviceListUpdater) + self.get_success( + device_list_updater.incoming_device_list_update( + origin=remote_origin, + edu_content={ + "deleted": False, + "device_display_name": "Mobile", + "device_id": "QBUAZIFURK", + "prev_id": [5], + "stream_id": 6, + "user_id": remote_user_id, + }, + ) + ) + + # The update should have been dropped without triggering a resync. + self.assertEqual(self.resync_attempts, 0) + + # The user should not have been marked as needing a device list resync. + need_resync = self.get_success( + self.store.get_user_ids_requiring_device_list_resync() + ) + self.assertNotIn(remote_user_id, need_resync) + def test_cross_signing_keys_retry(self) -> None: """Tests that resyncing a device list correctly processes cross-signing keys from the remote server. From 82aff09c4770b6252129372316700963f94e4801 Mon Sep 17 00:00:00 2001 From: guillermo <60621778+guillemo12@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:55:18 +0200 Subject: [PATCH 23/41] Optimize thumbnail file handling and add tests for state events (#20100) --- changelog.d/20100.bugfix | 1 + synapse/media/thumbnailer.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20100.bugfix diff --git a/changelog.d/20100.bugfix b/changelog.d/20100.bugfix new file mode 100644 index 0000000000..1c73ada4da --- /dev/null +++ b/changelog.d/20100.bugfix @@ -0,0 +1 @@ +Improve server concurrency by opening local media thumbnails asynchronously. Contributed by @guillemo12. \ No newline at end of file diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index a75a420750..8e90aeb8de 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -31,6 +31,7 @@ from synapse.config.repository import THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP from synapse.http.server import respond_with_json from synapse.http.site import SynapseRequest +from synapse.logging.context import defer_to_thread from synapse.logging.opentracing import trace from synapse.media._base import ( FileInfo, @@ -533,10 +534,11 @@ async def select_or_generate_local_thumbnail( assert file_info.thumbnail is not None if for_federation: + f = await defer_to_thread(self.hs.get_reactor(), open, file_path, "rb") await respond_with_multipart_responder( self.hs.get_clock(), request, - FileResponder(self.hs, open(file_path, "rb")), + FileResponder(self.hs, f), file_info.thumbnail.type, file_info.thumbnail.length, None, From d553b05e45817248617c12015d4c88d71e61b3e8 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Thu, 17 Sep 2026 12:16:25 +0200 Subject: [PATCH 24/41] Return `allowed_room_ids` in the client `/hierarchy` response (#20154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of #18731 (Support Matrix 1.15). Since Matrix 1.15, the [spec](https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy) defines `allowed_room_ids` on the room summaries returned by `GET /_matrix/client/v1/rooms/{roomId}/hierarchy`, so that clients can tell whether a restricted room can be joined or only knocked at: > **`allowed_room_ids`** — `[Room ID]` — If the room is a [restricted room](https://spec.matrix.org/v1.19/client-server-api/#restricted-rooms), these are the room IDs which are specified by the join rules. Empty or omitted otherwise. Added in `v1.15` > > — [Matrix Spec](https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy) Synapse currently strips the field from the client `/hierarchy` response before returning it — a guard added in matrix-org/synapse#12175 (2022), correct at the time, when the spec defined the field for federation only and it was leaking into client responses. The other two surfaces that define the field (`/room_summary` and the federation `/hierarchy`) already return it. This PR removes the strip in `_RoomEntry.as_json` (and the now-unused `for_client` parameter), so client hierarchy entries include `allowed_room_ids` for restricted rooms, both local and received over federation. The first commit adds the failing test coverage, the second removes the strip. --- ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20154.bugfix | 1 + synapse/handlers/room_summary.py | 18 ++---- tests/handlers/test_room_summary.py | 85 ++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 changelog.d/20154.bugfix diff --git a/changelog.d/20154.bugfix b/changelog.d/20154.bugfix new file mode 100644 index 0000000000..50de48799f --- /dev/null +++ b/changelog.d/20154.bugfix @@ -0,0 +1 @@ +Return `allowed_room_ids` in the client [`GET /_matrix/client/v1/rooms/{roomId}/hierarchy`](https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy) response, as required since Matrix 1.15. diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index c79a94e930..64330de558 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -369,7 +369,7 @@ async def _get_room_hierarchy( # inaccessible to the requesting user. if room_entry: # Add the room (including the stripped m.space.child events). - rooms_result.append(room_entry.as_json(for_client=True)) + rooms_result.append(room_entry.as_json()) # If this room is not at the max-depth, check if there are any # children to process. @@ -889,7 +889,8 @@ async def get_room_summary( remote_room_hosts: list[str] | None = None, ) -> JsonDict: """ - Implementation of the room summary C-S API from MSC3266 + Implementation of the room summary C-S API, see + https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1room_summaryroomidoralias Args: requester: user id of the user making this request, will be None @@ -997,25 +998,14 @@ class _RoomEntry: # This may not include all children. children_state_events: Sequence[JsonDict] = () - def as_json(self, for_client: bool = False) -> JsonDict: + def as_json(self) -> JsonDict: """ Returns a JSON dictionary suitable for the room hierarchy endpoint. It returns the room summary including the stripped m.space.child events as a sub-key. - - Args: - for_client: If true, any server-server only fields are stripped from - the result. - """ result = dict(self.room) - - # Before returning to the client, remove the allowed_room_ids key, if it - # exists. - if for_client: - result.pop("allowed_room_ids", False) - result["children_state"] = self.children_state_events return result diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index 0f8de6e7b9..febfce5328 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -31,6 +31,7 @@ JoinRules, Membership, RestrictedJoinRuleTypes, + RoomEncryptionAlgorithms, RoomTypes, ) from synapse.api.errors import AuthError, NotFoundError, SynapseError @@ -184,9 +185,6 @@ def _assert_hierarchy( result_room_ids = [] result_children_ids = [] for result_room in result["rooms"]: - # Ensure federation results are not leaking over the client-server API. - self.assertNotIn("allowed_room_ids", result_room) - result_room_ids.append(result_room["room_id"]) result_children_ids.append( [ @@ -489,6 +487,80 @@ def test_filtering(self) -> None: ) self._assert_hierarchy(result, expected) + def test_summary_fields_restricted(self) -> None: + """ + The room entries returned to clients include the `room_version`, + `encryption` and `allowed_room_ids` fields, added to the client + `/hierarchy` API in Matrix 1.15. + + See https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy + """ + restricted_room = self._create_room_with_join_rule( + JoinRules.RESTRICTED, + room_version=RoomVersions.V8.identifier, + allow=[ + { + "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP, + "room_id": self.space, + "via": [self.hs.hostname], + } + ], + ) + self.helper.send_state( + restricted_room, + event_type=EventTypes.RoomEncryption, + body={"algorithm": RoomEncryptionAlgorithms.DEFAULT}, + tok=self.token, + ) + + result = self.get_success( + self.handler.get_room_hierarchy(create_requester(self.user), self.space) + ) + room_entries = {entry["room_id"]: entry for entry in result["rooms"]} + entry = room_entries[restricted_room] + self.assertEqual(entry["join_rule"], JoinRules.RESTRICTED) + self.assertEqual(entry["room_version"], RoomVersions.V8.identifier) + self.assertEqual(entry["encryption"], RoomEncryptionAlgorithms.DEFAULT) + self.assertEqual(entry["allowed_room_ids"], [self.space]) + + # Rooms without restricted join rules should not have the field at all. + self.assertNotIn("allowed_room_ids", room_entries[self.room]) + + def test_summary_fields_knock_restricted(self) -> None: + """ + Same as `test_summary_fields_restricted`, for a `knock_restricted` room. + + Joining such a room without an invite applies the restricted join rule, so + it also returns `allowed_room_ids`. + """ + knock_restricted_room = self._create_room_with_join_rule( + JoinRules.KNOCK_RESTRICTED, + room_version=RoomVersions.V10.identifier, + allow=[ + { + "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP, + "room_id": self.space, + "via": [self.hs.hostname], + } + ], + ) + self.helper.send_state( + knock_restricted_room, + event_type=EventTypes.RoomEncryption, + body={"algorithm": RoomEncryptionAlgorithms.DEFAULT}, + tok=self.token, + ) + + result = self.get_success( + self.handler.get_room_hierarchy(create_requester(self.user), self.space) + ) + room_entries = {entry["room_id"]: entry for entry in result["rooms"]} + entry = room_entries[knock_restricted_room] + self.assertEqual(entry["join_rule"], JoinRules.KNOCK_RESTRICTED) + self.assertEqual(entry["room_version"], RoomVersions.V10.identifier) + self.assertEqual(entry["encryption"], RoomEncryptionAlgorithms.DEFAULT) + self.assertEqual(entry["allowed_room_ids"], [self.space]) + def test_complex_space(self) -> None: """ Create a "complex" space to see how it handles things like loops and subspaces. @@ -960,6 +1032,13 @@ async def summarize_remote_room_hierarchy( ) self._assert_hierarchy(result, expected) + # `allowed_room_ids` returned over federation should be passed through + # to the client. + room_entries = {entry["room_id"]: entry for entry in result["rooms"]} + self.assertEqual( + room_entries[restricted_accessible_room]["allowed_room_ids"], [self.room] + ) + def test_fed_invited(self) -> None: """ A room which the user was invited to should be included in the response. From cf02d0d1d70fa26b415d30c792cbf1a3f44e90be Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 17 Sep 2026 10:09:17 -0500 Subject: [PATCH 25/41] Align comments in `.dockerignore` with `synapse-private` `pro` branch (#20224) See https://github.com/element-hq/synapse-private/blob/pro/.dockerignore Spawning from https://github.com/element-hq/synapse-private/issues/169 where because we now have 'Synapse Variants' builds for the Synapse Pro container images, we can revert our special `docker/` changes back to match this repo. In this case, the new comments here make some sense so let's keep it :fast_forward: --- .dockerignore | 3 ++- changelog.d/20224.misc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20224.misc diff --git a/.dockerignore b/.dockerignore index 0b51345cbd..7b09e18b43 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ # ignore everything by default * -# things to include +# Things to include !docker !synapse !rust @@ -12,6 +12,7 @@ !Cargo.toml !build_rust.py +# Things to re-ignore rust/target synapse/*.so diff --git a/changelog.d/20224.misc b/changelog.d/20224.misc new file mode 100644 index 0000000000..829f713cf6 --- /dev/null +++ b/changelog.d/20224.misc @@ -0,0 +1 @@ +Align comments in `.dockerignore` with what we have in the private Synapse Pro branch. From 1d9aa4823ad131764c15b6f500572997ba987ac4 Mon Sep 17 00:00:00 2001 From: Haris Date: Thu, 17 Sep 2026 19:34:02 +0200 Subject: [PATCH 26/41] docs: add docs about firewall configuration (#20225) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) Replaces #20198 to remove git issues --- changelog.d/20225.doc | 1 + docs/setup/security.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 changelog.d/20225.doc diff --git a/changelog.d/20225.doc b/changelog.d/20225.doc new file mode 100644 index 0000000000..cba91aef6f --- /dev/null +++ b/changelog.d/20225.doc @@ -0,0 +1 @@ +Added docs regarding firewall configuration. Contributed by @HarisDotParis. diff --git a/docs/setup/security.md b/docs/setup/security.md index 2c21b494e5..237476a1a6 100644 --- a/docs/setup/security.md +++ b/docs/setup/security.md @@ -39,3 +39,19 @@ mentioned in MXIDs hosted on that server. Following this advice ensures that even if an XSS is found in Synapse, the impact to other applications will be minimal. + +## Firewall configuration + +If your Matrix server is secured by a firewall, firewall rules for two different purposes are required. + +Client access to a Synapse server runs via the client port which is TCP port 443 by default. The client port has to be allowed inbound. + +Federation to exchange messages with other Matrix servers runs via TCP port 8448 by default or alternatively via regular HTTPS TCP port 443 (usually if run with [delegation](../delegate.md) and/or behind a [reverse proxy](../reverse_proxy.md)). So to allow federation in a firewall, make sure to allow the federation port of your Synapse server inbound. + +If you have outbound access limited by a firewall allowlist, make sure both TCP ports 443 **and** 8448 are allowed outbound. + +Direction | Port | Protocol | Description +--------- | ---------------- | -------- | ----------- +Inbound | 443 | TCP | Client access +Inbound | 8448 and/or 443 | TCP | Federation inbound, depending on the federation port used +Outbound | 8448 **and** 443 | TCP | Federation outbound, both ports are usually used From 7530874a1250d6ad975b39582a784c594d29a505 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Thu, 17 Sep 2026 13:32:51 -0500 Subject: [PATCH 27/41] Raise default room version to "12" (#20130) Requires * #19768 * #19782 * https://github.com/matrix-org/complement/pull/915 To meet the requirements for bumping Synapse to support [Matrix spec to 1.16](https://github.com/element-hq/synapse/issues/19414) the default room version should be incremented to "12". Other than the two separate Synapse PRs above(which are included here but marked as "[diverted]" and should be removed prior to review) there is only [one other real change](https://github.com/element-hq/synapse/commit/c005e96c665b5a098a9b95d10f33faf224564be9) to the code base itself to fix a `KeyError` during logging for a `/sync` test against unknown room versions. Everything else should be on the unit tests themselves. Standard unit test running applies, should be nothing special to test this outright. `poetry run trial -jN tests` and similar for Postgresql. Probably ok to review commit-by-commit I took the liberty of writing a [room creating helper](https://github.com/element-hq/synapse/commit/5ba81ad5d4a0a6787b7626bdb3293f633ea4097a) for the two test series that try and test the sharding of the `event_persister` workers. I'm not certain it stands up to scrutiny, but at least does not do any funny mocking when producing room v12 appropriate room IDs. I also took the liberty of writing an [assertion helper](https://github.com/element-hq/synapse/pull/20130/commits/e408cef241d1b9c6a88f069eb5832de053790f9b) for comparing lists of dicts for a select subset of keys/values. This is used to compare stripped state selections while waiting on https://github.com/element-hq/synapse/pull/19723 to be completed. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Paul Chobert --- changelog.d/20130.feature | 1 + .../configuration/config_documentation.md | 3 +- schema/synapse-config.schema.yaml | 6 +- synapse/config/server.py | 2 +- .../storage/databases/main/events_worker.py | 8 +- tests/federation/test_federation_client.py | 75 +++++++++------ .../test_federation_out_of_band_membership.py | 15 +-- tests/handlers/test_device.py | 36 ++++---- tests/handlers/test_federation_event.py | 22 ++--- tests/handlers/test_message.py | 9 -- tests/handlers/test_presence.py | 1 - tests/handlers/test_register.py | 5 +- tests/handlers/test_room_member.py | 50 +++++----- tests/handlers/test_sliding_sync.py | 38 ++------ tests/module_api/test_api.py | 28 +++++- tests/replication/_base.py | 73 +++++++++++++++ .../test_federation_sender_shard.py | 1 - .../test_sharded_event_persister.py | 36 ++------ tests/rest/admin/test_room.py | 14 ++- .../client/sliding_sync/test_rooms_invites.py | 88 ++++++++++++++---- .../sliding_sync/test_rooms_required_state.py | 2 +- tests/rest/client/test_rooms.py | 92 ++++++++----------- tests/rest/client/test_sync.py | 2 +- tests/rest/client/test_third_party_rules.py | 1 - tests/rest/client/test_upgrade_room.py | 14 ++- tests/storage/test_cleanup_extrems.py | 64 +++++++++---- tests/storage/test_sticky_events.py | 4 +- tests/storage/test_stream.py | 73 +++++++-------- tests/test_utils/event_builders.py | 14 +++ tests/unittest.py | 7 ++ 30 files changed, 468 insertions(+), 316 deletions(-) create mode 100644 changelog.d/20130.feature diff --git a/changelog.d/20130.feature b/changelog.d/20130.feature new file mode 100644 index 0000000000..82c8faa015 --- /dev/null +++ b/changelog.d/20130.feature @@ -0,0 +1 @@ +Raise default room version to "12". Contributed by @jason-famedly @famedly. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 74ac85256f..c3bc8ebcb2 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -375,8 +375,9 @@ For example, for room version 1, `default_room_version` should be set to "1". _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). _Changed in Synapse 1.157:_ the default version room version was increased from [10](https://spec.matrix.org/v1.12/rooms/v10/) to [11](https://spec.matrix.org/v1.12/rooms/v11/). +_Changed in Synapse 1.162:_ the default room version was increased from [11](https://spec.matrix.org/v1.16/rooms/v11/) to [12](https://spec.matrix.org/v1.16/rooms/v12/) -Defaults to `"11"`. +Defaults to `"12"`. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index b34bc374f3..ad5e87ef18 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -330,7 +330,11 @@ properties: _Changed in Synapse 1.157:_ the default version room version was increased from [10](https://spec.matrix.org/v1.12/rooms/v10/) to [11](https://spec.matrix.org/v1.12/rooms/v11/). - default: "11" + + _Changed in Synapse 1.162:_ the default room version was increased from + [11](https://spec.matrix.org/v1.16/rooms/v11/) to + [12](https://spec.matrix.org/v1.16/rooms/v12/) + default: "12" examples: - "8" gc_thresholds: diff --git a/synapse/config/server.py b/synapse/config/server.py index 9c5a42fc52..6c9345e5c2 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -176,7 +176,7 @@ def generate_ip_set( "fec0::/10", ] -DEFAULT_ROOM_VERSION = "11" +DEFAULT_ROOM_VERSION = "12" # Defaults for the presence state machine timers, in milliseconds. Overridden # by the corresponding options in the `presence` config section. diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 215c8e44e3..c5d0c35259 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1482,7 +1482,7 @@ async def _fetch_event_ids_and_get_outstanding_redactions( # if d["type"] != EventTypes.Member: raise InvalidEventError( - "Room %s for event %s is unknown" % (d["room_id"], event_id) + "Room %s for event %s is unknown" % (d.get("room_id"), event_id) ) # so, assuming this is an out-of-band-invite that arrived before @@ -1513,7 +1513,7 @@ async def _fetch_event_ids_and_get_outstanding_redactions( logger.warning( "Event %s in room %s has unknown room version %s", event_id, - d["room_id"], + d.get("room_id"), room_version_id, ) continue @@ -1523,7 +1523,7 @@ async def _fetch_event_ids_and_get_outstanding_redactions( "Event %s in room %s with version %s has wrong format: " "expected %s, was %s", event_id, - d["room_id"], + d.get("room_id"), room_version_id, room_version.event_format, format_version, @@ -1552,7 +1552,7 @@ async def _fetch_event_ids_and_get_outstanding_redactions( # it's difficult to see what to do here. Pretty much all bets are off # if Synapse cannot rely on the consistency of its database. raise DatabaseCorruptionError( - d["room_id"], event_id, original_ev.event_id + d.get("room_id"), event_id, original_ev.event_id ) event_map[event_id] = original_ev diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index 656665cb42..47a4df942d 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -18,14 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # - +import urllib from unittest import mock import twisted.web.client from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.api.room_versions import RoomVersions from synapse.events import EventBase from synapse.rest import admin from synapse.rest.client import login, room @@ -33,6 +32,7 @@ from synapse.util.clock import Clock from tests.test_utils import FakeResponse, event_injection +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import FederatingHomeserverTestCase @@ -58,28 +58,42 @@ def prepare( self.reactor.advance(1000000000) self.creator = f"@creator:{self.OTHER_SERVER_NAME}" - self.test_room_id = "!room_id" + self.room_version = self.hs.config.server.default_room_version def test_get_room_state(self) -> None: # mock up some events to use in the response. - # In real life, these would have things in `prev_events` and `auth_events`, but that's - # a bit annoying to mock up, and the code under test doesn't care, so we don't bother. + # In real life, these would have things in `prev_events` and `auth_events`, but + # that's a bit annoying to mock up, and the code under test doesn't care, so we + # don't bother. We do bother with the room_id though, since it has to be + # generated by the hash of the creation event. + create_event_source = { + "type": "m.room.create", + "state_key": "", + "sender": self.creator, + "content": {"room_version": self.room_version.identifier}, + "prev_events": [], + "auth_events": [], + "depth": 1, + "origin_server_ts": 500, + } + if not self.room_version.msc4291_room_ids_as_hashes: + # Do not bother saving this now, we will pull it out of the calculated event + # later. + create_event_source["room_id"] = f"!room:{self.OTHER_SERVER_NAME}" + create_event_dict = self.add_hashes_and_signatures_from_other_server( - { - "room_id": self.test_room_id, - "type": "m.room.create", - "state_key": "", - "sender": self.creator, - "content": {"creator": self.creator}, - "prev_events": [], - "auth_events": [], - "depth": 1, - "origin_server_ts": 500, - } + create_event_source, + self.room_version, + ) + create_event = make_test_pdu_event( + create_event_source, + self.room_version, ) + room_id = create_event.room_id + member_event_dict = self.add_hashes_and_signatures_from_other_server( { - "room_id": self.test_room_id, + "room_id": room_id, "type": "m.room.member", "sender": self.creator, "state_key": self.creator, @@ -88,11 +102,12 @@ def test_get_room_state(self) -> None: "auth_events": [], "depth": 2, "origin_server_ts": 600, - } + }, + self.room_version, ) pl_event_dict = self.add_hashes_and_signatures_from_other_server( { - "room_id": self.test_room_id, + "room_id": room_id, "type": "m.room.power_levels", "sender": self.creator, "state_key": "", @@ -101,7 +116,8 @@ def test_get_room_state(self) -> None: "auth_events": [], "depth": 3, "origin_server_ts": 700, - } + }, + self.room_version, ) # mock up the response, and have the agent return it @@ -125,16 +141,18 @@ def test_get_room_state(self) -> None: state_resp, auth_resp = self.get_success( self.hs.get_federation_client().get_room_state( "yet.another.server", - self.test_room_id, + room_id, "event_id", - RoomVersions.V9, + self.room_version, ) ) # check the right call got made to the agent self._mock_agent.request.assert_called_once_with( b"GET", - b"matrix-federation://yet.another.server/_matrix/federation/v1/state/%21room_id?event_id=event_id", + f"matrix-federation://yet.another.server/_matrix/federation/v1/state/{urllib.parse.quote(room_id, '')}?event_id=event_id".encode( + "ascii" + ), headers=mock.ANY, bodyProducer=None, ) @@ -158,7 +176,7 @@ def test_get_pdu_returns_nothing_when_event_does_not_exist(self) -> None: self.hs.get_federation_client().get_pdu( ["yet.another.server"], "event_should_not_exist", - RoomVersions.V9, + self.room_version, ) ) self.assertEqual(pulled_pdu_info, None) @@ -185,7 +203,7 @@ def test_get_pdu_event_from_cache_is_pristine(self) -> None: self.hs.get_federation_client().get_pdu( ["yet.another.server"], remote_pdu.event_id, - RoomVersions.V9, + self.room_version, ) ) assert pulled_pdu_info2 is not None @@ -204,7 +222,7 @@ def _get_pdu_once(self) -> EventBase: """ message_event_dict = self.add_hashes_and_signatures_from_other_server( { - "room_id": self.test_room_id, + "room_id": f"!bogus_made_up_room_id:{self.OTHER_SERVER_NAME}", "type": "m.room.message", "sender": self.creator, "state_key": "", @@ -213,7 +231,8 @@ def _get_pdu_once(self) -> EventBase: "auth_events": [], "origin_server_ts": 700, "depth": 10, - } + }, + self.room_version, ) # mock up the response, and have the agent return it @@ -233,7 +252,7 @@ def _get_pdu_once(self) -> EventBase: self.hs.get_federation_client().get_pdu( ["yet.another.server"], "event_id", - RoomVersions.V9, + self.room_version, ) ) assert pulled_pdu_info is not None diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index 85e221a64a..3adbf4e35b 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -32,7 +32,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes, Membership -from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.api.room_versions import RoomVersion from synapse.events import EventBase from synapse.events.utils import strip_event from synapse.federation.transport.client import SendJoinResponse @@ -188,13 +188,11 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: # Create a remote room room_creator_user_id = f"@remote-user:{self.OTHER_SERVER_NAME}" - remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" - room_version = RoomVersions.V11 + room_version = self.hs.config.server.default_room_version room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { - "room_id": remote_room_id, "sender": room_creator_user_id, "depth": 1, "origin_server_ts": 1, @@ -210,6 +208,8 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: room_version=room_version, ) + remote_room_id = room_create_event.room_id + creator_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { @@ -220,7 +220,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: "type": EventTypes.Member, "state_key": room_creator_user_id, "content": {"membership": Membership.JOIN}, - "auth_events": [room_create_event.event_id], + "auth_events": [], "prev_events": [room_create_event.event_id], } ), @@ -239,7 +239,6 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: "state_key": local_user1_id, "content": {"membership": Membership.INVITE}, "auth_events": [ - room_create_event.event_id, creator_membership_event.event_id, ], "prev_events": [creator_membership_event.event_id], @@ -300,7 +299,6 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: "state_key": local_user1_id, "content": {"membership": Membership.JOIN}, "auth_events": [ - room_create_event.event_id, user1_invite_membership_event.event_id, ], "prev_events": [user1_invite_membership_event.event_id], @@ -542,9 +540,6 @@ async def put_json( "state_key": local_user2_id, "content": {"membership": Membership.INVITE}, "auth_events": [ - remote_room_join_result.state_map[ - (EventTypes.Create, "") - ].event_id, remote_room_join_result.state_map[ ( EventTypes.Member, diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index ce1bd06b6d..abc6c01d4e 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -33,10 +33,9 @@ from synapse.api.constants import EventTypes, JoinRules, RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError -from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.federation.federation_client import SendJoinResult from synapse.federation.transport.client import ( StateRequestResponse, @@ -60,6 +59,7 @@ from synapse.util.task_scheduler import TaskScheduler from tests import unittest +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import override_config user1 = "@boris:aaa" @@ -671,8 +671,7 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: def _build_public_room(self) -> StateMap[EventBase]: """Build a public room DAG that has REMOTE1 in it""" - room_id = f"!room:{self.REMOTE1_SERVER_NAME}" - room_version = RoomVersions.V10 + room_version = self.hs.config.server.default_room_version events: list[EventBase] = [] @@ -680,13 +679,11 @@ def _build_public_room(self) -> StateMap[EventBase]: create_event_dict: JsonDict = { "auth_events": [], "content": { - "creator": self.REMOTE1_USER, "room_version": room_version.identifier, }, - "depth": 0, + "depth": 1, "origin_server_ts": 0, "prev_events": [], - "room_id": room_id, "sender": self.REMOTE1_USER, "state_key": "", "type": EventTypes.Create, @@ -699,16 +696,15 @@ def _build_public_room(self) -> StateMap[EventBase]: self.REMOTE1_SERVER_SIGNATURE_KEY, ) - create_event = make_event_from_dict(create_event_dict, room_version) + create_event = make_test_pdu_event(create_event_dict, room_version) + # This will always be the correct room_id + room_id = create_event.room_id events.append(create_event) - room_version = self.hs.config.server.default_room_version join_event_dict: JsonDict = { - "auth_events": [ - create_event.event_id, - ], + "auth_events": [], "content": {"membership": "join"}, - "depth": 1, + "depth": 2, "origin_server_ts": 100, "prev_events": [create_event.event_id], "sender": self.REMOTE1_USER, @@ -716,20 +712,21 @@ def _build_public_room(self) -> StateMap[EventBase]: "room_id": room_id, "type": EventTypes.Member, } + add_hashes_and_signatures( room_version, join_event_dict, self.hs.hostname, self.hs.signing_key, ) - join_event = make_event_from_dict(join_event_dict, room_version) + join_event = make_test_pdu_event(join_event_dict, room_version) events.append(join_event) # Then set the join rules to public join_rules_event_dict: JsonDict = { - "auth_events": [create_event.event_id, join_event.event_id], + "auth_events": [join_event.event_id], "content": {"join_rule": JoinRules.PUBLIC}, - "depth": 2, + "depth": 3, "origin_server_ts": 200, "prev_events": [join_event.event_id], "room_id": room_id, @@ -744,7 +741,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.REMOTE1_SERVER_NAME, self.REMOTE1_SERVER_SIGNATURE_KEY, ) - join_rules_event = make_event_from_dict(join_rules_event_dict, room_version) + join_rules_event = make_test_pdu_event(join_rules_event_dict, room_version) events.append(join_rules_event) return {(event.type, event.state_key): event for event in events} @@ -763,7 +760,6 @@ def _build_signed_join_event( room_version = self.hs.config.server.default_room_version join_event_dict: JsonDict = { "auth_events": [ - state[(EventTypes.Create, "")].event_id, state[(EventTypes.JoinRules, "")].event_id, ], "content": {"membership": "join"}, @@ -781,7 +777,7 @@ def _build_signed_join_event( get_domain_from_id(user), signing_key, ) - return make_event_from_dict(join_event_dict, room_version) + return make_test_pdu_event(join_event_dict, room_version) @parameterized.expand([("not_pruned", False), ("pruned", True)]) @patch( @@ -795,7 +791,7 @@ def test_local_device_changes_sent_to_new_servers_on_un_partial_state( partial state period should be sent to remote servers that were NOT known at the time of the partial join. - We do this by creating a room with one remote server, partialling + We do this by creating a room with one remote server, partially joining it, then receiving a join event from a second remote server. The second remote server should receive a device list update EDU for any local device changes that happened during the partial state period. diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 9da9f52fb6..b423948096 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -120,7 +120,6 @@ def _test_process_pulled_event_with_missing_state( ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -416,7 +415,6 @@ def test_process_pulled_event_clears_backfill_attempts_after_being_successfully_ ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -517,7 +515,6 @@ def test_backfill_signature_failure_does_not_fetch_same_prev_event_later( ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -714,7 +711,6 @@ def test_backfill_process_previously_failed_pull_attempt_event_in_the_background ) auth_event_ids = [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], member_event.event_id, ] @@ -834,6 +830,12 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: ) room_version = self.get_success(main_store.get_room_version(room_id)) + # This test requires a user that is not the room creator. Infinite power levels + # from MSC4289 change the dynamics of the power level state resolution later to + # not fallback on `origin_server_ts` as described in more detail below. + ernie_user_id = self.register_user("ernie", "test") + ernie_tok = self.login("ernie", "test") + self.helper.join(room_id, user=ernie_user_id, tok=ernie_tok) # Add another local user to the room. This user is going to be kicked in a # rejected event. bert_user_id = self.register_user("bert", "test") @@ -842,13 +844,13 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # Allow the remote user to kick bert. # The remote user is going to send a rejected power levels event later on and we - # need state resolution to order it before another power levels event kermit is + # need state resolution to order it before another power levels event ernie is # going to send later on. Hence we give both users the same power level, so that # ties are broken by `origin_server_ts`. self.helper.send_state( room_id, "m.room.power_levels", - {"users": {kermit_user_id: 100, OTHER_USER: 100}}, + {"users": {ernie_user_id: 100, OTHER_USER: 100}}, tok=kermit_tok, ) @@ -887,7 +889,6 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: "sender": OTHER_USER, "prev_events": [other_member_event.event_id], "auth_events": [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], # The event will be rejected because of the duplicated auth # event. @@ -936,7 +937,6 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: "sender": OTHER_USER, "prev_events": [rejected_power_levels_event.event_id], "auth_events": [ - initial_state_map[("m.room.create", "")], rejected_power_levels_event.event_id, initial_state_map[("m.room.member", bert_user_id)], initial_state_map[("m.room.member", OTHER_USER)], @@ -1014,8 +1014,8 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: self.helper.send_state( room_id, "m.room.power_levels", - {"users": {kermit_user_id: 100, OTHER_USER: 100, bert_user_id: 1}}, - tok=kermit_tok, + {"users": {ernie_user_id: 100, OTHER_USER: 100, bert_user_id: 1}}, + tok=ernie_tok, )["event_id"] ) ) @@ -1050,7 +1050,6 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: "sender": OTHER_USER, "prev_events": [rejected_kick_event.event_id], "auth_events": [ - initial_state_map[("m.room.create", "")], initial_state_map[("m.room.power_levels", "")], initial_state_map[("m.room.member", OTHER_USER)], ], @@ -1078,7 +1077,6 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: missing_event.event_id, ], "auth_events": [ - initial_state_map[("m.room.create", "")], new_power_levels_event.event_id, initial_state_map[("m.room.member", OTHER_USER)], ], diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py index 6450a90444..1160a66938 100644 --- a/tests/handlers/test_message.py +++ b/tests/handlers/test_message.py @@ -234,11 +234,6 @@ def test_reject_event_with_empty_prev_events( ) def test_call_invite_event_creation_fails_in_public_room(self) -> None: - # get prev_events for room - prev_events = self.get_success( - self.store.get_prev_events_for_room(self.room_id) - ) - # the invite in a public room should fail self.get_failure( self.handler.create_event( @@ -248,8 +243,6 @@ def test_call_invite_event_creation_fails_in_public_room(self) -> None: "room_id": self.room_id, "sender": self.requester.user.to_string(), }, - prev_event_ids=prev_events, - auth_event_ids=prev_events, ), SynapseError, ) @@ -263,8 +256,6 @@ def test_call_invite_event_creation_fails_in_public_room(self) -> None: "room_id": self.private_room_id, "sender": self.requester.user.to_string(), }, - prev_event_ids=prev_events, - auth_event_ids=prev_events, ) ) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index a562050842..61cf78730f 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -2382,7 +2382,6 @@ def _add_new_user(self, room_id: str, user_id: str) -> None: "origin_server_ts": 1, "content": {"membership": Membership.JOIN}, "auth_events": [ - state_map[(EventTypes.Create, "")].event_id, state_map[(EventTypes.JoinRules, "")].event_id, ], "prev_events": list(forward_extremity_event_ids), diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 182ff7a8fc..46cd2ed83b 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -477,6 +477,7 @@ def test_auto_create_auto_join_room_preset_guest(self) -> None: "auto_join_rooms": ["#room:test"], "autocreate_auto_join_room_preset": "private_chat", "auto_join_mxid_localpart": "support", + "default_room_version": "11", } ) def test_auto_create_auto_join_room_preset_invalid_permissions(self) -> None: @@ -485,6 +486,8 @@ def test_auto_create_auto_join_room_preset_invalid_permissions(self) -> None: registration doesn't completely break if the inviter doesn't have proper permissions. """ + # This test is limited to room version 11, as after that the room creator will + # always have permission to invite users and make any other changes they wish. inviter = "@support:test" # Register an initial user to create the room and such (essentially this @@ -528,7 +531,7 @@ def test_auto_create_auto_join_room_preset_invalid_permissions(self) -> None: ) ) - # Register a second user, which won't be be in the room (or even have an invite) + # Register a second user, which won't be in the room (or even have an invite) # since the inviter no longer has the proper permissions. user_id = self.get_success(self.handler.register_user(localpart="bob")) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index a4a57b64d9..1a61dcd8b0 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -8,10 +8,9 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer -from synapse.types import UserID, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -43,10 +42,10 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.chris_token = self.login("chris", "pass") # Create a room on this homeserver. Note that this counts as a join: it - # contributes to the rate limter's count of actions + # contributes to the rate limiter's count of actions self.room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) - self.intially_unjoined_room_id = f"!example:{self.OTHER_SERVER_NAME}" + self.room_version = self.hs.config.server.default_room_version @override_config({"rc_joins_per_room": {"per_second": 0.1, "burst_count": 2}}) def test_local_user_local_joins_contribute_to_limit_and_are_limited(self) -> None: @@ -105,60 +104,55 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: # We also patch out a bunch of event checks on our end. All we're really # trying to check here is that remote joins will bump the rate limter when # they are persisted. - create_event_source = { + create_event_source: JsonDict = { "auth_events": [], "content": { - "creator": f"@creator:{self.OTHER_SERVER_NAME}", - "room_version": self.hs.config.server.default_room_version.identifier, + "room_version": self.room_version.identifier, }, - "depth": 0, + "depth": 1, "origin_server_ts": 0, "prev_events": [], - "room_id": self.intially_unjoined_room_id, "sender": f"@creator:{self.OTHER_SERVER_NAME}", "state_key": "", "type": EventTypes.Create, } - self.add_hashes_and_signatures_from_other_server( - create_event_source, - self.hs.config.server.default_room_version, - ) - create_event = make_event_from_dict( + + self.add_hashes_and_signatures_from_other_server(create_event_source) + create_event = make_test_pdu_event( create_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) + # Extract the room_id for use below + initially_unjoined_room_id = create_event.room_id + join_event_source = { - "auth_events": [create_event.event_id], + "auth_events": [], "content": {"membership": "join"}, - "depth": 1, + "depth": 2, "origin_server_ts": 100, "prev_events": [create_event.event_id], "sender": self.bob, "state_key": self.bob, - "room_id": self.intially_unjoined_room_id, + "room_id": initially_unjoined_room_id, "type": EventTypes.Member, } add_hashes_and_signatures( - self.hs.config.server.default_room_version, + self.room_version, join_event_source, self.hs.hostname, self.hs.signing_key, ) - join_event = make_event_from_dict( + join_event = make_test_pdu_event( join_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) mock_make_membership_event = AsyncMock( return_value=( self.OTHER_SERVER_NAME, join_event, - self.hs.config.server.default_room_version, + self.room_version, ) ) mock_send_join = AsyncMock( @@ -197,7 +191,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: self.handler.update_membership( requester=create_requester(self.bob), target=UserID.from_string(self.bob), - room_id=self.intially_unjoined_room_id, + room_id=initially_unjoined_room_id, action=Membership.JOIN, remote_room_hosts=[self.OTHER_SERVER_NAME], ) @@ -208,7 +202,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: self.handler.update_membership( requester=create_requester(self.chris), target=UserID.from_string(self.chris), - room_id=self.intially_unjoined_room_id, + room_id=initially_unjoined_room_id, action=Membership.JOIN, remote_room_hosts=[self.OTHER_SERVER_NAME], ), diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 2293988036..f91468db1e 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -19,7 +19,6 @@ # import logging from typing import AbstractSet -from unittest.mock import patch import attr from parameterized import parameterized, parameterized_class @@ -3018,21 +3017,6 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() - def _create_room(self, room_id: str, user_id: str, tok: str) -> None: - """ - Create a room with a specific room_id. We use this so that that we have a - consistent room_id across test runs that hashes to the same value and will be - sharded to a known worker in the tests. - """ - - # We control the room ID generation by patching out the - # `_generate_room_id` method - with patch( - "synapse.handlers.room.RoomCreationHandler._generate_room_id" - ) as mock: - mock.side_effect = lambda: room_id - self.helper.create_room_as(user_id, tok=tok) - def test_sharded_event_persisters(self) -> None: """ This test should catch bugs that would come from flawed stream position @@ -3070,19 +3054,15 @@ def test_sharded_event_persisters(self) -> None: {"worker_name": "worker3"}, ) - # Specially crafted room IDs that get persisted on different workers. - # - # Sharded to worker1 - room_id1 = "!fooo:test" - # Sharded to worker2 - room_id2 = "!bar:test" - # Sharded to worker3 - room_id3 = "!quux:test" - - # Create rooms on the different workers. - self._create_room(room_id1, user2_id, user2_tok) - self._create_room(room_id2, user2_id, user2_tok) - self._create_room(room_id3, user2_id, user2_tok) + # 3 different rooms, each on a different worker. Match them up based on the + # trailing number. + room_results = self._generate_rooms_on_worker( + user2_id, user2_tok, ["worker1", "worker2", "worker3"] + ) + room_id1 = room_results["worker1"] + room_id2 = room_results["worker2"] + room_id3 = room_results["worker3"] + join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) join_response2 = self.helper.join(room_id2, user1_id, tok=user1_tok) # Leave room2 diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index b4b14c87b9..79b3d27d11 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -282,11 +282,11 @@ def test_sending_events_into_room(self) -> None: # Create and send a state event content = { "events_default": 0, - "users": {user_id: 100}, "state_default": 50, "users_default": 0, "events": {"test.event.type": 25}, } + event_dict = { "room_id": room_id, "type": "m.room.power_levels", @@ -743,6 +743,7 @@ def test_create_room(self) -> None: # Now do the happy path. user_id = self.register_user("user", "password") + access_token = self.login(user_id, "password") room_id, room_alias = self.get_success( self.module_api.create_room( @@ -750,6 +751,18 @@ def test_create_room(self) -> None: ) ) + # Check room creator. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", + access_token=access_token, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual( + channel.json_body["room_version"], + self.hs.config.server.default_room_version.identifier, + ) + # Check room alias. self.assertEqual(room_alias, f"#foo-bar:{self.module_api.server_name}") @@ -758,6 +771,17 @@ def test_create_room(self) -> None: self.module_api.create_room(user_id=user_id, config={}, ratelimit=False) ) + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", + access_token=access_token, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual( + channel.json_body["room_version"], + self.hs.config.server.default_room_version.identifier, + ) + # Check room alias. self.assertIsNone(room_alias) @@ -825,7 +849,7 @@ def test_event_deprecated_methods(self) -> None: self.assertEqual(create_event.user_id, user_id) # type: ignore[attr-defined] # The event supports looking up keys via `__getitem__` although deprecated - self.assertEqual(create_event["room_id"], room_id) # type: ignore[index] + self.assertEqual(create_event["type"], EventTypes.Create) # type: ignore[index] class ModuleApiWorkerTestCase(BaseModuleApiTestCase, BaseMultiWorkerStreamTestCase): diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 791a6beab5..20ed84eac0 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -39,6 +39,7 @@ from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.server import HomeServer from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.server import FakeTransport @@ -483,6 +484,78 @@ def connect_any_redis_attempts(self) -> None: ) server_protocol.makeConnection(server_to_client_transport) + def _generate_rooms_on_worker( + self, + user_id: str, + user_tok: str, + list_of_worker_names: list[str] | None = None, + try_at_most_count: int | None = None, + ) -> dict[str, str]: + """ + Given a list of worker names, generate rooms until there is at least one on each + of the named workers. + + Args: + user_id: The user_id of the user making the room. + user_tok: The token of the user making the room. + list_of_worker_names: A list of worker names that need to have rooms. By + default, each `events` worker provided in the homeserver config will be used. + try_at_most_count: A given number of iterations to try and produce rooms. If + not provided, use the number of workers multiplied by 3 for the count. + Returns: + A mapping of `worker_name`->`room_id` + """ + # Save a shorter reference to the `RoutableShardedWorkerHandlingConfig` that + # contains the information needed to not only identify the full list of workers + # (in case the default for `list_of_workernames` is used) and provides the + # routing hash function that decides which worker a given room id should go to. + events_writers_config = self.hs.config.worker.events_shard_config + + if list_of_worker_names is None: + _set_of_worker_names = set(events_writers_config.instances) + else: + _set_of_worker_names = set(list_of_worker_names) + + assert len(_set_of_worker_names) > 0 + # Save a copy of this to use now, we can use the original to assert expectations + # before returning. + set_of_workernames = set(_set_of_worker_names) + + results_mapping = {} + + # Maintain a count, in case of a runaway process. The number 3 has no magical + # significance other than at the time of writing it allowed all tests that used + # this function to pass. The count is on a one based index(counts down to 1 and + # doesn't go past), so this will allow for (3 * num_of_workers) attempts before + # giving up if `try_at_most_count` does not override. + count = try_at_most_count or len(set_of_workernames) * 3 + + while set_of_workernames: + # Rooms created at the same millisecond will have the same room_id for + # MSC4291 rooms. Bump the reactor by that much so a different room_id will + # be tried on the next iteration. + self.reactor.advance(Duration(milliseconds=1).as_secs()) + _room_id = self.helper.create_room_as(user_id, tok=user_tok) + + _worker_responsible = events_writers_config.get_instance(_room_id) + + if _worker_responsible in set_of_workernames: + results_mapping[_worker_responsible] = _room_id + # Remember to remove the worker now that it is found + set_of_workernames.remove(_worker_responsible) + + count -= 1 + if count == 1: + raise AssertionError( + "Count exhausted attempting to generate rooms. Aborting and failing test" + ) + + # Since this *IS* part of a test, lets make sure all worker names requested are + # accounted for + assert results_mapping.keys() == _set_of_worker_names + + return results_mapping + class TestReplicationDataHandler(ReplicationDataHandler): """Drop-in for ReplicationDataHandler which just collects RDATA rows""" diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 3896e0ce8a..929b1d028e 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -338,7 +338,6 @@ def create_room_with_remote_server( "origin_server_ts": 1, "content": {"membership": Membership.JOIN}, "auth_events": [ - state_map[(EventTypes.Create, "")].event_id, state_map[(EventTypes.JoinRules, "")].event_id, ], "prev_events": list(prev_event_ids), diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index f37394c1f5..1b9b9aa630 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -18,8 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import logging -from unittest.mock import patch from twisted.internet.testing import MemoryReactor @@ -32,8 +30,6 @@ from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import make_request -logger = logging.getLogger(__name__) - class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): """Checks event persisting sharding works""" @@ -63,17 +59,6 @@ def default_config(self) -> dict: } return conf - def _create_room(self, room_id: str, user_id: str, tok: str) -> None: - """Create a room with given room_id""" - - # We control the room ID generation by patching out the - # `_generate_room_id` method - with patch( - "synapse.handlers.room.RoomCreationHandler._generate_room_id" - ) as mock: - mock.side_effect = lambda: room_id - self.helper.create_room_as(user_id, tok=tok) - def test_basic(self) -> None: """Simple test to ensure that multiple rooms can be created and joined, and that different rooms get handled by different instances. @@ -145,10 +130,16 @@ def test_vector_clock_token(self) -> None: ) sync_hs_site = self._hs_to_site[sync_hs] - # Specially selected room IDs that get persisted on different workers. - room_id1 = "!foo:test" - room_id2 = "!baz:test" + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + store = self.hs.get_datastores().main + + room_generation_results = self._generate_rooms_on_worker(user_id, access_token) + room_id1 = room_generation_results["worker1"] + room_id2 = room_generation_results["worker2"] + # Verify that the expected worker is responsible for the specific room self.assertEqual( self.hs.config.worker.events_shard_config.get_instance(room_id1), "worker1" ) @@ -156,15 +147,6 @@ def test_vector_clock_token(self) -> None: self.hs.config.worker.events_shard_config.get_instance(room_id2), "worker2" ) - user_id = self.register_user("user", "pass") - access_token = self.login("user", "pass") - - store = self.hs.get_datastores().main - - # Create two room on the different workers. - self._create_room(room_id1, user_id, access_token) - self._create_room(room_id2, user_id, access_token) - # The other user joins self.helper.join( room=room_id1, user=self.other_user_id, tok=self.other_access_token diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 4deb3c29f4..b776929e21 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -3318,15 +3318,25 @@ def test_not_enough_power(self) -> None: self.creator, tok=self.creator_tok, is_public=True ) - # The creator drops admin rights in the room. + # In the case that this is a msc4289 compliant room, we can not use the room + # creator as the "puppeted" user. Join another user to use as a sentinel + # instead. The room's creator will leave the room before the command runs. + self.helper.join(room_id, self.second_user_id, tok=self.second_tok) + + # The creator gives a placeholder of rights in the room. The power level can be + # 0, as long as it is present it will be considered. Having it be missing will + # produce a different error: "No local admin user in room" which is not what + # this test is supposed to check pl = self.helper.get_state( room_id, EventTypes.PowerLevels, tok=self.creator_tok ) - pl["users"][self.creator] = 0 + pl["users"][self.second_user_id] = 0 self.helper.send_state( room_id, EventTypes.PowerLevels, body=pl, tok=self.creator_tok ) + self.helper.leave(room_id, self.creator, tok=self.creator_tok) + channel = self.make_request( "POST", f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin", diff --git a/tests/rest/client/sliding_sync/test_rooms_invites.py b/tests/rest/client/sliding_sync/test_rooms_invites.py index 85628833f5..09a6674f53 100644 --- a/tests/rest/client/sliding_sync/test_rooms_invites.py +++ b/tests/rest/client/sliding_sync/test_rooms_invites.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.types import UserID +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -66,6 +66,48 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.storage_controllers = hs.get_storage_controllers() super().prepare(reactor, clock, hs) + self.room_version = hs.config.server.default_room_version + + def assertHasSubset( + self, + container: list[JsonDict], + contains_these: list[JsonDict], + ) -> None: + """ + Fail the test if the count of elements in both `container` and `contains_these` + do not match, and if the individual elements in `container` is not a super set + of the individual elements of `contains_these`. + + Useful for comparison of lists of PDU dictionaries that may or may not have + stripped state, so the minimum required data is asserted to be present. + + (Note: maybe revert after https://github.com/element-hq/synapse/pull/19723 if + this is still necessary) + """ + assert contains_these, "`contains_these` was empty" + assert container, "`container` was empty" + + self.assertEqual( + len(container), + len(contains_these), + f"container must have the same number of elements as contains_these:\n\n{container=}\n\n{contains_these=}", + ) + + for dict_to_search_for in contains_these: + for container_entry in container: + # The <= operator is a subset comparison operator when used on + # 'set-like' containers + if dict_to_search_for.items() <= container_entry.items(): + # The searched for item was found, move on. Break will skip the + # else statement below + break + else: + # Searching the 'container' for this subset yielded nothing, that is + # an error. + raise AssertionError( + "The searched for dict was not present in the container:\n\n" + f"Searched for {dict_to_search_for}\n\nContainer: {container}" + ) def test_rooms_invite_shared_history_initial_sync(self) -> None: """ @@ -73,7 +115,7 @@ def test_rooms_invite_shared_history_initial_sync(self) -> None: initial sync. This is an `invite` room so we should only have `stripped_state` (no `timeline`) - but we also shouldn't see any timeline events because the history visiblity is + but we also shouldn't see any timeline events because the history visibility is `shared` and we haven't joined the room yet. """ user1_id = self.register_user("user1", "pass") @@ -85,7 +127,7 @@ def test_rooms_invite_shared_history_initial_sync(self) -> None: room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) # Ensure we're testing with a room with `shared` history visibility which means - # history visible until you actually join the room. + # history won't be visible until you actually join the room. history_visibility_response = self.helper.get_state( room_id1, EventTypes.RoomHistoryVisibility, tok=user2_tok ) @@ -138,12 +180,15 @@ def test_rooms_invite_shared_history_initial_sync(self) -> None: response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -167,7 +212,6 @@ def test_rooms_invite_shared_history_initial_sync(self) -> None: "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) def test_rooms_invite_shared_history_incremental_sync(self) -> None: @@ -248,12 +292,15 @@ def test_rooms_invite_shared_history_incremental_sync(self) -> None: response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -277,7 +324,6 @@ def test_rooms_invite_shared_history_incremental_sync(self) -> None: "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) def test_rooms_invite_world_readable_history_initial_sync(self) -> None: @@ -369,12 +415,15 @@ def test_rooms_invite_world_readable_history_initial_sync(self) -> None: response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -398,7 +447,6 @@ def test_rooms_invite_world_readable_history_initial_sync(self) -> None: "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) def test_rooms_invite_world_readable_history_incremental_sync(self) -> None: @@ -495,12 +543,15 @@ def test_rooms_invite_world_readable_history_incremental_sync(self) -> None: response_body["rooms"][room_id1], ) # We should have some `stripped_state` so the potential joiner can identify the - # room (we don't care about the order). - self.assertCountEqual( + # room (we don't care about the order). The exception is the creation event + # which is a full PDU format from room v12 and newer + self.assertHasSubset( response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"room_version": "11"}, + "content": { + "room_version": self.room_version.identifier, + }, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -524,5 +575,4 @@ def test_rooms_invite_world_readable_history_incremental_sync(self) -> None: "type": "m.room.member", }, ], - response_body["rooms"][room_id1]["invite_state"], ) diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index 901f22a35d..5aff19c7fd 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -1289,7 +1289,7 @@ def test_rooms_required_state_me(self) -> None: self.helper.send_state( room_id1, event_type=EventTypes.PowerLevels, - body={"users": {user1_id: 50, user2_id: 100}}, + body={"users": {user1_id: 50}}, tok=user2_tok, ) self.helper.send_state( diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index d0df8809c3..c5a01a9a7b 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -45,7 +45,6 @@ RoomTypes, ) from synapse.api.errors import Codes, HttpResponseException -from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext @@ -4804,6 +4803,9 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.creator = self.register_user("creator", "test") self.creator_tok = self.login("creator", "test") + self.good_admin = self.register_user("good_admin", "test") + self.good_admin_tok = self.login("good_admin", "test") + self.bad_user_id = self.register_user("bad", "test") self.bad_tok = self.login("bad", "test") @@ -4816,6 +4818,8 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.hs.config.experimental.msc4293_enabled = True + self.room_version = self.hs.config.server.default_room_version + def _check_redactions( self, original_events: list[EventBase], @@ -4923,16 +4927,13 @@ def test_banning_remote_member_with_flag_redacts_their_events(self) -> None: bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -4947,7 +4948,6 @@ def test_banning_remote_member_with_flag_redacts_their_events(self) -> None: self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -4966,7 +4966,7 @@ def test_banning_remote_member_with_flag_redacts_their_events(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5017,7 +5017,7 @@ def test_banning_remote_member_with_flag_redacts_their_events(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5042,16 +5042,13 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5066,7 +5063,6 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5085,7 +5081,7 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5126,16 +5122,13 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: # user should be able to join again channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V10, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5153,7 +5146,6 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: self._storage_controllers.state.get_current_state(self.room_id) ) auth_ids = [ - new_state[("m.room.create", "")].event_id, new_state[("m.room.power_levels", "")].event_id, new_state[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5174,7 +5166,7 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5196,13 +5188,16 @@ def test_unbanning_remote_user_stops_redaction_action(self) -> None: def test_redaction_flag_ignored_for_user_if_banner_lacks_redaction_power( self, ) -> None: - # change power levels so creator can ban but not redact + # change power levels so room admin can ban but not redact. Do not use the + # room's creator, as the power level they hold is not consistently definable + # after msc4289(creator has infinite power level). self.helper.send_state( self.room_id, "m.room.power_levels", - {"events_default": 0, "redact": 100, "users": {self.creator: 75}}, + {"events_default": 0, "redact": 100, "users": {self.good_admin: 75}}, tok=self.creator_tok, ) + self.helper.join(self.room_id, self.good_admin, tok=self.good_admin_tok) self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) # bad user sends some messages @@ -5224,11 +5219,11 @@ def test_redaction_flag_ignored_for_user_if_banner_lacks_redaction_power( } self.helper.change_membership( self.room_id, - self.creator, + self.good_admin, self.bad_user_id, "ban", content, - self.creator_tok, + self.good_admin_tok, ) filter = json.dumps({"types": [EventTypes.Message]}) @@ -5288,16 +5283,13 @@ def test_kicking_remote_member_with_flag_redacts_their_events(self) -> None: bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5312,7 +5304,6 @@ def test_kicking_remote_member_with_flag_redacts_their_events(self) -> None: self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5331,7 +5322,7 @@ def test_kicking_remote_member_with_flag_redacts_their_events(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5382,7 +5373,7 @@ def test_kicking_remote_member_with_flag_redacts_their_events(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5404,16 +5395,13 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5428,7 +5416,6 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: self.assertEqual(r[("m.room.member", bad_user)].membership, "join") auth_ids = [ - r[("m.room.create", "")].event_id, r[("m.room.power_levels", "")].event_id, r[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5447,7 +5434,7 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5483,16 +5470,13 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: # user re-joins after kick channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver={self.room_version.identifier}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body join_event_dict = join_result["event"] - self.add_hashes_and_signatures_from_other_server( - join_event_dict, - RoomVersions.V11, - ) + self.add_hashes_and_signatures_from_other_server(join_event_dict) channel = self.make_signed_federation_request( "PUT", f"/_matrix/federation/v2/send_join/{self.room_id}/x", @@ -5510,7 +5494,6 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: self._storage_controllers.state.get_current_state(self.room_id) ) auth_ids = [ - new_state[("m.room.create", "")].event_id, new_state[("m.room.power_levels", "")].event_id, new_state[("m.room.member", "@remote_bad_user:other.example.com")].event_id, ] @@ -5531,7 +5514,7 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: "prev_events": auth_ids, } ), - room_version=RoomVersions.V11, + room_version=self.room_version, ) self.get_success( @@ -5553,13 +5536,16 @@ def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: def test_redaction_flag_ignored_for_user_if_kicker_lacks_redaction_power( self, ) -> None: - # change power levels so creator can kick but not redact + # change power levels so room admin can kick but not redact. Do not use the + # room's creator, as the power level they hold is not consistently definable + # after msc4289(creator has infinite power level). self.helper.send_state( self.room_id, "m.room.power_levels", - {"events_default": 0, "redact": 100, "users": {self.creator: 75}}, + {"events_default": 0, "redact": 100, "users": {self.good_admin: 75}}, tok=self.creator_tok, ) + self.helper.join(self.room_id, self.good_admin, tok=self.good_admin_tok) self.helper.join(self.room_id, self.bad_user_id, tok=self.bad_tok) # bad user sends some messages @@ -5581,11 +5567,11 @@ def test_redaction_flag_ignored_for_user_if_kicker_lacks_redaction_power( } self.helper.change_membership( self.room_id, - self.creator, + self.good_admin, self.bad_user_id, "kick", content, - self.creator_tok, + self.good_admin_tok, ) filter = json.dumps({"types": [EventTypes.Message]}) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 2da5348492..6e281e5e73 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -502,7 +502,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.room_id, EventTypes.PowerLevels, { - "users": {self.user_id: 100, self.user2: 100}, + "users": {self.user2: 100}, "users_default": 0, "events": { "m.room.name": 50, diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index 5eaa6f9fb2..0b0fb6c4f7 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -576,7 +576,6 @@ def _update_power_levels(self, event_default: int = 0) -> None: "kick": 50, "redact": 50, "state_default": 50, - "users": {self.user_id: 100}, "users_default": 0, }, tok=self.tok, diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index 6cb85c94c4..cc4073cafc 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -134,7 +134,9 @@ def test_power_levels(self) -> None: "m.room.power_levels", tok=self.creator_token, ) - power_levels["users"][self.other] = 100 + # In the Matrix Spec v1.16, m.room.tombstone events are MUST explicitly be + # higher than `state_default` per MSC4289. + power_levels["users"][self.other] = 150 self.helper.send_state( self.room_id, "m.room.power_levels", @@ -160,7 +162,9 @@ def test_power_levels_user_default(self) -> None: "m.room.power_levels", tok=self.creator_token, ) - power_levels["users_default"] = 100 + # In the Matrix Spec v1.16, m.room.tombstone events are MUST explicitly be + # higher than `state_default` per MSC4289. + power_levels["users_default"] = 150 self.helper.send_state( self.room_id, "m.room.power_levels", @@ -220,8 +224,8 @@ def test_stringy_power_levels(self) -> None: tok=self.creator_token, ) - # Set creator's power level to the string "100" instead of the integer `100`. - power_levels["users"][self.creator] = "100" + # Set other's power level to the string "100" instead of the integer `100`. + power_levels["users"][self.other] = "100" # Synapse refuses to accept new stringy power level events. Bypass this by # neutering the validation. @@ -252,7 +256,7 @@ def test_stringy_power_levels(self) -> None: ) # We should now have an integer power level. - self.assertEqual(new_power_levels["users"][self.creator], 100, new_power_levels) + self.assertEqual(new_power_levels["users"][self.other], 100, new_power_levels) def test_events_field_missing(self) -> None: """Regression test for https://github.com/matrix-org/synapse/issues/16715.""" diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py index 054e4a4d0b..3437b6dd77 100644 --- a/tests/storage/test_cleanup_extrems.py +++ b/tests/storage/test_cleanup_extrems.py @@ -283,52 +283,76 @@ def prepare( self.user = UserID.from_string(self.register_user("user1", "password")) self.token1 = self.login("user1", "password") self.requester = create_requester(self.user) - self.room_id, _, _ = self.get_success( - self.room_creator.create_room(self.requester, {"visibility": "public"}) - ) self.event_creator = homeserver.get_event_creation_handler() homeserver.config.consent.user_consent_version = self.CONSENT_VERSION def test_send_dummy_event(self) -> None: - self._create_extremity_rich_graph() + room_id, _, _ = self.get_success( + self.room_creator.create_room(self.requester, {"visibility": "public"}) + ) + self._create_extremity_rich_graph(room_id, self.user) # Pump the reactor repeatedly so that the background updates have a # chance to run. self.pump(20) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0) def test_send_dummy_events_when_insufficient_power(self) -> None: - self._create_extremity_rich_graph() - # Criple power levels + # The Plan: + # create a new room with main user(creator). + # join with second user(user2). + # creator changes power levels of user2 so cannot send any events. + # create extremities graph. + # creators leaves - this should remove the user with potentially infinite power + # level. Side effect: this event will capture 10 extremities. + # move time(pump()), try and resolve with dummy events. Should fail. + # creator rejoins. Side effect: this will take tie up 10 more extremities. + # move time(pump()), try and resolve with dummy events again. Should succeed. + + room_id, _, _ = self.get_success( + self.room_creator.create_room(self.requester, {"visibility": "public"}) + ) + + user2 = self.register_user("user2", "password") + token2 = self.login("user2", "password") + self.helper.join(room_id, user2, tok=token2) + + # Cripple power levels for this new user self.helper.send_state( - self.room_id, + room_id, EventTypes.PowerLevels, - body={"users": {str(self.user): -1}}, + body={"users": {user2: -1}}, tok=self.token1, ) + + self._create_extremity_rich_graph(room_id, self.user) + + # The room creator leaves now. They are the only member of the room with a power + # level that can not be tweaked + self.helper.leave(room_id, self.user.to_string(), tok=self.token1) + # Pump the reactor repeatedly so that the background updates have a # chance to run. self.pump(10 * 60) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) # Check that the room has not been pruned self.assertTrue(len(latest_event_ids) > 10) - # New user with regular levels - user2 = self.register_user("user2", "password") - token2 = self.login("user2", "password") - self.helper.join(self.room_id, user2, tok=token2) + # Rejoin the room creator + self.helper.join(room_id, self.user.to_string(), tok=self.token1) + # Running the background updates again should clean up the extremities self.pump(10 * 60) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) @@ -373,18 +397,20 @@ def test_expiry_logic(self) -> None: 0, ) - def _create_extremity_rich_graph(self) -> None: + def _create_extremity_rich_graph( + self, room_id: str, user_for_sending: UserID + ) -> None: """Helper method to create bushy graph on demand""" - event_id_start = self.create_and_send_event(self.room_id, self.user) + event_id_start = self.create_and_send_event(room_id, user_for_sending) for _ in range(self.EXTREMITIES_COUNT): self.create_and_send_event( - self.room_id, self.user, prev_event_ids=[event_id_start] + room_id, user_for_sending, prev_event_ids=[event_id_start] ) latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) + self.store.get_latest_event_ids_in_room(room_id) ) self.assertEqual(len(latest_event_ids), 50) diff --git a/tests/storage/test_sticky_events.py b/tests/storage/test_sticky_events.py index 6335553f0a..1a61a6e066 100644 --- a/tests/storage/test_sticky_events.py +++ b/tests/storage/test_sticky_events.py @@ -485,7 +485,7 @@ def test_soft_failure_cleared_when_state_changes(self) -> None: self.helper.send_state( self.room_id, EventTypes.PowerLevels, - body={"users": {self.user_id: 100, user2_id: -1}, "events_default": 0}, + body={"users": {user2_id: -1}, "events_default": 0}, tok=self.token, ) @@ -517,7 +517,7 @@ def test_soft_failure_cleared_when_state_changes(self) -> None: self.helper.send_state( self.room_id, EventTypes.PowerLevels, - body={"users": {self.user_id: 100, user2_id: 0}, "events_default": 0}, + body={"users": {user2_id: 0}, "events_default": 0}, tok=self.token, ) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 4ece16e9a0..c5e880d7b2 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -35,7 +35,6 @@ ) from synapse.api.filtering import Filter from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -50,6 +49,7 @@ ) from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.test_utils.event_injection import create_event from tests.unittest import FederatingHomeserverTestCase, HomeserverTestCase @@ -690,7 +690,6 @@ def test_server_left_room_after_us(self) -> None: extra_content={ "power_level_content_override": { "users": { - user2_id: 100, # Allow user1 to send state in the room user1_id: 100, } @@ -861,7 +860,6 @@ def test_we_cause_server_left_room(self) -> None: extra_content={ "power_level_content_override": { "users": { - user2_id: 100, # Allow user1 to send state in the room user1_id: 100, } @@ -1350,6 +1348,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() self.room_member_handler = hs.get_room_member_handler() + self.room_version = self.hs.config.server.default_room_version def test_remote_join(self) -> None: """ @@ -1361,98 +1360,96 @@ def test_remote_join(self) -> None: before_join_token = self.event_sources.get_current_token() - intially_unjoined_room_id = f"!example:{self.OTHER_SERVER_NAME}" - # Remotely join a room on another homeserver. # # To do this we have to mock the responses from the remote homeserver. We also # patch out a bunch of event checks on our end. - create_event_source = { + creation_event_source = { "auth_events": [], "content": { - "creator": f"@creator:{self.OTHER_SERVER_NAME}", - "room_version": self.hs.config.server.default_room_version.identifier, + "room_version": self.room_version.identifier, }, - "depth": 0, + "depth": 1, "origin_server_ts": 0, "prev_events": [], - "room_id": intially_unjoined_room_id, "sender": f"@creator:{self.OTHER_SERVER_NAME}", "state_key": "", "type": EventTypes.Create, } + self.add_hashes_and_signatures_from_other_server( - create_event_source, - self.hs.config.server.default_room_version, + creation_event_source, + self.room_version, ) - create_event = make_event_from_dict( - create_event_source, - self.hs.config.server.default_room_version, - {}, - None, + # This handles FrozenEvent* creation for us, and will create the room_id if this + # is a msc4291 room + creation_event = make_test_pdu_event( + creation_event_source, + self.room_version, ) + + initially_unjoined_room_id = creation_event.room_id + creator_join_event_source = { - "auth_events": [create_event.event_id], + "auth_events": [], "content": { "membership": "join", }, - "depth": 1, + "depth": 2, "origin_server_ts": 1, "prev_events": [], - "room_id": intially_unjoined_room_id, + "room_id": initially_unjoined_room_id, "sender": f"@creator:{self.OTHER_SERVER_NAME}", "state_key": f"@creator:{self.OTHER_SERVER_NAME}", "type": EventTypes.Member, } + self.add_hashes_and_signatures_from_other_server( creator_join_event_source, - self.hs.config.server.default_room_version, + self.room_version, ) - creator_join_event = make_event_from_dict( + creator_join_event = make_test_pdu_event( creator_join_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) # Our local user is going to remote join the room join_event_source = { - "auth_events": [create_event.event_id], + "auth_events": [], "content": {"membership": "join"}, - "depth": 1, + "depth": 3, "origin_server_ts": 100, "prev_events": [creator_join_event.event_id], "sender": user1_id, "state_key": user1_id, - "room_id": intially_unjoined_room_id, + "room_id": initially_unjoined_room_id, "type": EventTypes.Member, } + add_hashes_and_signatures( - self.hs.config.server.default_room_version, + self.room_version, join_event_source, self.hs.hostname, self.hs.signing_key, ) - join_event = make_event_from_dict( + join_event = make_test_pdu_event( join_event_source, - self.hs.config.server.default_room_version, - {}, - None, + self.room_version, ) mock_make_membership_event = AsyncMock( return_value=( self.OTHER_SERVER_NAME, join_event, - self.hs.config.server.default_room_version, + self.room_version, ) ) mock_send_join = AsyncMock( return_value=SendJoinResult( join_event, self.OTHER_SERVER_NAME, - state=[create_event, creator_join_event], - auth_chain=[create_event, creator_join_event], + state=[creation_event, creator_join_event], + auth_chain=[creation_event, creator_join_event], partial_state=False, servers_in_room=frozenset(), state_dag=None, @@ -1483,7 +1480,7 @@ def test_remote_join(self) -> None: self.room_member_handler.update_membership( requester=create_requester(user1_id), target=UserID.from_string(user1_id), - room_id=intially_unjoined_room_id, + room_id=initially_unjoined_room_id, action=Membership.JOIN, remote_room_hosts=[self.OTHER_SERVER_NAME], ) @@ -1520,7 +1517,7 @@ def test_remote_join(self) -> None: membership_changes, [ CurrentStateDeltaMembership( - room_id=intially_unjoined_room_id, + room_id=initially_unjoined_room_id, event_id=join_event.event_id, event_pos=join_pos, membership="join", diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py index a5d686801d..f81468eb0b 100644 --- a/tests/test_utils/event_builders.py +++ b/tests/test_utils/event_builders.py @@ -112,6 +112,20 @@ def make_test_pdu_event( defaults as `make_test_event` before delegating. """ pdu = {**default_event_fields(room_version), **pdu} + + # For room versions where the create event's room_id is derived from its + # event ID (v11+ format), omit the default room_id on create events so each + # create event ends up with a distinct room_id. + # + # We can't do this in the `default_event_fields` as we don't know the event + # type at that point. + if ( + room_version.msc4291_room_ids_as_hashes + and pdu["type"] == "m.room.create" + and pdu["state_key"] == "" + ): + pdu.pop("room_id", None) + return event_from_pdu_json(pdu, room_version, received_time=received_time) diff --git a/tests/unittest.py b/tests/unittest.py index 4881a6650a..51d0514095 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -19,6 +19,7 @@ # [This file includes modifications made by New Vector Limited] # # +import copy import functools import gc import hashlib @@ -163,8 +164,14 @@ def deepcopy_config(config: _TConfig) -> _TConfig: if attr_name.startswith("__") or attr_name == "root": continue attr = getattr(config, attr_name) + new_attr: Any if isinstance(attr, Config): new_attr = deepcopy_config(attr) + elif isinstance(attr, (list, dict, set)): + # Copy mutable containers so that tests which modify config values + # in place (e.g. appending to a list) don't leak those changes into + # the cached config object, and thus into every other test. + new_attr = copy.deepcopy(attr) else: new_attr = attr From 88e5b6221deb4c9ac0c8100b9f83ec87fbf40879 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:32:27 +0100 Subject: [PATCH 28/41] Update stale minimum poetry required in contributing docs (#20219) --- changelog.d/20219.doc | 1 + docs/development/contributing_guide.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20219.doc diff --git a/changelog.d/20219.doc b/changelog.d/20219.doc new file mode 100644 index 0000000000..7bfa49e480 --- /dev/null +++ b/changelog.d/20219.doc @@ -0,0 +1 @@ +Replace the stale minimum `poetry` version in the contributing docs with a pointer to the minimum defined in `pyproject.toml`. \ No newline at end of file diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index f967dda6f2..c3f6a236c2 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -72,7 +72,7 @@ pipx install poetry but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) for other installation methods. -Developing Synapse requires Poetry version 1.3.2 or later. +The minimum poetry version is specified by the `requires-poetry` line in [pyproject.toml](https://github.com/element-hq/synapse/blob/develop/pyproject.toml). Next, open a terminal and install dependencies as follows: From 15624be2799e1cb83e4b600b658cd1ea9a29bab4 Mon Sep 17 00:00:00 2001 From: Paul Chobert Date: Fri, 18 Sep 2026 13:00:45 +0200 Subject: [PATCH 29/41] Profile endpoint rate limit (#20218) This provides configurable (via `rc_profile`) rate limits for profile endpoints: - `GET /profile/{username}` - `GET /profile/{username}/{keyName}` including (`displayname` & `avatar_url`) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Erik Johnston --- changelog.d/20218.feature | 1 + demo/start.sh | 3 + .../conf/workers-shared-extra.yaml.j2 | 4 ++ .../configuration/config_documentation.md | 21 +++++++ schema/synapse-config.schema.yaml | 14 +++++ synapse/api/auth/__init__.py | 14 +++++ synapse/api/auth/base.py | 38 +++++++++++- synapse/config/ratelimiting.py | 6 ++ synapse/rest/client/profile.py | 42 +++++++++---- synapse/server.py | 8 +++ tests/api/test_auth.py | 35 +++++++++++ tests/rest/client/test_profile.py | 61 +++++++++++++++++++ 12 files changed, 235 insertions(+), 12 deletions(-) create mode 100644 changelog.d/20218.feature diff --git a/changelog.d/20218.feature b/changelog.d/20218.feature new file mode 100644 index 0000000000..f8e85a88f8 --- /dev/null +++ b/changelog.d/20218.feature @@ -0,0 +1 @@ +Add a rate limit on the client profile lookup endpoints, configurable via `rc_profile`. diff --git a/demo/start.sh b/demo/start.sh index 471e881f46..5e4e1c10b9 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -151,6 +151,9 @@ for port in 8080 8081 8082; do rc_user_directory: per_second: 1000 burst_count: 1000 + rc_profile: + per_second: 1000 + burst_count: 1000 RC ) echo "${ratelimiting}" >> "$port.config" diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 4dc4eb932b..251018e1d2 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -108,6 +108,10 @@ rc_user_directory: per_second: 9999 burst_count: 9999 +rc_profile: + per_second: 9999 + burst_count: 9999 + federation_rr_transactions_per_room_per_second: 9999 allow_device_name_lookup_over_federation: true diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index c3bc8ebcb2..566596dc8a 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2116,6 +2116,27 @@ rc_user_directory: burst_count: 200.0 ``` --- +### `rc_profile` + +*(object)* This option allows admins to ratelimit profile lookups by clients. + +Requests are limited per user when the request is authenticated, otherwise per client IP address. + +_Added in Synapse 1.162.0._ + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_profile: + per_second: 1.0 + burst_count: 500.0 +``` +--- ### `federation_rr_transactions_per_room_per_second` *(integer)* Sets outgoing federation transaction frequency for sending read-receipts, per-room. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index ad5e87ef18..4550ad3e06 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2391,6 +2391,20 @@ properties: default: per_second: 0.016 burst_count: 200.0 + rc_profile: + $ref: "#/$defs/rc" + description: >- + This option allows admins to ratelimit profile lookups by clients. + + + Requests are limited per user when the request is authenticated, + otherwise per client IP address. + + + _Added in Synapse 1.162.0._ + default: + per_second: 1.0 + burst_count: 500.0 federation_rr_transactions_per_room_per_second: type: integer description: >- diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index d8d3b31b9d..dd0f05b1d3 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -98,6 +98,20 @@ async def get_user_by_req( AuthError if access is denied for the user in the access token """ + async def get_optional_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester | None: + """Like `get_user_by_req`, except returns None when the request carries + no access token at all. A token that is present but invalid still + raises, as with `get_user_by_req`. + + For endpoints where authentication is optional. + """ + async def get_user_by_req_experimental_feature( self, request: SynapseRequest, diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 7b8214fabb..93f579c69a 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,6 +19,7 @@ # # import logging +from abc import ABC, abstractmethod from http import HTTPStatus from typing import TYPE_CHECKING @@ -49,7 +50,7 @@ logger = logging.getLogger(__name__) -class BaseAuth: +class BaseAuth(ABC): """Common base class for all auth implementations.""" def __init__(self, hs: "HomeServer"): @@ -187,6 +188,7 @@ async def validate_appservice_can_control_user_id( 403, "Application service has not registered this user (%s)" % user_id ) + @abstractmethod async def is_server_admin(self, requester: Requester) -> bool: """Check if the given user is a local server admin. @@ -240,6 +242,40 @@ async def check_can_change_room_list( return user_level >= send_level + @abstractmethod + async def get_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester: + """Get a registered user's ID. See `Auth.get_user_by_req`.""" + raise NotImplementedError() + + async def get_optional_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool = False, + allow_expired: bool = False, + allow_locked: bool = False, + ) -> Requester | None: + """Like `get_user_by_req`, except returns None when the request carries + no access token at all. A token that is present but invalid still + raises, as with `get_user_by_req`. + + For endpoints where authentication is optional. + """ + if not self.has_access_token(request): + return None + + return await self.get_user_by_req( + request, + allow_guest=allow_guest, + allow_expired=allow_expired, + allow_locked=allow_locked, + ) + @staticmethod def has_access_token(request: Request) -> bool: """Checks if the request has an access_token. diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 13c9c4dba0..e5f29b6dc1 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -258,3 +258,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "rc_user_directory", defaults={"per_second": 0.016, "burst_count": 200}, ) + + self.rc_profile = RatelimitSettings.parse( + config, + "rc_profile", + defaults={"per_second": 1, "burst_count": 500}, + ) diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 58a6c7c4e2..aadda870fd 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -36,7 +36,7 @@ ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, Requester, UserID from synapse.util.stringutils import is_namedspaced_grammar if TYPE_CHECKING: @@ -57,6 +57,32 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: return propagate +async def _auth_and_ratelimit_profile_lookup( + hs: "HomeServer", request: SynapseRequest +) -> Requester | None: + """Authenticate a profile lookup request and apply the `rc_profile` rate + limit to it. + + Authentication is optional unless `require_auth_for_profile_requests` is + set. The rate limit is applied per user if credentials were supplied, else + per client IP address. + + Returns: + The requester if the request was authenticated, else None. + """ + auth = hs.get_auth() + requester: Requester | None + if hs.config.server.require_auth_for_profile_requests: + requester = await auth.get_user_by_req(request) + else: + requester = await auth.get_optional_user_by_req(request, allow_guest=True) + + await hs.get_profile_lookup_ratelimiter().ratelimit( + requester, key=None if requester else request.getClientAddress().host + ) + return requester + + class ProfileRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)$", v1=True) CATEGORY = "Event sending requests" @@ -70,11 +96,8 @@ def __init__(self, hs: "HomeServer"): async def on_GET( self, request: SynapseRequest, user_id: str ) -> tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user + requester = await _auth_and_ratelimit_profile_lookup(self.hs, request) + requester_user = requester.user if requester else None if not UserID.is_valid(user_id): raise SynapseError( @@ -119,11 +142,8 @@ def __init__(self, hs: "HomeServer"): async def on_GET( self, request: SynapseRequest, user_id: str, field_name: str ) -> tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user + requester = await _auth_and_ratelimit_profile_lookup(self.hs, request) + requester_user = requester.user if requester else None if not UserID.is_valid(user_id): raise SynapseError( diff --git a/synapse/server.py b/synapse/server.py index 9422882167..9bea208974 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -722,6 +722,14 @@ def get_registration_ratelimiter(self) -> Ratelimiter: cfg=self.config.ratelimiting.rc_registration, ) + @cache_in_self + def get_profile_lookup_ratelimiter(self) -> Ratelimiter: + return Ratelimiter( + store=self.get_datastores().main, + clock=self.get_clock(), + cfg=self.config.ratelimiting.rc_profile, + ) + @cache_in_self def get_federation_client(self) -> FederationClient: return FederationClient(self) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index e7ee7eec91..5604f41437 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -106,6 +106,41 @@ def test_get_user_by_req_user_missing_token(self) -> None: self.assertEqual(f.code, 401) self.assertEqual(f.errcode, "M_MISSING_TOKEN") + def test_get_optional_user_by_req_valid_token(self) -> None: + user_info = TokenLookupResult( + user_id=self.test_user_id.to_string(), token_id=5, device_id="device" + ) + self.store.get_user_by_access_token = AsyncMock(return_value=user_info) + self.store.mark_access_token_as_used = AsyncMock(return_value=None) + self.store.get_user_locked_status = AsyncMock(return_value=False) + + request = Mock(args={}) + request.args[b"access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_optional_user_by_req(request)) + assert requester is not None + self.assertEqual(requester.user, self.test_user_id) + + def test_get_optional_user_by_req_bad_token(self) -> None: + """A token that is present but invalid is still rejected.""" + self.store.get_user_by_access_token = AsyncMock(return_value=None) + + request = Mock(args={}) + request.args[b"access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + f = self.get_failure( + self.auth.get_optional_user_by_req(request), InvalidClientTokenError + ).value + self.assertEqual(f.code, 401) + self.assertEqual(f.errcode, Codes.UNKNOWN_TOKEN) + + def test_get_optional_user_by_req_missing_token(self) -> None: + """A request without any token yields no requester rather than an error.""" + request = Mock(args={}) + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + requester = self.get_success(self.auth.get_optional_user_by_req(request)) + self.assertIsNone(requester) + def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( id="as_id", diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 30f58f1cc3..b3a719d851 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -1037,3 +1037,64 @@ def test_can_lookup_own_profile(self) -> None: access_token=self.requester_tok, ) self.assertEqual(channel.code, 200, channel.result) + + +class ProfileRatelimitTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + profile.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.owner = self.register_user("owner", "pass") + self.owner_tok = self.login("owner", "pass") + self.other = self.register_user("other", "pass", displayname="Bob") + self.other_tok = self.login("other", "pass") + + @unittest.override_config({"rc_profile": {"per_second": 0.1, "burst_count": 3}}) + def test_ratelimit_authenticated(self) -> None: + """Profile lookups from an authenticated user are rate limited per user, + with the limit shared across the profile endpoints. + """ + channel = self.make_request( + "GET", f"/profile/{self.other}", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", f"/profile/{self.other}/displayname", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", f"/profile/{self.other}/avatar_url", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "GET", f"/profile/{self.other}", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 429, channel.result) + + # Another user is not affected by the first user's limit. + channel = self.make_request( + "GET", f"/profile/{self.owner}", access_token=self.other_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + @unittest.override_config({"rc_profile": {"per_second": 0.1, "burst_count": 3}}) + def test_ratelimit_unauthenticated(self) -> None: + """Unauthenticated profile lookups are rate limited per IP address.""" + for _ in range(3): + channel = self.make_request("GET", f"/profile/{self.other}") + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request("GET", f"/profile/{self.other}") + self.assertEqual(channel.code, 429, channel.result) + + # An authenticated user is not affected by the per-IP limit. + channel = self.make_request( + "GET", f"/profile/{self.other}", access_token=self.owner_tok + ) + self.assertEqual(channel.code, 200, channel.result) From c595869fb70b38a69cd3243c85ce4f71572fcdad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 18 Sep 2026 12:11:09 +0100 Subject: [PATCH 30/41] Add cache to `get_partial_filtered_current_state_ids` (#20160) For state filters that ask for concrete types. This allows us to cache the common case of asking for a specific type/state key. I noticed a bunch of queries in the jaeger traces that could be cached. --- changelog.d/20160.misc | 1 + synapse/replication/tcp/client.py | 21 +++-- synapse/storage/_base.py | 2 + synapse/storage/controllers/state.py | 4 +- synapse/storage/databases/main/state.py | 105 +++++++++++++++++++++-- tests/storage/test_state.py | 108 ++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 14 deletions(-) create mode 100644 changelog.d/20160.misc diff --git a/changelog.d/20160.misc b/changelog.d/20160.misc new file mode 100644 index 0000000000..476608a767 --- /dev/null +++ b/changelog.d/20160.misc @@ -0,0 +1 @@ +Add a cache for looking up individual pieces of current room state. diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index c0896b83e7..8a60f8df6f 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -50,6 +50,8 @@ ) from synapse.replication.tcp.streams.events import ( EventsStream, + EventsStreamAllStateRow, + EventsStreamCurrentStateRow, EventsStreamEventRow, EventsStreamRow, ) @@ -191,6 +193,20 @@ async def on_rdata( # We shouldn't get multiple rows per token for events stream, so # we don't need to optimise this for multiple rows. for row in rows: + # If this is a server ACL event, clear the cache in the storage controller. + if row.type in ( + EventsStreamEventRow.TypeId, + EventsStreamCurrentStateRow.TypeId, + ): + if row.data.type == EventTypes.ServerACL: + self._state_storage_controller.get_server_acl_for_room.invalidate( + (row.data.room_id,) + ) + elif row.type == EventsStreamAllStateRow.TypeId: + self._state_storage_controller.get_server_acl_for_room.invalidate( + (row.data.room_id,) + ) + if row.type != EventsStreamEventRow.TypeId: # The row's data is an `EventsStreamCurrentStateRow`. # When we recompute the current state of a room based on forward @@ -238,11 +254,6 @@ async def on_rdata( row.data.event_id, row.data.room_id ) - # If this is a server ACL event, clear the cache in the storage controller. - if row.data.type == EventTypes.ServerACL: - self._state_storage_controller.get_server_acl_for_room.invalidate( - (row.data.room_id,) - ) elif stream_name == UnPartialStatedRoomStream.NAME: for row in rows: assert isinstance(row, UnPartialStatedRoomStreamRow) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1df7f70b71..bc13a54dd1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -137,6 +137,7 @@ def _invalidate_state_caches( # Purge other caches based on room state. self._attempt_to_invalidate_cache("get_room_summary", (room_id,)) self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,)) + self._attempt_to_invalidate_cache("_get_current_state_event_id", (room_id,)) self._attempt_to_invalidate_cache("get_room_type", (room_id,)) self._attempt_to_invalidate_cache("get_room_encryption", (room_id,)) self._attempt_to_invalidate_cache( @@ -154,6 +155,7 @@ def _invalidate_state_caches_all(self, room_id: str) -> None: room_id: Room where state changed """ self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,)) + self._attempt_to_invalidate_cache("_get_current_state_event_id", (room_id,)) self._attempt_to_invalidate_cache("get_users_in_room", (room_id,)) self._attempt_to_invalidate_cache("is_host_invited", None) self._attempt_to_invalidate_cache("is_host_joined", None) diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 4885268305..b2e1d61a33 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -580,8 +580,8 @@ async def get_current_state_ids( """Get the current state event ids for a room based on the current_state_events table. - If a state filter is given (that is not `StateFilter.all()`) the query - result is *not* cached. + If a wildcard state filter is given (that is not `StateFilter.all()`) + the query result is *not* cached. Args: room_id: The room to get the state IDs of. state_filter: The state diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 87523e6f18..3e42d46084 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -49,6 +49,7 @@ LoggingDatabaseConnection, LoggingTransaction, make_in_list_sql_clause, + make_tuple_in_list_sql_clause, ) from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.roommember import RoomMemberWorkerStore @@ -536,7 +537,84 @@ async def check_if_events_in_current_state( return frozenset(event_id for (event_id,) in rows) - # FIXME: how should this be cached? + @cached(max_entries=100000, tree=True) + async def _get_current_state_event_id( + self, room_id: str, event_type_and_state_key: tuple[str, str] + ) -> str | None: + """Get the event ID of the given piece of current state in the room. + + Returns None if there is no such event in the current state. + """ + return await self.db_pool.simple_select_one_onecol( + table="current_state_events", + keyvalues={ + "room_id": room_id, + "type": event_type_and_state_key[0], + "state_key": event_type_and_state_key[1], + }, + retcol="event_id", + allow_none=True, + desc="_get_current_state_event_id", + ) + + @cachedList( + cached_method_name="_get_current_state_event_id", + list_name="event_types_and_state_keys", + num_args=2, + ) + async def _get_current_state_event_ids( + self, room_id: str, event_types_and_state_keys: Collection[tuple[str, str]] + ) -> Mapping[tuple[str, str], str | None]: + """Bulk version of `_get_current_state_event_id`. + + Types/state keys that aren't in the room's current state map to None, so + that their absence gets cached too. + """ + if not event_types_and_state_keys: + return {} + + # Check if the room_id is in `get_partial_current_state_ids` cache, if + # so, we can use that to avoid a DB query. + room_state = self.get_partial_current_state_ids.cache.get_immediate( + room_id, None, update_metrics=False + ) + if room_state is not None: + return { + (intern_string(typ), intern_string(state_key)): room_state.get( + (typ, state_key) + ) + for typ, state_key in event_types_and_state_keys + } + + def _get_current_state_event_ids_txn( + txn: LoggingTransaction, + ) -> dict[tuple[str, str], str | None]: + results: dict[tuple[str, str], str | None] = { + (intern_string(typ), intern_string(state_key)): None + for typ, state_key in event_types_and_state_keys + } + + for batch in batch_iter(event_types_and_state_keys, 500): + clause, args = make_tuple_in_list_sql_clause( + self.database_engine, ("type", "state_key"), batch + ) + + sql = f""" + SELECT type, state_key, event_id FROM current_state_events + WHERE room_id = ? AND {clause} + """ + + txn.execute(sql, [room_id, *args]) + + for typ, state_key, event_id in txn: + results[(intern_string(typ), intern_string(state_key))] = event_id + + return results + + return await self.db_pool.runInteraction( + "_get_current_state_event_ids", _get_current_state_event_ids_txn + ) + @cancellable async def get_partial_filtered_current_state_ids( self, room_id: str, state_filter: StateFilter | None = None @@ -555,14 +633,27 @@ async def get_partial_filtered_current_state_ids( Returns: Map from type/state_key to event ID. """ - if state_filter is None: - state_filter = StateFilter.all() + # First we check if we can delegate to one of the cached functions. + if state_filter is None or state_filter.is_full(): + return await self.get_partial_current_state_ids(room_id) - where_clause, where_args = (state_filter).make_sql_filter_clause() + if not state_filter.has_wildcards(): + results = StateMapWrapper(state_filter=state_filter) - if not where_clause: - # We delegate to the cached version - return await self.get_partial_current_state_ids(room_id) + concrete_types = state_filter.concrete_types() + if not concrete_types: + # The filter matches nothing. + return results + + ids = await self._get_current_state_event_ids(room_id, concrete_types) + results.update( + (type_and_state_key, event_id) + for type_and_state_key, event_id in ids.items() + if event_id is not None + ) + return results + + where_clause, where_args = (state_filter).make_sql_filter_clause() def _get_filtered_current_state_ids_txn( txn: LoggingTransaction, diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index 6c3506f348..46019c2e7b 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -30,6 +30,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions from synapse.events import EventBase +from synapse.logging.context import LoggingContext from synapse.server import HomeServer from synapse.types import JsonDict, RoomID, StateMap, UserID from synapse.types.state import StateFilter @@ -644,6 +645,113 @@ def test_batched_state_group_storing(self) -> None: ) self.assertEqual(context.state_group_before_event, groups[0][0]) + def test_get_partial_filtered_current_state_ids_concrete(self) -> None: + """A filter with no wildcards is served from the per-key cache, and the + absence of a key is cached too.""" + room_id = self.room.to_string() + + create = self.inject_state_event( + self.room, self.u_alice, EventTypes.Create, "", {} + ) + name = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"} + ) + + state_filter = StateFilter.from_types( + [(EventTypes.Create, ""), (EventTypes.Name, ""), (EventTypes.Topic, "")] + ) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + + self.assertEqual( + dict(state), + { + (EventTypes.Create, ""): create.event_id, + (EventTypes.Name, ""): name.event_id, + }, + ) + + # The room has no topic, and that fact is cached, so asking again does + # not go back to the database. + sentinel = object() + cache = self.store._get_current_state_event_id.cache + self.assertIsNone( + cache.get_immediate((room_id, (EventTypes.Topic, "")), sentinel) + ) + self.assertEqual( + cache.get_immediate((room_id, (EventTypes.Name, "")), sentinel), + name.event_id, + ) + + def test_get_partial_filtered_current_state_ids_invalidation(self) -> None: + """Persisting a new state event invalidates the cached entries for the + room.""" + room_id = self.room.to_string() + + self.inject_state_event(self.room, self.u_alice, EventTypes.Create, "", {}) + + state_filter = StateFilter.from_types([(EventTypes.Name, "")]) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {}) + + name = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"} + ) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {(EventTypes.Name, ""): name.event_id}) + + name2 = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "renamed"} + ) + + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {(EventTypes.Name, ""): name2.event_id}) + + def test_get_partial_filtered_current_state_ids_uses_full_cache(self) -> None: + """Test that fetching a single state key from a room with a full cache + hits the full cache and does not go to the database.""" + + room_id = self.room.to_string() + + create = self.inject_state_event( + self.room, self.u_alice, EventTypes.Create, "", {} + ) + name = self.inject_state_event( + self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"} + ) + + # prime the full cache + state_filter = StateFilter.all() + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual( + dict(state), + { + (EventTypes.Create, ""): create.event_id, + (EventTypes.Name, ""): name.event_id, + }, + ) + + # now fetch a single key and check that it hits the full cache + with LoggingContext(name="test", server_name=self.hs.hostname) as ctx: + state_filter = StateFilter.from_types([(EventTypes.Name, "")]) + state = self.get_success( + self.store.get_partial_filtered_current_state_ids(room_id, state_filter) + ) + self.assertEqual(dict(state), {(EventTypes.Name, ""): name.event_id}) + self.assertEqual(ctx.get_resource_usage().db_txn_count, 0) + class CurrentStateDeltaStreamTestCase(HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: From 26e46786b32e9ff48142e963422d106557df8aff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 21 Sep 2026 10:00:53 +0100 Subject: [PATCH 31/41] Port `LogContext` to Rust (#19979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the logcontext classes to Rust, and gives tokio tasks a captured logcontext so that work running in (or spawned from) Rust is attributed to the request that caused it. 1. **Add characterization tests for logcontext error messages and the filter** — pins the exact `logcontext_error` message shapes, the abuse-detection code paths and `LoggingContextFilter`'s observable behaviour, *against the existing Python implementation* (this commit is green on its own). These are the behavioural contract the port has to satisfy. 2. **Port `ContextResourceUsage` to a Rust pyclass** — self-contained: the new `synapse_rust.logcontext` module, the class, its stub and the re-export. 3. **Move the logcontext storage and `LoggingContext` to Rust** — the core change; the commit message carries detailed design notes. Highlights: - The slot is typed `Option>`, with `None` representing the sentinel. `_Sentinel`/`SENTINEL_CONTEXT` stay pure Python (unchanged); thin wrappers on `current_context`/`set_current_context` convert at the boundary, and pyo3's extraction enforces the type (`TypeError` otherwise). - The accounting is native: one `getrusage(RUSAGE_THREAD)` read per switch via libc, inline `stop`/`start` bookkeeping for base `LoggingContext`s, Python dispatch only for subclasses (`BackgroundProcessLoggingContext`) so their overrides run. The thread id comes from `PyThread_get_thread_ident` (the exact `threading.get_ident()` value) without calling into Python. - The hot paths avoid per-operation allocation: names are `Py` (the per-log-record `str(context)`/`server_name` reads are INCREF-only), error branches materialise strings only when hit. 4. **Attribute Rust-spawned work to the caller's logcontext** — `create_deferred` captures the caller's context and scopes it onto the spawned task via a tokio task-local (`LogContextHandle`); `current_context()` gives the task-local read precedence, so `LoggingContextFilter`/`pyo3-log` resolve the right context on worker threads with no per-record stamping. `run_python_awaitable` restores the captured context (via a `with_logcontext` helper, the Rust `PreserveLoggingContext`) around Python called back from Rust, so e.g. `runInteraction` from the Rust `/versions` handler accounts its DB usage against the right request. Integration tests exercise both guarantees through real production code paths. Follow-up work on top of this (separate PR): porting `BackgroundProcessLoggingContext` natively and removing further `Py<_>` indirections. The fact that `BackgroundProcessLoggingContext` is a subclass is what forces some of the warts in this PR: e.g. having to use `Py` everywhere, etc. We don't try (yet) to make this pure Rust, instead we see this as simply maintaining the Python logcontext machinery when crossing, rather than trying to make a Rust equivalent that can be used by pure Rust dependencies. We probably do want to do that in future, as well as wire up e.g. CPU recording on Rust side, but that is unnecessary for now. --- Cargo.lock | 1 + changelog.d/19979.misc | 1 + docs/log_contexts.md | 59 +- rust/Cargo.toml | 1 + rust/src/deferred.rs | 86 +- rust/src/lib.rs | 2 + rust/src/logging/context.rs | 1136 +++++++++++++++++ rust/src/logging/mod.rs | 23 + synapse/logging/context.py | 487 +------ synapse/metrics/background_process_metrics.py | 4 +- synapse/synapse_rust/logcontext.pyi | 177 +++ tests/handlers/test_appservice.py | 1 + tests/storage/test_base.py | 2 +- tests/synapse_rust/test_logcontext.py | 293 +++++ tests/test_state.py | 1 + tests/util/test_logcontext.py | 227 ++++ 16 files changed, 2025 insertions(+), 476 deletions(-) create mode 100644 changelog.d/19979.misc create mode 100644 rust/src/logging/context.rs create mode 100644 rust/src/logging/mod.rs create mode 100644 synapse/synapse_rust/logcontext.pyi create mode 100644 tests/synapse_rust/test_logcontext.py diff --git a/Cargo.lock b/Cargo.lock index 13228a5542..111b78e7b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,6 +1417,7 @@ dependencies = [ "icu_segmenter", "itertools", "lazy_static", + "libc", "log", "mime", "once_cell", diff --git a/changelog.d/19979.misc b/changelog.d/19979.misc new file mode 100644 index 0000000000..3210329a56 --- /dev/null +++ b/changelog.d/19979.misc @@ -0,0 +1 @@ +Port the logcontext machinery (`LoggingContext`, `ContextResourceUsage` and the current-context storage) to Rust. diff --git a/docs/log_contexts.md b/docs/log_contexts.md index 76710e10e0..bb2dede66d 100644 --- a/docs/log_contexts.md +++ b/docs/log_contexts.md @@ -2,9 +2,15 @@ To help track the processing of individual requests, synapse uses a '`log context`' to track which request it is handling at any given -moment. This is done via a thread-local variable; a `logging.Filter` is -then used to fish the information back out of the thread-local variable -and add it to each log record. +moment. The "current" log context is stored in the Rust extension +(`synapse.synapse_rust.logcontext`), which resolves it from the current +tokio task (if we are running inside one) and otherwise from the current OS +thread; a `logging.Filter` is then used to fish the information back out and +add it to each log record. Storing it in Rust means a single source of truth is +visible from both Python (the reactor and its thread pool) and Rust (tokio +tasks), so log records emitted from either — including from Rust code polled on +a worker thread — are attributed to the right request. See +[the Rust side](#the-rust-side) below. Logcontexts are also used for CPU and database accounting, so that we can track which requests were responsible for high CPU use or database @@ -18,7 +24,7 @@ Asynchronous functions make the whole thing complicated, so this document descri how it all works, and how to write code which follows the rules. In this document, "awaitable" refers to any object which can be `await`ed. In the context of -Synapse, that normally means either a coroutine or a Twisted +Synapse, that normally means either a coroutine or a Twisted [`Deferred`](https://twistedmatrix.com/documents/current/api/twisted.internet.defer.Deferred.html). ## Logcontexts without asynchronous code @@ -268,7 +274,7 @@ async def main(): await d # Good: This will be logged against the "main" logcontext logger.debug("phew") -``` +``` **Solution 2:** We could also fix this by surrounding the call to `d.callback` with a `PreserveLoggingContext`, which will reset the logcontext to the sentinel before calling @@ -534,9 +540,9 @@ def reset_listener_queue(): So, both ends of the awaitable chain have now dropped their references, and the awaitable chain is now orphaned, and will be garbage-collected at -some point. Note that `await_something_interesting` is a coroutine, +some point. Note that `await_something_interesting` is a coroutine, which Python implements as a generator function. When Python -garbage-collects generator functions, it gives them a chance to +garbage-collects generator functions, it gives them a chance to clean up by making the `await` (or `yield`) raise a `GeneratorExit` exception. In our case, that means that the `__exit__` handler of `PreserveLoggingContext` will carefully restore the request context, but @@ -550,6 +556,30 @@ actually happen too much. Unfortunately, when it does happen, it will lead to leaked logcontexts which are incredibly hard to track down. +## The Rust side + +The "current" logcontext is stored in the Rust extension rather than in a +Python thread-local, so that it is visible from both Python and Rust. +`current_context()` and `set_current_context()` are imported from +`synapse.logging.context` as usual — the Rust storage is an implementation +detail that Python code does not need to care about. + +`set_current_context` only ever runs on a Python thread, i.e. the reactor or one +of its thread pools, where it does the `getrusage` CPU accounting. It is never +called from a tokio worker thread. Therefore, CPU accounting is not captured +when executing purely Rust work. + +What Rust code *does* need to be aware of is that when spawning a future onto +the tokio runtime, the current logcontext must be captured and carried along, so +that log records emitted while the future is polled (including any `log::` +records from dependencies, and any Python invoked back from Rust) are attributed +correctly. Don't use a bare `tokio::spawn`. Instead use +`LogContextHandle::capture(py)` plus `LogContextHandle::scope`, which capture +the caller's logcontext from the Python side and record it on the spawned task; +this is what `create_deferred` does. `current_context()` returns the task's +captured context first, so `LoggingContextFilter` — and therefore `pyo3-log` — +sees the right context on worker threads without any per-log-record work. + ## Debugging logcontext issues Debugging logcontext issues can be tricky as leaking or losing a logcontext will surface @@ -564,3 +594,18 @@ loggers: synapse.logging.context.debug: level: DEBUG ``` + +Note that some of these traces (`LoggingContext(...).__enter__`/`__exit__`) are +emitted from Rust via `pyo3-log`, which caches logger levels for performance. +Configuring the logger in the logging config (as above) works — the cache is +flushed whenever the config is (re)loaded, including on `SIGHUP` — but enabling +the logger *at runtime* (e.g. `setLevel(logging.DEBUG)` from the manhole) will +not surface the Rust-emitted traces until you also flush the cache: + +```python +import logging +from synapse.synapse_rust import reset_logging_config + +logging.getLogger("synapse.logging.context.debug").setLevel(logging.DEBUG) +reset_logging_config() +``` diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f6b12c4c1b..36af85aae1 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,6 +29,7 @@ bytes = "1.6.0" headers = "0.4.0" http = "1.1.0" lazy_static = "1.4.0" +libc = "0.2.174" log = "0.4.17" mime = "0.3.17" pyo3 = { version = "0.29.0", features = [ diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 3da5defb83..48ce283959 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -25,6 +25,7 @@ use pyo3::{ }; use tokio::sync::oneshot; +use crate::logging::context::with_logcontext; use crate::tokio_runtime::runtime; create_exception!( @@ -71,7 +72,13 @@ fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { /// Creates a twisted deferred from the given future, spawning the task on the /// tokio runtime. /// -/// Does not handle deferred cancellation or contextvars. +/// Does not handle contextvars. +/// +/// TODO: propagate deferred cancellation to the tokio task (via +/// `JoinHandle::abort`). Until then a cancelled request leaves its task +/// running, so the task can outlive the request's logcontext — +/// `run_python_awaitable` defends against the resulting finished-context case, +/// but the work itself is wasted. pub fn create_deferred<'py, F, O>( py: Python<'py>, reactor: &Bound<'py, PyAny>, @@ -85,9 +92,15 @@ where let deferred_callback = deferred.getattr("callback")?.unbind(); let deferred_errback = deferred.getattr("errback")?.unbind(); + // Capture the caller's logcontext here (on the reactor thread) and record + // it in the spawned task. Log records emitted while the future is polled + // (on the tokio threads) are then attributed to the context that was + // current when the caller invoked us. See `crate::logging::context`. + let logcontext = crate::logging::context::LogContextHandle::capture(py); + let rt = runtime(reactor)?; let handle = rt.handle()?; - let task = handle.spawn(fut); + let task = handle.spawn(logcontext.scope(fut)); // Unbind the reactor so that we can pass it to the task let reactor = reactor.clone().unbind(); @@ -148,7 +161,16 @@ where // Shared between the success and error callbacks (only one ever fires). let sender = Arc::new(Mutex::new(Some(tx))); - Python::attach(|py| -> PyResult<()> { + // Capture the logcontext of the calling tokio task (if any). We restore it + // on the reactor thread before driving the awaitable, so Python code + // invoked from Rust (e.g. `DatabasePool.runInteraction`) runs in the same + // logcontext that was current when Python originally called into Rust. Its + // logging and DB-metrics accounting are then attributed to the right + // request. If we are not inside a scoped task, this is `None` and the + // awaitable runs in the sentinel. + let logcontext = crate::logging::context::LogContextHandle::task_current(); + + Python::attach(move |py| -> PyResult<()> { // Create some deferred success/error callback functions that we will use to get // the result from Python to Rust. let success_sender = Arc::clone(&sender); @@ -216,21 +238,29 @@ where move |args, _kwargs| -> PyResult> { let py = args.py(); - // We fire-and-forget using `run_in_background`. Re-using - // `run_in_background` also makes sure the awaitable gets run with the - // current logcontext while following the logcontext rules. - // - // FIXME: Currently runs in the sentinel logcontext because we don't manage it here - let deferred = logging_context_module(py)?.call_method1( - intern!(py, "run_in_background"), - (awaitable_factory.bind(py),), - ); - - let deferred = deferred?; - deferred.call_method1( - intern!(py, "addCallbacks"), - (on_success.bind(py), on_error.bind(py)), - )?; + // Drive the awaitable in the captured logcontext. Restored here + // as we're on the reactor thread (the only thread where the + // context's `main_thread` check passes). + let context = match &logcontext { + Some(handle) => handle.logging_context().map(|ctx| ctx.clone_ref(py)), + // Called from outside any scoped task: the sentinel. + None => None, + }; + + // Kick off the awaitable, fire-and-forget, via + // `run_in_background`. It calls the factory in the current + // logcontext and follows the logcontext rules from there. In + // particular, it arranges for the reactor to be back at the + // sentinel when the awaitable later completes. + with_logcontext(py, context, || { + let deferred = run_in_background(py, awaitable_factory.bind(py))?; + deferred.call_method1( + intern!(py, "addCallbacks"), + (on_success.bind(py), on_error.bind(py)), + )?; + Ok(()) + })?; + Ok(py.None()) }, )?; @@ -270,6 +300,26 @@ fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { } } +/// A reference to `synapse.logging.context.run_in_background`. +static RUN_IN_BACKGROUND: OnceCell> = OnceCell::new(); + +/// Call `synapse.logging.context.run_in_background(f)`, which calls `f` in the +/// current logcontext and drives the awaitable it returns to completion, +/// following the logcontext rules. Returns the resulting `Deferred`. +fn run_in_background<'py>(py: Python<'py>, f: &Bound<'py, PyAny>) -> PyResult> { + let run_in_background = RUN_IN_BACKGROUND.get_or_try_init(|| { + logging_context_module(py)? + .getattr("run_in_background") + .map(Into::into) + })?; + + run_in_background + .call1(py, (f,))? + .extract(py) + .map_err(Into::into) +} + +/// A reference to `synapse.logging.context.make_deferred_yieldable`. static MAKE_DEFERRED_YIELDABLE: OnceCell> = OnceCell::new(); /// Given a deferred, make it follow the Synapse logcontext rules diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 28783afbba..f68b7c17d7 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -16,6 +16,7 @@ pub mod http; pub mod http_client; pub mod identifier; pub mod json; +pub mod logging; pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; @@ -70,6 +71,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; + logging::context::register_module(py, m)?; deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; diff --git a/rust/src/logging/context.rs b/rust/src/logging/context.rs new file mode 100644 index 0000000000..d360f57cf6 --- /dev/null +++ b/rust/src/logging/context.rs @@ -0,0 +1,1136 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust storage for the Synapse "current logcontext". +//! +//! The storage lives in Rust rather than in a Python `threading.local` because +//! a Python thread-local is invisible to the Rust tokio threads. +//! +//! There are two pieces of storage, and [`current_context`] combines them so +//! that it answers correctly from both Python and Rust: +//! +//! 1. [`THREAD_LOCAL_CONTEXT`], one value per OS thread. Used by the Python +//! reactor thread and any reactor-managed threadpool threads. +//! 2. [`TASK_LOCAL_CONTEXT`], one value per tokio task, set when the task is +//! spawned. It stays with the task as the task moves between worker threads +//! across `.await` points. +//! +//! Both hold an `Option>`, where `None` means "no context", +//! i.e. the sentinel. The `_Sentinel` object itself stays pure Python +//! (`synapse.logging.context.SENTINEL_CONTEXT`): the wrappers there convert +//! between it and `None`, so Rust code never sees the sentinel object. +//! +//! [`current_context`] returns the task-local value when called from inside a +//! tokio task, and the thread-local value otherwise. `LoggingContextFilter` +//! (and therefore `pyo3-log`) calls [`current_context`] for each log record, so +//! records emitted while a task runs are attributed to that task's context +//! without having to call into Python. +//! +//! [`set_current_context`] also does the CPU accounting. It reads the thread +//! rusage via libc, calls `stop` on the old context and `start` on the new one, +//! and writes the new context with [`swap_current_context`]. It is only ever +//! called on reactor or threadpool threads, never on tokio worker threads, so +//! it always writes the thread-local. Therefore, CPU accounting is not captured +//! when executing purely Rust work. +//! +//! The task-local is written exactly once, at spawn time, by +//! [`LogContextHandle::scope`]. [`swap_current_context`] checks that invariant +//! rather than trusting it. + +use std::{cell::RefCell, future::Future}; + +use log::{debug, error, log_enabled, Level}; +use pyo3::call::PyCallArgs; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{intern, PyTraverseError, PyVisit}; + +/// Name of the opt-in logger for debugging when the logcontext switches. +/// +/// Used as the `debug!` `target:` for the messages emitted below, and exported +/// to Python via [`register_module`] so that `synapse.logging.context` builds +/// exactly this logger (see `logcontext_debug_logger` there). Sharing one +/// constant keeps the two names in sync. The messages only appear when this +/// logger is explicitly configured: `ExplicitlyConfiguredLogger` on the Python +/// side implements that, and pyo3-log honours its `isEnabledFor`, so the +/// opt-in applies to messages from Rust too. +pub const DEBUG_LOGGER_NAME: &str = "synapse.logging.context.debug"; + +thread_local! { + /// The current logcontext for this OS thread, a [`LoggingContext`] + /// (possibly a Python subclass instance), or `None` for the sentinel. + static THREAD_LOCAL_CONTEXT: RefCell>> = const { RefCell::new(None) }; +} + +tokio::task_local! { + /// The logcontext captured for the current tokio task, set by + /// [`LogContextHandle::scope`] when the task is spawned. Only present + /// inside such a task. Readable during any poll of the task, whichever + /// worker thread runs it. + static TASK_LOCAL_CONTEXT: LogContextHandle; +} + +/// A captured logcontext that can be cloned and read without the GIL. Holds a +/// [`LoggingContext`] (possibly a Python subclass instance), or `None` for the +/// sentinel. +#[derive(Clone)] +pub struct LogContextHandle { + context: std::sync::Arc>>, +} + +impl LogContextHandle { + /// Capture the calling thread's current logcontext. + pub fn capture(py: Python<'_>) -> Self { + LogContextHandle { + context: std::sync::Arc::new(current_context(py)), + } + } + + /// Create a handle to the logcontext of the current tokio task, if we are + /// running inside one that was spawned through [`LogContextHandle::scope`]. + pub fn task_current() -> Option { + TASK_LOCAL_CONTEXT.try_with(|c| c.clone()).ok() + } + + /// Run `fut` with this logcontext active for the duration of the task. + pub fn scope(self, fut: F) -> impl Future + where + F: Future, + { + TASK_LOCAL_CONTEXT.scope(self, fut) + } + + /// The captured [`LoggingContext`], or `None` if the sentinel was captured. + pub fn logging_context(&self) -> Option<&Py> { + self.context.as_ref().as_ref() + } +} + +/// Tracks the resources used by a log context. +/// +/// The attributes, operators and `repr` format are relied on by Python callers +/// (`Measure`, request and background-process metrics, the task scheduler, +/// etc). Implemented in Rust so that [`set_current_context`] can update the +/// counters without allocating a Python object each time. +#[pyclass(skip_from_py_object, get_all, set_all)] +#[derive(Clone, Default)] +pub struct ContextResourceUsage { + /// System CPU time, in seconds. + pub ru_stime: f64, + /// User CPU time, in seconds. + pub ru_utime: f64, + /// Number of database transactions done. + pub db_txn_count: i64, + /// Time spent doing database transactions (excluding scheduling), in seconds. + pub db_txn_duration_sec: f64, + /// Time spent waiting for a database connection, in seconds. + pub db_sched_duration_sec: f64, + /// Number of events requested from the database. + pub evt_db_fetch_count: i64, +} + +impl ContextResourceUsage { + /// Add the stats from `other` to `self`, mutating in place. Equivalent to + /// `self += other`. + fn add_assign(&mut self, other: &ContextResourceUsage) { + self.ru_utime += other.ru_utime; + self.ru_stime += other.ru_stime; + self.db_txn_count += other.db_txn_count; + self.db_txn_duration_sec += other.db_txn_duration_sec; + self.db_sched_duration_sec += other.db_sched_duration_sec; + self.evt_db_fetch_count += other.evt_db_fetch_count; + } + + /// Subtract the stats from `other` from `self`, mutating in place. + /// Equivalent to `self -= other`. + fn sub_assign(&mut self, other: &ContextResourceUsage) { + self.ru_utime -= other.ru_utime; + self.ru_stime -= other.ru_stime; + self.db_txn_count -= other.db_txn_count; + self.db_txn_duration_sec -= other.db_txn_duration_sec; + self.db_sched_duration_sec -= other.db_sched_duration_sec; + self.evt_db_fetch_count -= other.evt_db_fetch_count; + } +} + +#[pymethods] +impl ContextResourceUsage { + /// If `copy_from` is given, copy its stats; otherwise start at zero. + #[new] + #[pyo3(signature = (copy_from=None))] + fn new(copy_from: Option<&ContextResourceUsage>) -> Self { + copy_from.cloned().unwrap_or_default() + } + + /// Return a copy of this object. + fn copy(&self) -> ContextResourceUsage { + self.clone() + } + + /// Reset all stats to zero. + fn reset(&mut self) { + *self = ContextResourceUsage::default(); + } + + fn __repr__(&self) -> String { + // `Measure` logs this string in its "Failed to save metrics!" warning, + // and log scrapers may match on it, so keep the format stable. + format!( + "", + self.ru_stime, + self.ru_utime, + self.db_txn_count, + self.db_txn_duration_sec, + self.db_sched_duration_sec, + self.evt_db_fetch_count, + ) + } + + /// `self += other`, mutating in place. + fn __iadd__(&mut self, other: &ContextResourceUsage) { + self.add_assign(other); + } + + /// `self -= other`, mutating in place. + fn __isub__(&mut self, other: &ContextResourceUsage) { + self.sub_assign(other); + } + + /// `self + other`, returning a new object. + fn __add__(&self, other: &ContextResourceUsage) -> ContextResourceUsage { + let mut res = self.clone(); + res.add_assign(other); + res + } + + /// `self - other`, returning a new object. + fn __sub__(&self, other: &ContextResourceUsage) -> ContextResourceUsage { + let mut res = self.clone(); + res.sub_assign(other); + res + } +} + +/// Call `logcontext_error(msg)` in `synapse.logging.context`. +/// +/// Looked up at call time rather than cached, so tests can patch it. +fn logcontext_error(py: Python<'_>, msg: String) -> PyResult<()> { + let module = py.import("synapse.logging.context")?; + module.getattr("logcontext_error")?.call1((msg,))?; + Ok(()) +} + +extern "C" { + /// CPython's thread identifier (`pythread.h`) — the exact value + /// `threading.get_ident()` returns. Part of the stable ABI, but not bound + /// by pyo3-ffi. + fn PyThread_get_thread_ident() -> std::os::raw::c_ulong; +} + +/// This thread's `threading.get_ident()` value, read without calling into +/// Python. +/// +/// Note that `get_ident` is *not* an OS-level tid. On Linux it returns the same +/// value either side of a `fork()` call. Synapse forks in exactly one place, so +/// contexts created before the fork still pass the `main_thread` check after +/// it. +fn get_thread_id() -> u64 { + // SAFETY: no preconditions; returns an identifier for the calling thread. + (unsafe { PyThread_get_thread_ident() }) as u64 +} + +/// Propagate a usage update to the parent context, if there is a (truthy, i.e. +/// non-sentinel) one. +/// +/// Handles the parent being a [`LoggingContext`] subclass (a Python object), +/// but has a fast-path for the common case that the parent is a base +/// [`LoggingContext`]. +fn forward_to_parent<'py, N>( + parent: &Option>, + py: Python<'py>, + method: &str, + args: impl PyCallArgs<'py>, + native: N, +) -> PyResult<()> +where + N: FnOnce(&Bound<'py, LoggingContext>) -> PyResult<()>, +{ + if let Some(parent) = parent { + let parent = parent.bind(py); + + // Check if the parent is a base LoggingContext, and if so call the + // native Rust method rather than going through Python. + if let Ok(base) = parent.cast_exact::() { + return native(base); + } + + // Otherwise call the method on the parent, if it is truthy. This + // handles Python subclasses of `LoggingContext`. + if parent.is_truthy()? { + parent.call_method1(method, args)?; + } + } + Ok(()) +} + +/// The current thread's CPU usage as `(ru_utime, ru_stime)` in seconds, read +/// via `getrusage(RUSAGE_THREAD)`. +/// +/// Returns `None` where per-thread rusage isn't available, which we take to +/// be any non-Linux target: `RUSAGE_THREAD` is Linux-only, so e.g. macOS gets +/// no per-context CPU accounting. +#[cfg(target_os = "linux")] +fn get_thread_rusage() -> Option<(f64, f64)> { + fn timeval_to_secs(tv: libc::timeval) -> f64 { + tv.tv_sec as f64 + tv.tv_usec as f64 / 1_000_000.0 + } + + // SAFETY: `getrusage` only writes into `usage`, and we only read it once it + // reports success. + let mut usage = std::mem::MaybeUninit::::uninit(); + let ret = unsafe { libc::getrusage(libc::RUSAGE_THREAD, usage.as_mut_ptr()) }; + if ret != 0 { + return None; + } + let usage = unsafe { usage.assume_init() }; + Some(( + timeval_to_secs(usage.ru_utime), + timeval_to_secs(usage.ru_stime), + )) +} + +#[cfg(not(target_os = "linux"))] +fn get_thread_rusage() -> Option<(f64, f64)> { + None +} + +/// The `(user, system)` CPU seconds elapsed between `start` and `current`. +/// +/// Guards against the clock going backwards (clamping to zero and logging, as +/// the accounting must never go negative). +fn cputime_delta(current: (f64, f64), start: (f64, f64)) -> (f64, f64) { + let mut utime_delta = current.0 - start.0; + let mut stime_delta = current.1 - start.1; + + // sanity check + if utime_delta < 0.0 { + error!("utime went backwards! {} < {}", current.0, start.0); + utime_delta = 0.0; + } + if stime_delta < 0.0 { + error!("stime went backwards! {} < {}", current.1, start.1); + stime_delta = 0.0; + } + + (utime_delta, stime_delta) +} + +/// Additional context for log formatting, tracking which request a unit of work +/// belongs to and accounting CPU/DB usage against it. Contexts are scoped +/// within a `with` block. +/// +/// The attributes and methods are relied on by Python callers and subclasses +/// (notably `BackgroundProcessLoggingContext`). +/// +/// Construction is split between `__new__`, which allocates a blank instance, +/// and `__init__`, which does the real initialisation, to support subclasses. +#[pyclass(subclass)] +pub struct LoggingContext { + /// Name for the context, used in logging. Stored as a Python string. + /// `LoggingContextFilter` calls `str(context)` on every log record, and + /// returning the stored `Py` only bumps a reference count where a + /// `String` field would allocate a fresh `str` per record. + #[pyo3(get, set)] + name: Py, + /// The homeserver name this context is associated with. Stored as a Python + /// string for the same reason as `name` (read per log record). + #[pyo3(get, set)] + server_name: Py, + /// The `threading.get_ident()` value of the thread this context was + /// created on (see [`get_thread_id`] for why it is not a real OS tid). + /// Activity on any other thread is an error. Settable only so tests can + /// simulate activity on the wrong thread. + #[pyo3(get)] + main_thread: u64, + /// Whether `__exit__` has run. Re-activating a finished context is an error. + #[pyo3(get, set)] + finished: bool, + /// The thread CPU usage `(ru_utime, ru_stime)` in seconds captured when + /// this context became active, or `None` if it is not currently active. + /// Kept as a plain `(f64, f64)` rather than a Python `struct_rusage` so + /// that switching contexts allocates no Python object. Nothing outside + /// this module reads it, so it is not exposed to Python. + usage_start: Option<(f64, f64)>, + /// A short human-readable tag (e.g. the sync type). Initialised to `""` and + /// treated as a `str` by everything in-tree, but `Option` so that assigning + /// `None` (which the sentinel's `tag` reports, and which out-of-tree + /// callers may assign) is allowed. + #[pyo3(get, set)] + tag: Option, + /// The resources used by this context so far. + resource_usage: Py, + /// The context that was current when this one was created. Restored on + /// exit. + #[pyo3(get, set)] + previous_context: Option>, + /// The parent context, if any. + #[pyo3(get, set)] + parent_context: Option>, + /// The `ContextRequest` this work belongs to, if any. + #[pyo3(get, set)] + request: Option>, + /// The opentracing scope associated with this context, if any. + #[pyo3(get, set)] + scope: Option>, +} + +#[pymethods] +impl LoggingContext { + /// Allocate a blank context. The real initialisation happens in `__init__`, + /// the same as for a normal Python class. Extra positional/keyword + /// arguments are accepted and ignored. `type.__call__` feeds the same + /// arguments to both `__new__` and `__init__`, so a subclass constructor's + /// arguments must not be rejected here. + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn __new__( + py: Python<'_>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + Ok(LoggingContext { + name: intern!(py, "").clone().unbind(), + server_name: intern!(py, "").clone().unbind(), + main_thread: 0, + finished: false, + usage_start: None, + tag: Some(String::new()), + resource_usage: Py::new(py, ContextResourceUsage::default())?, + previous_context: None, + parent_context: None, + request: None, + scope: None, + }) + } + + #[pyo3(signature = (*, name, server_name, parent_context=None, request=None))] + fn __init__( + &mut self, + py: Python<'_>, + name: Bound<'_, PyString>, + server_name: Bound<'_, PyString>, + parent_context: Option>, + request: Option>, + ) -> PyResult<()> { + self.previous_context = current_context(py); + + // The resource-usage tracker was already allocated (zeroed) by `__new__`, + // which `type.__call__` runs immediately before this. + + self.usage_start = None; + + self.name = name.unbind(); + self.server_name = server_name.unbind(); + self.main_thread = get_thread_id(); + self.request = None; + self.tag = Some(String::new()); + self.scope = None; + + // keep track of whether we have hit the __exit__ block for this context + self.finished = false; + + // Inherit some fields from the parent context (read before we move it + // into `self`, so no borrow of `self.parent_context` is held). + if let Some(parent) = &parent_context { + let parent = parent.bind(py); + // which request this corresponds to + self.request = parent.getattr("request")?.extract()?; + // we also track the current scope + self.scope = parent.getattr("scope")?.extract()?; + } + + if let Some(request) = request { + // the request param overrides the request from the parent context + self.request = Some(request); + } + + self.parent_context = parent_context; + + Ok(()) + } + + /// Returns the stored name object itself, without copying the string. + /// This runs for every log record via `LoggingContextFilter`. + fn __str__(&self, py: Python<'_>) -> Py { + self.name.clone_ref(py) + } + + /// Enter this logging context, making it the current context. + fn __enter__<'py>(slf: Bound<'py, Self>) -> PyResult> { + let py = slf.py(); + // Clone the reference rather than holding a borrow across the call. + // `set_current_context` re-borrows `slf` (borrow_mut in `start_inner`). + let previous = slf + .borrow() + .previous_context + .as_ref() + .map(|p| p.clone_ref(py)); + + if log_enabled!(target: DEBUG_LOGGER_NAME, Level::Debug) { + // Only build the name string when the opt-in debug logger is + // enabled. This runs on every context entry. + debug!( + target: DEBUG_LOGGER_NAME, + "LoggingContext({}).__enter__", + slf.borrow().name_string(py) + ); + } + + let old_context = set_current_context(py, Some(slf.clone().unbind()))?; + + if !are_contexts_identical(&previous, &old_context) { + let previous_repr = context_repr(py, &previous)?; + let old_repr = context_repr(py, &old_context)?; + logcontext_error( + py, + format!("Expected previous context {previous_repr}, found {old_repr}"), + )?; + } + + Ok(slf) + } + + /// Restore the previous logging context. Returns `None` (does not suppress + /// exceptions). + fn __exit__( + slf: Bound<'_, Self>, + _exc_type: Bound<'_, PyAny>, + _exc_value: Bound<'_, PyAny>, + _traceback: Bound<'_, PyAny>, + ) -> PyResult<()> { + let py = slf.py(); + + let previous = slf + .borrow() + .previous_context + .as_ref() + .map(|p| p.clone_ref(py)); + + if log_enabled!(target: DEBUG_LOGGER_NAME, Level::Debug) { + // The `str()` of the previous context, or `"sentinel"` for + // `None`. Only built (along with the name) when the opt-in debug + // logger is enabled. This runs on every context exit. + let previous_str = match &previous { + Some(p) => p.bind(py).str()?.extract::()?, + None => "sentinel".to_owned(), + }; + debug!( + target: DEBUG_LOGGER_NAME, + "LoggingContext({}).__exit__ --> {previous_str}", + slf.borrow().name_string(py) + ); + } + + let current = set_current_context(py, previous)?; + + let restored_self = current.as_ref().is_some_and(|c| c.bind(py).is(&slf)); + if !restored_self { + // Error path: the name string is only built for the message. + let name = slf.borrow().name_string(py); + match ¤t { + None => logcontext_error(py, format!("Expected logging context {name} was lost"))?, + Some(current) => { + let current_str: String = current.bind(py).str()?.extract()?; + logcontext_error( + py, + format!("Expected logging context {name} but found {current_str}"), + )?; + } + } + } + + slf.borrow_mut().finished = true; + + Ok(()) + } + + /// Record that this logcontext is currently running. + /// + /// This should not be called directly from Python, use + /// `set_current_context`. + /// + /// `rusage` is the thread CPU usage `(ru_utime, ru_stime)` at the point of + /// switching to this context (`None` if the platform doesn't track it). + /// + /// Note that the this takes a `Bound<..>` rather than `&self` as it can + /// call into Python, which may re-enter the context and try mutable borrow + /// it. + fn start(slf: &Bound<'_, Self>, rusage: Option<(f64, f64)>) -> PyResult<()> { + let py = slf.py(); + let main_thread = slf.borrow().main_thread; + + if get_thread_id() != main_thread { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Started logcontext {name} on different thread"))?; + return Ok(()); + } + + if slf.borrow().finished { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Re-starting finished log context {name}"))?; + return Ok(()); + } + + if slf.borrow().usage_start.is_some() { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Re-starting already-active log context {name}"))?; + return Ok(()); + } + + // If we haven't already started, record the thread resource usage so far. + slf.borrow_mut().usage_start = rusage; + + Ok(()) + } + + /// Record that this logcontext is no longer running. + /// + /// This should not be called directly from Python: use + /// `set_current_context`. + /// + /// Note that the this takes a `Bound<..>` rather than `&self` as it can + /// call into Python, which may re-enter the context and try mutable borrow + /// it. + fn stop(slf: &Bound<'_, Self>, rusage: Option<(f64, f64)>) -> PyResult<()> { + // `usage_start` must be cleared even if `stop_inner` fails, so don't + // use `?` here. + let result = Self::stop_inner(slf, rusage); + + slf.borrow_mut().usage_start = None; + + result + } + + /// Get a *copy* of the resources used by this logcontext so far. + /// + /// Should only be called from the main thread (it will return None otherwise). + fn get_resource_usage(slf: Bound<'_, Self>) -> PyResult { + let py = slf.py(); + + let mut res = slf.borrow().resource_usage.borrow(py).clone(); + + let (usage_start, main_thread) = { + let this = slf.borrow(); + (this.usage_start, this.main_thread) + }; + + // If we are on the correct thread and we're currently running then we can + // include resource usage so far. + if let Some(start) = usage_start { + if get_thread_id() == main_thread { + if let Some(current) = get_thread_rusage() { + let (utime_delta, stime_delta) = cputime_delta(current, start); + res.ru_utime += utime_delta; + res.ru_stime += stime_delta; + } + } + } + + Ok(res) + } + + /// Update the CPU time usage of this context (and any parents, recursively). + fn add_cputime(&self, py: Python<'_>, utime_delta: f64, stime_delta: f64) -> PyResult<()> { + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.ru_utime += utime_delta; + usage.ru_stime += stime_delta; + } + forward_to_parent( + &self.parent_context, + py, + "add_cputime", + (utime_delta, stime_delta), + |p| p.borrow().add_cputime(py, utime_delta, stime_delta), + ) + } + + /// Record the use of a database transaction and how long it took. + fn add_database_transaction(&self, py: Python<'_>, duration_sec: f64) -> PyResult<()> { + if duration_sec < 0.0 { + return Err(PyValueError::new_err( + "DB txn time can only be zero or non-negative", + )); + } + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.db_txn_count += 1; + usage.db_txn_duration_sec += duration_sec; + } + forward_to_parent( + &self.parent_context, + py, + "add_database_transaction", + (duration_sec,), + |p| p.borrow().add_database_transaction(py, duration_sec), + ) + } + + /// Record a use of the database pool (the time taken to get a connection). + fn add_database_scheduled(&self, py: Python<'_>, sched_sec: f64) -> PyResult<()> { + if sched_sec < 0.0 { + return Err(PyValueError::new_err( + "DB scheduling time can only be zero or non-negative", + )); + } + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.db_sched_duration_sec += sched_sec; + } + forward_to_parent( + &self.parent_context, + py, + "add_database_scheduled", + (sched_sec,), + |p| p.borrow().add_database_scheduled(py, sched_sec), + ) + } + + /// Record a number of events being fetched from the db. + fn record_event_fetch(&self, py: Python<'_>, event_count: i64) -> PyResult<()> { + { + let mut usage = self.resource_usage.borrow_mut(py); + usage.evt_db_fetch_count += event_count; + } + forward_to_parent( + &self.parent_context, + py, + "record_event_fetch", + (event_count,), + |p| p.borrow().record_event_fetch(py, event_count), + ) + } + + /// Traverse referenced Python objects for the cyclic garbage collector. + /// `scope` and the context can reference each other, forming a real cycle. + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some(previous_context) = &self.previous_context { + visit.call(previous_context)?; + } + if let Some(parent_context) = &self.parent_context { + visit.call(parent_context)?; + } + if let Some(request) = &self.request { + visit.call(request)?; + } + if let Some(scope) = &self.scope { + visit.call(scope)?; + } + Ok(()) + } + + fn __clear__(&mut self) { + self.previous_context = None; + self.parent_context = None; + self.request = None; + self.scope = None; + } +} + +impl LoggingContext { + /// The context name as an owned Rust string. + /// + /// This copies the string data, so it is for error/debug paths only. + /// Switching contexts should not allocate in the common case. + fn name_string(&self, py: Python<'_>) -> String { + self.name.bind(py).to_string_lossy().into_owned() + } + + /// Inner implementation of [`Self::stop`], which does not clear + /// `usage_start`. + /// + /// Note that the this takes a `Bound<..>` rather than `&self` as it can + /// call into Python, which may re-enter the context and try mutable borrow + /// it. + fn stop_inner(slf: &Bound<'_, Self>, rusage: Option<(f64, f64)>) -> PyResult<()> { + let py = slf.py(); + let main_thread = slf.borrow().main_thread; + + if get_thread_id() != main_thread { + let name = slf.borrow().name_string(py); + logcontext_error(py, format!("Stopped logcontext {name} on different thread"))?; + return Ok(()); + } + + // No rusage means this platform doesn't track per-thread CPU, so + // there is nothing to account. + let Some(current) = rusage else { + return Ok(()); + }; + + // Record the cpu used since we started. + let Some(start) = slf.borrow().usage_start else { + let name = slf.borrow().name_string(py); + logcontext_error( + py, + format!("Called stop on logcontext {name} without recording a start rusage"), + )?; + return Ok(()); + }; + + let (utime_delta, stime_delta) = cputime_delta(current, start); + slf.borrow().add_cputime(py, utime_delta, stime_delta)?; + Ok(()) + } +} + +/// Which way a [`switch_context`] dispatch goes. +#[derive(Clone, Copy)] +enum SwitchDirection { + Start, + Stop, +} + +/// Call `stop` or `start` on a context as the current context changes, +/// respecting subclass overrides. +/// +/// `None` (the sentinel) is a no-op. +/// +/// Handles the context being a Python subclass of [`LoggingContext`], but has +/// a fast-path for the common case that it is a base [`LoggingContext`]. +fn switch_context( + py: Python<'_>, + ctx: Option<&Py>, + direction: SwitchDirection, + rusage: Option<(f64, f64)>, +) -> PyResult<()> { + let Some(ctx) = ctx else { + return Ok(()); + }; + let ctx = ctx.bind(py); + if ctx.as_any().is_exact_instance_of::() { + match direction { + SwitchDirection::Start => LoggingContext::start(ctx, rusage), + SwitchDirection::Stop => LoggingContext::stop(ctx, rusage), + } + } else { + let method = match direction { + SwitchDirection::Start => intern!(py, "start"), + SwitchDirection::Stop => intern!(py, "stop"), + }; + ctx.call_method1(method, (rusage,))?; + Ok(()) + } +} + +/// Whether two stored values are the same context, or both the sentinel. +/// `LoggingContext` has no `__eq__`, so contexts compare by identity. +fn are_contexts_identical(a: &Option>, b: &Option>) -> bool { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => a.is(b), + _ => false, + } +} + +/// `repr()` of a stored context, for error messages only. `None` renders as +/// `"None"`. +fn context_repr(py: Python<'_>, slot: &Option>) -> PyResult { + match slot { + Some(ctx) => Ok(ctx.bind(py).repr()?.extract()?), + None => Ok("None".to_owned()), + } +} + +/// Set the current logging context, returning the context that was previously +/// current. `None` means the sentinel, in both directions. +/// +/// `context` must be a [`LoggingContext`] (or subclass) or `None`. +/// +/// This is not the Python-facing API. +/// `synapse.logging.context.set_current_context` wraps this, converting between +/// `SENTINEL_CONTEXT` and `None`. +/// +/// Reads the thread rusage once via libc, calls `stop` on the old context and +/// `start` on the new one. +#[pyfunction] +#[pyo3(signature = (context))] +pub fn set_current_context( + py: Python<'_>, + context: Option>, +) -> PyResult>> { + let current = current_context(py); + + if !are_contexts_identical(¤t, &context) { + let rusage = get_thread_rusage(); + switch_context(py, current.as_ref(), SwitchDirection::Stop, rusage)?; + // We already hold `current`, so ignore the previous value the swap + // returns. The clone_ref keeps a reference for the `start` below. + let new_ref = context.as_ref().map(|ctx| ctx.clone_ref(py)); + swap_current_context(context); + switch_context(py, new_ref.as_ref(), SwitchDirection::Start, rusage)?; + } + + Ok(current) +} + +/// Run `f` with `context` as the current logcontext (`None` meaning the +/// sentinel), restoring the previously-current context afterwards. This is the +/// Rust equivalent of Python's `with PreserveLoggingContext(context):`. +/// +/// The restore runs whether or not `f` fails. If both `f` and the restore fail, +/// `f`'s error is reported. +pub(crate) fn with_logcontext( + py: Python<'_>, + context: Option>, + f: impl FnOnce() -> PyResult, +) -> PyResult { + let previous = set_current_context(py, context)?; + let result = f(); + // Run before checking success/failure of `f()`. + let restored = set_current_context(py, previous); + + let value = result?; + restored?; + Ok(value) +} + +/// Get the current logging context, or `None` for the sentinel. +/// +/// Returns the tokio task-local value when called from inside a task that has +/// one (even if what it captured was `None`/the sentinel) and this OS thread's +/// value otherwise. Logging emitted while a task runs is therefore attributed +/// to the context that was current when the task was spawned. +/// +/// This is not the Python-facing API. +/// `synapse.logging.context.current_context` wraps this and returns +/// `SENTINEL_CONTEXT` instead of `None`. +#[pyfunction] +pub fn current_context(py: Python<'_>) -> Option> { + if let Some(handle) = LogContextHandle::task_current() { + return handle.logging_context().map(|ctx| ctx.clone_ref(py)); + } + + THREAD_LOCAL_CONTEXT.with(|slot| slot.borrow().as_ref().map(|ctx| ctx.clone_ref(py))) +} + +/// Replace this OS thread's current logging context, returning the previous one +/// (`None` is the sentinel, in both directions). +/// +/// This is the raw write only. It does no resource-usage accounting and no +/// thread checks; [`set_current_context`] wraps it with the `getrusage` +/// `stop`/`start` bookkeeping. +/// +/// It never touches the tokio task-local. The current context is only ever +/// changed on reactor/threadpool threads, while the task-local is written +/// exactly once, at spawn time, by [`LogContextHandle::scope`]. If this is +/// nonetheless called while a task-local is in scope, it logs an error, leaves +/// the thread-local unchanged and returns `None`. +fn swap_current_context(context: Option>) -> Option> { + // A task-local in scope would shadow the write (`current_context` prefers + // it), so refuse rather than misattribute later reads. + if TASK_LOCAL_CONTEXT.try_with(|_| ()).is_ok() { + error!( + "swap_current_context called during a tokio-scoped poll; the switch is \ + invisible to current_context() and will misattribute logs and metrics" + ); + return None; + } + + THREAD_LOCAL_CONTEXT.with(|slot| std::mem::replace(&mut *slot.borrow_mut(), context)) +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module: Bound<'_, PyModule> = PyModule::new(py, "logcontext")?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_function(wrap_pyfunction!(current_context, &child_module)?)?; + child_module.add_function(wrap_pyfunction!(set_current_context, &child_module)?)?; + child_module.add("DEBUG_LOGGER_NAME", DEBUG_LOGGER_NAME)?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import logcontext` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.logcontext", child_module)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use pyo3::types::PyString; + + use super::*; + + /// A minimal `LoggingContext` for these tests, built directly (bypassing + /// `__init__`, which would capture the current context and thread id). + fn test_context(py: Python<'_>, name: &str) -> Py { + Py::new( + py, + LoggingContext { + name: PyString::new(py, name).unbind(), + server_name: PyString::new(py, "test_server").unbind(), + main_thread: 0, + finished: false, + usage_start: None, + tag: Some(String::new()), + resource_usage: Py::new(py, ContextResourceUsage::default()) + .expect("failed to allocate ContextResourceUsage"), + previous_context: None, + parent_context: None, + request: None, + scope: None, + }, + ) + .expect("failed to allocate LoggingContext") + } + + #[test] + fn thread_local_defaults_to_sentinel() { + Python::initialize(); + Python::attach(|py| { + // Nothing has been set on this fresh test thread, so the current + // context is `None`: the sentinel. + assert!(current_context(py).is_none()); + }); + } + + #[test] + fn swap_returns_previous_and_updates_thread_local() { + Python::initialize(); + Python::attach(|py| { + let a = test_context(py, "A"); + let b = test_context(py, "B"); + + // Swapping in A returns the previous value (`None` / the sentinel) + // and makes A current. + let prev = swap_current_context(Some(a.clone_ref(py))); + assert!(prev.is_none()); + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(a.bind(py))); + + // Swapping in B returns A. + let prev = swap_current_context(Some(b.clone_ref(py))); + assert!(prev + .expect("expected previous context") + .bind(py) + .is(a.bind(py))); + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(b.bind(py))); + + // Reset to the sentinel so we don't leak into another test that + // reuses this OS thread from the test harness's pool. + swap_current_context(None); + }); + } + + #[test] + fn task_local_takes_precedence_over_thread_local() { + Python::initialize(); + Python::attach(|py| { + let task_ctx = test_context(py, "TASKCTX"); + + // Outside any task, `current_context` reads the thread-local, which + // here is `None` / the sentinel. + assert!(current_context(py).is_none()); + assert!(LogContextHandle::task_current().is_none()); + + let log_context = LogContextHandle { + context: Arc::new(Some(task_ctx.clone_ref(py))), + }; + + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(log_context.scope(async { + // Inside the scope, both the Rust handle and the + // `current_context` (what the log filter calls) return the + // task-local context, even though the thread-local is still + // unset. + assert!(LogContextHandle::task_current().is_some()); + Python::attach(|py| { + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(task_ctx.bind(py))); + }); + })); + + // Once the scope ends, we fall back to the thread-local again. + assert!(LogContextHandle::task_current().is_none()); + assert!(current_context(py).is_none()); + }); + } + + #[test] + fn swap_during_scoped_poll_leaves_thread_local_untouched() { + Python::initialize(); + Python::attach(|py| { + let thread_ctx = test_context(py, "THREAD"); + let task_ctx = test_context(py, "TASK"); + let stray = test_context(py, "STRAY"); + + // Give this thread a current context. + swap_current_context(Some(thread_ctx.clone_ref(py))); + + let log_context = LogContextHandle { + context: Arc::new(Some(task_ctx.clone_ref(py))), + }; + + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(log_context.scope(async { + Python::attach(|py| { + // Swapping while a task-local is in scope is refused: it + // returns `None` and leaves the thread-local unchanged. + assert!(swap_current_context(Some(stray.clone_ref(py))).is_none()); + + // The task-local still governs reads inside the scope. + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(task_ctx.bind(py))); + }); + })); + + // The scope has ended, the thread-local still holds the original + // context, not the stray value from the invalid swap. + assert!(current_context(py) + .expect("expected a current context") + .bind(py) + .is(thread_ctx.bind(py))); + + // Reset to the sentinel so we don't leak into another test that + // reuses this OS thread from the test harness's pool. + swap_current_context(None); + }); + } +} diff --git a/rust/src/logging/mod.rs b/rust/src/logging/mod.rs new file mode 100644 index 0000000000..1d6d898cb4 --- /dev/null +++ b/rust/src/logging/mod.rs @@ -0,0 +1,23 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust counterparts to the `synapse.logging` Python package. +//! +//! Submodules live at the Rust module path that mirrors their Python logging +//! namespace (e.g. `synapse::logging::context` -> `synapse.logging.context`), so +//! that log records emitted from here via the `log` crate land under the matching +//! Python logger without an explicit `target:`. + +pub mod context; diff --git a/synapse/logging/context.py b/synapse/logging/context.py index b6535be388..c7121cf260 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -31,7 +31,6 @@ """ import logging -import threading import typing from types import TracebackType from typing import ( @@ -42,6 +41,7 @@ Literal, TypeVar, Union, + cast, overload, ) @@ -52,17 +52,24 @@ from twisted.python.threadpool import ThreadPool from synapse.logging.loggers import ExplicitlyConfiguredLogger +from synapse.synapse_rust.logcontext import ( + DEBUG_LOGGER_NAME, + # Not used in this module, but re-exported: callers import it from here. + ContextResourceUsage, # noqa: F401 + LoggingContext, + current_context as _rust_current_context, + set_current_context as _rust_set_current_context, +) from synapse.util.stringutils import random_string_insecure_fast if TYPE_CHECKING: - from synapse.logging.scopecontextmanager import _LogContextScope from synapse.types import ISynapseReactor logger = logging.getLogger(__name__) original_logger_class = logging.getLoggerClass() logging.setLoggerClass(ExplicitlyConfiguredLogger) -logcontext_debug_logger = logging.getLogger("synapse.logging.context.debug") +logcontext_debug_logger = logging.getLogger(DEBUG_LOGGER_NAME) """ A logger for debugging when the logcontext switches. @@ -74,146 +81,12 @@ # Restore the original logger class logging.setLoggerClass(original_logger_class) -try: - import resource - - # Python doesn't ship with a definition of RUSAGE_THREAD but it's defined - # to be 1 on linux so we hard code it. - RUSAGE_THREAD = 1 - - # If the system doesn't support RUSAGE_THREAD then this should throw an - # exception. - resource.getrusage(RUSAGE_THREAD) - - is_thread_resource_usage_supported = True - - def get_thread_resource_usage() -> "resource.struct_rusage | None": - return resource.getrusage(RUSAGE_THREAD) - -except Exception: - # If the system doesn't support resource.getrusage(RUSAGE_THREAD) then we - # won't track resource usage. - is_thread_resource_usage_supported = False - - def get_thread_resource_usage() -> "resource.struct_rusage | None": - return None - # a hook which can be set during testing to assert that we aren't abusing logcontexts. def logcontext_error(msg: str) -> None: logger.warning(msg) -# get an id for the current thread. -# -# threading.get_ident doesn't actually return an OS-level tid, and annoyingly, -# on Linux it actually returns the same value either side of a fork() call. However -# we only fork in one place, so it's not worth the hoop-jumping to get a real tid. -# -get_thread_id = threading.get_ident - - -class ContextResourceUsage: - """Object for tracking the resources used by a log context - - Attributes: - ru_utime (float): user CPU time (in seconds) - ru_stime (float): system CPU time (in seconds) - db_txn_count (int): number of database transactions done - db_sched_duration_sec (float): amount of time spent waiting for a - database connection - db_txn_duration_sec (float): amount of time spent doing database - transactions (excluding scheduling time) - evt_db_fetch_count (int): number of events requested from the database - """ - - __slots__ = [ - "ru_stime", - "ru_utime", - "db_txn_count", - "db_txn_duration_sec", - "db_sched_duration_sec", - "evt_db_fetch_count", - ] - - def __init__(self, copy_from: "ContextResourceUsage | None" = None) -> None: - """Create a new ContextResourceUsage - - Args: - copy_from: if not None, an object to copy stats from - """ - if copy_from is None: - self.reset() - else: - # FIXME: mypy can't infer the types set via reset() above, so specify explicitly for now - self.ru_utime: float = copy_from.ru_utime - self.ru_stime: float = copy_from.ru_stime - self.db_txn_count: int = copy_from.db_txn_count - - self.db_txn_duration_sec: float = copy_from.db_txn_duration_sec - self.db_sched_duration_sec: float = copy_from.db_sched_duration_sec - self.evt_db_fetch_count: int = copy_from.evt_db_fetch_count - - def copy(self) -> "ContextResourceUsage": - return ContextResourceUsage(copy_from=self) - - def reset(self) -> None: - self.ru_stime = 0.0 - self.ru_utime = 0.0 - self.db_txn_count = 0 - - self.db_txn_duration_sec = 0.0 - self.db_sched_duration_sec = 0.0 - self.evt_db_fetch_count = 0 - - def __repr__(self) -> str: - return ( - "" - ) % ( - self.ru_stime, - self.ru_utime, - self.db_txn_count, - self.db_txn_duration_sec, - self.db_sched_duration_sec, - self.evt_db_fetch_count, - ) - - def __iadd__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - """Add another ContextResourceUsage's stats to this one's. - - Args: - other: the other resource usage object - """ - self.ru_utime += other.ru_utime - self.ru_stime += other.ru_stime - self.db_txn_count += other.db_txn_count - self.db_txn_duration_sec += other.db_txn_duration_sec - self.db_sched_duration_sec += other.db_sched_duration_sec - self.evt_db_fetch_count += other.evt_db_fetch_count - return self - - def __isub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - self.ru_utime -= other.ru_utime - self.ru_stime -= other.ru_stime - self.db_txn_count -= other.db_txn_count - self.db_txn_duration_sec -= other.db_txn_duration_sec - self.db_sched_duration_sec -= other.db_sched_duration_sec - self.evt_db_fetch_count -= other.evt_db_fetch_count - return self - - def __add__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - res = ContextResourceUsage(copy_from=self) - res += other - return res - - def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": - res = ContextResourceUsage(copy_from=self) - res -= other - return res - - @attr.s(slots=True, auto_attribs=True) class ContextRequest: """ @@ -237,9 +110,6 @@ class ContextRequest: user_agent: str -LoggingContextOrSentinel = Union["LoggingContext", "_Sentinel"] - - class _Sentinel: """ Sentinel to represent the root context @@ -273,10 +143,10 @@ def __init__(self) -> None: def __str__(self) -> str: return "sentinel" - def start(self, rusage: "resource.struct_rusage | None") -> None: + def start(self, rusage: "tuple[float, float] | None") -> None: pass - def stop(self, rusage: "resource.struct_rusage | None") -> None: + def stop(self, rusage: "tuple[float, float] | None") -> None: pass def add_database_transaction(self, duration_sec: float) -> None: @@ -294,285 +164,41 @@ def __bool__(self) -> Literal[False]: SENTINEL_CONTEXT = _Sentinel() +LoggingContextOrSentinel = Union[LoggingContext, _Sentinel] -class LoggingContext: - """Additional context for log formatting. Contexts are scoped within a - "with" block. - - If a parent is given when creating a new context, then: - - logging fields are copied from the parent to the new context on entry - - when the new context exits, the cpu usage stats are copied from the - child to the parent - Args: - name: Name for the context for logging. - server_name: The name of the server this context is associated with - (`config.server.server_name` or `hs.hostname`) - parent_context (LoggingContext|None): The parent of the new context - request: Synapse Request Context object. Useful to associate all the logs - happening to a given request. +def current_context() -> LoggingContextOrSentinel: + """Get the current logging context. + The storage lives in the Rust extension, which represents "no context" as + `None`. This wrapper maps that to `SENTINEL_CONTEXT`, so callers never see + `None`. """ + context = _rust_current_context() + return SENTINEL_CONTEXT if context is None else context - __slots__ = [ - "previous_context", - "name", - "server_name", - "parent_context", - "_resource_usage", - "usage_start", - "main_thread", - "finished", - "request", - "tag", - "scope", - ] - - def __init__( - self, - *, - name: str, - server_name: str, - parent_context: "LoggingContext | None" = None, - request: ContextRequest | None = None, - ) -> None: - self.previous_context = current_context() - - # track the resources used by this context so far - self._resource_usage = ContextResourceUsage() - - # The thread resource usage when the logcontext became active. None - # if the context is not currently active. - self.usage_start: resource.struct_rusage | None = None - - self.name = name - self.server_name = server_name - self.main_thread = get_thread_id() - self.request = None - self.tag = "" - self.scope: "_LogContextScope" | None = None - - # keep track of whether we have hit the __exit__ block for this context - # (suggesting that the the thing that created the context thinks it should - # be finished, and that re-activating it would suggest an error). - self.finished = False - - self.parent_context = parent_context - - # Inherit some fields from the parent context - if self.parent_context is not None: - # which request this corresponds to - self.request = self.parent_context.request - - # we also track the current scope: - self.scope = self.parent_context.scope - - if request is not None: - # the request param overrides the request from the parent context - self.request = request - - def __str__(self) -> str: - return self.name - - def __enter__(self) -> "LoggingContext": - """Enters this logging context into thread local storage""" - logcontext_debug_logger.debug("LoggingContext(%s).__enter__", self.name) - old_context = set_current_context(self) - if self.previous_context != old_context: - logcontext_error( - "Expected previous context %r, found %r" - % ( - self.previous_context, - old_context, - ) - ) - return self - - def __exit__( - self, - type: type[BaseException] | None, - value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """Restore the logging context in thread local storage to the state it - was before this context was entered. - Returns: - None to avoid suppressing any exceptions that were thrown. - """ - logcontext_debug_logger.debug( - "LoggingContext(%s).__exit__ --> %s", self.name, self.previous_context - ) - current = set_current_context(self.previous_context) - if current is not self: - if current is SENTINEL_CONTEXT: - logcontext_error("Expected logging context %s was lost" % (self,)) - else: - logcontext_error( - "Expected logging context %s but found %s" % (self, current) - ) - - # the fact that we are here suggests that the caller thinks that everything - # is done and dusted for this logcontext, and further activity will not get - # recorded against the correct metrics. - self.finished = True - - def start(self, rusage: "resource.struct_rusage | None") -> None: - """ - Record that this logcontext is currently running. - - This should not be called directly: use set_current_context - - Args: - rusage: the resources used by the current thread, at the point of - switching to this logcontext. May be None if this platform doesn't - support getrusuage. - """ - if get_thread_id() != self.main_thread: - logcontext_error("Started logcontext %s on different thread" % (self,)) - return - - if self.finished: - logcontext_error("Re-starting finished log context %s" % (self,)) - - # If we haven't already started record the thread resource usage so - # far - if self.usage_start: - logcontext_error("Re-starting already-active log context %s" % (self,)) - else: - self.usage_start = rusage - - def stop(self, rusage: "resource.struct_rusage | None") -> None: - """ - Record that this logcontext is no longer running. - - This should not be called directly: use set_current_context - - Args: - rusage: the resources used by the current thread, at the point of - switching away from this logcontext. May be None if this platform - doesn't support getrusuage. - """ - - try: - if get_thread_id() != self.main_thread: - logcontext_error("Stopped logcontext %s on different thread" % (self,)) - return - - if not rusage: - return - - # Record the cpu used since we started - if not self.usage_start: - logcontext_error( - "Called stop on logcontext %s without recording a start rusage" - % (self,) - ) - return - - utime_delta, stime_delta = self._get_cputime(rusage) - self.add_cputime(utime_delta, stime_delta) - finally: - self.usage_start = None - - def get_resource_usage(self) -> ContextResourceUsage: - """Get resources used by this logcontext so far. - - Returns: - A *copy* of the object tracking resource usage so far - """ - # we always return a copy, for consistency - res = self._resource_usage.copy() - - # If we are on the correct thread and we're currently running then we - # can include resource usage so far. - is_main_thread = get_thread_id() == self.main_thread - if self.usage_start and is_main_thread: - rusage = get_thread_resource_usage() - assert rusage is not None - utime_delta, stime_delta = self._get_cputime(rusage) - res.ru_utime += utime_delta - res.ru_stime += stime_delta - - return res - - def _get_cputime(self, current: "resource.struct_rusage") -> tuple[float, float]: - """Get the cpu usage time between start() and the given rusage - - Args: - rusage: the current resource usage - - Returns: tuple[float, float]: seconds in user mode, seconds in system mode - """ - assert self.usage_start is not None - - utime_delta = current.ru_utime - self.usage_start.ru_utime - stime_delta = current.ru_stime - self.usage_start.ru_stime - - # sanity check - if utime_delta < 0: - logger.error( - "utime went backwards! %f < %f", - current.ru_utime, - self.usage_start.ru_utime, - ) - utime_delta = 0 - - if stime_delta < 0: - logger.error( - "stime went backwards! %f < %f", - current.ru_stime, - self.usage_start.ru_stime, - ) - stime_delta = 0 - return utime_delta, stime_delta - - def add_cputime(self, utime_delta: float, stime_delta: float) -> None: - """Update the CPU time usage of this context (and any parents, recursively). - - Args: - utime_delta: additional user time, in seconds, spent in this context. - stime_delta: additional system time, in seconds, spent in this context. - """ - self._resource_usage.ru_utime += utime_delta - self._resource_usage.ru_stime += stime_delta - if self.parent_context: - self.parent_context.add_cputime(utime_delta, stime_delta) - - def add_database_transaction(self, duration_sec: float) -> None: - """Record the use of a database transaction and the length of time it took. - - Args: - duration_sec: The number of seconds the database transaction took. - """ - if duration_sec < 0: - raise ValueError("DB txn time can only be non-negative") - self._resource_usage.db_txn_count += 1 - self._resource_usage.db_txn_duration_sec += duration_sec - if self.parent_context: - self.parent_context.add_database_transaction(duration_sec) - - def add_database_scheduled(self, sched_sec: float) -> None: - """Record a use of the database pool - - Args: - sched_sec: number of seconds it took us to get a connection - """ - if sched_sec < 0: - raise ValueError("DB scheduling time can only be non-negative") - self._resource_usage.db_sched_duration_sec += sched_sec - if self.parent_context: - self.parent_context.add_database_scheduled(sched_sec) - - def record_event_fetch(self, event_count: int) -> None: - """Record a number of events being fetched from the db +def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSentinel: + """Set the current logging context, returning the context that was + previously current. + + The implementation lives in the Rust extension: it reads the thread CPU + usage once (`getrusage(RUSAGE_THREAD)`) and does the `stop`/`start` + accounting in Rust. Rust represents the sentinel as `None`. This wrapper + converts in both directions, so callers only ever see + `LoggingContextOrSentinel`. + """ + # everything blows up if we allow current_context to be set to None, so + # sanity-check that now. + if context is None: + raise TypeError("'context' argument may not be None") - Args: - event_count: number of events being fetched - """ - self._resource_usage.evt_db_fetch_count += event_count - if self.parent_context: - self.parent_context.record_event_fetch(event_count) + # The cast is needed because mypy cannot narrow the Union via the + # `is SENTINEL_CONTEXT` identity check; Rust enforces the type at runtime. + previous = _rust_set_current_context( + None if context is SENTINEL_CONTEXT else cast(LoggingContext, context) + ) + return SENTINEL_CONTEXT if previous is None else previous class LoggingContextFilter(logging.Filter): @@ -733,39 +359,6 @@ def __exit__( ) -_thread_local = threading.local() -_thread_local.current_context = SENTINEL_CONTEXT - - -def current_context() -> LoggingContextOrSentinel: - """Get the current logging context from thread local storage""" - return getattr(_thread_local, "current_context", SENTINEL_CONTEXT) - - -def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSentinel: - """Set the current logging context in thread local storage - Args: - context: The context to activate. - - Returns: - The context that was previously active - """ - # everything blows up if we allow current_context to be set to None, so sanity-check - # that now. - if context is None: - raise TypeError("'context' argument may not be None") - - current = current_context() - - if current is not context: - rusage = get_thread_resource_usage() - current.stop(rusage) - _thread_local.current_context = context - context.start(rusage) - - return current - - def nested_logging_context(suffix: str) -> LoggingContext: """Creates a new logging context as a child of another. diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 8ff2803455..d51e08aae5 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -57,8 +57,6 @@ from synapse.metrics._types import Collector if TYPE_CHECKING: - import resource - # Old versions don't have `LiteralString` from typing_extensions import LiteralString @@ -506,7 +504,7 @@ def __init__( desc=name, server_name=server_name, ctx=self ) - def start(self, rusage: "resource.struct_rusage | None") -> None: + def start(self, rusage: "tuple[float, float] | None") -> None: """Log context has started running (again).""" super().start(rusage) diff --git a/synapse/synapse_rust/logcontext.pyi b/synapse/synapse_rust/logcontext.pyi new file mode 100644 index 0000000000..7e10128e3e --- /dev/null +++ b/synapse/synapse_rust/logcontext.pyi @@ -0,0 +1,177 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from types import TracebackType +from typing import TYPE_CHECKING, Optional + +from synapse.logging.context import ContextRequest + +if TYPE_CHECKING: + from synapse.logging.scopecontextmanager import _LogContextScope + +DEBUG_LOGGER_NAME: str +"""Name of the opt-in logger for debugging when the logcontext switches +(`synapse.logging.context.debug`). Shared with the Rust `debug!` target so the +names stay in sync.""" + +class ContextResourceUsage: + """Tracks the resources used by a log context. + + The attributes, operators and `repr` format are relied on by Python callers + (`Measure`, request and background-process metrics, the task scheduler, + etc). Implemented in Rust so that `set_current_context` can update the + counters without allocating a Python object each time. + """ + + ru_stime: float + """System CPU time, in seconds.""" + ru_utime: float + """User CPU time, in seconds.""" + db_txn_count: int + """Number of database transactions done.""" + db_txn_duration_sec: float + """Time spent doing database transactions (excluding scheduling), in seconds.""" + db_sched_duration_sec: float + """Time spent waiting for a database connection, in seconds.""" + evt_db_fetch_count: int + """Number of events requested from the database.""" + + def __init__(self, copy_from: "Optional[ContextResourceUsage]" = None) -> None: + """If `copy_from` is given, copy its stats; otherwise start at zero.""" + + def copy(self) -> "ContextResourceUsage": + """Return a copy of this object.""" + + def reset(self) -> None: + """Reset all stats to zero.""" + + def __iadd__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self += other`, mutating in place.""" + + def __isub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self -= other`, mutating in place.""" + + def __add__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self + other`, returning a new object.""" + + def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage": + """`self - other`, returning a new object.""" + +class LoggingContext: + """Additional context for log formatting. Contexts are scoped within a + "with" block. + + If a parent is given when creating a new context, then: + - logging fields are copied from the parent to the new context on entry + - when the new context exits, the cpu usage stats are copied from the + child to the parent + """ + + # The context that was current when this one was created. None means the + # sentinel. + previous_context: Optional[LoggingContext] + name: str + server_name: str + parent_context: "Optional[LoggingContext]" + main_thread: int + finished: bool + request: Optional[ContextRequest] + # Narrower than the runtime: the setter also accepts None (see the Rust + # field docs for why), but all in-tree code treats this as str. + tag: str + scope: "Optional[_LogContextScope]" + + def __init__( + self, + *, + name: str, + server_name: str, + parent_context: "Optional[LoggingContext]" = None, + request: Optional[ContextRequest] = None, + ) -> None: + """ + Args: + name: Name for the context for logging. + server_name: The name of the server this context is associated with + (`config.server.server_name` or `hs.hostname`). + parent_context: The parent of the new context. + request: Synapse Request Context object. Useful to associate all the + logs happening to a given request. + """ + + def __str__(self) -> str: ... + def __enter__(self) -> "LoggingContext": ... + def __exit__( + self, + type: Optional[type[BaseException]], + value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: ... + def start(self, rusage: "Optional[tuple[float, float]]") -> None: + """Record that this logcontext is currently running. + + Should not be called directly: use `set_current_context`. + + Args: + rusage: The thread CPU usage `(ru_utime, ru_stime)` at the point of + switching to this context, or None if the platform doesn't + track it. + """ + + def stop(self, rusage: "Optional[tuple[float, float]]") -> None: + """Record that this logcontext is no longer running. + + Should not be called directly: use `set_current_context`. + + Args: + rusage: The thread CPU usage `(ru_utime, ru_stime)` at the point of + switching away from this context, or None if the platform + doesn't track it. + """ + + def get_resource_usage(self) -> ContextResourceUsage: + """Get the resources used by this logcontext so far. + + Returns: + A *copy* of the object tracking resource usage so far. + """ + + def add_cputime(self, utime_delta: float, stime_delta: float) -> None: + """Update the CPU time usage of this context (and any parents, recursively).""" + + def add_database_transaction(self, duration_sec: float) -> None: + """Record the use of a database transaction and how long it took.""" + + def add_database_scheduled(self, sched_sec: float) -> None: + """Record a use of the database pool (the time taken to get a connection).""" + + def record_event_fetch(self, event_count: int) -> None: + """Record a number of events being fetched from the db.""" + +def current_context() -> Optional[LoggingContext]: + """Get the current logging context, or None for the sentinel. + + This is not the Python-facing API. + `synapse.logging.context.current_context` wraps this and returns + `SENTINEL_CONTEXT` instead of `None`. + """ + +def set_current_context( + context: Optional[LoggingContext], +) -> Optional[LoggingContext]: + """Set the current logging context, returning the context that was + previously current. `None` means the sentinel, in both directions. + + This is not the Python-facing API. + `synapse.logging.context.set_current_context` wraps this, converting between + `SENTINEL_CONTEXT` and `None`. + """ diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 011d0552a5..78a186ee4e 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -77,6 +77,7 @@ def setUp(self) -> None: self.reactor, self.clock = get_clock() hs = Mock() + hs.hostname = "test_server" def test_run_as_background_process( desc: "LiteralString", diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 577229c119..e67d36cf63 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -105,7 +105,7 @@ def runWithConnection(func, *args, **kwargs): # type: ignore[no-untyped-def] # To fix isinstance(...) checks. fake_engine.__class__ = engine.__class__ # type: ignore[assignment] - db = DatabasePool(Mock(), Mock(config=db_config), fake_engine) + db = DatabasePool(Mock(hostname="test"), Mock(config=db_config), fake_engine) db._db_pool = conn_pool self.datastore = SQLBaseStore(db, None, hs) # type: ignore[arg-type] diff --git a/tests/synapse_rust/test_logcontext.py b/tests/synapse_rust/test_logcontext.py new file mode 100644 index 0000000000..0be82f0caa --- /dev/null +++ b/tests/synapse_rust/test_logcontext.py @@ -0,0 +1,293 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +"""Tests for logcontext attribution across the Python/Rust boundary. + +The current logcontext is stored on the Rust side +(`synapse.synapse_rust.logcontext`), visible +from both Python (reactor/threadpool threads) and Rust (tokio tasks). These +tests exercise the two guarantees that gives us: + +1. Log records emitted from Rust while a task is being polled (e.g. reqwest + connecting) are attributed to the logcontext that was current when Python + called into Rust — not the sentinel. +2. When Rust calls back into Python (`run_python_awaitable`, as the Rust + `/versions` handler does for its per-user feature DB lookup), the Python code + runs in that same logcontext, so its DB-transaction accounting lands on the + right request. +""" + +import logging +import time +from typing import Callable + +from twisted.internet.testing import MemoryReactor + +from synapse.logging.context import ( + LoggingContext, + LoggingContextFilter, + PreserveLoggingContext, + _Sentinel, + current_context, + run_in_background, +) +from synapse.rest import admin +from synapse.rest.client import login +from synapse.server import HomeServer +from synapse.synapse_rust import reset_logging_config +from synapse.synapse_rust.http_client import HttpClient +from synapse.util.clock import Clock + +from tests.unittest import HomeserverTestCase + +logger = logging.getLogger(__name__) + +# Log-target roots that Rust code emits under while running on tokio worker +# threads: the reqwest dependency stack, plus "synapse"/"synapse_rust" because +# the Rust crate is itself named `synapse` (see rust/Cargo.toml). Anything +# emitted under these while a task is being polled should be attributed to the +# caller's logcontext, never the sentinel. +# +# NB: bare "synapse" also matches every *Python* `synapse.*` record, so the +# attribution assertion below implicitly relies on nothing else logging during +# the pump (MemoryReactor with `advance(0)`, so no timed background work fires). +# If this test starts flaking on unrelated records, tighten this filter rather +# than weakening the assertion. +_RUST_LOGGER_ROOTS = frozenset( + {"reqwest", "hyper", "hyper_util", "h2", "rustls", "synapse_rust", "synapse"} +) + + +class RustLogContextTestCase(HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` + # below. Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` + # callbacks if it's already running and we rely on that to start the Tokio + # thread pool in Rust. + self._http_client = hs.get_proxied_http_client() + self._rust_http_client = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the Tokio + # thread pool to be stopped. + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.user_id = self.register_user("user1", "pass") + + def _check_current_logcontext(self, expected: str) -> None: + context = current_context() + assert isinstance(context, (LoggingContext, _Sentinel)), context + self.assertEqual(str(context), expected, f"expected {expected}, saw {context}") + + def _run_in_logcontext_and_pump( + self, name: str, body: Callable[[dict[str, object]], None] + ) -> dict[str, object]: + """Run `body` inside a fresh `LoggingContext(name)`, then pump the + reactor (yielding to the Tokio pool) until it sets `result["done"]`. + + Returns the `result` dict that `body` populated. Asserts that the + caller's logcontext is intact afterwards and that we end back in the + sentinel. + """ + self._check_current_logcontext("sentinel") + result: dict[str, object] = {} + + with LoggingContext(name=name, server_name="test_server"): + body(result) + + with PreserveLoggingContext(): + # Generous upper bound (the work is a real HTTP round-trip or DB + # hop on a possibly-loaded CI box); the loop exits early via + # `result["done"]`, and we fail below if it never gets set. + for _ in range(50000): + if result.get("done"): + break + # Let the Tokio worker threads make progress... + time.sleep(0) + # ...and run anything they scheduled back on the reactor. + self.reactor.advance(0) + + # The caller's logcontext must be intact after firing off the work. + self._check_current_logcontext(name) + + # ...and we must not have leaked it into the reactor. + self._check_current_logcontext("sentinel") + + self.assertTrue( + result.get("done"), + "work never finished; the test probably didn't pump long enough", + ) + return result + + def test_rust_log_records_attributed_to_caller_logcontext(self) -> None: + """A log record emitted from Rust on a tokio thread (reqwest connecting) + is attributed to the caller's logcontext via `LoggingContextFilter`, not + the sentinel.""" + records: list[tuple[str, object]] = [] + + class CapturingHandler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + records.append((record.name, getattr(record, "request", ""))) + + handler = CapturingHandler() + # The global filter is what copies `str(current_context())` onto the + # record as `record.request`; attach it so we observe what Synapse would. + handler.addFilter(LoggingContextFilter()) + + root = logging.getLogger() + root.addHandler(handler) + + # Turn up the Rust-side loggers so reqwest actually emits, and refresh + # pyo3-log's cached levels so it forwards them. + saved_levels = { + name: logging.getLogger(name).level for name in _RUST_LOGGER_ROOTS + } + for name in _RUST_LOGGER_ROOTS: + logging.getLogger(name).setLevel(logging.DEBUG) + reset_logging_config() + + try: + server = _StubServer() + self.addCleanup(server.shutdown) + + def body(result: dict[str, object]) -> None: + async def do() -> None: + try: + await self._rust_http_client.get( + url=server.endpoint, + response_limit=1 * 1024 * 1024, + ) + finally: + result["done"] = True + + run_in_background(do) + + self._run_in_logcontext_and_pump("http-caller", body) + finally: + root.removeHandler(handler) + for name, level in saved_levels.items(): + logging.getLogger(name).setLevel(level) + reset_logging_config() + + rust_records = [ + (name, req) + for (name, req) in records + if name.split(".", 1)[0] in _RUST_LOGGER_ROOTS + ] + self.assertTrue( + rust_records, + "expected at least one Rust-origin log record (e.g. reqwest connecting); " + f"captured loggers: {sorted({name for name, _ in records})}", + ) + for name, req in rust_records: + self.assertEqual( + req, + "http-caller", + f"Rust log record from {name!r} was attributed to {req!r}, " + "not the caller's logcontext", + ) + + def test_db_callback_runs_in_caller_logcontext(self) -> None: + """The Rust `/versions` handler's per-user feature lookup calls back into + Python via `run_python_awaitable`; the DB transaction it runs must be + accounted against the caller's logcontext. The failure mode is the + awaitable running in the sentinel instead, silently losing the + accounting.""" + versions_handler = self.hs.get_rust_handlers().versions + + def body(result: dict[str, object]) -> None: + async def do() -> None: + try: + context = current_context() + assert isinstance(context, LoggingContext) + before = context.get_resource_usage().db_txn_count + + # Passing a user id makes the Rust handler do a per-user + # feature DB lookup (msc3881/msc3575 default to off), which + # goes Rust -> run_python_awaitable -> runInteraction. + await versions_handler.get_versions(self.user_id) + + after = context.get_resource_usage().db_txn_count + result["db_txn_delta"] = after - before + finally: + result["done"] = True + + run_in_background(do) + + result = self._run_in_logcontext_and_pump("db-caller", body) + + db_txn_delta = result.get("db_txn_delta", 0) + assert isinstance(db_txn_delta, int) + self.assertGreaterEqual( + db_txn_delta, + 1, + "the Rust handler's DB work was not accounted against the caller's " + "logcontext — run_python_awaitable is not restoring it (ran in the " + "sentinel instead)", + ) + + +class _StubServer: + """A real HTTP server on a random port, served from a background thread.""" + + def __init__(self) -> None: + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"ok": true}') + + def log_message(self, format: str, *args: object) -> None: + pass + + self._server = HTTPServer(("127.0.0.1", 0), Handler) + self._thread = threading.Thread( + target=self._server.serve_forever, + name="StubServer", + kwargs={"poll_interval": 0.01}, + daemon=True, + ) + self._thread.start() + + @property + def endpoint(self) -> str: + return f"http://127.0.0.1:{self._server.server_port}/" + + def shutdown(self) -> None: + self._server.shutdown() + self._thread.join() diff --git a/tests/test_state.py b/tests/test_state.py index b69a09dab5..d2e50cf94b 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -245,6 +245,7 @@ def setUp(self) -> None: ) reactor, clock = get_clock() hs.config = default_config(server_name="tesths", parse=True) + hs.hostname = "tesths" hs.get_datastores.return_value = Mock( main=self.dummy_store, state_deletion=dummy_deletion_store, diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index a4114cdfcc..230257105d 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -20,13 +20,19 @@ # import logging +import threading +from contextlib import contextmanager from typing import Callable, Generator, cast +from unittest.mock import patch from twisted.internet import defer, reactor as _reactor +import synapse.logging.context as context_module from synapse.logging.context import ( SENTINEL_CONTEXT, + ContextRequest, LoggingContext, + LoggingContextFilter, PreserveLoggingContext, _Sentinel, current_context, @@ -34,6 +40,7 @@ nested_logging_context, run_coroutine_in_background, run_in_background, + set_current_context, ) from synapse.types import ISynapseReactor from synapse.util.clock import Clock @@ -700,6 +707,226 @@ def test_nested_logging_context(self) -> None: self.assertEqual(nested_context.name, "foo-bar") +# A stand-in `(ru_utime, ru_stime)` value to pass to `start()`/`stop()`. +# +# The code under test only checks whether an rusage was supplied at all +# (`None` means the platform doesn't track per-thread CPU); the values +# themselves don't matter here. A constant also avoids depending on +# `RUSAGE_THREAD` support on the platform running the tests. +_TRUTHY_RUSAGE = (0.0, 0.0) + + +@contextmanager +def _capture_logcontext_errors() -> Generator[list[str], None, None]: + """Capture the messages passed to `logcontext_error` while inside the block. + + `logcontext_error` is looked up as a module global at call time, so patching + the module attribute intercepts every call site (`LoggingContext`, + `PreserveLoggingContext`, `run_in_background`, ...). + """ + messages: list[str] = [] + with patch.object(context_module, "logcontext_error", messages.append): + yield messages + + +class LogContextErrorMessageTestCase(unittest.TestCase): + """Tests asserting the exact messages passed to `logcontext_error`, and + the conditions that trigger each one. + """ + + def setUp(self) -> None: + # These tests call `__enter__`/`__exit__`/`start`/`stop` by hand. Let's + # make sure the current context ends up back at the sentinel regardless + # of what a test does. + self.assertIs(current_context(), SENTINEL_CONTEXT) + self.addCleanup(set_current_context, SENTINEL_CONTEXT) + + def test_enter_wrong_previous_context(self) -> None: + with _capture_logcontext_errors() as messages: + # `ctx` records the sentinel as its `previous_context` (we are at the + # sentinel now), but we enter it while a *different* context is active. + ctx = LoggingContext(name="ctx", server_name="s") + other = LoggingContext(name="other", server_name="s") + other.__enter__() + + ctx.__enter__() + + self.assertEqual( + messages, + [ + "Expected previous context %r, found %r" + % (ctx.previous_context, other) + ], + ) + + def test_exit_context_was_lost(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.__enter__() + # Simulate the context being lost from under us before exit. + set_current_context(SENTINEL_CONTEXT) + + ctx.__exit__(None, None, None) + + self.assertEqual(messages, ["Expected logging context ctx was lost"]) + self.assertTrue(ctx.finished) + + def test_exit_found_different_context(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.__enter__() + other = LoggingContext(name="other", server_name="s") + other.__enter__() + + ctx.__exit__(None, None, None) + + self.assertEqual(messages, ["Expected logging context ctx but found other"]) + + def test_preserve_logging_context_was_lost(self) -> None: + with _capture_logcontext_errors() as messages: + target = LoggingContext(name="target", server_name="s") + preserve = PreserveLoggingContext(target) + preserve.__enter__() + set_current_context(SENTINEL_CONTEXT) + + preserve.__exit__(None, None, None) + + self.assertEqual(messages, ["Expected logging context target was lost"]) + + def test_preserve_logging_context_found_different_context(self) -> None: + with _capture_logcontext_errors() as messages: + target = LoggingContext(name="target", server_name="s") + preserve = PreserveLoggingContext(target) + preserve.__enter__() + other = LoggingContext(name="other", server_name="s") + other.__enter__() + + preserve.__exit__(None, None, None) + + self.assertEqual( + messages, ["Expected logging context target but found other"] + ) + + def test_start_on_different_thread(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + + # Call `start` on a different thread + thread = threading.Thread(target=ctx.start, args=(None,)) + thread.start() + thread.join() + + self.assertEqual(messages, ["Started logcontext ctx on different thread"]) + + def test_stop_on_different_thread(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + + # Call `stop` on a different thread + thread = threading.Thread(target=ctx.stop, args=(None,)) + thread.start() + thread.join() + + self.assertEqual(messages, ["Stopped logcontext ctx on different thread"]) + + def test_restart_finished_context(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.finished = True + + ctx.start(None) + + self.assertEqual(messages, ["Re-starting finished log context ctx"]) + + def test_restart_already_active_context(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + ctx.start(_TRUTHY_RUSAGE) + + ctx.start(_TRUTHY_RUSAGE) + + self.assertEqual(messages, ["Re-starting already-active log context ctx"]) + + def test_stop_without_start(self) -> None: + with _capture_logcontext_errors() as messages: + ctx = LoggingContext(name="ctx", server_name="s") + + ctx.stop(_TRUTHY_RUSAGE) + + self.assertEqual( + messages, + ["Called stop on logcontext ctx without recording a start rusage"], + ) + + +class LoggingContextFilterTestCase(unittest.TestCase): + """Tests for `LoggingContextFilter`, which is how logcontexts reach log + output. Checks that it fills record attributes from a real logcontext, and + that under the sentinel it only sets defaults for attributes that aren't + already present.""" + + def setUp(self) -> None: + self.assertIs(current_context(), SENTINEL_CONTEXT) + self.filter = LoggingContextFilter() + + def _make_record(self, **attrs: object) -> logging.LogRecord: + record = logging.LogRecord("test", logging.INFO, "path", 1, "msg", None, None) + for key, value in attrs.items(): + setattr(record, key, value) + return record + + def test_fills_record_from_real_context(self) -> None: + request = ContextRequest( + request_id="req-1", + ip_address="1.2.3.4", + site_tag="site", + requester="@u:test", + authenticated_entity="@u:test", + method="GET", + url="/path", + protocol="1.1", + user_agent="UA", + ) + with LoggingContext(name="ctx", server_name="myserver", request=request): + record = self._make_record() + self.assertTrue(self.filter.filter(record)) + + # Read via `__dict__` since these attributes are only ever set + # dynamically (they aren't declared on `logging.LogRecord`). + attrs = record.__dict__ + self.assertEqual(attrs["server_name"], "myserver") + # For backwards compatibility, `str(context)` is stored as `request`. + self.assertEqual(attrs["request"], "ctx") + self.assertEqual(attrs["ip_address"], "1.2.3.4") + self.assertEqual(attrs["site_tag"], "site") + self.assertEqual(attrs["requester"], "@u:test") + self.assertEqual(attrs["authenticated_entity"], "@u:test") + self.assertEqual(attrs["method"], "GET") + self.assertEqual(attrs["url"], "/path") + self.assertEqual(attrs["protocol"], "1.1") + self.assertEqual(attrs["user_agent"], "UA") + + def test_sentinel_sets_defaults_when_absent(self) -> None: + record = self._make_record() + self.assertTrue(self.filter.filter(record)) + + self.assertEqual( + record.__dict__["server_name"], "unknown_server_from_sentinel_context" + ) + self.assertEqual(record.__dict__["request"], "sentinel") + + def test_sentinel_does_not_clobber_preset_attributes(self) -> None: + # Simulate a 3rd-party log record that already carries its own attributes; + # under the sentinel the filter must leave them untouched. + record = self._make_record( + server_name="third-party-server", request="third-party-request" + ) + self.assertTrue(self.filter.filter(record)) + + self.assertEqual(record.__dict__["server_name"], "third-party-server") + self.assertEqual(record.__dict__["request"], "third-party-request") + + # a function which returns a deferred which has been "called", but # which had a function which returned another incomplete deferred on # its callback list, so won't yet call any other new callbacks. From 218fc912105c44d229fd6faf32e4b5efc0167fc2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 21 Sep 2026 13:40:31 +0100 Subject: [PATCH 32/41] Add a cache to state res to cache more cases (#20185) This is an attempt to better cache the cases where there are a large number of extremities to resolve over, which keep slightly changing. This spawns from seeing issues on matrix.org. We already have a cache over the exact state groups being resolved. However, we can do better by caching the inputs into state res (i.e. the conflicted sets), which are more likely to be constant across repeated state res in a room. We key this cache based on a sha256 hash, on the assumption that this will never conflict. Also includes a commit that removes needless copying of the state. --- changelog.d/20185.misc | 1 + synapse/state/__init__.py | 56 +++++- synapse/state/v2.py | 171 +++++++++++++++-- synapse/util/__init__.py | 40 ++-- synapse/util/caches/expiringcache.py | 48 ++--- tests/state/test_v2.py | 208 ++++++++++++++++++++- tests/test_state.py | 34 +++- tests/util/test_expiring_cache.py | 43 +++++ tests/util/test_mutable_overlay_mapping.py | 73 ++++++++ 9 files changed, 604 insertions(+), 70 deletions(-) create mode 100644 changelog.d/20185.misc diff --git a/changelog.d/20185.misc b/changelog.d/20185.misc new file mode 100644 index 0000000000..69125baad8 --- /dev/null +++ b/changelog.d/20185.misc @@ -0,0 +1 @@ +Add a cache to state resolution keyed off the conflicted events. diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index b22f91b37c..e5cd386213 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -32,7 +32,6 @@ ) import attr -from immutabledict import immutabledict from prometheus_client import Counter, Histogram from synapse.api.constants import EventTypes @@ -53,6 +52,7 @@ from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import StateMap, StrCollection from synapse.types.state import StateFilter +from synapse.util import MutableOverlayMapping from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.duration import Duration @@ -113,18 +113,13 @@ def __init__( # # This can be None if we have a `state_group` (as then we can fetch the # state from the DB.) - self._state: StateMap[str] | None = ( - immutabledict(state) if state is not None else None - ) - + self._state = state # the ID of a state group if one and only one is involved. # otherwise, None otherwise? self.state_group = state_group self.prev_group = prev_group - self.delta_ids: StateMap[str] | None = ( - immutabledict(delta_ids) if delta_ids is not None else None - ) + self.delta_ids = delta_ids async def get_state( self, @@ -174,14 +169,29 @@ def __len__(self) -> int: length = 0 if self._state: - length += len(self._state) + length += _state_map_size(self._state) if self.delta_ids: - length += len(self.delta_ids) + length += _state_map_size(self.delta_ids) return length or 1 # Make sure its not 0. +def _state_map_size(state_map: Mapping[Any, Any]) -> int: + """Estimate a proxy for the memory a state map holds, for sizing caches. + + Since state maps are often combinations of `ChainMap` and + `MutableOverlayMapping`, we look at the total number of entries across all + layers rather than just the number of distinct keys. This is both faster and + a more accurate proxy for memory usage. + """ + if isinstance(state_map, ChainMap): + return sum(_state_map_size(layer) for layer in state_map.maps) + if isinstance(state_map, MutableOverlayMapping): + return state_map.total_entries() + return len(state_map) + + class StateHandler: """Fetches bits of state from the stores, and does state resolution where necessary @@ -636,6 +646,31 @@ def __init__(self, hs: "HomeServer"): ) ) + # The result of resolving a conflicted set of state, keyed on a digest + # of the inputs to `_resolve_conflicted_set`. See + # `v2._conflict_cache_key`. + # + # This is different to `_state_cache` above, which caches the resolved + # state based on the state groups. This cache aims to address the case + # where resolving across different state groups often produces the same + # conflicted set, which we can then cache. + # + # We bound the size of the cache based on the size calculated by + # `_state_map_size`, which calculates a proxy for a rough estimate of + # the memory footprint of a state map. + self._conflict_resolution_cache: ExpiringCache[bytes, StateMap[str]] = ( + ExpiringCache( + cache_name="state_conflict_resolution_cache", + server_name=self.server_name, + hs=hs, + clock=self.clock, + max_len=100000, + expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, + size_callback=_state_map_size, + reset_expiry_on_get=True, + ) + ) + # # stuff for tracking time spent on state-res by room # @@ -798,6 +833,7 @@ async def resolve_events_with_store( state_sets, event_map, state_res_store, + conflict_cache=self._conflict_resolution_cache, ) finally: self._record_state_res_metrics(room_id, m.get_resource_usage()) diff --git a/synapse/state/v2.py b/synapse/state/v2.py index 1241a4d66e..b384964bac 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -18,18 +18,24 @@ # # +import hashlib import heapq import itertools +import json import logging +from collections import ChainMap from typing import ( + AbstractSet, Any, Awaitable, Callable, Generator, Iterable, Literal, + MutableMapping, Protocol, Sequence, + cast, overload, ) @@ -39,7 +45,8 @@ from synapse.api.room_versions import RoomVersion, StateResolutionVersions from synapse.events import EventBase, is_creator from synapse.storage.databases.main.event_federation import StateDifference -from synapse.types import MutableStateMap, StateMap, StrCollection +from synapse.types import MutableStateMap, StateKey, StateMap, StrCollection +from synapse.util import MutableOverlayMapping from synapse.util.duration import Duration logger = logging.getLogger(__name__) @@ -86,6 +93,7 @@ async def resolve_events_with_store( state_sets: Sequence[StateMap[str]], event_map: dict[str, EventBase] | None, state_res_store: StateResolutionStore, + conflict_cache: "ConflictCache | None" = None, ) -> StateMap[str]: """Resolves the state using the v2 state resolution algorithm @@ -104,6 +112,10 @@ async def resolve_events_with_store( If None, all events will be fetched via state_res_store. state_res_store: + conflict_cache: + if given, the resolution of the conflicted set is looked up here + before it is computed, and stored here afterwards. See + `_conflict_cache_key` for what the key covers. Returns: A map from (type, state_key) to event_id. @@ -163,6 +175,145 @@ async def resolve_events_with_store( logger.debug("%d full_conflicted_set entries", len(full_conflicted_set)) + # Calculate the base state. + # + # v2 uses the unconflicted state as the base state, but v2.1 uses the empty + # set. + base_state: StateMap[str] = {} + if room_version.state_res != StateResolutionVersions.V2_1: + # Resolving conflicted sets requires the following types from the base + # state: + # - the `auth_types_for_event(..)` of the conflicted events for + # `_iterative_auth_checks` + # - the power levels for `_mainline_sort` + # + # We can therefore safely restrict the base state to those keys, which + # keeps keys that cannot affect the outcome out of the cache key. The + # rest of the unconflicted state is layered back on below. + base_state_keys = {(EventTypes.PowerLevels, "")} + for event_id in full_conflicted_set: + base_state_keys.update( + event_auth.auth_types_for_event(room_version, event_map[event_id]) + ) + + base_state = { + key: unconflicted_state[key] + for key in base_state_keys + if key in unconflicted_state + } + + resolved_state: StateMap[str] | None = None + if conflict_cache is not None: + cache_key = _conflict_cache_key( + room_id, room_version, full_conflicted_set, base_state + ) + resolved_state = conflict_cache.get(cache_key) + + if resolved_state is None: + resolved_state = await _resolve_conflicted_set( + clock, + room_id, + room_version, + full_conflicted_set, + base_state, + event_map, + state_res_store, + ) + if conflict_cache is not None: + conflict_cache[cache_key] = resolved_state + else: + logger.debug( + "Reusing the resolution of %d conflicted events", + len(full_conflicted_set), + ) + + logger.debug("done") + + # Finally, we copy the unconflicted state over the resolved state. + # + # We use a `ChainMap` here to avoid a copy. + # + # `ChainMap` expects mutable mappings as it is a mutable mapping. However, + # the return type of this function is an immutable mapping so it is safe to + # cast the underlying mappings to mutable mappings to satify `ChainMap`'s + # signature. + return ChainMap( + cast(MutableMapping[StateKey, str], unconflicted_state), + cast(MutableMapping[StateKey, str], resolved_state), + ) + + +class ConflictCache(Protocol): + """What `resolve_events_with_store` needs from a `conflict_cache`. + + Satisfied by a plain `dict` and by `ExpiringCache`. + + Mainly used to allow unit tests to pass a `dict` rather than building a full + cache. + """ + + def get(self, key: bytes) -> StateMap[str] | None: ... + + def __setitem__(self, key: bytes, value: StateMap[str]) -> None: ... + + +def _conflict_cache_key( + room_id: str, + room_version: RoomVersion, + full_conflicted_set: AbstractSet[str], + base_state: StateMap[str], +) -> bytes: + """Create a key for the conflict cache. Incorporates everything + that would affect the result of `_resolve_conflicted_set`. + + We use a digest as the key as the inputs can be very large. + """ + + # We use JSON as the serialization format for ease, we could use a + # hand-rolled format but one has to be careful to ensure that you can't get + # collisions (given in some room versions the room ID is arbitrary bytes + # rather than a hash). + key_material = json.dumps( + { + "room_id": room_id, + "room_version": room_version.identifier, + "conflicted_set": sorted(full_conflicted_set), + "base_state": sorted(base_state.values()), + } + ) + return hashlib.sha256(key_material.encode("utf-8")).digest() + + +async def _resolve_conflicted_set( + clock: Clock, + room_id: str, + room_version: RoomVersion, + full_conflicted_set: set[str], + base_state: StateMap[str], + event_map: dict[str, EventBase], + state_res_store: StateResolutionStore, +) -> MutableStateMap[str]: + """Apply the conflicted events to the base state. + + This is the expensive part of `resolve_events_with_store`: the reverse + topological power sort, the two iterative auth check passes and the + mainline sort. + + Args: + clock + room_id: the room we are working in + room_version: the room version + full_conflicted_set: the conflicted events plus the auth chain + difference. All must be present in `event_map`. + base_state: the state to apply the conflicted events to + event_map: updated in place with the events fetched along the way + state_res_store + + Returns: + The base state with the conflicted events that passed auth applied + over it. The unconflicted state is not merged in. + """ + # Get and sort all the power events (kicks/bans/etc) power_events = ( eid for eid in full_conflicted_set if _is_power_event(event_map[eid]) @@ -174,17 +325,6 @@ async def resolve_events_with_store( logger.debug("sorted %d power events", len(sorted_power_events)) - # v2.1 starts iterative auth checks from the empty set and not the unconflicted state. - # It relies on IAC behaviour which populates the base state with the events from auth_events - # if the state tuple is missing from the base state. This ensures the base state is only - # populated from auth_events rather than whatever the unconflicted state is (which could be - # completely bogus). - base_state = ( - {} - if room_version.state_res == StateResolutionVersions.V2_1 - else unconflicted_state - ) - # Now sequentially auth each one resolved_state = await _iterative_auth_checks( clock, @@ -227,11 +367,6 @@ async def resolve_events_with_store( logger.debug("resolved") - # We make sure that unconflicted state always still applies. - resolved_state.update(unconflicted_state) - - logger.debug("done") - return resolved_state @@ -691,7 +826,7 @@ async def _iterative_auth_checks( Returns: Returns the final updated state """ - resolved_state = dict(base_state) + resolved_state = MutableOverlayMapping(base_state) for idx, event_id in enumerate(event_ids, start=1): event = event_map[event_id] diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 977a662d7a..38e57e572e 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -151,19 +151,35 @@ def __iter__(self) -> Iterator[K]: yield key def __len__(self) -> int: - count = len(self._underlying_map) - for key in self._deletions: - if key in self._underlying_map: - count -= 1 - - for key in self._mutable_map: - # `key` should not be in both _mutable_map and _deletions - assert key not in self._deletions - - if key not in self._underlying_map: - count += 1 + # The distinct keys can be calculated via `(underlying ∪ mutable) − + # deletions`. A key is never in both `_mutable_map` and `_deletions`, so + # the only deletions to subtract are those of keys in the underlying + # map: + # + # |underlying| + |mutable| − |mutable ∩ underlying| − |underlying ∩ deletions| + # + # The intersections run at C speed and iterates over the smaller + # operand, so this is much cheaper than a Python loop over every key. + underlying_keys = self._underlying_map.keys() + return ( + len(self._underlying_map) + + len(self._mutable_map) + - len(self._mutable_map.keys() & underlying_keys) + - len(underlying_keys & self._deletions) + ) - return count + def total_entries(self) -> int: + """The number of entries held across the underlying map, the + overrides and the deletions, following nested overlays down. + + Useful for estimating the memory usage of the overlay mapping. + """ + underlying = self._underlying_map + if isinstance(underlying, MutableOverlayMapping): + underlying_size = underlying.total_entries() + else: + underlying_size = len(underlying) + return underlying_size + len(self._mutable_map) + len(self._deletions) def clear(self) -> None: self._underlying_map = {} diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 87870f4223..367ec6b413 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -24,8 +24,8 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Generic, - Iterable, Literal, TypeVar, overload, @@ -66,6 +66,7 @@ def __init__( expiry_ms: int = 0, reset_expiry_on_get: bool = False, iterable: bool = False, + size_callback: Callable[[VT], int] | None = None, ): """ Args: @@ -83,7 +84,18 @@ def __init__( an item on access. Defaults to False. iterable: If true, the size is calculated by summing the sizes of all entries, rather than the number of entries. + Shorthand for `size_callback=len`. + size_callback: If given, the size of the cache is the sum of this + function over all values, rather than the number of entries. """ + if iterable and size_callback is not None: + raise ValueError("`iterable` and `size_callback` are exclusive") + if iterable: + # type-ignore: if `iterable` is true, then the value type VT should + # be Sized (i.e. have a `__len__` method). We don't enforce this via + # the type system at present. + size_callback = len # type: ignore[assignment] + self._cache_name = cache_name self._original_max_size = max_len @@ -97,7 +109,7 @@ def __init__( self._cache: OrderedDict[KT, _CacheEntry[VT]] = OrderedDict() - self.iterable = iterable + self._size_callback = size_callback self.metrics = register_cache( cache_type="expiring", @@ -124,13 +136,7 @@ def evict(self) -> None: # Evict if there are now too many items while self._max_size and len(self) > self._max_size: _key, value = self._cache.popitem(last=False) - if self.iterable: - # type-ignore, here and below: if self.iterable is true, then the value - # type VT should be Sized (i.e. have a __len__ method). We don't enforce - # this via the type system at present. - self.metrics.inc_evictions(EvictionReason.size, len(value.value)) # type: ignore[arg-type] - else: - self.metrics.inc_evictions(EvictionReason.size) + self.metrics.inc_evictions(EvictionReason.size, self._size_of(value.value)) def __getitem__(self, key: KT) -> VT: try: @@ -161,10 +167,9 @@ def pop(self, key: KT, default: T = SENTINEL) -> VT | T: raise KeyError(key) return default - if self.iterable: - self.metrics.inc_evictions(EvictionReason.invalidation, len(value.value)) - else: - self.metrics.inc_evictions(EvictionReason.invalidation) + self.metrics.inc_evictions( + EvictionReason.invalidation, self._size_of(value.value) + ) return value.value @@ -207,10 +212,7 @@ async def _prune_cache(self) -> None: for k in keys_to_delete: value = self._cache.pop(k) - if self.iterable: - self.metrics.inc_evictions(EvictionReason.time, len(value.value)) # type: ignore[arg-type] - else: - self.metrics.inc_evictions(EvictionReason.time) + self.metrics.inc_evictions(EvictionReason.time, self._size_of(value.value)) logger.debug( "[%s] _prune_cache before: %d, after len: %d", @@ -219,12 +221,16 @@ async def _prune_cache(self) -> None: len(self), ) + def _size_of(self, value: VT) -> int: + """How much a value counts towards `max_len`.""" + if self._size_callback is None: + return 1 + return self._size_callback(value) + def __len__(self) -> int: - if self.iterable: - g: Iterable[int] = (len(entry.value) for entry in self._cache.values()) # type: ignore[arg-type] - return sum(g) - else: + if self._size_callback is None: return len(self._cache) + return sum(self._size_callback(entry.value) for entry in self._cache.values()) def set_cache_factor(self, factor: float) -> bool: """ diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index a9924e28b1..36208bb82e 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -23,18 +23,21 @@ Collection, Iterable, Mapping, + Sequence, TypeVar, ) import attr +from parameterized import parameterized from twisted.internet import defer from synapse.api.constants import EventTypes, JoinRules, Membership -from synapse.api.room_versions import RoomVersions +from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.event_auth import auth_types_for_event from synapse.events import EventBase from synapse.state.v2 import ( + ConflictCache, _get_auth_chain_difference, _get_power_level_for_sender, lexicographical_topological_sort, @@ -181,6 +184,59 @@ def to_event(self, auth_events: list[str], prev_events: list[str]) -> EventBase: INITIAL_EDGES = ["START", "IMZ", "IMC", "IMB", "IJR", "IPOWER", "IMA", "CREATE"] +ZARA_KEY = (EventTypes.Member, ZARA) +TOPIC_KEY = (EventTypes.Topic, "") + + +def _member(node_id: str, sender: str, state_key: str, content: dict) -> FakeEvent: + return FakeEvent( + id=node_id, + sender=sender, + type=EventTypes.Member, + state_key=state_key, + content=content, + ) + + +# Events for the conflict cache tests. All branch off START. +# +# PA is a power levels event that gives Bob PL 50. T1 and T2 are topic changes +# by Bob. Bob has no power under IPOWER, so they only pass auth if PA is in the +# state. +# +# ZJ1 and ZJ2 are Zara re-joining on two branches, and INV1 and INV2 are +# invites she sends to Evelyn on each branch. The invites pull the joins into +# the auth chain difference (see `test_conflict_cache_key_repartitioned`). +CACHE_TEST_CASE_EVENTS = [ + FakeEvent( + id="PA", + sender=ALICE, + type=EventTypes.PowerLevels, + state_key="", + content={"users": {ALICE: 100, BOB: 50}}, + ), + FakeEvent(id="T1", sender=BOB, type=EventTypes.Topic, state_key="", content={}), + FakeEvent(id="T2", sender=BOB, type=EventTypes.Topic, state_key="", content={}), + _member("ZJ1", ZARA, ZARA, MEMBERSHIP_CONTENT_JOIN), + _member("ZJ2", ZARA, ZARA, MEMBERSHIP_CONTENT_JOIN), + _member("INV1", ZARA, EVELYN, {"membership": Membership.INVITE}), + _member("INV2", ZARA, EVELYN, {"membership": Membership.INVITE}), +] + +CACHE_TEST_CASE_EDGES = [ + ["PA", "START"], + ["T1", "START"], + ["T2", "START"], + ["INV1", "ZJ1", "START"], + ["INV2", "ZJ2", "START"], +] + +# Room versions that use v2 and v2.1 state resolution respectively. Both have +# the same auth rules. The difference is that v2.1 starts the iterative auth +# checks from empty state rather than from the unconflicted state. +V2_ROOM = RoomVersions.V11 +V21_ROOM = RoomVersions.HydraV11 + class StateTestCase(unittest.TestCase): def test_ban_vs_pl(self) -> None: @@ -453,21 +509,137 @@ def test_mainline_sort(self) -> None: self.do_check(events, edges, expected_state_ids) - def do_check( + # Helpers for the conflict cache tests. These use a plain dict as the + # cache, so `len(conflict_cache)` after a call tells us whether it was a + # hit or a miss. + + def _build_cache_scenario(self) -> None: + self.event_map, self.state_at_event = self.build_event_graph( + CACHE_TEST_CASE_EVENTS, CACHE_TEST_CASE_EDGES + ) + + def _state(self, *node_ids: str) -> StateMap[str]: + """The state at START with the given events applied on top.""" + state = dict(self.state_at_event["START"]) + for node_id in node_ids: + event = self.event_map[EventID(node_id, "example.com").to_string()] + state[(event.type, event.state_key)] = event.event_id + return state + + def _resolve_with_cache( + self, + room_version: RoomVersion, + state_sets: Sequence[StateMap[str]], + conflict_cache: ConflictCache, + ) -> StateMap[str]: + return self.successResultOf( + defer.ensureDeferred( + resolve_events_with_store( + FakeClock(), + ROOM_ID, + room_version, + state_sets, + event_map=None, + state_res_store=TestStateResolutionStore(self.event_map), + conflict_cache=conflict_cache, + ) + ) + ) + + @parameterized.expand((V2_ROOM, V21_ROOM)) + def test_conflict_cache_shared_across_unconflicted_state( + self, room_version: RoomVersion + ) -> None: + """Test that two resolutions with the same conflicted set but different + unconflicted state share a cache entry, and that each result still + includes its own unconflicted state. + + The unconflicted state differs on Zara's membership. The topic events + aren't authed against that, so under v2 it isn't part of the cache key. + """ + self._build_cache_scenario() + conflict_cache: dict[bytes, StateMap[str]] = {} + + with_zara = [self._state("PA", "T1"), self._state("PA", "T2")] + without_zara = [ + {key: value for key, value in state.items() if key != ZARA_KEY} + for state in with_zara + ] + + first = self._resolve_with_cache(room_version, with_zara, conflict_cache) + second = self._resolve_with_cache(room_version, without_zara, conflict_cache) + self.assertEqual(len(conflict_cache), 1, "expected a cache hit") + + self.assertIn(ZARA_KEY, first) + self.assertNotIn(ZARA_KEY, second) + + # Everything else agrees. + self.assertEqual({k: v for k, v in first.items() if k != ZARA_KEY}, second) + + def test_conflict_cache_keys_on_base_state(self) -> None: + """Test that under v2 the cache key includes the unconflicted state the + auth checks depend on. Changing the power levels is a cache miss and + gives a different result.""" + self._build_cache_scenario() + + powerless = [self._state("T1"), self._state("T2")] + powered = [self._state("PA", "T1"), self._state("PA", "T2")] + + conflict_cache: dict[bytes, StateMap[str]] = {} + under_ipower = self._resolve_with_cache(V2_ROOM, powerless, conflict_cache) + under_pa = self._resolve_with_cache(V2_ROOM, powered, conflict_cache) + self.assertEqual(len(conflict_cache), 2, "expected a cache miss") + + # Bob's topics fail auth under IPOWER and pass under PA. + self.assertNotIn(TOPIC_KEY, under_ipower) + self.assertIn(TOPIC_KEY, under_pa) + + def test_conflict_cache_key_repartitioned(self) -> None: + """Test that a cached result is correct when the same conflicted set is + split differently between conflicted and unconflicted keys. + + In the first call Zara's membership is unconflicted, but ZJ1 and ZJ2 are + in the auth chain difference (via the invites) and so are in the + conflicted set. In the second call Zara's membership is itself + conflicted. Both calls have the same cache key, so the cached result + must include the resolved Zara membership, even though the first call + overrides it with its unconflicted state. + """ + self._build_cache_scenario() + + agreed = [self._state("INV1"), self._state("INV2")] + conflicting = [self._state("ZJ1", "INV1"), self._state("ZJ2", "INV2")] + + conflict_cache: dict[bytes, StateMap[str]] = {} + agreed_result = self._resolve_with_cache(V21_ROOM, agreed, conflict_cache) + warm = self._resolve_with_cache(V21_ROOM, conflicting, conflict_cache) + self.assertEqual(len(conflict_cache), 1, "expected a cache hit") + + # The first call's unconflicted state takes precedence over the cached + # resolution. + self.assertEqual(agreed_result[ZARA_KEY], self._state()[ZARA_KEY]) + + # The second call gets the winner from the cached resolution, the same + # one it computes from cold. + conflict_cache.clear() + cold = self._resolve_with_cache(V21_ROOM, conflicting, conflict_cache) + self.assertNotEqual(warm[ZARA_KEY], agreed_result[ZARA_KEY]) + self.assertEqual(warm, cold) + + def build_event_graph( self, events: list[FakeEvent], edges: list[list[str]], - expected_state_ids: list[str], - ) -> None: - """Take a list of events and edges and calculate the state of the - graph at END, and asserts it matches `expected_state_ids` + ) -> tuple[dict[str, EventBase], dict[str, StateMap[str]]]: + """Build the graph of `INITIAL_EVENTS` plus `events`. Args: events edges: A list of chains of event edges, e.g. `[[A, B, C]]` are edges A->B and B->C. - expected_state_ids: The expected state at END, (excluding - the keys that haven't changed since START). + + Returns: + The events by event ID, and the state after each node ID. """ # We want to sort the events into topological order for processing. graph: dict[str, set[str]] = {} @@ -539,6 +711,26 @@ def do_check( state_at_event[node_id] = state_after event_map[event_id] = event + return event_map, state_at_event + + def do_check( + self, + events: list[FakeEvent], + edges: list[list[str]], + expected_state_ids: list[str], + ) -> None: + """Take a list of events and edges and calculate the state of the + graph at END, and asserts it matches `expected_state_ids` + + Args: + events + edges: A list of chains of event edges, e.g. + `[[A, B, C]]` are edges A->B and B->C. + expected_state_ids: The expected state at END, (excluding + the keys that haven't changed since START). + """ + event_map, state_at_event = self.build_event_graph(events, edges) + expected_state = {} for node_id in expected_state_ids: # expected_state_ids are node IDs rather than event IDs, diff --git a/tests/test_state.py b/tests/test_state.py index d2e50cf94b..bb0b7f8878 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +from collections import ChainMap from typing import ( Any, Collection, @@ -34,9 +35,15 @@ from synapse.api.room_versions import RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry +from synapse.state import ( + StateHandler, + StateResolutionHandler, + _make_state_cache_entry, + _state_map_size, +) from synapse.types import JsonDict, MutableStateMap, StateMap from synapse.types.state import StateFilter +from synapse.util import MutableOverlayMapping from synapse.util.macaroons import MacaroonGenerator from tests import unittest @@ -864,6 +871,31 @@ def _get_context( result = yield defer.ensureDeferred(self.state.compute_event_context(event)) return result + def test_state_map_size(self) -> None: + "Cache sizing counts every held entry, not the distinct keys" + + base: StateMap[str] = {("a", ""): "A", ("b", ""): "B", ("c", ""): "C"} + self.assertEqual(_state_map_size(base), 3) + + # Overriding and deleting keys leaves `len()` alone but holds entries. + overlay = MutableOverlayMapping(base) + overlay[("a", "")] = "A2" + overlay[("d", "")] = "D" + del overlay[("b", "")] + self.assertEqual(len(overlay), 3) + self.assertEqual(_state_map_size(overlay), 3 + 2 + 1) + + # Nested overlays are followed down. + outer = MutableOverlayMapping(overlay) + outer[("e", "")] = "E" + self.assertEqual(len(outer), 4) + self.assertEqual(_state_map_size(outer), 6 + 1) + + # A `ChainMap` holds every layer in full, however much they overlap. + chain = ChainMap({("a", ""): "A3", ("f", ""): "F"}, outer) + self.assertEqual(len(chain), 5) + self.assertEqual(_state_map_size(chain), 2 + 7) + def test_make_state_cache_entry(self) -> None: "Test that calculating a prev_group and delta is correct" diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index 8964359a6e..8346cab4f0 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -87,6 +87,49 @@ def test_iterable_eviction(self) -> None: self.assertEqual(cache.get("key3"), [4, 5]) self.assertEqual(cache.get("key4"), [6, 7]) + def test_size_callback_eviction(self) -> None: + reactor, clock = get_clock() + cache: ExpiringCache[str, list[int]] = ExpiringCache( + cache_name="test", + server_name="testserver", + hs=self.hs, + clock=clock, + max_len=5, + # Size each value by its sum, so that `len()` and the reported + # size disagree. + size_callback=sum, + ) + + cache["key"] = [1] + cache["key2"] = [2] + cache["key3"] = [2] + + self.assertEqual(cache.get("key"), [1]) + self.assertEqual(cache.get("key2"), [2]) + self.assertEqual(cache.get("key3"), [2]) + self.assertEqual(len(cache), 5) + + # Three entries of `len()` 1 each, but a size of 6, so the oldest two + # go. + cache["key4"] = [3] + self.assertEqual(cache.get("key"), None) + self.assertEqual(cache.get("key2"), None) + self.assertEqual(cache.get("key3"), [2]) + self.assertEqual(cache.get("key4"), [3]) + self.assertEqual(len(cache), 5) + + def test_iterable_and_size_callback_are_exclusive(self) -> None: + reactor, clock = get_clock() + with self.assertRaises(ValueError): + ExpiringCache( + cache_name="test", + server_name="testserver", + hs=self.hs, + clock=clock, + iterable=True, + size_callback=len, + ) + def test_time_eviction(self) -> None: reactor, clock = get_clock() cache: ExpiringCache[str, int] = ExpiringCache( diff --git a/tests/util/test_mutable_overlay_mapping.py b/tests/util/test_mutable_overlay_mapping.py index ed738919e4..85444635db 100644 --- a/tests/util/test_mutable_overlay_mapping.py +++ b/tests/util/test_mutable_overlay_mapping.py @@ -187,3 +187,76 @@ def test_key_presence(self) -> None: self.assertNotIn("c", mapping) self.assertIn("d", mapping) self.assertNotIn("e", mapping) + + def test_len_after_reset_and_redelete(self) -> None: + """len() must follow keys that move between the underlying map, the + overrides and the deletions.""" + underlying = {"a": 1, "b": 2, "c": 3} + mapping = MutableOverlayMapping(underlying) + + # Delete an underlying key, then set it again. + del mapping["a"] + self.assertEqual(len(mapping), 2) + mapping["a"] = 10 + self.assertEqual(len(mapping), 3) + + # Add a key only the overlay knows about, delete it, and add it back. + mapping["d"] = 4 + self.assertEqual(len(mapping), 4) + del mapping["d"] + self.assertEqual(len(mapping), 3) + mapping["d"] = 40 + self.assertEqual(len(mapping), 4) + + # Override then delete an underlying key. + mapping["b"] = 20 + del mapping["b"] + self.assertEqual(len(mapping), 3) + + self.assertEqual(len(mapping), len(dict(mapping))) + + def test_len_nested(self) -> None: + """An overlay over an overlay reports the right length.""" + inner = MutableOverlayMapping({"a": 1, "b": 2}) + inner["c"] = 3 + del inner["a"] + + outer = MutableOverlayMapping(inner) + self.assertEqual(len(outer), 2) + + outer["a"] = 10 # deleted in inner, so new to outer + outer["b"] = 20 # present in inner's underlying map + outer["c"] = 30 # present in inner's overrides + outer["d"] = 40 # new + self.assertEqual(len(outer), 4) + + del outer["b"] + del outer["d"] + self.assertEqual(len(outer), 2) + self.assertEqual(len(outer), len(dict(outer))) + + def test_total_entries(self) -> None: + """total_entries() counts the base map plus every override and + deletion, unlike len().""" + mapping = MutableOverlayMapping({"a": 1, "b": 2, "c": 3}) + self.assertEqual(mapping.total_entries(), 3) + + mapping["a"] = 10 # override: +1 entry, same length + mapping["d"] = 4 # new key: +1 entry, +1 length + del mapping["b"] # deletion: +1 entry, -1 length + self.assertEqual(len(mapping), 3) + self.assertEqual(mapping.total_entries(), 6) + + # Deleting an override drops it from the overrides and records the + # deletion, so the entry count is unchanged. + del mapping["d"] + self.assertEqual(len(mapping), 2) + self.assertEqual(mapping.total_entries(), 6) + + # Nested overlays are counted all the way down. + outer = MutableOverlayMapping(mapping) + outer["e"] = 5 + self.assertEqual(outer.total_entries(), 7) + + mapping.clear() + self.assertEqual(mapping.total_entries(), 0) From 9b5697d37805ffd2dd477c2e2da8796eac9d692c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 21 Sep 2026 18:48:37 +0100 Subject: [PATCH 33/41] Move per-homeserver Rust state into a `RustRuntime` object on the `HomeServer` (#20011) Previously the tokio runtime was stashed in a hidden attribute on the reactor object, installed lazily by whichever Rust code first needed it, and started via `callWhenRunning`. Instead, we create a `RustRuntime` (accessible via `HomeServer.get_rust_runtime()`) that holds any per-reactor Rust state, such as the tokio runtime. It is constructed lazily on use. Rust consumers (`HttpClient`, `VersionsHandler`, the Python DB pool wrapper) now receive the runtime or reactor handle explicitly, and the `reactor.run()` / manual-startup workarounds in tests are no longer needed. We also add helper wrappers in Rust for `Reactor` and `HomeServer` that exposes the needed functionality. The aim is to allow us to have a Rust-side clock (mainly to get the current time), that respects the unit test per-reactor time management. --------- Co-authored-by: Claude Opus 5 (1M context) --- changelog.d/20011.misc | 1 + rust/clippy.toml | 10 + rust/src/config/mod.rs | 1 + rust/src/deferred.rs | 32 ++- rust/src/handlers/mod.rs | 24 +- rust/src/handlers/versions.rs | 9 +- rust/src/homeserver.rs | 102 ++++++++ rust/src/http_client.rs | 15 +- rust/src/lib.rs | 5 +- rust/src/msc4388_rendezvous/mod.rs | 22 +- rust/src/reactor.rs | 55 +++++ rust/src/rendezvous/mod.rs | 28 +-- rust/src/runtime.rs | 217 ++++++++++++++++++ rust/src/storage/db/python_db_pool.rs | 5 +- rust/src/tokio_runtime.rs | 107 --------- synapse/api/auth/mas.py | 2 +- synapse/server.py | 13 ++ synapse/synapse_rust/http_client.pyi | 6 +- synapse/synapse_rust/runtime.pyi | 15 ++ tests/handlers/test_oauth_delegation.py | 2 - tests/push/test_http.py | 19 -- tests/rest/client/test_delayed_events.py | 19 -- tests/rest/client/test_login_token_request.py | 14 -- tests/rest/client/test_matrixrtc.py | 19 -- tests/rest/client/test_versions.py | 19 -- tests/synapse_rust/test_http_client.py | 16 +- tests/synapse_rust/test_logcontext.py | 2 +- 27 files changed, 482 insertions(+), 297 deletions(-) create mode 100644 changelog.d/20011.misc create mode 100644 rust/clippy.toml create mode 100644 rust/src/homeserver.rs create mode 100644 rust/src/reactor.rs create mode 100644 rust/src/runtime.rs delete mode 100644 rust/src/tokio_runtime.rs create mode 100644 synapse/synapse_rust/runtime.pyi diff --git a/changelog.d/20011.misc b/changelog.d/20011.misc new file mode 100644 index 0000000000..9582106675 --- /dev/null +++ b/changelog.d/20011.misc @@ -0,0 +1 @@ +Refactor the Rust code to have a single place to store per-homeserver state. diff --git a/rust/clippy.toml b/rust/clippy.toml new file mode 100644 index 0000000000..bf2f715c8e --- /dev/null +++ b/rust/clippy.toml @@ -0,0 +1,10 @@ +# Blocking work spawned onto the tokio threadpool is leaked if it is still +# running when the per-homeserver `RustRuntime` is shut down (see +# `rust/src/runtime.rs`). We should avoid spawning blocking work until we figure +# out how to cancel/stop such work on shutdown. +disallowed-methods = [ + { path = "tokio::task::spawn_blocking", reason = "leaks past RustRuntime shutdown; see rust/src/runtime.rs" }, + { path = "tokio::runtime::Runtime::spawn_blocking", reason = "leaks past RustRuntime shutdown; see rust/src/runtime.rs" }, + { path = "tokio::runtime::Handle::spawn_blocking", reason = "leaks past RustRuntime shutdown; see rust/src/runtime.rs" }, + { path = "tokio::task::block_in_place", reason = "blocks a tokio worker thread; see rust/src/runtime.rs" }, +] diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index db00197107..aba29046db 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -48,6 +48,7 @@ pub struct AuthConfig { #[derive(FromPyObject, Clone)] pub struct ServerConfig { pub msc4140_enabled: bool, + pub public_baseurl: String, pub include_profile_updates_in_sync: bool, } diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 48ce283959..9f438993fb 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -26,7 +26,8 @@ use pyo3::{ use tokio::sync::oneshot; use crate::logging::context::with_logcontext; -use crate::tokio_runtime::runtime; +use crate::reactor::Reactor; +use crate::runtime::RustRuntime; create_exception!( synapse.synapse_rust.http_client, @@ -81,7 +82,7 @@ fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { /// but the work itself is wasted. pub fn create_deferred<'py, F, O>( py: Python<'py>, - reactor: &Bound<'py, PyAny>, + runtime: &RustRuntime, fut: F, ) -> PyResult> where @@ -98,12 +99,12 @@ where // current when the caller invoked us. See `crate::logging::context`. let logcontext = crate::logging::context::LogContextHandle::capture(py); - let rt = runtime(reactor)?; - let handle = rt.handle()?; + let handle = runtime.tokio_handle()?; let task = handle.spawn(logcontext.scope(fut)); - // Unbind the reactor so that we can pass it to the task - let reactor = reactor.clone().unbind(); + // Keep the runtime state (and, through it, the reactor) alive while the + // task is in flight. + let runtime = runtime.clone(); handle.spawn(async move { let res = task.await; @@ -117,19 +118,18 @@ where }, }; - // Re-bind the reactor - let reactor = reactor.bind(py); - // Send the result to the deferred, via `.callback(..)` or `.errback(..)` match res { Ok(obj) => { - reactor - .call_method("callFromThread", (deferred_callback, obj), None) + runtime + .reactor() + .call_from_thread(py, (deferred_callback, obj)) .expect("callFromThread should not fail"); // There's nothing we can really do with errors here } Err(err) => { - reactor - .call_method("callFromThread", (deferred_errback, err), None) + runtime + .reactor() + .call_from_thread(py, (deferred_errback, err)) .expect("callFromThread should not fail"); // There's nothing we can really do with errors here } } @@ -150,7 +150,7 @@ where /// the Twisted reactor and runs to completion regardless of whether the returned Rust /// future is ever polled; awaiting it only observes the result. pub(crate) async fn run_python_awaitable( - reactor: Py, + reactor: Reactor, make_awaitable: F, ) -> PyResult> where @@ -265,9 +265,7 @@ where }, )?; - reactor - .bind(py) - .call_method1(intern!(py, "callFromThread"), (starter,))?; + reactor.call_from_thread(py, (starter,))?; Ok(()) })?; diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index 5df019adff..3119113a0e 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -21,7 +21,7 @@ use pyo3::{ Bound, PyResult, Python, }; -use crate::config::SynapseHomeServerConfig; +use crate::homeserver::HomeServer; use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::Store; @@ -36,19 +36,17 @@ struct RustHandlers { impl RustHandlers { #[new] #[pyo3(signature = (homeserver))] - pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { - let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; + pub fn py_new(py: Python<'_>, homeserver: HomeServer) -> PyResult { + let config = homeserver.config(py)?; - // The Twisted reactor, used both to drive our Tokio runtime and to - // marshal database work back onto the reactor thread. - let reactor = homeserver.call_method0("get_reactor")?.unbind(); + // The per-homeserver Rust state, which gives us the tokio runtime + // and the Twisted reactor. + let runtime = homeserver.get_rust_runtime(py)?; - // hs.get_datastores().main.db_pool - let db_pool_py = homeserver - .call_method0("get_datastores")? - .getattr("main")? - .getattr("db_pool")?; - let db_pool = PythonDatabasePoolWrapper::new(&db_pool_py, reactor.clone_ref(py))?; + let db_pool = PythonDatabasePoolWrapper::new( + &homeserver.main_database_pool(py)?, + runtime.reactor().clone_ref(py), + )?; // Store is shared across all of the handlers so let's use an `Arc` let store = Arc::new(Store { @@ -64,7 +62,7 @@ impl RustHandlers { versions::VersionsHandler { global_unstable_feature_map: Arc::clone(&global_unstable_feature_map), store: Arc::clone(&store), - reactor: reactor.clone_ref(py), + runtime: runtime.clone(), }, )?; diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index ab4bd9a160..f3e6ad8f20 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -21,6 +21,7 @@ use serde::Serialize; use crate::config::{types::RoomCreationPreset, SynapseHomeServerConfig}; use crate::deferred::create_deferred; +use crate::runtime::RustRuntime; use crate::storage::store::{PerUserExperimentalFeature, Store}; /// `GET /_matrix/client/versions` response @@ -45,9 +46,9 @@ impl<'py> IntoPyObject<'py> for VersionsResponse { pub struct VersionsHandler { pub global_unstable_feature_map: Arc, pub store: Arc, - /// The Twisted reactor, used to bridge our `async` response back into a - /// Twisted deferred that Python can `await`. - pub reactor: Py, + /// The per-homeserver Rust state, used to bridge our `async` response + /// back into a Twisted deferred that Python can `await`. + pub runtime: RustRuntime, } #[pymethods] @@ -63,7 +64,7 @@ impl VersionsHandler { let store = Arc::clone(&self.store); let global_unstable_feature_map = Arc::clone(&self.global_unstable_feature_map); - create_deferred(py, self.reactor.bind(py), async move { + create_deferred(py, &self.runtime, async move { build_versions_response(&store, &global_unstable_feature_map, user_id.as_deref()) .await .map_err(|err| { diff --git a/rust/src/homeserver.rs b/rust/src/homeserver.rs new file mode 100644 index 0000000000..44f8ba905e --- /dev/null +++ b/rust/src/homeserver.rs @@ -0,0 +1,102 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! A typed wrapper around the Python `HomeServer`. + +use pyo3::{intern, prelude::*, types::PyDict}; + +use crate::config::SynapseHomeServerConfig; +use crate::reactor::Reactor; +use crate::runtime::RustRuntime; + +/// The Python `HomeServer`, as seen from Rust. +/// +/// This is a wrapper around a foreign Python object. No validation is +/// performed, on the assumption the Python side is correctly typed. +pub struct HomeServer(Py); + +impl<'a, 'py> FromPyObject<'a, 'py> for HomeServer { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult { + Ok(HomeServer(obj.to_owned().unbind())) + } +} + +impl HomeServer { + /// Fetch the Twisted reactor in use by this HomeServer. + pub fn get_reactor(&self, py: Python<'_>) -> PyResult { + self.0 + .bind(py) + .call_method0(intern!(py, "get_reactor"))? + .extract() + } + + /// Register a system event trigger with the HomeServer so it can be cleanly + /// removed when the HomeServer is shutdown. + pub fn register_sync_shutdown_handler( + &self, + py: Python<'_>, + callable: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let kwargs = PyDict::new(py); + kwargs.set_item(intern!(py, "phase"), intern!(py, "after"))?; + kwargs.set_item(intern!(py, "eventType"), intern!(py, "shutdown"))?; + kwargs.set_item(intern!(py, "shutdown_func"), callable)?; + self.0.bind(py).call_method( + intern!(py, "register_sync_shutdown_handler"), + (), + Some(&kwargs), + )?; + + Ok(()) + } + + /// The per-homeserver Rust state (`hs.get_rust_runtime()`), which gives + /// access to the tokio runtime and the reactor. + pub fn get_rust_runtime(&self, py: Python<'_>) -> PyResult { + Ok(self + .0 + .bind(py) + .call_method0(intern!(py, "get_rust_runtime"))? + .cast::()? + .get() + .clone()) + } + + /// The Rust-side view of `hs.config`. + pub fn config(&self, py: Python<'_>) -> PyResult { + self.0.bind(py).getattr(intern!(py, "config"))?.extract() + } + + /// The Synapse `Clock` (`hs.get_clock()`). + // TODO: give the clock a typed wrapper of its own. + pub fn get_clock(&self, py: Python<'_>) -> PyResult> { + Ok(self + .0 + .bind(py) + .call_method0(intern!(py, "get_clock"))? + .unbind()) + } + + /// The main database pool (`hs.get_datastores().main.db_pool`). + pub fn main_database_pool<'py>(&self, py: Python<'py>) -> PyResult> { + self.0 + .bind(py) + .call_method0(intern!(py, "get_datastores"))? + .getattr(intern!(py, "main"))? + .getattr(intern!(py, "db_pool")) + } +} diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index aaa1066a67..802f93b2d0 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -21,7 +21,7 @@ use reqwest::RequestBuilder; use crate::deferred::create_deferred; use crate::errors::HttpResponseException; -use crate::tokio_runtime::runtime; +use crate::runtime::RustRuntime; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -42,21 +42,18 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> #[pyclass] struct HttpClient { client: reqwest::Client, - reactor: Py, + runtime: RustRuntime, } #[pymethods] impl HttpClient { #[new] - #[pyo3(signature = (reactor, user_agent, http2_only = false))] + #[pyo3(signature = (runtime, user_agent, http2_only = false))] pub fn py_new( - reactor: Bound, + runtime: &RustRuntime, user_agent: &str, http2_only: bool, ) -> PyResult { - // Make sure the runtime gets installed - let _ = runtime(&reactor)?; - let mut builder = reqwest::Client::builder().user_agent(user_agent); if http2_only { @@ -69,7 +66,7 @@ impl HttpClient { Ok(HttpClient { client, - reactor: reactor.unbind(), + runtime: runtime.clone(), }) } @@ -107,7 +104,7 @@ impl HttpClient { builder: RequestBuilder, response_limit: usize, ) -> PyResult> { - create_deferred(py, self.reactor.bind(py), async move { + create_deferred(py, &self.runtime, async move { let response = builder.send().await.context("sending request")?; let status = response.status(); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f68b7c17d7..3d965daf51 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,6 +12,7 @@ pub mod duration; pub mod errors; pub mod events; pub mod handlers; +pub mod homeserver; pub mod http; pub mod http_client; pub mod identifier; @@ -20,11 +21,12 @@ pub mod logging; pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; +pub mod reactor; pub mod rendezvous; pub mod room_versions; +pub mod runtime; pub mod segmenter; pub mod storage; -pub mod tokio_runtime; pub mod types; lazy_static! { @@ -78,6 +80,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { handlers::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; + runtime::register_module(py, m)?; msc4388_rendezvous::register_module(py, m)?; segmenter::register_module(py, m)?; room_versions::register_module(py, m)?; diff --git a/rust/src/msc4388_rendezvous/mod.rs b/rust/src/msc4388_rendezvous/mod.rs index 2f1b004ec1..ffd4f240d9 100644 --- a/rust/src/msc4388_rendezvous/mod.rs +++ b/rust/src/msc4388_rendezvous/mod.rs @@ -21,7 +21,7 @@ use http::StatusCode; use pyo3::{ pyclass, pymethods, types::{PyAnyMethods, PyModule, PyModuleMethods}, - Bound, IntoPyObject, Py, PyAny, PyResult, Python, + Bound, Py, PyAny, PyResult, Python, }; use serde::Deserialize; use ulid::Ulid; @@ -30,9 +30,9 @@ use self::session::Session; use crate::{ duration::SynapseDuration, errors::{NotFoundError, SynapseError}, + homeserver::HomeServer, http::http_request_from_twisted, msc4388_rendezvous::session::{GetResponse, PostResponse, PutResponse}, - UnwrapInfallible, }; mod session; @@ -92,25 +92,21 @@ impl MSC4388RendezvousHandler { #[pyo3(signature = (homeserver, /, soft_limit=100, hard_limit=200,max_content_length=4*1024, eviction_interval=60*1000, ttl=2*60*1000))] fn new( py: Python<'_>, - homeserver: &Bound<'_, PyAny>, + homeserver: HomeServer, soft_limit: usize, hard_limit: usize, max_content_length: u64, eviction_interval: u64, ttl: u64, ) -> PyResult> { - let clock = homeserver - .call_method0("get_clock")? - .into_pyobject(py) - .unwrap_infallible() - .unbind(); + let clock = homeserver.get_clock(py)?; // Construct a Python object so that we can get a reference to the // evict method and schedule it to run. let self_ = Py::new( py, Self { - clock, + clock: clock.clone_ref(py), sessions: BTreeMap::new(), soft_limit, hard_limit, @@ -122,11 +118,9 @@ impl MSC4388RendezvousHandler { let eviction_duration = SynapseDuration::from_milliseconds(eviction_interval); let evict = self_.getattr(py, "_evict")?; - homeserver.call_method0("get_clock")?.call_method( - "looping_call", - (evict, &eviction_duration), - None, - )?; + clock + .bind(py) + .call_method("looping_call", (evict, &eviction_duration), None)?; Ok(self_) } diff --git a/rust/src/reactor.rs b/rust/src/reactor.rs new file mode 100644 index 0000000000..7522cd3390 --- /dev/null +++ b/rust/src/reactor.rs @@ -0,0 +1,55 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! A typed wrapper around the Twisted reactor. + +use pyo3::{call::PyCallArgs, intern, prelude::*}; + +/// The Twisted reactor, as seen from Rust. +/// +/// This is a wrapper around a foreign Python object. No validation is +/// performed, on the assumption the Python side is correctly typed. +pub struct Reactor(Py); + +impl<'a, 'py> FromPyObject<'a, 'py> for Reactor { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult { + Ok(Reactor(obj.to_owned().unbind())) + } +} + +impl Reactor { + /// `reactor.callFromThread(f, *args)`: schedule a call on the reactor + /// thread. This is the only reactor method that is safe to call from + /// other threads (e.g. tokio workers). + /// + /// `args` is the full argument tuple, starting with the callable itself. + pub fn call_from_thread<'py>( + &self, + py: Python<'py>, + args: impl PyCallArgs<'py>, + ) -> PyResult<()> { + self.0 + .bind(py) + .call_method1(intern!(py, "callFromThread"), args)?; + + Ok(()) + } + + pub fn clone_ref(&self, py: Python<'_>) -> Reactor { + Reactor(self.0.clone_ref(py)) + } +} diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index 9a6da9fcc3..64a5123f31 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -29,7 +29,7 @@ use pyo3::{ exceptions::PyValueError, pyclass, pymethods, types::{PyAnyMethods, PyModule, PyModuleMethods}, - Bound, IntoPyObject, Py, PyAny, PyResult, Python, + Bound, Py, PyAny, PyResult, Python, }; use ulid::Ulid; @@ -37,8 +37,8 @@ use self::session::Session; use crate::{ duration::SynapseDuration, errors::{NotFoundError, SynapseError}, + homeserver::HomeServer, http::{http_request_from_twisted, http_response_to_twisted, HeaderMapPyExt}, - UnwrapInfallible, }; mod session; @@ -113,25 +113,17 @@ impl RendezvousHandler { #[pyo3(signature = (homeserver, /, capacity=100, max_content_length=4*1024, eviction_interval=60*1000, ttl=60*1000))] fn new( py: Python<'_>, - homeserver: &Bound<'_, PyAny>, + homeserver: HomeServer, capacity: usize, max_content_length: u64, eviction_interval: u64, ttl: u64, ) -> PyResult> { - let base: String = homeserver - .getattr("config")? - .getattr("server")? - .getattr("public_baseurl")? - .extract()?; + let base = homeserver.config(py)?.server.public_baseurl; let base = Uri::try_from(format!("{base}_synapse/client/rendezvous")) .map_err(|_| PyValueError::new_err("Invalid base URI"))?; - let clock = homeserver - .call_method0("get_clock")? - .into_pyobject(py) - .unwrap_infallible() - .unbind(); + let clock = homeserver.get_clock(py)?; let eviction_duration = SynapseDuration::from_milliseconds(eviction_interval); @@ -141,7 +133,7 @@ impl RendezvousHandler { py, Self { base, - clock, + clock: clock.clone_ref(py), sessions: BTreeMap::new(), capacity, max_content_length, @@ -150,11 +142,9 @@ impl RendezvousHandler { )?; let evict = self_.getattr(py, "_evict")?; - homeserver.call_method0("get_clock")?.call_method( - "looping_call", - (evict, &eviction_duration), - None, - )?; + clock + .bind(py) + .call_method("looping_call", (evict, &eviction_duration), None)?; Ok(self_) } diff --git a/rust/src/runtime.rs b/rust/src/runtime.rs new file mode 100644 index 0000000000..46ceeec1a8 --- /dev/null +++ b/rust/src/runtime.rs @@ -0,0 +1,217 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! The per-homeserver state for the Rust side of Synapse. +//! +//! A [`RustRuntime`] is created once per homeserver (`hs.get_rust_runtime()`) +//! and holds everything the Rust side keeps for the lifetime of that +//! homeserver: currently the tokio thread pool and a handle to the Twisted +//! reactor. Rust consumers (e.g. the HTTP client) clone the inner +//! [`Arc`] at construction time and don't need the GIL (or +//! the Python-facing object) to reach it afterwards. +//! +//! The tokio runtime is shut down with the homeserver, via a handler +//! registered with `HomeServer.register_sync_shutdown_handler`. + +use std::ops::Deref; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; + +use anyhow::Context; +use pyo3::{exceptions::PyRuntimeError, prelude::*}; +use tokio::runtime::{Handle, Runtime}; + +use crate::homeserver::HomeServer; +use crate::reactor::Reactor; + +/// How long to wait for in-flight tokio tasks to be cancelled when shutting +/// down with the reactor. +/// +/// Note that any [`Runtime::spawn_blocking`] work that is still running when +/// the timeout expires is leaked, along with the worker thread running it. +/// +/// See [`tokio::runtime`] for details. +const SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); + +/// State of the lazily-started tokio runtime. +enum TokioState { + /// Not started yet; the runtime is built on first use. + NotStarted, + Running(Runtime), + /// Shut down with the homeserver. Cannot be restarted. + Shutdown, +} + +/// The state shared between the Python-facing [`RustRuntime`] handle and any +/// Rust-side consumers holding an `Arc` of this. +pub struct RustRuntimeInner { + reactor: Reactor, + tokio: Mutex, + worker_threads: usize, +} + +impl RustRuntimeInner { + /// The Twisted reactor this homeserver runs on. + pub fn reactor(&self) -> &Reactor { + &self.reactor + } + + /// Get a handle to the tokio runtime, starting the runtime if it hasn't + /// been started yet. + pub fn tokio_handle(&self) -> PyResult { + let mut state = self + .tokio + .lock() + .map_err(|_| PyRuntimeError::new_err("tokio runtime lock poisoned"))?; + + match &*state { + TokioState::Running(runtime) => Ok(runtime.handle().clone()), + TokioState::NotStarted => { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(self.worker_threads) + .enable_all() + .build() + .context("building tokio runtime")?; + let handle = runtime.handle().clone(); + *state = TokioState::Running(runtime); + Ok(handle) + } + TokioState::Shutdown => Err(PyRuntimeError::new_err( + "the tokio runtime has been shut down", + )), + } + } + + /// Shut the tokio runtime down, cancelling all in-flight tasks and waiting + /// for up to [`SHUTDOWN_TIMEOUT`] for them to finish. Called via + /// [`ShutdownHook`] when the reactor shuts down. + /// + /// Note that any [`Runtime::spawn_blocking`] work is leaked until it is + /// finished, along with the worker thread running it. + fn shutdown(&self, py: Python<'_>) -> PyResult<()> { + let mut state = self + .tokio + .lock() + .map_err(|_| PyRuntimeError::new_err("tokio runtime lock poisoned"))?; + let previous_state = std::mem::replace(&mut *state, TokioState::Shutdown); + // Don't hold the lock while blocking on the shutdown below. + drop(state); + + if let TokioState::Running(runtime) = previous_state { + // Shutdown the runtime, waiting for a small grace period for + // in-flight tasks to be cancelled. + // + // See [`tokio::runtime`] for details. + py.detach(|| runtime.shutdown_timeout(SHUTDOWN_TIMEOUT)); + } + + Ok(()) + } +} + +impl Drop for RustRuntimeInner { + fn drop(&mut self) { + // Backstop for homeservers whose shutdown trigger never fires (e.g. in + // tests). We use `shutdown_background` rather than a blocking shutdown, + // because the last `Arc` may be dropped from a task running on this + // very runtime, where blocking would panic. + if let Ok(state) = self.tokio.get_mut() { + if let TokioState::Running(runtime) = std::mem::replace(state, TokioState::Shutdown) { + runtime.shutdown_background(); + } + } + } +} + +/// A cheaply-clonable handle to the per-homeserver Rust state, and the +/// Python-facing class for it. +/// +/// One instance is constructed per homeserver by +/// `HomeServer.get_rust_runtime()`. Rust classes that need it take it as a +/// constructor argument and store their own clone, which is just an `Arc` +/// refcount bump. Derefs to [`RustRuntimeInner`]. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct RustRuntime { + inner: Arc, +} + +impl Deref for RustRuntime { + type Target = RustRuntimeInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[pymethods] +impl RustRuntime { + #[new] + #[pyo3(signature = (hs, worker_threads = 4))] + fn py_new(py: Python<'_>, hs: HomeServer, worker_threads: usize) -> PyResult { + let inner = Arc::new(RustRuntimeInner { + reactor: hs.get_reactor(py)?, + tokio: Mutex::new(TokioState::NotStarted), + worker_threads, + }); + + // Shut the tokio runtime down when the homeserver is shut down. The + // trigger holds only a `Weak` reference, as otherwise we risk a + // reference cycle passing through a Rust field that Python's GC cannot + // see into. + let hook = Py::new( + py, + ShutdownHook { + inner: Arc::downgrade(&inner), + }, + )?; + hs.register_sync_shutdown_handler(py, hook.bind(py).as_any())?; + + Ok(RustRuntime { inner }) + } +} + +/// The callable registered with `HomeServer.register_sync_shutdown_handler`, +/// which runs it on `HomeServer.shutdown()` or when the reactor stops. +#[pyclass(frozen)] +struct ShutdownHook { + inner: Weak, +} + +#[pymethods] +impl ShutdownHook { + fn __call__(&self, py: Python<'_>) -> PyResult<()> { + if let Some(inner) = self.inner.upgrade() { + inner.shutdown(py)?; + } + + Ok(()) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "runtime")?; + + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.runtime", child_module)?; + + Ok(()) +} diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 89003d574b..b683a9c00e 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -40,6 +40,7 @@ use pyo3::{ }; use crate::deferred::run_python_awaitable; +use crate::reactor::Reactor; use crate::storage::db::{ DatabasePool, DbRow, DbValue, ErasedInteraction, ErasedResult, Transaction, }; @@ -112,13 +113,13 @@ pub struct PythonDatabasePoolWrapper { /// never gets garbage collected and never points back at the homeserver, so it is /// not part of any reference cycle. Ideally, we could worry about it but /// practically probably doesn't matter. - reactor: Py, + reactor: Reactor, } impl PythonDatabasePoolWrapper { /// Build a wrapper around the Python `DatabasePool` (e.g. /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. - pub fn new(database_pool: &Bound<'_, PyAny>, reactor: Py) -> PyResult { + pub fn new(database_pool: &Bound<'_, PyAny>, reactor: Reactor) -> PyResult { Ok(Self { database_pool_py_ref: PyWeakrefReference::new(database_pool)?.unbind(), reactor, diff --git a/rust/src/tokio_runtime.rs b/rust/src/tokio_runtime.rs deleted file mode 100644 index f239040f30..0000000000 --- a/rust/src/tokio_runtime.rs +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is licensed under the Affero General Public License (AGPL) version 3. - * - * Copyright (C) 2026 Element Creations Ltd - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * See the GNU Affero General Public License for more details: - * . - * - */ - -use anyhow::Context; -use pyo3::prelude::*; -use tokio::runtime::Runtime; - -/// This is the name of the attribute where we store the runtime on the reactor -static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; - -/// A Python wrapper around a Tokio runtime. -/// -/// This allows us to 'store' the runtime on the reactor instance, starting it -/// when the reactor starts, and stopping it when the reactor shuts down. -#[pyclass] -pub struct PyTokioRuntime { - runtime: Option, -} - -#[pymethods] -impl PyTokioRuntime { - fn start(&mut self) -> PyResult<()> { - // TODO: allow customization of the runtime like the number of threads - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build()?; - - self.runtime = Some(runtime); - - Ok(()) - } - - fn shutdown(&mut self) -> PyResult<()> { - let runtime = self - .runtime - .take() - .context("Runtime was already shutdown")?; - - // Dropping the runtime will shut it down - drop(runtime); - - Ok(()) - } -} - -impl PyTokioRuntime { - /// Get the handle to the Tokio runtime, if it is running. - pub fn handle(&self) -> PyResult<&tokio::runtime::Handle> { - let handle = self - .runtime - .as_ref() - .context("Tokio runtime is not running")? - .handle(); - - Ok(handle) - } -} - -/// Get a handle to the Tokio runtime stored on the reactor instance, or create -/// a new one. -pub fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { - install_runtime(reactor)?; - } - - get_runtime(reactor) -} - -/// Install a new Tokio runtime on the reactor instance. -fn install_runtime(reactor: &Bound) -> PyResult<()> { - let py = reactor.py(); - let runtime = PyTokioRuntime { runtime: None }; - let runtime = runtime.into_pyobject(py)?; - - // Attach the runtime to the reactor, starting it when the reactor is - // running, stopping it when the reactor is shutting down - reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; - reactor.call_method1( - "addSystemEventTrigger", - ("after", "shutdown", runtime.getattr("shutdown")?), - )?; - reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; - - Ok(()) -} - -/// Get a reference to a Tokio runtime handle stored on the reactor instance. -fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is - // not a `Runtime`. Careful that this could happen if the user sets it - // manually, or if multiple versions of `pyo3-twisted` are used! - let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; - Ok(runtime.borrow()) -} diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index ed0427d6f3..d094410ff9 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -109,7 +109,7 @@ def __init__(self, hs: "HomeServer"): self._http_client = hs.get_proxied_http_client() self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), + runtime=hs.get_rust_runtime(), user_agent=self._http_client.user_agent.decode("utf8"), http2_only=self._config.force_http2, ) diff --git a/synapse/server.py b/synapse/server.py index 9bea208974..266440534c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -177,6 +177,7 @@ from synapse.synapse_rust.handlers import RustHandlers from synapse.synapse_rust.msc4388_rendezvous import MSC4388RendezvousHandler from synapse.synapse_rust.rendezvous import RendezvousHandler +from synapse.synapse_rust.runtime import RustRuntime from synapse.types import DomainSpecificString, ISynapseReactor from synapse.util import SYNAPSE_VERSION from synapse.util.caches import CACHE_METRIC_REGISTRY @@ -972,6 +973,18 @@ def get_set_password_handler(self) -> SetPasswordHandler: def get_rust_handlers(self) -> RustHandlers: return RustHandlers(self) + @cache_in_self + def get_rust_runtime(self) -> RustRuntime: + """The per-homeserver state for the Rust side of Synapse: the tokio + thread pool, plus anything else Rust code keeps for the lifetime of the + homeserver. + + The tokio runtime is started lazily on first use, and shut down when + this homeserver is shut down. + """ + # TODO: make the number of worker threads configurable + return RustRuntime(hs=self, worker_threads=4) + @cache_in_self def get_event_sources(self) -> EventSources: return EventSources(self) diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi index 1814daec73..e6468bc46b 100644 --- a/synapse/synapse_rust/http_client.pyi +++ b/synapse/synapse_rust/http_client.pyi @@ -14,7 +14,7 @@ from typing import Mapping from twisted.internet.defer import Deferred -from synapse.types import ISynapseReactor +from synapse.synapse_rust.runtime import RustRuntime class HttpClient: """ @@ -23,7 +23,7 @@ class HttpClient: def __init__( self, - reactor: ISynapseReactor, + runtime: RustRuntime, user_agent: str, http2_only: bool = False, ) -> None: @@ -31,7 +31,7 @@ class HttpClient: Create a new HTTP client backed by reqwest. Args: - reactor: The Twisted reactor to coordinate with + runtime: The per-homeserver Rust state (`hs.get_rust_runtime()`) user_agent: The user agent to use for requests http2_only: Whether to use HTTP/2 only, even on unencrypted connections. By default, it will always use HTTP/1.1 over unencrypted connections, and diff --git a/synapse/synapse_rust/runtime.pyi b/synapse/synapse_rust/runtime.pyi new file mode 100644 index 0000000000..32ae77ec28 --- /dev/null +++ b/synapse/synapse_rust/runtime.pyi @@ -0,0 +1,15 @@ +from synapse.server import HomeServer + +class RustRuntime: + """The per-homeserver state for the Rust side of Synapse. + + Holds the tokio thread pool (started lazily on first use, shut down with the + Homeserver). Rust classes that need them take this as a constructor + argument. Get it from `hs.get_rust_runtime()`. + """ + + def __init__( + self, + hs: HomeServer, + worker_threads: int = 4, + ) -> None: ... diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 71f83607ff..0fc1e17d6b 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -216,8 +216,6 @@ def default_config(self) -> dict[str, Any]: def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: self.server = FakeMasServer() hs = self.setup_test_homeserver() - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() self._auth = checked_cast(MasDelegatedAuth, hs.get_auth()) return hs diff --git a/tests/push/test_http.py b/tests/push/test_http.py index 47521a773f..bd2ada9155 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -31,7 +31,6 @@ from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import login, push_rule, pusher, receipts, room, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock @@ -1233,24 +1232,6 @@ class MSC3881VersionsTestCase(HomeserverTestCase): versions.register_servlets, ] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 3af7d13858..644b373f36 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -24,7 +24,6 @@ from synapse.rest import admin from synapse.rest.client import delayed_events, login, room, sync, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -41,24 +40,6 @@ class DelayedEventsUnstableSupportTestCase(HomeserverTestCase): servlets = [versions.register_servlets] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/rest/client/test_login_token_request.py b/tests/rest/client/test_login_token_request.py index d6b2cf054e..b387cb98fb 100644 --- a/tests/rest/client/test_login_token_request.py +++ b/tests/rest/client/test_login_token_request.py @@ -24,7 +24,6 @@ from synapse.rest import admin from synapse.rest.client import login, login_token_request, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -48,19 +47,6 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: self.hs.config.registration.auto_join_rooms = [] self.hs.config.captcha.enable_registration_captcha = False - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = self.hs.get_proxied_http_client() - _ = HttpClient( - reactor=self.hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - return self.hs def tearDown(self) -> None: diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index 63c7632a88..545a5a8e8c 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -27,7 +27,6 @@ from synapse.rest import admin from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -157,24 +156,6 @@ class MatrixRtcVersionsTestCase(HomeserverTestCase): servlets = [versions.register_servlets] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index 1ed6bb145b..6ecff9de6f 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -17,7 +17,6 @@ from synapse.rest import admin from synapse.rest.client import login, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock @@ -37,24 +36,6 @@ class VersionsTestCase(unittest.HomeserverTestCase): versions.register_servlets, ] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 - self._http_client = hs.get_proxied_http_client() - _ = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - def tearDown(self) -> None: # MemoryReactor doesn't trigger the shutdown phases, and we want the # Tokio thread pool to be stopped diff --git a/tests/synapse_rust/test_http_client.py b/tests/synapse_rust/test_http_client.py index 845fe2b503..330fbeeba6 100644 --- a/tests/synapse_rust/test_http_client.py +++ b/tests/synapse_rust/test_http_client.py @@ -86,25 +86,13 @@ def endpoint(self) -> str: class HttpClientTestCase(HomeserverTestCase): - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - hs = self.setup_test_homeserver() - - # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. - # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's - # already running and we rely on that to start the Tokio thread pool in Rust. In - # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self._http_client = hs.get_proxied_http_client() self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), + runtime=hs.get_rust_runtime(), user_agent=self._http_client.user_agent.decode("utf8"), ) - # This triggers the server startup hooks, which starts the Tokio thread pool - reactor.run() - - return hs - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.server = StubServer() def tearDown(self) -> None: diff --git a/tests/synapse_rust/test_logcontext.py b/tests/synapse_rust/test_logcontext.py index 0be82f0caa..5e1cefaff9 100644 --- a/tests/synapse_rust/test_logcontext.py +++ b/tests/synapse_rust/test_logcontext.py @@ -82,7 +82,7 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: # thread pool in Rust. self._http_client = hs.get_proxied_http_client() self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), + runtime=hs.get_rust_runtime(), user_agent=self._http_client.user_agent.decode("utf8"), ) From 7bbf1a5a48e7c319ba7a88687b41b5340d15e32f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:38:31 +0000 Subject: [PATCH 34/41] Bump libc from 0.2.174 to 0.2.189 in the patches group (#20249) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 111b78e7b7..6583a18d69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -732,9 +732,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" From f3ae5648778ae6dabb39dfdb8b3d47789ebe3a98 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 Sep 2026 13:16:35 +0100 Subject: [PATCH 35/41] Report each ID generator's current position as a metric (#20097) When a stream advances in the database but stops being replicated to a process, that process's view of the stream freezes. Requests that wait for it to catch up to a token issued by another worker then time out and return empty responses indefinitely (see #20080), and nothing exported said so. Report `get_current_token` from every ID generator, on every process. The value is comparable between processes, so a stream that has stopped reaching one of them shows up as divergence with no client traffic needed. It is also the position that `wait_for_stream_token` waits on, so its divergence is the failure itself rather than a proxy for it. Being a watermark over gapless runs of persisted IDs, it also catches a single writer of a sharded stream going quiet, which a maximum across writers would hide behind the writers still being replicated. --- changelog.d/20097.misc | 1 + synapse/storage/util/id_generators.py | 37 +++++++++++ tests/storage/test_id_generators.py | 31 ++++++++- tests/unittest.py | 93 +++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20097.misc diff --git a/changelog.d/20097.misc b/changelog.d/20097.misc new file mode 100644 index 0000000000..3c9cd6c86e --- /dev/null +++ b/changelog.d/20097.misc @@ -0,0 +1 @@ +Add a `synapse_storage_stream_current_position` metric, reporting each stream's current position as each worker process sees it. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index d294119bb2..fbf75d5a2a 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -36,9 +36,11 @@ ) import attr +from prometheus_client import Gauge from sortedcontainers import SortedList, SortedSet from synapse.logging import issue9533_logger +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.database import ( DatabasePool, @@ -55,6 +57,16 @@ logger = logging.getLogger(__name__) +# We report the calculated current position as seen by this worker. Note that +# this is always positive (even for negative streams like backfill) to make +# monitoring/alerting easier. +stream_current_position_gauge = Gauge( + "synapse_storage_stream_current_position", + "The stream's current position as this process sees it." + "Note, these are always positive, even for negative streams.", + labelnames=["stream_name", SERVER_NAME_LABEL], +) + T = TypeVar("T") @@ -232,6 +244,15 @@ def __init__( self._writers = writers self._return_factor = 1 if positive else -1 + # The `instance_name` label is what separates one worker's metrics from + # another's when there are multiple Synapse workers for the same homeserver + # running in the same process. In production the scrape labels would do that + # too, but tests run several homeservers in one process. + self._current_position_gauge = stream_current_position_gauge.labels( + stream_name=stream_name, + **{SERVER_NAME_LABEL: server_name}, + ) + # We lock as some functions may be called from DB threads. self._lock = threading.Lock() @@ -340,6 +361,20 @@ def __init__( # position with the current minimum. self._current_positions[self._instance_name] = self._persisted_upto_position + with self._lock: + self._report_current_position() + + def _report_current_position(self) -> None: + """Report the current position as a metric. + + Must be called with `_lock` held. + """ + assert self._lock.locked() + + # Note that we always report this as a positive value, so we don't + # multiply by the `_return_factor`. + self._current_position_gauge.set(self._persisted_upto_position) + def _load_current_ids( self, db_conn: LoggingDatabaseConnection, @@ -828,6 +863,8 @@ def _add_persisted_position(self, new_id: int) -> None: # do. break + self._report_current_position() + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795. # If this is the to-device stream, and we are a writer for that stream, log some stats if ( diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index a42247b498..0666eec00c 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -32,7 +32,10 @@ LoggingTransaction, ) from synapse.storage.types import Cursor -from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.storage.util.id_generators import ( + MultiWriterIdGenerator, + stream_current_position_gauge, +) from synapse.storage.util.sequence import ( LocalSequenceGenerator, PostgresSequenceGenerator, @@ -109,6 +112,14 @@ def _create(conn: LoggingDatabaseConnection) -> MultiWriterIdGenerator: ) return self.instances[instance_name] + def _get_reported_metric_position(self) -> int: + """The position the metric reports for the test stream.""" + + return self.get_prometheus_metric_current_value( + stream_current_position_gauge, + stream_name="test_stream", + ) + def _replicate(self, instance_name: str) -> None: """Similate a replication event for the given instance.""" @@ -229,6 +240,24 @@ async def _get_next_async() -> None: self.assertEqual(id_gen.get_positions(), {"master": 8}) self.assertEqual(id_gen.get_current_token_for_writer("master"), 8) + def test_current_position_metric(self) -> None: + """The reported position follows `get_current_token`.""" + + self._insert_rows("master", 7) + + id_gen = self._create_id_generator() + + self.assertEqual(self._get_reported_metric_position(), 7) + + async def _get_next_async() -> None: + async with id_gen.get_next(): + pass + + self.get_success(_get_next_async()) + + self.assertEqual(id_gen.get_current_token(), 8) + self.assertEqual(self._get_reported_metric_position(), 8) + def test_cancelled_enter_does_not_wedge_position(self) -> None: """Reproduces presence getting stuck. diff --git a/tests/unittest.py b/tests/unittest.py index 51d0514095..fee928bc4c 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -32,6 +32,7 @@ import time from collections.abc import Set from typing import ( + TYPE_CHECKING, Any, Awaitable, Callable, @@ -76,6 +77,7 @@ current_context, set_current_context, ) +from synapse.metrics import SERVER_NAME_LABEL from synapse.rest import RegisterServletsFunc from synapse.server import HomeServer from synapse.storage.keys import FetchKeyResult @@ -95,6 +97,9 @@ from tests.test_utils.logging_setup import setup_logging from tests.utils import checked_cast, default_config, setupdb +if TYPE_CHECKING: + from prometheus_client.registry import Collector + setupdb() setup_logging() @@ -1225,6 +1230,72 @@ def inject_room_member(self, room: str, user: str, membership: str) -> None: event_injection.inject_member_event(self.hs, room, user, membership) ) + def get_prometheus_metric_current_value( + self, metric: "Collector", **labels: str + ) -> int: + """Get the value of a prometheus metric with the given labels. + + This function will raise an AssertionError if there is not exactly one + sample with the given labels. + + Note that the metrics outlives each individual test, so it may hold + values from previous tests. + + Automatically includes SERVER_NAME_LABEL. + """ + + labels = dict(labels) + labels[SERVER_NAME_LABEL] = self.hs.hostname + + # Matching samples for the given labels. + found_samples = [] + + for collected in metric.collect(): + for sample in collected.samples: + # Check that all the labels match. If any label doesn't match, + # we skip this sample. + for label, value in labels.items(): + if sample.labels.get(label) != value: + break + else: + # We didn't break, so all the labels matched. Return this + # sample's value. + found_samples.append(sample) + + # The caller expects there to be exactly one sample with the given + # labels. If there are multiple (or zero) samples, we error. + if len(found_samples) == 1: + # We found exactly one sample with the given labels, so return its + # value. + return int(found_samples[0].value) + elif len(found_samples) > 1: + # We found multiple samples with the given labels, so we error. We + # helpfully include the differences in labels between the samples to + # help the caller figure out why they got multiple samples. + labels_differences_dicts = _get_dict_differences( + [sample.labels for sample in found_samples] + ) + differences_str = "\n".join(f" {diff}" for diff in labels_differences_dicts) + + raise AssertionError( + f"Multiple metrics found for '{metric}' with labels {labels}\n\n" + "`get_prometheus_metric_current_value(...)` expects you to be specific enough" + "with labels that only one metric matches. Either, the metrics changed and" + "that's wrong in and of itself or you need to update the test to be more" + "specific with the labels. The extra labels you can match with are:\n" + f"{differences_str}" + ) + else: + all_metrics = "\n".join( + f" {sample.labels}" + for collected in metric.collect() + for sample in collected.samples + ) + raise AssertionError( + f"No metric found for {metric} with labels {labels}\n" + f"All metrics:\n{all_metrics}" + ) + class FederatingHomeserverTestCase(HomeserverTestCase): """ @@ -1407,3 +1478,25 @@ def decorator(f: TV) -> TV: return f return decorator + + +def _get_dict_differences(dicts: list[dict[str, str]]) -> list[dict[str, str]]: + """Return a list of dicts where each dict has the common key/values removed. + + Useful for printing comparisons of prometheus metrics with different labels. + """ + if not dicts: + return [] + + # Find the common key/values across all dicts + common_items = set(dicts[0].items()) + for d in dicts[1:]: + common_items.intersection_update(d.items()) + + # Remove the common items from each dict + differences = [] + for d in dicts: + diff = {k: v for k, v in d.items() if (k, v) not in common_items} + differences.append(diff) + + return differences From 625b331768d3d0027a0d020310a648f1e40247e5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 22 Sep 2026 11:57:54 -0500 Subject: [PATCH 36/41] MSC4311: Use full PDU's in stripped state (like `invite_room_state`) over federation and always include `m.room.create` event (#19723) ### Background This PR was originally just trying to remove the flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation as client side API's like `/sync` should still use stripped events. But it turns out we were just re-using the client logic for the federation side and things might break if we didn't include the full `m.room.create` event so this PR now introduces MSC4311 support to use full PDU's in the `invite_room_state`/`knock_room_state` in the federation API's. The flawed implementation was originally introduced in https://github.com/element-hq/synapse/commit/0eb7252a230811e59679cc7e55b92dac26532efc (no PR I assume because part of Hydra security fix) which was part of [Synapse v1.136.0](https://github.com/element-hq/synapse/blob/7530874a1250d6ad975b39582a784c594d29a505/CHANGES.md#synapse-11360-2025-08-12). Spawning from reviewing https://github.com/element-hq/synapse/pull/19722 and noticing that we have [`TestMSC4311FullCreateEventOnStrippedState`](https://github.com/matrix-org/complement/blob/1e2e12eebc1edb27bbf12108ec849a8254b6ddcd/tests/v12_test.go#L1341-L1376) in Complement which already passes even though that test looks [flawed](https://github.com/matrix-org/complement/pull/791#discussion_r3132468346): > I think this test is mixing up what [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) proposes. Perhaps these were changes to the MSC that came after? > > For the client API's like `/sync`, it only proposes that `m.room.create` is a required *stripped* state event. > > For the federation API's, alongside requiring `m.room.create`, it also mandates using the full event PDU format for all events in the `invite_room_state`/`knock_room_state` on `m.room.member` events (in `unsigned`) ### What does this PR do? 1. Always use stripped state for client API's 1. Remove flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation (as explained above) 1. Sanitize stripped state when we receive events over federation 1. Use full PDU's when sending `invite_room_state`/`knock_room_state` over federation 1. Validate PDU's and warn when receiving `invite_room_state`/`knock_room_state` over federation 1. In the future, we will strictly validate and reject Complement tests: https://github.com/matrix-org/complement/pull/796 --- Part of https://github.com/element-hq/synapse/issues/19414 --- changelog.d/19723.bugfix | 1 + rust/src/room_versions.rs | 22 + synapse/events/__init__.py | 40 ++ synapse/events/utils.py | 39 +- synapse/federation/federation_client.py | 124 ++++- synapse/federation/federation_server.py | 73 ++- .../federation/transport/server/federation.py | 18 +- synapse/handlers/federation.py | 437 +++++++++++++++++- synapse/handlers/message.py | 9 +- synapse/handlers/sliding_sync/room_lists.py | 3 +- synapse/rest/client/sync.py | 11 +- synapse/storage/databases/main/events.py | 3 +- .../storage/databases/main/events_worker.py | 69 ++- synapse/synapse_rust/room_versions.pyi | 22 + tests/events/test_auto_accept_invites.py | 12 +- tests/events/test_utils.py | 28 +- tests/federation/transport/test_knocking.py | 5 +- tests/handlers/test_federation.py | 12 +- tests/handlers/test_room_member.py | 18 +- tests/handlers/test_room_summary.py | 4 +- tests/rest/client/test_sync.py | 3 + tests/test_visibility.py | 8 +- 22 files changed, 792 insertions(+), 169 deletions(-) create mode 100644 changelog.d/19723.bugfix diff --git a/changelog.d/19723.bugfix b/changelog.d/19723.bugfix new file mode 100644 index 0000000000..c445a79b06 --- /dev/null +++ b/changelog.d/19723.bugfix @@ -0,0 +1 @@ +Fix flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation introduced in Synapse v1.136.0 for invites/knocks: client-side API's like `/sync` use [stripped state events](https://spec.matrix.org/v1.18/client-server-api/#stripped-state) and we now send full PDUs on the federation-side. To allow some time for the ecosystem to adapt and support MSC4311, Synapse will only apply strict validation when receiving invites/knocks after 2027-06-01. diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index 2860bfdbd7..1f8570a03e 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -161,6 +161,26 @@ pub struct RoomVersion { /// This is similar to how doubly-linked lists can potentially not refer to previous items correctly /// without verifying the list's integrity, but doing it on every insert is too expensive. pub msc4242_state_dags: bool, + /// Determines whether a room version *SHOULD* rather than *MAY* reject invites/knocks + /// with invalid stripped state events. + /// + /// According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /// /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + /// > The `invite_room_state` has additional validation, which servers MAY apply to + /// > room versions 1 through 11 and SHOULD apply to all other room versions. As with + /// > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + /// > - The `m.room.create` event is missing from `invite_room_state`. + /// > - One or more entries in `invite_room_state` are not formatted according to the room's version. + /// > - One or more events fails a signature check. + /// > - One or more events does not reside in the same room as the invite. + /// + /// Regardless of room version (we should always do these things): + /// 1. The `m.room.create` event *MUST* be included in + /// `invite_room_state`/`knock_room_state` when sending invites/knocks over the + /// federation API's. + /// 2. Use full PDU's in the `invite_room_state`/`knock_room_state` in the federation + /// API. The client API still uses stripped state. + msc4311_stripped_state: bool, } impl RoomVersion { @@ -187,6 +207,7 @@ impl RoomVersion { msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, msc4242_state_dags: false, + msc4311_stripped_state: false, }; pub const V2: RoomVersion = RoomVersion { @@ -304,6 +325,7 @@ impl RoomVersion { state_res: StateResolutionVersions::V2_1, msc4289_creator_power_enabled: true, msc4291_room_ids_as_hashes: true, + msc4311_stripped_state: true, ..Self::V11 }; diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index b01e786550..cab2fe1a46 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -199,6 +199,46 @@ class StrippedStateEvent: sender: str content: dict[str, Any] + def as_json_dict(self) -> JsonDict: + """ + Serialize to the JSON representation of 'Stripped State' according to the Matrix + spec + """ + + return { + "type": self.type, + "state_key": self.state_key, + "sender": self.sender, + "content": self.content, + } + + @staticmethod + def from_json_dict(raw_stripped_event: JsonDict) -> "StrippedStateEvent | None": + """ + Given a raw value from an event's `unsigned` field, attempt to parse it into a + `StrippedStateEvent`. + """ + if isinstance(raw_stripped_event, dict): + # All of these fields are required + type = raw_stripped_event.get("type") + state_key = raw_stripped_event.get("state_key") + sender = raw_stripped_event.get("sender") + content = raw_stripped_event.get("content") + if ( + isinstance(type, str) + and isinstance(state_key, str) + and isinstance(sender, str) + and isinstance(content, dict) + ): + return StrippedStateEvent( + type=type, + state_key=state_key, + sender=sender, + content=content, + ) + + return None + @attr.s(slots=True, frozen=True, auto_attribs=True) class EventMetadata: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 55891f9b15..c3bc6fa9da 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -37,7 +37,6 @@ CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT, MAX_PDU_SIZE, - EventTypes, ) from synapse.api.errors import Codes, SynapseError from synapse.logging.opentracing import SynapseTags, set_tag, trace @@ -55,7 +54,7 @@ from synapse.synapse_rust.types import Requester from synapse.types import JsonDict -from . import EventBase, StrippedStateEvent +from . import EventBase # These are imported only to re-export them (callers import them from this # module); listing them in __all__ stops the unused-import lint flagging them @@ -549,45 +548,9 @@ def strip_event(event: EventBase) -> JsonDict: Stripped state events can only have the `sender`, `type`, `state_key` and `content` properties present. """ - # MSC4311: Ensure the create event is available on invites and knocks. - # TODO: Implement the rest of MSC4311 - if ( - event.room_version.msc4291_room_ids_as_hashes - and event.type == EventTypes.Create - and event.get_state_key() == "" - ): - return event.get_pdu_json() - return { "type": event.type, "state_key": event.state_key, "content": dict(event.content), "sender": event.sender, } - - -def parse_stripped_state_event(raw_stripped_event: Any) -> StrippedStateEvent | None: - """ - Given a raw value from an event's `unsigned` field, attempt to parse it into a - `StrippedStateEvent`. - """ - if isinstance(raw_stripped_event, dict): - # All of these fields are required - type = raw_stripped_event.get("type") - state_key = raw_stripped_event.get("state_key") - sender = raw_stripped_event.get("sender") - content = raw_stripped_event.get("content") - if ( - isinstance(type, str) - and isinstance(state_key, str) - and isinstance(sender, str) - and isinstance(content, dict) - ): - return StrippedStateEvent( - type=type, - state_key=state_key, - sender=sender, - content=content, - ) - - return None diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index e46392944b..214f88e0e1 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -23,6 +23,7 @@ import itertools import logging +from http import HTTPStatus from typing import ( TYPE_CHECKING, AbstractSet, @@ -59,6 +60,7 @@ RoomVersions, ) from synapse.events import EventBase, builder, make_event_from_dict +from synapse.events.snapshot import EventContext from synapse.federation.federation_base import ( FederationBase, InvalidEventSignatureError, @@ -70,7 +72,12 @@ from synapse.http.types import QueryParams from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace from synapse.metrics import SERVER_NAME_LABEL -from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id +from synapse.types import ( + JsonDict, + StrCollection, + UserID, + get_domain_from_id, +) from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.duration import Duration @@ -1352,12 +1359,12 @@ async def send_invite( self, destination: str, room_id: str, - event_id: str, pdu: EventBase, + context: EventContext, ) -> EventBase: room_version = await self.store.get_room_version(room_id) - content = await self._do_send_invite(destination, pdu, room_version) + content = await self._do_send_invite(destination, pdu, context, room_version) pdu_dict = content["event"] @@ -1378,11 +1385,21 @@ async def send_invite( return pdu async def _do_send_invite( - self, destination: str, pdu: EventBase, room_version: RoomVersion + self, + destination: str, + pdu: EventBase, + context: EventContext, + room_version: RoomVersion, ) -> JsonDict: """Actually sends the invite, first trying v2 API and falling back to v1 API if necessary. + Args: + destination: + pdu: Invite event + context: + room_version: + Returns: The event as a dict as returned by the remote server @@ -1393,6 +1410,19 @@ async def _do_send_invite( """ time_now = self._clock.time_msec() + # MSC4311: For the federation API, format events in `invite_room_state` as full + # PDU's + # + # Find the full events based on the state at the time of the invite + state_ids = await self.store.get_stripped_room_state_ids_from_event_context( + pdu, + context, + ) + state_events = await self.store.get_events(state_ids) + assert set(state_ids) == set(state_events.keys()), ( + "We should have all events available that were set as stripped state." + ) + try: return await self.transport_layer.send_invite_v2( destination=destination, @@ -1401,7 +1431,11 @@ async def _do_send_invite( content={ "event": pdu.get_pdu_json(time_now), "room_version": room_version.identifier, - "invite_room_state": pdu.unsigned.get("invite_room_state", []), + "invite_room_state": [ + # Use full PDU's according to MSC4311 + state_event.get_pdu_json(time_now) + for state_event in state_events.values() + ], }, ) except HttpResponseException as e: @@ -1416,18 +1450,86 @@ async def _do_send_invite( "User's homeserver does not support this room version", Codes.UNSUPPORTED_ROOM_VERSION, ) + # Matrix v1.18 (introduced in MSC4311) says that for a `400` response, "If + # `M_MISSING_PARAM` or `M_INVALID_PARAM` is returned and the request is + # associated with a Client-Server API request, the Client-Server API request + # SHOULD fail with a 5xx error rather than being passed through." (see + # https://spec.matrix.org/v1.18/server-server-api/#put_matrixfederationv2inviteroomideventid) + # + # But this was actually further clarified in MSC4528 that it should be + # translated to a `400` with `M_INCOMPATIBLE_SERVER`. + elif err.code == HTTPStatus.BAD_REQUEST and err.errcode in ( + Codes.MISSING_PARAM, + Codes.INVALID_PARAM, + ): + raise SynapseError( + 400, + f"Invite was rejected by the recipient's server.\n\n" + f"The remote homeserver ({destination}) returned {err.code} {err.errcode} " + "which indicates a compatibility problem between your homeserver and the " + "homeserver you're trying to send the invite to (either one could be at fault).", + # FIXME(MSC4528): Use `M_INCOMPATIBLE_SERVER` stable error code once + # the MSC is merged + Codes.UNKNOWN, + additional_fields={ + "cause": err.msg, + "destination_server": destination, + }, + ) else: raise err # Didn't work, try v1 API. # Note the v1 API returns a tuple of `(200, content)` - _, content = await self.transport_layer.send_invite_v1( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) + try: + # Use full PDU's for `invite_room_state` according to MSC4311 + # + # With the v1 invite API, `invite_room_state` is carried inside the event + # instead of a separate field like in v2 so we must munge it in ourselves + event_json = pdu.get_pdu_json(time_now) + event_json.setdefault("unsigned", {})["invite_room_state"] = [ + # Use full PDU's according to MSC4311 + state_event.get_pdu_json(time_now) + for state_event in state_events.values() + ] + + _, content = await self.transport_layer.send_invite_v1( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=event_json, + ) + except HttpResponseException as e: + # Matrix v1.18 (introduced in MSC4311) says that for a `400` response, "If + # `M_MISSING_PARAM` or `M_INVALID_PARAM` is returned and the request is + # associated with a Client-Server API request, the Client-Server API request + # SHOULD fail with a 5xx error rather than being passed through." (see + # https://spec.matrix.org/v1.18/server-server-api/#put_matrixfederationv2inviteroomideventid) + # + # But this was actually further clarified in MSC4528 that it should be + # translated to a `400` with `M_INCOMPATIBLE_SERVER`. + err = e.to_synapse_error() + if err.code == HTTPStatus.BAD_REQUEST and err.errcode in ( + Codes.MISSING_PARAM, + Codes.INVALID_PARAM, + ): + raise SynapseError( + 400, + f"Invite was rejected by the recipient's server.\n\n" + f"The remote homeserver ({destination}) returned {err.code} {err.errcode} " + "which indicates a compatibility problem between your homeserver and the " + "homeserver you're trying to send the invite to (either one could be at fault).", + # FIXME(MSC4528): Use `M_INCOMPATIBLE_SERVER` stable error code once + # the MSC is merged + Codes.UNKNOWN, + additional_fields={ + "cause": err.msg, + "destination_server": destination, + }, + ) + else: + raise err return content async def send_leave(self, destinations: Iterable[str], pdu: EventBase) -> None: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b90bb67ba2..95aff08ddf 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -198,8 +198,6 @@ def __init__(self, hs: "HomeServer"): hs.config.federation.federation_metrics_domains ) - self._room_prejoin_state_types = hs.config.api.room_prejoin_state - # Whether we have started handling old events in the staging area. self._started_handling_of_staged_events = False @@ -817,8 +815,27 @@ async def on_make_join_request( return {"event": pdu.get_templated_pdu_json(), "room_version": room_version} async def on_invite_request( - self, origin: str, content: JsonDict, room_version_id: str + self, + *, + origin: str, + expected_room_id: str, + expected_event_id: str, + event_json: JsonDict, + room_version_id: str, ) -> dict[str, Any]: + """ + Args: + origin: + expected_room_id: The room ID specified in the + `/_matrix/federation/v1/invite/{roomId}/{eventId}` request that we expect to + match in the actual event itself. + expected_event_id: The event ID specified in the + `/_matrix/federation/v1/invite/{roomId}/{eventId}` request that we expect to + match in the actual event itself. + event_json: + room_version_id: + """ + room_version = KNOWN_ROOM_VERSIONS.get(room_version_id) if not room_version: raise SynapseError( @@ -827,9 +844,21 @@ async def on_invite_request( Codes.UNSUPPORTED_ROOM_VERSION, ) - pdu = event_from_pdu_json(content, room_version) + pdu = event_from_pdu_json(event_json, room_version) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, pdu.room_id) + if pdu.event_id != expected_event_id: + raise SynapseError( + 400, + "Invite event ID must match event ID specified in the federation `/invite` request", + Codes.INVALID_PARAM, + ) + if pdu.room_id != expected_room_id: + raise SynapseError( + 400, + "The room_id specified in the invite event must match room ID specified in the federation `/invite` request", + Codes.INVALID_PARAM, + ) if await self._spam_checker_module_callbacks.should_drop_federated_event(pdu): logger.info( "Federated event contains spam, dropping %s", @@ -842,7 +871,11 @@ async def on_invite_request( errmsg = f"event id {pdu.event_id}: {e}" logger.warning("%s", errmsg) raise SynapseError(403, errmsg, Codes.FORBIDDEN) - ret_pdu = await self.handler.on_invite_request(origin, pdu, room_version) + ret_pdu = await self.handler.on_invite_request( + origin=origin, + event=pdu, + room_version=room_version, + ) time_now = self._clock.time_msec() return {"event": ret_pdu.get_pdu_json(time_now)} @@ -1061,19 +1094,31 @@ async def on_send_knock_request( Returns: The stripped room state. """ - _, context = await self._on_send_membership_event( + time_now = self._clock.time_msec() + + event, context = await self._on_send_membership_event( origin, content, Membership.KNOCK, room_id ) - # Retrieve stripped state events from the room and send them back to the remote - # server. This will allow the remote server's clients to display information - # related to the room while the knock request is pending. - stripped_room_state = ( - await self.store.get_stripped_room_state_from_event_context( - context, self._room_prejoin_state_types - ) + # MSC4311: For the federation API, format events in `knock_room_state` as full + # PDU's + # + # Find the full events based on the state at the time of the knock + state_ids = await self.store.get_stripped_room_state_ids_from_event_context( + event, context + ) + state_events = await self.store.get_events(state_ids) + assert set(state_ids) == set(state_events.keys()), ( + "We should have all events available that were set as stripped state." ) - return {"knock_room_state": stripped_room_state} + + return { + "knock_room_state": [ + # Use full PDU's according to MSC4311 + state_event.get_pdu_json(time_now) + for state_event in state_events.values() + ] + } async def _on_send_membership_event( self, origin: str, content: JsonDict, membership_type: str, room_id: str diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index af6c733029..6d2f3f0e43 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -491,7 +491,11 @@ async def on_PUT( # state resolution algorithm, and we don't use that for processing # invites result = await self.handler.on_invite_request( - origin, content, room_version_id=RoomVersions.V1.identifier + origin=origin, + expected_room_id=room_id, + expected_event_id=event_id, + event_json=content, + room_version_id=RoomVersions.V1.identifier, ) # V1 federation API is defined to return a content of `[200, {...}]` @@ -513,9 +517,6 @@ async def on_PUT( room_id: str, event_id: str, ) -> tuple[int, JsonDict]: - # TODO(paul): assert that room_id/event_id parsed from path actually - # match those given in content - room_version = content["room_version"] event = content["event"] invite_room_state = content.get("invite_room_state", []) @@ -524,12 +525,15 @@ async def on_PUT( invite_room_state = [] # Synapse expects invite_room_state to be in unsigned, as it is in v1 - # API - + # API. We will sanitize this inside `on_invite_request(...)` event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state result = await self.handler.on_invite_request( - origin, event, room_version_id=room_version + origin=origin, + expected_room_id=room_id, + expected_event_id=event_id, + event_json=event, + room_version_id=room_version, ) # We only store invite_room_state for internal use, so remove it before diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 655e8d244a..52efbe92cc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -30,15 +30,19 @@ from typing import ( TYPE_CHECKING, AbstractSet, + Any, Iterable, ) import attr -from prometheus_client import Histogram +from prometheus_client import Counter, Histogram from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json +from typing_extensions import assert_never from unpaddedbase64 import decode_base64 +from twisted.internet.defer import CancelledError + from synapse import event_auth from synapse.api.constants import ( MAX_DEPTH, @@ -65,10 +69,14 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, event_exists_in_state_dag +from synapse.events import EventBase, StrippedStateEvent, event_exists_in_state_dag from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import EventContext, UnpersistedEventContextBase from synapse.events.validator import EventValidator +from synapse.federation.federation_base import ( + InvalidEventSignatureError, + event_from_pdu_json, +) from synapse.federation.federation_client import InvalidResponseError from synapse.handlers.pagination import PURGE_PAGINATION_LOCK_NAME from synapse.http.servlet import assert_params_in_dict @@ -80,7 +88,7 @@ from synapse.storage.invite_rule import InviteRule from synapse.types import JsonDict, StrCollection, get_domain_from_id from synapse.types.state import StateFilter -from synapse.util.async_helpers import Linearizer +from synapse.util.async_helpers import Linearizer, concurrently_execute from synapse.util.duration import Duration from synapse.util.retryutils import NotRetryingDestination from synapse.visibility import filter_events_for_server @@ -113,6 +121,14 @@ ), ) +# So we have some sort of measure of Matrix ecosystem adoption so we know better how +# safe it is to make the migration to strict validation after 2027-06-01. +invalid_stripped_state_counter = Counter( + "synapse_federation_saw_invalid_stripped_state", + "The number of invites/knocks we saw with invalid stripped state.", + labelnames=[SERVER_NAME_LABEL], +) + NUMBER_OF_EVENTS_TO_BACKFILL = 100 """ @@ -136,6 +152,22 @@ class _BackfillPoint: type: _BackfillPointType +class InvalidStrippedStateBehaviour(Enum): + """ + What to do when parsing stripped state events + """ + + remove_invalid = enum.auto() + """ + Remove the individual event from the set of stripped state. + """ + + reject_all = enum.auto() + """ + Reject the entire set of stripped state events if there is a single invalid event + """ + + class FederationHandler: """Handles general incoming federation requests @@ -574,7 +606,9 @@ async def try_backfill(domains: StrCollection) -> bool: return False - async def send_invite(self, target_host: str, event: EventBase) -> EventBase: + async def send_invite( + self, target_host: str, event: EventBase, context: EventContext + ) -> EventBase: """Sends the invite to the remote server for signing. Invites must be signed by the invitee's server before distribution. @@ -583,8 +617,8 @@ async def send_invite(self, target_host: str, event: EventBase) -> EventBase: pdu = await self.federation_client.send_invite( destination=target_host, room_id=event.room_id, - event_id=event.event_id, pdu=event, + context=context, ) except RequestSendFailed: raise SynapseError(502, f"Can't connect to server {target_host}") @@ -952,15 +986,53 @@ async def do_knock( # This is a bit of a hack and is cribbing off of invites. Basically we # store the room state here and retrieve it again when this event appears # in the invitee's sync stream. It is stripped out for all other local users. - stripped_room_state = knock_response.get("knock_room_state") - - if stripped_room_state is None: - raise KeyError("Missing 'knock_room_state' field in send_knock response") - - if not isinstance(stripped_room_state, list): - raise TypeError("'knock_room_state' has wrong type") - - event.unsigned["knock_room_state"] = stripped_room_state + # + # Parse/validate `knock_room_state` + try: + stripped_room_state = await self._parse_stripped_room_state( + stripped_room_state=knock_response.get("knock_room_state"), + room_id=event.room_id, + room_version=event_format_version, + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/send_knock/{roomId}/{eventId}` endpoint: + # > Entries which are improperly signed or formatted SHOULD be removed by the + # > server prior to supplying them over the Client-Server API. + # + # FIXME(MSC4311): This should be + # `InvalidStrippedStateBehaviour.remove_invalid` but that means this + # never raises and we'd always assign empty `knock_room_state` for some + # server that's still using the legacy stripped-state format. After + # 2027-06-01 (to allow some time for the ecosystem to adapt and support + # MSC4311), update this to be + # `InvalidStrippedStateBehaviour.remove_invalid` alongside removing the + # whole `_minimal_parse_stripped_room_state` fallback. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.reject_all, + ) + # Replace with our sanitized `knock_room_state` + event.unsigned["knock_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state + ] + except CancelledError: + raise + except Exception: + # FIXME(MSC4311): Remove this whole block after we always enforce the + # validation above. The only reason this is here is because the validation + # can fail for non-compliant servers but we should still use stripped state. + stripped_room_state_for_client = self._minimal_parse_stripped_room_state( + stripped_room_state=knock_response.get("knock_room_state"), + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/send_knock/{roomId}/{eventId}` endpoint: + # > Entries which are improperly signed or formatted SHOULD be removed by the + # > server prior to supplying them over the Client-Server API. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.remove_invalid, + ) + if stripped_room_state_for_client is not None: + # Replace with our sanitized `knock_room_state` + event.unsigned["knock_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state_for_client + ] context = EventContext.for_outlier(self._storage_controllers) stream_id = await self._federation_event_handler.persist_events_and_notify( @@ -1127,8 +1199,260 @@ async def on_make_join_request( await self._event_auth_handler.check_auth_rules_from_context(event) return event + def _minimal_parse_stripped_room_state( + self, + *, + stripped_room_state: Any, + invalid_stripped_state_behaviour: InvalidStrippedStateBehaviour, + ) -> list[StrippedStateEvent] | None: + """ + The goal of this function is to sanitize whatever we got from federation and + make it presentable to the client. The minimum amount of parsing necessary to + ensure `invite_room_state`/`knock_room_state` is at-least a list of stripped + state events (compared to `_parse_stripped_room_state`). + + Args: + stripped_room_state: The raw `invite_room_state`/`knock_room_state` JSON + invalid_stripped_state_behaviour: How to handle the scenario where we see a + single invalid stripped state event + """ + parsed_stripped_room_state: list[StrippedStateEvent] = [] + + # Scrutinize JSON values + # + # In previous versions of the Matrix spec, + # `invite_room_state`/`knock_room_state` was an optional list of stripped state + # events which means we can't strictly enforce that this is always present. + if stripped_room_state is None: + return None + # We're going to strictly enforce that they at-least gave us a list. + elif not isinstance(stripped_room_state, list): + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise TypeError("Stripped state must be a list of PDU's") + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + return parsed_stripped_room_state + else: + assert_never(invalid_stripped_state_behaviour) + + for raw_stripped_event in stripped_room_state: + # Parse each stripped event + parsed_stripped_event = StrippedStateEvent.from_json_dict( + raw_stripped_event + ) + if parsed_stripped_event is None: + # Drop any invalid stripped state events as this is spec'ed and we + # might as well save the client from dealing with anything crazy. + continue + parsed_stripped_room_state.append(parsed_stripped_event) + + return parsed_stripped_room_state + + async def _parse_stripped_room_state( + self, + *, + stripped_room_state: Any, + room_id: str, + room_version: RoomVersion, + invalid_stripped_state_behaviour: InvalidStrippedStateBehaviour, + ) -> list[StrippedStateEvent]: + """ + Parse and validate `invite_room_state`/`knock_room_state` according to the + Matrix spec (c.f. MSC4311). + + According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + > The `invite_room_state` has additional validation, which servers MAY apply to + > room versions 1 through 11 and SHOULD apply to all other room versions. As with + > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + > - [...] + > - One or more entries in `invite_room_state` are not formatted according to the room's version. + > - One or more events fails a signature check. + > - One or more events does not reside in the same room as the invite. + + According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /_matrix/federation/v1/send_knock/{roomId}/{eventId}` endpoint: + > Entries which are improperly signed or formatted SHOULD be removed by the + > server prior to supplying them over the Client-Server API. + + We refer to `invite_room_state`/`knock_room_state` as `stripped_room_state` but + the events contained within can be full PDU's or stripped state events (older + version of the Matrix spec). + + Args: + stripped_room_state: The raw `invite_room_state`/`knock_room_state` JSON + room_id: The room ID the invite/knock is happening in + room_version: The version of the room the invite/knock is happening in + invalid_stripped_state_behaviour: How to handle the scenario where we see a + single invalid stripped state event + + Returns: + A list of parsed `StrippedStateEvent` + + Raises: + When `InvalidStrippedStateBehaviour.reject_all`, raises + `TypeError`/`ValueError` when one of the stripped room state events is + invalid + """ + parsed_stripped_room_state: list[StrippedStateEvent] = [] + + # Scrutinize JSON values + if not isinstance(stripped_room_state, list): + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise TypeError( + "Stripped state must be a list of PDU's that includes the `m.room.create` event" + ) + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + return parsed_stripped_room_state + else: + assert_never(invalid_stripped_state_behaviour) + + # First assemble an ordered list of PDUs (remember the order of the stripped + # state) as we want to concurrently process below and then re-assemble in the + # original order. + pdus: list[EventBase] = [] + already_counted_fail_metric = False + for raw_stripped_event in stripped_room_state: + try: + # Scrutinize JSON values + if not isinstance(raw_stripped_event, dict): + raise ValueError( + "PDU from stripped state must be a JSON dictionary" + ) + + # Validate PDU + try: + pdu = event_from_pdu_json(raw_stripped_event, room_version) + pdus.append(pdu) + except Exception as exc: + raise ValueError( + "Unable to parse one of the stripped state events as a PDU" + ) from exc + except ValueError as exc: + # Count how many times we saw invalid stripped state. We're counting the + # number of times we saw invalid state overall, not each individual + # invalid stripped state event. + if not already_counted_fail_metric: + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + already_counted_fail_metric = True + + # React to invalid event + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise exc + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + continue + else: + assert_never(invalid_stripped_state_behaviour) + + # Because we're processing things in parallel, we need to worry about assembling + # things in the correct order after we're done. + # + # Map from `event_id` to `StrippedStateEvent` + parsed_stripped_room_state_map: dict[str, StrippedStateEvent] = {} + + async def _validate_stripped_state_pdu_event(pdu: EventBase) -> None: + nonlocal already_counted_fail_metric + + try: + # Validate that it's from the same room + if pdu.room_id != room_id: + raise ValueError( + "PDU from stripped state must be from the room ID specified in the request" + ) + # Validate signature/hashes + try: + # Note: `_check_sigs_and_hash` will redact the event in the case of + # a hash mismatch (proper thing to use according to the spec)) so + # let's be sure to use the new updated PDU returned from this call. + pdu = await self.federation_client._check_sigs_and_hash( + room_version, pdu + ) + except InvalidEventSignatureError as exc: + raise ValueError( + "PDU from stripped state must pass signature/hash checks" + ) from exc + + # Parse the stripped events to ensure it has all of the fields necessary + parsed_stripped_event = StrippedStateEvent.from_json_dict( + pdu.get_dict() + ) + if parsed_stripped_event is None: + raise ValueError("Unable to parse as stripped event") + parsed_stripped_room_state_map[pdu.event_id] = parsed_stripped_event + except ValueError as exc: + # Count how many times we saw invalid stripped state. We're counting the + # number of times we saw invalid state overall, not each individual + # invalid stripped state event. + if not already_counted_fail_metric: + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + already_counted_fail_metric = True + + # React to invalid event + if ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.reject_all + ): + raise exc + elif ( + invalid_stripped_state_behaviour + == InvalidStrippedStateBehaviour.remove_invalid + ): + pass + else: + assert_never(invalid_stripped_state_behaviour) + + # Check in parallel as `_check_sigs_and_hash` can take some time if we don't + # already have the server signatures. + await concurrently_execute( + _validate_stripped_state_pdu_event, + pdus, + # Arbitrary concurrency + 1000, + ) + + # Create an ordered list that matches the original `stripped_room_state` + parsed_stripped_room_state = [ + parsed_stripped_room_state_map[pdu.event_id] + # Iterate in the original order + for pdu in pdus + # But check to make sure we it ended up being parsed correctly as a stripped + # event + if pdu.event_id in parsed_stripped_room_state_map + ] + + return parsed_stripped_room_state + async def on_invite_request( - self, origin: str, event: EventBase, room_version: RoomVersion + self, + *, + origin: str, + event: EventBase, + room_version: RoomVersion, ) -> EventBase: """We've got an invite event. Process and persist it. Sign it. @@ -1201,6 +1525,89 @@ async def on_invite_request( room_id=event.room_id, room_version=room_version ) + # Parse/validate `invite_room_state` + try: + stripped_room_state = await self._parse_stripped_room_state( + stripped_room_state=event.unsigned.get("invite_room_state"), + room_id=event.room_id, + room_version=room_version, + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + # > The `invite_room_state` has additional validation, which servers MAY apply to + # > room versions 1 through 11 and SHOULD apply to all other room versions. As with + # > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + # > - The `m.room.create` event is missing from `invite_room_state`. + # > - One or more entries in `invite_room_state` are not formatted according to the room's version. + # > - One or more events fails a signature check. + # > - One or more events does not reside in the same room as the invite. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.reject_all, + ) + # Validate `m.room.create` event is included (see spec blurb above) + includes_create_event = any( + (stripped_state_event.type, stripped_state_event.state_key) + == (EventTypes.Create, "") + for stripped_state_event in stripped_room_state + ) + if not includes_create_event: + invalid_stripped_state_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).inc(1) + raise ValueError( + "Stripped state must include `m.room.create` event (MSC4311)" + ) + # Replace with our sanitized `invite_room_state` + event.unsigned["invite_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state + ] + except CancelledError: + raise + except Exception as exc: + # FIXME(MSC4311): Apply this validation for all room versions after + # 2027-06-01 (to allow some time for the ecosystem to adapt and support + # MSC4311), see https://github.com/element-hq/synapse/issues/19943 + # + # The Matrix spec says that for version 12+ rooms, servers SHOULD rather than + # MAY respond to such requests with 400 M_INVALID_PARAM. Given we have the + # lee-way to enforce this in all room versions, we might as well. + # + # For now, we'll only log in room versions 12+ where this SHOULD be the case + # already. + if room_version.msc4311_stripped_state: + # FIXME(MSC4311): Instead of logging, reject with 400 `M_INVALID_PARAM` + # after 2027-06-01. Given Synapse claimed to support room version 12 but + # didn't adhere to this behavior until 2026-07, we will only warn for + # now. + logger.warning( + "Continuing anyway but failed to validate `invite_room_state` on invite %s (room_version=%s): %s", + event, + room_version, + exc, + ) + + # FIXME(MSC4311): Remove this whole block after we always enforce the + # validation above. The only reason this is here is because the validation + # can fail for non-compliant servers but we should still use stripped state. + stripped_room_state_for_client = self._minimal_parse_stripped_room_state( + stripped_room_state=event.unsigned.get("invite_room_state"), + # According to Matrix v1.18 (introduced in MSC4311) for the `PUT + # /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + # > The `invite_room_state` has additional validation, which servers MAY apply to + # > room versions 1 through 11 and SHOULD apply to all other room versions. As with + # > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + # > - The `m.room.create` event is missing from `invite_room_state`. + # > - One or more entries in `invite_room_state` are not formatted according to the room's version. + # > - One or more events fails a signature check. + # > - One or more events does not reside in the same room as the invite. + invalid_stripped_state_behaviour=InvalidStrippedStateBehaviour.reject_all, + ) + if stripped_room_state_for_client is not None: + # Replace with our sanitized `invite_room_state` + event.unsigned["invite_room_state"] = [ + stripped_state_event.as_json_dict() + for stripped_state_event in stripped_room_state_for_client + ] + event.internal_metadata.outlier = True event.internal_metadata.out_of_band_membership = True diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5230772e49..7a8866c8ed 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -509,8 +509,6 @@ def __init__(self, hs: "HomeServer"): self._worker_lock_handler = hs.get_worker_locks_handler() self._policy_handler = hs.get_room_policy_handler() - self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state - self.send_events = ReplicationSendEventsRestServlet.make_client(hs) self.request_ratelimiter = hs.get_request_ratelimiter() @@ -2081,9 +2079,8 @@ async def persist_and_notify_client_events( event.unsigned, "invite_room_state", await self.store.get_stripped_room_state_from_event_context( + event, context, - self.room_prejoin_state_types, - membership_user_id=event.sender, ), ) @@ -2094,7 +2091,7 @@ async def persist_and_notify_client_events( # to get them to sign the event. returned_invite = await federation_handler.send_invite( - invitee.domain, event + invitee.domain, event, context ) # TODO: Make sure the signatures actually are correct. @@ -2106,8 +2103,8 @@ async def persist_and_notify_client_events( event.unsigned, "knock_room_state", await self.store.get_stripped_room_state_from_event_context( + event, context, - self.room_prejoin_state_types, ), ) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 836cee6c20..f4d1f484e7 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -37,7 +37,6 @@ from synapse.api.errors import SlidingSyncUnknownPosition from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import StrippedStateEvent -from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import start_active_span, trace from synapse.storage.databases.main.sliding_sync import UPDATE_INTERVAL_LAST_USED_TS from synapse.storage.databases.main.state import ( @@ -1618,7 +1617,7 @@ async def _bulk_get_stripped_state_for_rooms_from_sync_room_map( stripped_state_map = {} if isinstance(raw_stripped_state_events, list): for raw_stripped_event in raw_stripped_state_events: - stripped_state_event = parse_stripped_state_event( + stripped_state_event = StrippedStateEvent.from_json_dict( raw_stripped_event ) if stripped_state_event is not None: diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 08002a6708..83aa00d0b5 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -33,6 +33,7 @@ EventFormat, FilteredEvent, SerializeEventConfig, + strip_event, ) from synapse.handlers.presence import format_user_presence_state from synapse.handlers.sliding_sync import SlidingSyncConfig, SlidingSyncResult @@ -477,7 +478,8 @@ async def encode_invited( invited_state = [] invited_state = list(invited_state) - invited_state.append(invite) + # MSC4319: Add the invite itself + invited_state.append(strip_event(room.invite)) invited[room.room_id] = {"invite_state": {"events": invited_state}} return invited @@ -523,12 +525,11 @@ async def encode_knocked( knocked_state = [] knocked_state = list(knocked_state) - # Append the actual knock membership event itself as well. This provides - # the client with: + # MSC4319: Append the actual knock membership event itself as well. This + # provides the client with: # # * A knock state event that they can use for easier internal tracking - # * The rough timestamp of when the knock occurred contained within the event - knocked_state.append(knock) + knocked_state.append(strip_event(room.knock)) # Build the `knock_state` dictionary, which will contain the state of the # room that the client has knocked on diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 26a2b92f8d..b628b7fed7 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -57,7 +57,6 @@ ) from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventPersistencePair -from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import trace from synapse.metrics import SERVER_NAME_LABEL from synapse.storage._base import db_to_json, make_in_list_sql_clause @@ -2415,7 +2414,7 @@ def _get_sliding_sync_insert_values_from_stripped_state( stripped_state_map: MutableStateMap[StrippedStateEvent] = {} if isinstance(unsigned_stripped_state_events, list): for raw_stripped_event in unsigned_stripped_state_events: - stripped_state_event = parse_stripped_state_event( + stripped_state_event = StrippedStateEvent.from_json_dict( raw_stripped_event ) if stripped_state_event is not None: diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index c5d0c35259..efe77375da 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -41,7 +41,7 @@ from twisted.internet import defer -from synapse.api.constants import Direction, EventTypes +from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import NotFoundError, SynapseError from synapse.api.room_versions import ( KNOWN_ROOM_VERSIONS, @@ -390,6 +390,8 @@ def get_chain_id_txn(txn: Cursor) -> int: finished (so we don't have to keep querying it every time) """ + self._room_prejoin_state_types = hs.config.api.room_prejoin_state + def get_un_partial_stated_events_token(self, instance_name: str) -> int: return ( self._un_partial_stated_events_stream_id_gen.get_current_token_for_writer( @@ -1141,14 +1143,12 @@ async def _get_events_from_local_cache( async def get_stripped_room_state_from_event_context( self, + event: EventBase, context: EventContext, - state_keys_to_include: StateFilter, - membership_user_id: str | None = None, ) -> list[JsonDict]: """ Retrieve the stripped state from a room, given an event context to retrieve state - from as well as the state types to include. Optionally, include the membership - events from a specific user. + from as well as the state types to include. "Stripped" state means that only the `type`, `state_key`, `content` and `sender` keys are included from each state event. @@ -1156,35 +1156,60 @@ async def get_stripped_room_state_from_event_context( Args: context: The event context to retrieve state of the room from. state_keys_to_include: The state events to include, for each event type. - membership_user_id: An optional user ID to include the stripped membership state - events of. This is useful when generating the stripped state of a room for - invites. We want to send membership events of the inviter, so that the - invitee can display the inviter's profile information if the room lacks any. Returns: A list of dictionaries, each representing a stripped state event from the room. """ - if membership_user_id: + selected_state_ids = await self.get_stripped_room_state_ids_from_event_context( + event, context + ) + + state_to_include = await self.get_events(selected_state_ids) + + return [strip_event(e) for e in state_to_include.values()] + + async def get_stripped_room_state_ids_from_event_context( + self, + event: EventBase, + context: EventContext, + ) -> list[str]: + """ + Retrieve the stripped state IDs for an event, given an event context to retrieve state + from as well as the state types to include. + + Args: + context: The event context to retrieve state of the room from. + + Returns: + A list of event_ids, each representing the stripped state event to include for this event + """ + # Start with the configured default set of stripped state to include + state_filter = self._room_prejoin_state_types + + # MSC4319: We want to send membership events of the inviter, so that the invitee + # can display the inviter's profile information if the room lacks any. + is_invite_event = ( + event.type == EventTypes.Member and event.membership == Membership.INVITE + ) + if is_invite_event: types = chain( - state_keys_to_include.to_types(), - [(EventTypes.Member, membership_user_id)], + self._room_prejoin_state_types.to_types(), + [(EventTypes.Member, event.sender)], ) - filter = StateFilter.from_types(types) - else: - filter = state_keys_to_include - selected_state_ids = await context.get_current_state_ids(filter) + state_filter = StateFilter.from_types(types) + + # Get the relevant state + selected_state_ids = await context.get_current_state_ids(state_filter) # We know this event is not an outlier, so this must be # non-None. assert selected_state_ids is not None - # Confusingly, get_current_state_events may return events that are discarded by - # the filter, if they're in context._state_delta_due_to_event. Strip these away. - selected_state_ids = filter.filter_state(selected_state_ids) + # Confusingly, `get_current_state_ids` may return events that are discarded by + # the filter, if they're in `context._state_delta_due_to_event`. Strip these away. + selected_state_ids = state_filter.filter_state(selected_state_ids) - state_to_include = await self.get_events(selected_state_ids.values()) - - return [strip_event(e) for e in state_to_include.values()] + return list(selected_state_ids.values()) def _maybe_start_fetch_thread(self) -> None: """Starts an event fetch thread if we are not yet at the maximum number.""" diff --git a/synapse/synapse_rust/room_versions.pyi b/synapse/synapse_rust/room_versions.pyi index 9bbb538f18..1f99f6d29b 100644 --- a/synapse/synapse_rust/room_versions.pyi +++ b/synapse/synapse_rust/room_versions.pyi @@ -123,6 +123,28 @@ class RoomVersion: to the create event every time we insert an event would be prohibitively expensive. This is similar to how doubly-linked lists can potentially not refer to previous items correctly without verifying the list's integrity, but doing it on every insert is too expensive.""" + msc4311_stripped_state: bool + """ + Determines whether a room version *SHOULD* rather than *MAY* reject invites/knocks + with invalid stripped state events. + + According to Matrix v1.18 (introduced in MSC4311) for the `PUT + /_matrix/federation/v1/invite/{roomId}/{eventId}` endpoint: + > The `invite_room_state` has additional validation, which servers MAY apply to + > room versions 1 through 11 and SHOULD apply to all other room versions. As with + > the above errors, servers SHOULD return `M_INVALID_PARAM` if: + > - The `m.room.create` event is missing from `invite_room_state`. + > - One or more entries in `invite_room_state` are not formatted according to the room's version. + > - One or more events fails a signature check. + > - One or more events does not reside in the same room as the invite. + + Regardless of room version (we should always do these things): + 1. The `m.room.create` event *MUST* be included in + `invite_room_state`/`knock_room_state` when sending invites/knocks over the + federation API's. + 2. Use full PDU's in the `invite_room_state`/`knock_room_state` in the federation + API. The client API still uses stripped state. + """ class RoomVersions: V1: RoomVersion diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index 632d1dc4f6..c44b4e011e 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -198,9 +198,9 @@ def test_invite_from_remote_user(self) -> None: ) self.get_success( self.handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ) ) @@ -324,9 +324,9 @@ def test_accept_invite_local_user( ) self.get_success( self.handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ) ) else: diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 9fd85fc542..d018568798 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -35,7 +35,6 @@ format_event_for_client_v2_without_room_id, format_event_raw, maybe_upsert_event_field, - parse_stripped_state_event, prune_event, ) from synapse.types import JsonDict @@ -1004,7 +1003,7 @@ def test_invalid_nesting_raises_type_error(self) -> None: copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] -class TestParseStrippedStateEvent(stdlib_unittest.TestCase): +class TestStrippedStateEvent(stdlib_unittest.TestCase): def test_valid_dict(self) -> None: """A valid dict should be parsed into a StrippedStateEvent.""" raw = { @@ -1013,7 +1012,7 @@ def test_valid_dict(self) -> None: "sender": "@alice:example.com", "content": {"membership": "join"}, } - result = parse_stripped_state_event(raw) + result = StrippedStateEvent.from_json_dict(raw) self.assertEqual( result, StrippedStateEvent( @@ -1024,17 +1023,10 @@ def test_valid_dict(self) -> None: ), ) - def test_invalid_type(self) -> None: - """Non-dict inputs should return None.""" - self.assertIsNone(parse_stripped_state_event("string")) - self.assertIsNone(parse_stripped_state_event(123)) - self.assertIsNone(parse_stripped_state_event([])) - self.assertIsNone(parse_stripped_state_event(None)) - def test_missing_fields(self) -> None: """Dicts with missing required fields should return None.""" self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "state_key": "@alice:example.com", "sender": "@alice:example.com", @@ -1043,7 +1035,7 @@ def test_missing_fields(self) -> None: ) ) self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "sender": "@alice:example.com", @@ -1052,7 +1044,7 @@ def test_missing_fields(self) -> None: ) ) self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", @@ -1061,7 +1053,7 @@ def test_missing_fields(self) -> None: ) ) self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", @@ -1074,7 +1066,7 @@ def test_invalid_field_types(self) -> None: """Dicts with invalid field types should return None.""" # Type must be string self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": 123, "state_key": "@alice:example.com", @@ -1085,7 +1077,7 @@ def test_invalid_field_types(self) -> None: ) # State key must be string self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": 123, @@ -1096,7 +1088,7 @@ def test_invalid_field_types(self) -> None: ) # Sender must be string self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", @@ -1107,7 +1099,7 @@ def test_invalid_field_types(self) -> None: ) # Content must be dict self.assertIsNone( - parse_stripped_state_event( + StrippedStateEvent.from_json_dict( { "type": "m.room.member", "state_key": "@alice:example.com", diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index ec705676cc..cdaae5cfd7 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -184,14 +184,11 @@ def check_knock_room_state_against_room_state( expected_room_state[event_type]["content"], event["content"] ) - # Check the state key is correct + # Check the state_key is correct self.assertEqual( expected_room_state[event_type]["state_key"], event["state_key"] ) - # Ensure the event has been stripped - self.assertNotIn("signatures", event) - # Pop once we've found and processed a state event expected_room_state.pop(event_type) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 71f29be2ea..edb419744a 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -383,18 +383,18 @@ def create_invite() -> EventBase: event = create_invite() self.get_success( self.handler.on_invite_request( - other_server, - event, - event.room_version, + origin=other_server, + event=event, + room_version=event.room_version, ) ) event = create_invite() self.get_failure( self.handler.on_invite_request( - other_server, - event, - event.room_version, + origin=other_server, + event=event, + room_version=event.room_version, ), exc=LimitExceededError, ) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 1a61dcd8b0..1b62830571 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -555,9 +555,9 @@ def test_msc4155_block_invite_remote(self) -> None: f = self.get_failure( self.fed_handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ), SynapseError, ).value @@ -601,9 +601,9 @@ def test_msc4155_block_invite_remote_server(self) -> None: f = self.get_failure( self.fed_handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ), SynapseError, ).value @@ -716,9 +716,9 @@ def test_msc4380_block_invite_remote(self) -> None: f = self.get_failure( self.fed_handler.on_invite_request( - remote_server, - invite_event, - invite_event.room_version, + origin=remote_server, + event=invite_event, + room_version=invite_event.room_version, ), SynapseError, ).value diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index febfce5328..a3a0e62dcf 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -230,7 +230,9 @@ def _poke_fed_invite(self, room_id: str, from_user: str) -> None: } ) self.get_success( - fed_handler.on_invite_request(fed_hostname, event, RoomVersions.V6) + fed_handler.on_invite_request( + origin=fed_hostname, event=event, room_version=RoomVersions.V6 + ) ) def test_simple_space(self) -> None: diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 6e281e5e73..51ffabe75d 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -399,6 +399,9 @@ def test_knock_room_state(self) -> None: self.check_knock_room_state_against_room_state( room_state_events, self.expected_room_state ) + # Ensure the events have been stripped + for event in room_state_events: + self.assertNotIn("signatures", event) class SyncCreateEventInPrejoinStateTestCase(unittest.HomeserverTestCase): diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 9a5efbdd39..0654f84351 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -608,9 +608,11 @@ def test_out_of_band_invite_rejection(self) -> None: self.get_success( self.hs.get_federation_server().on_invite_request( - self.OTHER_SERVER_NAME, - invite_pdu, - "9", + origin=self.OTHER_SERVER_NAME, + expected_event_id=invite_event_id, + expected_room_id="!room:id", + event_json=invite_pdu, + room_version_id="9", ) ) From 66bfc418640593f93cc65655136970f526d42d83 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 22 Sep 2026 13:52:38 -0500 Subject: [PATCH 37/41] 1.162.0rc1 --- CHANGES.md | 50 +++++++++++++++++++++++++++++++ changelog.d/19723.bugfix | 1 - changelog.d/19768.bugfix | 1 - changelog.d/19979.misc | 1 - changelog.d/20011.misc | 1 - changelog.d/20097.misc | 1 - changelog.d/20100.bugfix | 1 - changelog.d/20115.bugfix | 1 - changelog.d/20130.feature | 1 - changelog.d/20133.misc | 1 - changelog.d/20145.bugfix | 1 - changelog.d/20154.bugfix | 1 - changelog.d/20160.misc | 1 - changelog.d/20161.misc | 1 - changelog.d/20162.feature | 1 - changelog.d/20166.misc | 1 - changelog.d/20171.bugfix | 1 - changelog.d/20181.bugfix | 1 - changelog.d/20182.bugfix | 1 - changelog.d/20185.misc | 1 - changelog.d/20187.feature | 1 - changelog.d/20193.misc | 1 - changelog.d/20200.bugfix | 1 - changelog.d/20204.bugfix | 1 - changelog.d/20205.misc | 1 - changelog.d/20207.misc | 1 - changelog.d/20209.doc | 1 - changelog.d/20210.doc | 1 - changelog.d/20216.misc | 1 - changelog.d/20217.doc | 1 - changelog.d/20218.feature | 1 - changelog.d/20219.doc | 1 - changelog.d/20224.misc | 1 - changelog.d/20225.doc | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 37 files changed, 58 insertions(+), 35 deletions(-) delete mode 100644 changelog.d/19723.bugfix delete mode 100644 changelog.d/19768.bugfix delete mode 100644 changelog.d/19979.misc delete mode 100644 changelog.d/20011.misc delete mode 100644 changelog.d/20097.misc delete mode 100644 changelog.d/20100.bugfix delete mode 100644 changelog.d/20115.bugfix delete mode 100644 changelog.d/20130.feature delete mode 100644 changelog.d/20133.misc delete mode 100644 changelog.d/20145.bugfix delete mode 100644 changelog.d/20154.bugfix delete mode 100644 changelog.d/20160.misc delete mode 100644 changelog.d/20161.misc delete mode 100644 changelog.d/20162.feature delete mode 100644 changelog.d/20166.misc delete mode 100644 changelog.d/20171.bugfix delete mode 100644 changelog.d/20181.bugfix delete mode 100644 changelog.d/20182.bugfix delete mode 100644 changelog.d/20185.misc delete mode 100644 changelog.d/20187.feature delete mode 100644 changelog.d/20193.misc delete mode 100644 changelog.d/20200.bugfix delete mode 100644 changelog.d/20204.bugfix delete mode 100644 changelog.d/20205.misc delete mode 100644 changelog.d/20207.misc delete mode 100644 changelog.d/20209.doc delete mode 100644 changelog.d/20210.doc delete mode 100644 changelog.d/20216.misc delete mode 100644 changelog.d/20217.doc delete mode 100644 changelog.d/20218.feature delete mode 100644 changelog.d/20219.doc delete mode 100644 changelog.d/20224.misc delete mode 100644 changelog.d/20225.doc diff --git a/CHANGES.md b/CHANGES.md index c8ea637bd3..1b98865216 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,53 @@ +# Synapse 1.162.0rc1 (2026-09-22) + +## Features + +- Raise default room version to "12". Contributed by @jason-famedly @famedly. ([\#20130](https://github.com/element-hq/synapse/issues/20130)) +- Limit the number of end-to-end encryption one-time keys stored per device to 500 per algorithm, rejecting uploads which would exceed the limit with a `400 Bad Request`. ([\#20162](https://github.com/element-hq/synapse/issues/20162)) +- Add support for configuring a `username` for Redis connections, for Redis 6+ ACL authentication. ([\#20187](https://github.com/element-hq/synapse/issues/20187)) +- Add a rate limit on the client profile lookup endpoints, configurable via `rc_profile`. ([\#20218](https://github.com/element-hq/synapse/issues/20218)) + +## Bugfixes + +- Fix flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation introduced in Synapse v1.136.0 for invites/knocks: client-side API's like `/sync` use [stripped state events](https://spec.matrix.org/v1.18/client-server-api/#stripped-state) and we now send full PDUs on the federation-side. To allow some time for the ecosystem to adapt and support MSC4311, Synapse will only apply strict validation when receiving invites/knocks after 2027-06-01. ([\#19723](https://github.com/element-hq/synapse/issues/19723)) +- Allow the Third Party Rules `check_event_allowed()` callback to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. ([\#19768](https://github.com/element-hq/synapse/issues/19768)) +- Improve server concurrency by opening local media thumbnails asynchronously. Contributed by @guillemo12. ([\#20100](https://github.com/element-hq/synapse/issues/20100)) +- Drop incoming federation device list updates from non-compliant (grandfathered historical) user IDs, so that such user IDs are not forwarded to clients outside the context of an event, as per [the spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids). ([\#20115](https://github.com/element-hq/synapse/issues/20115)) +- Stop treating unset display names and avatar URLs as profile fields with a `null` value. ([\#20145](https://github.com/element-hq/synapse/issues/20145)) +- Return `allowed_room_ids` in the client [`GET /_matrix/client/v1/rooms/{roomId}/hierarchy`](https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy) response, as required since Matrix 1.15. ([\#20154](https://github.com/element-hq/synapse/issues/20154)) +- Fix state events being omitted from the [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) `state_after` sync response when the client's `since` token falls inside an event persistence batch, as could happen on worker deployments. ([\#20171](https://github.com/element-hq/synapse/issues/20171)) +- Return the stable `M_UNKNOWN_DEVICE` error code, added in Matrix 1.17, instead of its unstable [MSC4326](https://github.com/matrix-org/matrix-spec-proposals/pull/4326)-prefixed identifier. ([\#20181](https://github.com/element-hq/synapse/issues/20181)) +- Fix slow recursive `/relations` requests in large rooms by joining events inside the recursive query. ([\#20182](https://github.com/element-hq/synapse/issues/20182)) +- Fix a bug introduced in Synapse 1.135.0 where fetching an unset `displayname` or `avatar_url` via `GET /_matrix/client/v3/profile/{userId}/{field}` returned the field with a `null` value instead of an empty object. ([\#20200](https://github.com/element-hq/synapse/issues/20200)) +- Add support for un-soft-failing [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) when room state changes, making federation support more reliable. ([\#20204](https://github.com/element-hq/synapse/issues/20204)) + +## Improved Documentation + +- Document the paths that can be handled on workers with stabilised delegated authentication. ([\#20209](https://github.com/element-hq/synapse/issues/20209)) +- Document the endpoint for getting a single delayed event is workerisable. ([\#20210](https://github.com/element-hq/synapse/issues/20210)) +- Fix small warnings in documentation building tooling output. ([\#20217](https://github.com/element-hq/synapse/issues/20217)) +- Replace the stale minimum `poetry` version in the contributing docs with a pointer to the minimum defined in `pyproject.toml`. ([\#20219](https://github.com/element-hq/synapse/issues/20219)) +- Added docs regarding firewall configuration. Contributed by @HarisDotParis. ([\#20225](https://github.com/element-hq/synapse/issues/20225)) + +## Internal Changes + +- Port the logcontext machinery (`LoggingContext`, `ContextResourceUsage` and the current-context storage) to Rust. ([\#19979](https://github.com/element-hq/synapse/issues/19979)) +- Refactor the Rust code to have a single place to store per-homeserver state. ([\#20011](https://github.com/element-hq/synapse/issues/20011)) +- Add a `synapse_storage_stream_current_position` metric, reporting each stream's current position as each worker process sees it. ([\#20097](https://github.com/element-hq/synapse/issues/20097)) +- Add HTTP serving functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. ([\#20133](https://github.com/element-hq/synapse/issues/20133)) +- Add a cache for looking up individual pieces of current room state. ([\#20160](https://github.com/element-hq/synapse/issues/20160)) +- Run the in-repo Complement test suite in CI, even when the standard Complement suite fails. ([\#20161](https://github.com/element-hq/synapse/issues/20161)) +- Refactor the federation transmission code to delineate transaction preparation and completion. ([\#20166](https://github.com/element-hq/synapse/issues/20166)) +- Add a cache to state resolution keyed off the conflicted events. ([\#20185](https://github.com/element-hq/synapse/issues/20185)) +- Improve the rendering of the set inequality errors produced by `assertEqual` in the tests. ([\#20193](https://github.com/element-hq/synapse/issues/20193)) +- Fix `/room_summary` returning stale `join_rules`. Contributed by @famedly. ([\#20205](https://github.com/element-hq/synapse/issues/20205)) +- Fix the `Schema Diff` CI check failing to post a comment on pull requests from forks. ([\#20207](https://github.com/element-hq/synapse/issues/20207)) +- Move documentation building python dependencies into `pyproject.toml` under a new `docs` dependency group. ([\#20216](https://github.com/element-hq/synapse/issues/20216)) +- Align comments in `.dockerignore` with what we have in the private Synapse Pro branch. ([\#20224](https://github.com/element-hq/synapse/issues/20224)) + + + + # Synapse 1.161.0 (2026-09-15) No significant changes since 1.161.0rc1. diff --git a/changelog.d/19723.bugfix b/changelog.d/19723.bugfix deleted file mode 100644 index c445a79b06..0000000000 --- a/changelog.d/19723.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation introduced in Synapse v1.136.0 for invites/knocks: client-side API's like `/sync` use [stripped state events](https://spec.matrix.org/v1.18/client-server-api/#stripped-state) and we now send full PDUs on the federation-side. To allow some time for the ecosystem to adapt and support MSC4311, Synapse will only apply strict validation when receiving invites/knocks after 2027-06-01. diff --git a/changelog.d/19768.bugfix b/changelog.d/19768.bugfix deleted file mode 100644 index 8314491ced..0000000000 --- a/changelog.d/19768.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow the Third Party Rules `check_event_allowed()` callback to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. diff --git a/changelog.d/19979.misc b/changelog.d/19979.misc deleted file mode 100644 index 3210329a56..0000000000 --- a/changelog.d/19979.misc +++ /dev/null @@ -1 +0,0 @@ -Port the logcontext machinery (`LoggingContext`, `ContextResourceUsage` and the current-context storage) to Rust. diff --git a/changelog.d/20011.misc b/changelog.d/20011.misc deleted file mode 100644 index 9582106675..0000000000 --- a/changelog.d/20011.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the Rust code to have a single place to store per-homeserver state. diff --git a/changelog.d/20097.misc b/changelog.d/20097.misc deleted file mode 100644 index 3c9cd6c86e..0000000000 --- a/changelog.d/20097.misc +++ /dev/null @@ -1 +0,0 @@ -Add a `synapse_storage_stream_current_position` metric, reporting each stream's current position as each worker process sees it. diff --git a/changelog.d/20100.bugfix b/changelog.d/20100.bugfix deleted file mode 100644 index 1c73ada4da..0000000000 --- a/changelog.d/20100.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve server concurrency by opening local media thumbnails asynchronously. Contributed by @guillemo12. \ No newline at end of file diff --git a/changelog.d/20115.bugfix b/changelog.d/20115.bugfix deleted file mode 100644 index 4cb368a1b3..0000000000 --- a/changelog.d/20115.bugfix +++ /dev/null @@ -1 +0,0 @@ -Drop incoming federation device list updates from non-compliant (grandfathered historical) user IDs, so that such user IDs are not forwarded to clients outside the context of an event, as per [the spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids). diff --git a/changelog.d/20130.feature b/changelog.d/20130.feature deleted file mode 100644 index 82c8faa015..0000000000 --- a/changelog.d/20130.feature +++ /dev/null @@ -1 +0,0 @@ -Raise default room version to "12". Contributed by @jason-famedly @famedly. diff --git a/changelog.d/20133.misc b/changelog.d/20133.misc deleted file mode 100644 index 9590db6934..0000000000 --- a/changelog.d/20133.misc +++ /dev/null @@ -1 +0,0 @@ -Add HTTP serving functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. diff --git a/changelog.d/20145.bugfix b/changelog.d/20145.bugfix deleted file mode 100644 index e2ce9d52db..0000000000 --- a/changelog.d/20145.bugfix +++ /dev/null @@ -1 +0,0 @@ -Stop treating unset display names and avatar URLs as profile fields with a `null` value. \ No newline at end of file diff --git a/changelog.d/20154.bugfix b/changelog.d/20154.bugfix deleted file mode 100644 index 50de48799f..0000000000 --- a/changelog.d/20154.bugfix +++ /dev/null @@ -1 +0,0 @@ -Return `allowed_room_ids` in the client [`GET /_matrix/client/v1/rooms/{roomId}/hierarchy`](https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1roomsroomidhierarchy) response, as required since Matrix 1.15. diff --git a/changelog.d/20160.misc b/changelog.d/20160.misc deleted file mode 100644 index 476608a767..0000000000 --- a/changelog.d/20160.misc +++ /dev/null @@ -1 +0,0 @@ -Add a cache for looking up individual pieces of current room state. diff --git a/changelog.d/20161.misc b/changelog.d/20161.misc deleted file mode 100644 index 4351f779c2..0000000000 --- a/changelog.d/20161.misc +++ /dev/null @@ -1 +0,0 @@ -Run the in-repo Complement test suite in CI, even when the standard Complement suite fails. \ No newline at end of file diff --git a/changelog.d/20162.feature b/changelog.d/20162.feature deleted file mode 100644 index a197d13664..0000000000 --- a/changelog.d/20162.feature +++ /dev/null @@ -1 +0,0 @@ -Limit the number of end-to-end encryption one-time keys stored per device to 500 per algorithm, rejecting uploads which would exceed the limit with a `400 Bad Request`. diff --git a/changelog.d/20166.misc b/changelog.d/20166.misc deleted file mode 100644 index 716e910f8f..0000000000 --- a/changelog.d/20166.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the federation transmission code to delineate transaction preparation and completion. \ No newline at end of file diff --git a/changelog.d/20171.bugfix b/changelog.d/20171.bugfix deleted file mode 100644 index ab54bf8df4..0000000000 --- a/changelog.d/20171.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix state events being omitted from the [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) `state_after` sync response when the client's `since` token falls inside an event persistence batch, as could happen on worker deployments. diff --git a/changelog.d/20181.bugfix b/changelog.d/20181.bugfix deleted file mode 100644 index 477902b97f..0000000000 --- a/changelog.d/20181.bugfix +++ /dev/null @@ -1 +0,0 @@ -Return the stable `M_UNKNOWN_DEVICE` error code, added in Matrix 1.17, instead of its unstable [MSC4326](https://github.com/matrix-org/matrix-spec-proposals/pull/4326)-prefixed identifier. diff --git a/changelog.d/20182.bugfix b/changelog.d/20182.bugfix deleted file mode 100644 index 637c3efb96..0000000000 --- a/changelog.d/20182.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix slow recursive `/relations` requests in large rooms by joining events inside the recursive query. diff --git a/changelog.d/20185.misc b/changelog.d/20185.misc deleted file mode 100644 index 69125baad8..0000000000 --- a/changelog.d/20185.misc +++ /dev/null @@ -1 +0,0 @@ -Add a cache to state resolution keyed off the conflicted events. diff --git a/changelog.d/20187.feature b/changelog.d/20187.feature deleted file mode 100644 index d5c53f1510..0000000000 --- a/changelog.d/20187.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for configuring a `username` for Redis connections, for Redis 6+ ACL authentication. diff --git a/changelog.d/20193.misc b/changelog.d/20193.misc deleted file mode 100644 index 03ab2b0f7e..0000000000 --- a/changelog.d/20193.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the rendering of the set inequality errors produced by `assertEqual` in the tests. \ No newline at end of file diff --git a/changelog.d/20200.bugfix b/changelog.d/20200.bugfix deleted file mode 100644 index 41eadbcf6e..0000000000 --- a/changelog.d/20200.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.135.0 where fetching an unset `displayname` or `avatar_url` via `GET /_matrix/client/v3/profile/{userId}/{field}` returned the field with a `null` value instead of an empty object. diff --git a/changelog.d/20204.bugfix b/changelog.d/20204.bugfix deleted file mode 100644 index 783bedee14..0000000000 --- a/changelog.d/20204.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add support for un-soft-failing [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) when room state changes, making federation support more reliable. \ No newline at end of file diff --git a/changelog.d/20205.misc b/changelog.d/20205.misc deleted file mode 100644 index 5a5d03b44a..0000000000 --- a/changelog.d/20205.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `/room_summary` returning stale `join_rules`. Contributed by @famedly. diff --git a/changelog.d/20207.misc b/changelog.d/20207.misc deleted file mode 100644 index 3709ad46f1..0000000000 --- a/changelog.d/20207.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the `Schema Diff` CI check failing to post a comment on pull requests from forks. diff --git a/changelog.d/20209.doc b/changelog.d/20209.doc deleted file mode 100644 index cfee70c67c..0000000000 --- a/changelog.d/20209.doc +++ /dev/null @@ -1 +0,0 @@ -Document the paths that can be handled on workers with stabilised delegated authentication. diff --git a/changelog.d/20210.doc b/changelog.d/20210.doc deleted file mode 100644 index 3b0f828036..0000000000 --- a/changelog.d/20210.doc +++ /dev/null @@ -1 +0,0 @@ -Document the endpoint for getting a single delayed event is workerisable. diff --git a/changelog.d/20216.misc b/changelog.d/20216.misc deleted file mode 100644 index 130a5b1339..0000000000 --- a/changelog.d/20216.misc +++ /dev/null @@ -1 +0,0 @@ -Move documentation building python dependencies into `pyproject.toml` under a new `docs` dependency group. \ No newline at end of file diff --git a/changelog.d/20217.doc b/changelog.d/20217.doc deleted file mode 100644 index a7d4590cfa..0000000000 --- a/changelog.d/20217.doc +++ /dev/null @@ -1 +0,0 @@ -Fix small warnings in documentation building tooling output. \ No newline at end of file diff --git a/changelog.d/20218.feature b/changelog.d/20218.feature deleted file mode 100644 index f8e85a88f8..0000000000 --- a/changelog.d/20218.feature +++ /dev/null @@ -1 +0,0 @@ -Add a rate limit on the client profile lookup endpoints, configurable via `rc_profile`. diff --git a/changelog.d/20219.doc b/changelog.d/20219.doc deleted file mode 100644 index 7bfa49e480..0000000000 --- a/changelog.d/20219.doc +++ /dev/null @@ -1 +0,0 @@ -Replace the stale minimum `poetry` version in the contributing docs with a pointer to the minimum defined in `pyproject.toml`. \ No newline at end of file diff --git a/changelog.d/20224.misc b/changelog.d/20224.misc deleted file mode 100644 index 829f713cf6..0000000000 --- a/changelog.d/20224.misc +++ /dev/null @@ -1 +0,0 @@ -Align comments in `.dockerignore` with what we have in the private Synapse Pro branch. diff --git a/changelog.d/20225.doc b/changelog.d/20225.doc deleted file mode 100644 index cba91aef6f..0000000000 --- a/changelog.d/20225.doc +++ /dev/null @@ -1 +0,0 @@ -Added docs regarding firewall configuration. Contributed by @HarisDotParis. diff --git a/debian/changelog b/debian/changelog index cebc8490cb..2d8b89895d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.162.0~rc1) stable; urgency=medium + + * New synapse release 1.162.0rc1. + + -- Synapse Packaging team Tue, 22 Sep 2026 18:51:00 +0000 + matrix-synapse-py3 (1.161.0) stable; urgency=medium * New synapse release 1.161.0. diff --git a/pyproject.toml b/pyproject.toml index e1965b71fd..b242148333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.161.0" +version = "1.162.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 4550ad3e06..3034db88b6 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.161/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.162/synapse-config.schema.json type: object properties: modules: From 5b3332bcf293264c758e6606ba90e9781c55730c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 22 Sep 2026 17:37:07 -0500 Subject: [PATCH 38/41] Better align case and canonical "third party rules" spelling --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1b98865216..f222da58b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,7 +10,7 @@ ## Bugfixes - Fix flawed [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) partial implementation introduced in Synapse v1.136.0 for invites/knocks: client-side API's like `/sync` use [stripped state events](https://spec.matrix.org/v1.18/client-server-api/#stripped-state) and we now send full PDUs on the federation-side. To allow some time for the ecosystem to adapt and support MSC4311, Synapse will only apply strict validation when receiving invites/knocks after 2027-06-01. ([\#19723](https://github.com/element-hq/synapse/issues/19723)) -- Allow the Third Party Rules `check_event_allowed()` callback to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. ([\#19768](https://github.com/element-hq/synapse/issues/19768)) +- Allow the third party rules callback `check_event_allowed()` to work with [MSC4291](https://github.com/matrix-org/matrix-spec-proposals/pull/4291) rooms. Contributed by @famedly. ([\#19768](https://github.com/element-hq/synapse/issues/19768)) - Improve server concurrency by opening local media thumbnails asynchronously. Contributed by @guillemo12. ([\#20100](https://github.com/element-hq/synapse/issues/20100)) - Drop incoming federation device list updates from non-compliant (grandfathered historical) user IDs, so that such user IDs are not forwarded to clients outside the context of an event, as per [the spec](https://spec.matrix.org/v1.14/appendices/#historical-user-ids). ([\#20115](https://github.com/element-hq/synapse/issues/20115)) - Stop treating unset display names and avatar URLs as profile fields with a `null` value. ([\#20145](https://github.com/element-hq/synapse/issues/20145)) From 63dc0cbfe514f60743212566374ffeb5e9e295c2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 29 Sep 2026 11:15:31 -0500 Subject: [PATCH 39/41] 1.162.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f222da58b0..d42c769e4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.162.0 (2026-09-29) + +No significant changes since 1.162.0rc1. + + + + # Synapse 1.162.0rc1 (2026-09-22) ## Features diff --git a/debian/changelog b/debian/changelog index 2d8b89895d..528e6b1d67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.162.0) stable; urgency=medium + + * New synapse release 1.162.0. + + -- Synapse Packaging team Tue, 29 Sep 2026 16:15:16 +0000 + matrix-synapse-py3 (1.162.0~rc1) stable; urgency=medium * New synapse release 1.162.0rc1. diff --git a/pyproject.toml b/pyproject.toml index b242148333..d6ef84360d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.162.0rc1" +version = "1.162.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 8e9756a743bb143af15f85ba9cec4f9deec2de4b Mon Sep 17 00:00:00 2001 From: Soyoung Kim Date: Fri, 2 Oct 2026 11:44:04 +0200 Subject: [PATCH 40/41] chore: update CI poetry version to 2.4.1 --- .github/workflows/famedly-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/famedly-tests.yml b/.github/workflows/famedly-tests.yml index 052a48f452..603c8e7b54 100644 --- a/.github/workflows/famedly-tests.yml +++ b/.github/workflows/famedly-tests.yml @@ -26,7 +26,7 @@ jobs: - uses: matrix-org/setup-python-poetry@v2 with: python-version: "3.13" - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "all" - run: poetry run scripts-dev/generate_sample_config.sh --check - run: poetry run scripts-dev/config-lint.sh @@ -62,7 +62,7 @@ jobs: - name: Setup Poetry uses: matrix-org/setup-python-poetry@v2 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" python-version: "3.13" install-project: "false" @@ -93,7 +93,7 @@ jobs: # To make CI green, err towards caution and install the project. install-project: "true" python-version: "3.13" - poetry-version: "2.2.1" + poetry-version: "2.4.1" # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 @@ -223,7 +223,7 @@ jobs: - uses: matrix-org/setup-python-poetry@v2 with: python-version: ${{ matrix.job.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: ${{ matrix.job.extras }} - name: Await PostgreSQL if: ${{ matrix.job.postgres-version }} @@ -336,7 +336,7 @@ jobs: # `poetry` is used in `complement.sh` - uses: matrix-org/setup-python-poetry@v2 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" # Matches the `path` where we checkout Synapse above working-directory: "synapse" From 8321b716d0362ff484140f8c05f604f1f13447a1 Mon Sep 17 00:00:00 2001 From: Soyoung Kim Date: Fri, 2 Oct 2026 13:38:57 +0200 Subject: [PATCH 41/41] Update CHANGES.md --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4a421c0133..7961e1d8bc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,9 @@ No significant changes since 1.162.0rc1. +## Famedly additions for v1.162.0_1 - +- chore: update CI poetry version to 2.4.1 (Soyoung Kim) # Synapse 1.162.0rc1 (2026-09-22)