Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/caching.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ Two rules keep this correct:
2. **Handle `CursorExpired`.** It means Microsoft no longer accepts your cursor, so clear that kind
and start again from `None`.

Cursors are called with the user's token, so `stream()` raises `ValueError` for one that doesn't point to
the expected Microsoft service. If your API accepts cursors from outside, also encrypt or sign them, and
check they belong to that user.

```python
from outnote import CursorExpired

Expand Down
31 changes: 29 additions & 2 deletions src/outnote/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import time
from concurrent.futures import ThreadPoolExecutor
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlencode
from urllib.parse import quote, urlencode, urlsplit
from urllib.request import Request

from entrance import HTTPError as ClientError
Expand Down Expand Up @@ -59,6 +59,27 @@ class CursorExpired(GraphError):
"""The server no longer accepts a `stream()` cursor: start again without one."""


def _check_link(url, scheme, hosts=(), suffixes=()):
"""Refuse a saved `stream()` or `delta()` link that doesn't point where it should. These links
are called with the user's token, so one from an untrusted source could otherwise send that token
to another server. Backslashes and whitespace are refused because URL parsers disagree on them."""
bad = not isinstance(url, str) or any(c == "\\" or c.isspace() or ord(c) < 32 for c in url)
if not bad:
try:
parts = urlsplit(url)
host = (parts.hostname or "").lower()
bad = (
parts.scheme != scheme
or parts.username is not None
or parts.password is not None
or not (host in hosts or host.endswith(tuple(suffixes)))
)
except ValueError:
bad = True
if bad:
raise ValueError("refusing a cursor that doesn't point to the expected Microsoft service")


def _retry_after(headers, attempt):
ra = (headers or {}).get("Retry-After")
return float(ra) if ra and str(ra).replace(".", "", 1).isdigit() else 2**attempt
Expand Down Expand Up @@ -146,6 +167,10 @@ def _url(self, path, params=None):
url += ("&" if "?" in url else "?") + q
return url

def _check_graph_link(self, url):
base = urlsplit(self.base)
_check_link(url, base.scheme, hosts=(base.hostname,))

def _send(self, req, raw=False, retry=True):
"""Send `req`; returns parsed JSON, or text (`raw=True`), or bytes
(`raw="bytes"`). Empty bodies (202/204) come back as None."""
Expand Down Expand Up @@ -362,7 +387,9 @@ def delta(self, path, delta_link=None, params=None, client=C_OFFICE):
"""Incremental sync of a collection. First call (no `delta_link`) GETs
`{path}/delta` and returns every item; pass the returned `delta_link`
next time to get only what changed since. Returns
`(items, delta_link)`."""
`(items, delta_link)`. Raises `ValueError` if `delta_link` isn't a Graph URL."""
if delta_link:
self._check_graph_link(delta_link)
url = delta_link or self._url(path.rstrip("/") + "/delta", params)
items = []
for _ in range(_MAX_PAGES * 4):
Expand Down
10 changes: 8 additions & 2 deletions src/outnote/_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,16 @@
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.parse import urlsplit

from ._base import _MAX_PAGES, CursorExpired, GraphError
from ._base import _MAX_PAGES, CursorExpired, GraphError, _check_link
from ._mail import LIST_SELECT
from ._native import _VIEW, NativeMixin, chat_from_native, message_from_native

_EXPIRED = (400, 404, 409, 410)
_GRAPH_PAGE = 999 # 1150 messages: 0.7 s at 999 per page, 2.7 s at 50
_CHAT_PAGE = 200
_CHAT_HOST_SUFFIXES = (".teams.microsoft.com",)


@dataclass
Expand All @@ -49,7 +51,8 @@ class StreamMixin(NativeMixin):
def stream(self, kind, cursor=None, **scope):
"""Yield `Change`s since `cursor` (None = everything). Stateless: nothing is kept between
calls but the cursor you pass back. Raises `CursorExpired` when the server no longer
accepts the cursor; start again from None.
accepts the cursor; start again from None. Raises `ValueError` for a cursor that doesn't
point to the expected Microsoft service, because cursors are called with the user's token.

`kind` and its options:
* `mail` - `folder="inbox"`, one folder's messages, with `select=` as in `emails`
Expand Down Expand Up @@ -84,6 +87,7 @@ def _delta_start(self, kind, folder="inbox", select=LIST_SELECT, start=None, end

def _stream_graph(self, kind, cursor, **scope):
if cursor:
self._check_graph_link(cursor)
url = cursor
else:
path, params = self._delta_start(kind, **scope)
Expand Down Expand Up @@ -117,6 +121,8 @@ def _stream_chatsvc(self, kind, cursor, chat_id=None):
if kind == "messages" and not chat_id:
raise ValueError("stream('messages') needs chat_id=")
if cursor:
service = urlsplit(self._chatsvc()["url"]).hostname
_check_link(cursor, "https", hosts=(service,), suffixes=_CHAT_HOST_SUFFIXES)
url, params = cursor, None
elif kind == "chats":
url, params = "conversations", {"view": _VIEW, "pageSize": _CHAT_PAGE}
Expand Down
84 changes: 78 additions & 6 deletions tests/test_native.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,17 +329,15 @@ def test_stream_graph_pages_deletes_and_final_cursor(monkeypatch):


def test_stream_resumes_and_reports_no_changes(monkeypatch):
c = client(
monkeypatch, [("https://g/done", {"value": [], "@odata.deltaLink": "https://g/done2"})]
)
got = list(c.stream("mail", "https://g/done"))
c = client(monkeypatch, [("/done", {"value": [], "@odata.deltaLink": "https://g/done2"})])
got = list(c.stream("mail", _base.GRAPH + "/done"))
assert [(x.op, x.cursor) for x in got] == [("sync", "https://g/done2")]


def test_expired_cursor_is_typed(monkeypatch):
c = client(monkeypatch, [("https://g/old", outnote.GraphError(410, "gone"))])
c = client(monkeypatch, [("/old", outnote.GraphError(410, "gone"))])
with pytest.raises(outnote.CursorExpired):
list(c.stream("mail", "https://g/old"))
list(c.stream("mail", _base.GRAPH + "/old"))


def test_a_failure_without_a_cursor_is_not_called_expired(monkeypatch):
Expand Down Expand Up @@ -474,3 +472,77 @@ def broken():
with pytest.raises(outnote.GraphError):
cache.sync("mail")
assert cache.cursor("mail") == "c1" and cache.item("mail", "1")


# --- cursors can't send the token elsewhere --------------------------------------
BAD_GRAPH_CURSORS = [
"https://evil.example/steal",
"http://graph.microsoft.com/v1.0/me",
"https://graph.microsoft.com.evil.example/x",
"https://graph.microsoft.com@evil.example/x",
"https://user:pw@graph.microsoft.com/x",
"https://evil.example\\@graph.microsoft.com/x",
"https://evil.example/ https://graph.microsoft.com/x",
"https://evil.example\t.graph.microsoft.com/x",
"/me",
"file:///etc/passwd",
"",
]


@pytest.mark.parametrize("cursor", [c for c in BAD_GRAPH_CURSORS if c])
def test_stream_refuses_cursors_for_other_hosts(monkeypatch, cursor):
c = client(monkeypatch, []) # any request at all would fail this test
with pytest.raises(ValueError, match="refusing a cursor"):
list(c.stream("mail", cursor))
assert c.fake.calls == []


@pytest.mark.parametrize("cursor", [c for c in BAD_GRAPH_CURSORS if c])
def test_delta_refuses_links_for_other_hosts(monkeypatch, cursor):
c = client(monkeypatch, [])
with pytest.raises(ValueError, match="refusing a cursor"):
c.delta("me/mailFolders", cursor)
assert c.fake.calls == []


@pytest.mark.parametrize(
"cursor",
[
"https://evil.example/x",
"https://evil.teams.microsoft.com.example/x",
"http://chat.test/x",
"https://graph.microsoft.com/v1.0/me",
"https://x@chat.test/y",
],
)
def test_chat_streams_refuse_cursors_for_other_hosts(monkeypatch, cursor):
c = client(monkeypatch, [("authz", AUTHZ)])
with pytest.raises(ValueError, match="refusing a cursor"):
list(c.stream("chats", cursor))
assert all("authz" in x["path"] for x in c.fake.calls) # only the sign-in, never the cursor


@pytest.mark.parametrize("host", ["chat.test", "de.ng.msg.teams.microsoft.com"])
def test_chat_cursors_for_the_chat_service_are_accepted(monkeypatch, host):
page = {"conversations": [], "_metadata": {"syncState": f"https://{host}/next"}}
c = client(monkeypatch, [("authz", AUTHZ), (host, page)])
got = list(c.stream("chats", f"https://{host}/s"))
assert got[-1].op == "sync"


def test_a_custom_base_is_the_trusted_graph_host(monkeypatch):
c = outnote.OutNote(lambda client_id, scope=None: "t", base="https://graph.example.us/v1.0")
fake = Fake([("/done", {"value": [], "@odata.deltaLink": "https://graph.example.us/v1.0/d2"})])
monkeypatch.setattr(c, "request", fake)
assert list(c.stream("mail", "https://graph.example.us/v1.0/done"))[-1].op == "sync"
with pytest.raises(ValueError):
list(c.stream("mail", _base.GRAPH + "/done"))


def test_sqlite_cache_does_not_hide_a_refused_cursor(monkeypatch):
c = client(monkeypatch, [])
cache = outnote.SQLiteCache(":memory:", c)
cache._db.execute("INSERT INTO cursors VALUES ('mail', '', 'https://evil.example/x')")
with pytest.raises(ValueError):
cache.sync("mail")
Loading