From 66022705f2954097bac116b99a3373ba827da89b Mon Sep 17 00:00:00 2001 From: Paolo Mazza Date: Tue, 22 Sep 2026 08:43:47 +0200 Subject: [PATCH] feat: preserve Substack widgets in Markdown (fixes #64) --- .pre-commit-config.yaml | 34 +- ROADMAP.md | 6 +- substack/api.py | 27 +- substack/cli.py | 5 + substack/mdrender.py | 553 ++++++++++++--------- tests/substack/test_widget_preservation.py | 159 ++++++ 6 files changed, 510 insertions(+), 274 deletions(-) create mode 100644 tests/substack/test_widget_preservation.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a5c0dff..c186ec0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,17 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 26.5.1 - hooks: - - id: black - - repo: https://github.com/pycqa/isort - rev: 8.0.1 - hooks: - - id: isort - name: isort (python) - args: ["--profile", "black"] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 26.5.1 + hooks: + - id: black + - repo: https://github.com/pycqa/isort + rev: 9.0.1 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black"] diff --git a/ROADMAP.md b/ROADMAP.md index 26b75ec..fa2385e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -111,8 +111,8 @@ Do not add telemetry to the package, CLI, SDK, or MCP server. | 0.4.0 | released | 2026-08-24 | v0.4.0 | - | - | - | - | - | [GitHub](https://github.com/ma2za/python-substack/releases/tag/v0.4.0) | | 0.5.0 | released | 2026-08-30 | v0.5.0 | 166 | 28 | 14,448 | 47 | 88 | [GitHub](https://github.com/ma2za/python-substack/releases/tag/v0.5.0) | | 0.6.0 | released | 2026-08-30 | v0.6.0 | 166 | 28 | 14,448 | 47 | 88 | [GitHub](https://github.com/ma2za/python-substack/releases/tag/v0.6.0); [Substack](https://mazzapaolo.substack.com/p/back-up-substack-draft-markdown-python-substack-060) | -| 0.7.0 | in progress | - | - | - | - | - | - | - | - | -| 0.8.0 | planned | - | - | - | - | - | - | - | - | +| 0.7.0 | released | 2026-09-22 | v0.7.0 | 166 | 28 | 14,448 | 47 | 88 | [GitHub](https://github.com/ma2za/python-substack/releases/tag/v0.7.0) | +| 0.8.0 | in progress | - | - | - | - | - | - | - | - | | 0.9.0 | planned | - | - | - | - | - | - | - | - | | 0.10.0 | planned | - | - | - | - | - | - | - | - | | 1.0.0 | planned | - | - | - | 300 target | - | 3x baseline target | - | - | - | @@ -449,7 +449,7 @@ without risking unsupported content. ## 0.8.0: Lossless unsupported widget preservation -**Status:** planned +**Status:** in progress **Objective:** Allow exported drafts containing Substack-only widgets to be updated without data loss. diff --git a/substack/api.py b/substack/api.py index 106a68c..39849de 100644 --- a/substack/api.py +++ b/substack/api.py @@ -478,6 +478,7 @@ def update_draft_from_markdown( draft_section_id: int = None, tags=None, dry_run: bool = False, + allow_unsupported_change: bool = False, ) -> dict: """ Update an existing draft body from Markdown, with optional metadata changes. @@ -497,12 +498,7 @@ def update_draft_from_markdown( if not isinstance(draft_body, dict): raise ValueError("Malformed draft body: draft_body must be a JSON object") - _, unsupported_nodes = document_to_markdown(draft_body) - if unsupported_nodes: - raise ValueError( - "Refusing to update: remote draft contains unsupported Substack nodes. " - "Export it first or remove the nodes manually to avoid data loss." - ) + _, remote_unsupported = document_to_markdown(draft_body) post = Post( title=draft.get("title", ""), @@ -515,6 +511,25 @@ def update_draft_from_markdown( ) post.from_markdown(markdown, api=self) + _, submitted_unsupported = document_to_markdown(post.draft_body) + + if not allow_unsupported_change: + remote_serialized = [ + json.dumps(n, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + for n in remote_unsupported + ] + submitted_serialized = [ + json.dumps(n, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + for n in submitted_unsupported + ] + if sorted(remote_serialized) != sorted(submitted_serialized): + raise ValueError( + "Refusing to update: remote unsupported nodes are missing, duplicated, stale, or altered. " + "Export the draft first or set allow_unsupported_change=True to proceed." + ) + + unsupported_nodes = remote_unsupported + update_payload = {"draft_body": json.dumps(post.draft_body)} if subtitle is not None: diff --git a/substack/cli.py b/substack/cli.py index cac1570..081ed05 100644 --- a/substack/cli.py +++ b/substack/cli.py @@ -310,6 +310,9 @@ def _drafts_create(api, args): def _drafts_update(api, args): + if args.allow_unsupported_change and not args.yes: + raise CLIUsageError("--yes is required when using --allow-unsupported-change") + if not args.yes and (args.json_output or not sys.stdin.isatty()): raise CLIUsageError("--yes is required in non-interactive or JSON mode") @@ -347,6 +350,7 @@ def _drafts_update(api, args): draft_section_id=args.draft_section_id, tags=args.tags, dry_run=args.dry_run, + allow_unsupported_change=args.allow_unsupported_change, ) if args.json_output: @@ -508,6 +512,7 @@ def _build_parser(): drafts_update.add_argument("--draft-section-id", type=int) drafts_update.add_argument("--tag", action="append", dest="tags", metavar="TAG") drafts_update.add_argument("--dry-run", action="store_true") + drafts_update.add_argument("--allow-unsupported-change", action="store_true") drafts_update.add_argument("--yes", action="store_true") drafts_update.set_defaults(handler=_drafts_update) diff --git a/substack/mdrender.py b/substack/mdrender.py index 5926f9c..59f373f 100644 --- a/substack/mdrender.py +++ b/substack/mdrender.py @@ -1,248 +1,305 @@ -"""Markdown -> Substack ProseMirror via markdown-it-py. - -Implements Post.from_markdown() using a real CommonMark parser (markdown-it-py) -plus the standard footnote plugin, with a small renderer that walks the syntax -tree into Substack's node schema. - -Node construction goes through ``substack.nodes`` so the (undocumented) schema -lives in exactly one place. - -Footnotes: Substack numbers footnote anchors by their position in the document -and pairs them one-to-one, in order, with the footnote blocks at the end (it -ignores any explicit number and does not support one block serving several -anchors). So each reference is emitted as its own sequentially-numbered anchor, -and a matching footnote block is appended for each -- a definition referenced -more than once is duplicated, which mirrors how Substack's own editor behaves. -""" - -from __future__ import annotations - -import copy -from typing import Dict, List, Optional - -from markdown_it import MarkdownIt -from markdown_it.tree import SyntaxTreeNode -from mdit_py_plugins.container import container_plugin -from mdit_py_plugins.dollarmath import dollarmath_plugin -from mdit_py_plugins.footnote import footnote_plugin -from mdit_py_plugins.subscript import sub_plugin -from mdit_py_plugins.superscript import superscript_plugin - -from substack import nodes -from substack.nodes import MarkType, NodeType - -_MARK_FOR = { - "strong": {"type": MarkType.STRONG}, - "em": {"type": MarkType.EM}, - "s": {"type": MarkType.STRIKETHROUGH}, - "sup": {"type": MarkType.SUPERSCRIPT}, - "sub": {"type": MarkType.SUBSCRIPT}, -} - - -def _make_parser() -> MarkdownIt: - return ( - MarkdownIt("commonmark") - .use(footnote_plugin) - # Pandoc-style delimiters: no whitespace just inside the dollars and no - # digit just outside them, so paired currency amounts ("$5 ... $10") - # stay plain text instead of becoming math. - .use(dollarmath_plugin, allow_space=False, allow_digits=False) - .use(sub_plugin) - .use(superscript_plugin) - .use(container_plugin, name="pullquote") - .use(container_plugin, name="callout") - .enable("strikethrough") - ) - - -def _coalesce(out_nodes: List[Dict]) -> List[Dict]: - """Merge adjacent text nodes that carry identical marks (e.g. softbreaks).""" - merged: List[Dict] = [] - for node in out_nodes: - if ( - merged - and node.get("type") == NodeType.TEXT - and merged[-1].get("type") == NodeType.TEXT - and node.get("marks") == merged[-1].get("marks") - ): - merged[-1]["text"] += node["text"] - else: - merged.append(node) - return merged - - -def _render_inline(node: SyntaxTreeNode, marks: List[Dict], ctx: Dict) -> List[Dict]: - """Render an inline subtree into a flat list of text / anchor nodes.""" - out: List[Dict] = [] - for child in node.children: - t = child.type - if t == "text": - if child.content: - out.append(nodes.text(child.content, marks)) - elif t == "code_inline": - out.append(nodes.text(child.content, marks + [nodes.code_mark()])) - elif t == "math_inline": - out.append(nodes.latex_inline(child.content.strip())) - elif t in _MARK_FOR: - out.extend(_render_inline(child, marks + [_MARK_FOR[t]], ctx)) - elif t == "link": - href = child.attrs.get("href", "") - out.extend(_render_inline(child, marks + [nodes.link_mark(href)], ctx)) - elif t in ("softbreak", "hardbreak"): - out.append(nodes.text(" ", marks)) - elif t == "footnote_ref": - # Number anchors by document position and record which definition each - # one points to, so matching blocks can be emitted 1:1 afterwards. - ctx["order"].append(child.meta["id"]) - out.append(nodes.footnote_anchor(len(ctx["order"]))) - elif t == "image": - # Inline images are rare in this schema; fall back to alt text. - alt = child.attrs.get("alt") or "".join( - c.content for c in child.children if c.type == "text" - ) - if alt: - out.append(nodes.text(alt, marks)) - return _coalesce(out) - - -def _only_image(inline: SyntaxTreeNode) -> Optional[SyntaxTreeNode]: - """If an inline node is just an image (optionally wrapped in a link), return it.""" - kids = [c for c in inline.children if c.type != "softbreak"] - if len(kids) == 1 and kids[0].type == "image": - return kids[0] - if len(kids) == 1 and kids[0].type == "link": - inner = [c for c in kids[0].children if c.type != "softbreak"] - if len(inner) == 1 and inner[0].type == "image": - img = inner[0] - img._link_href = kids[0].attrs.get("href") # type: ignore[attr-defined] - return img - return None - - -def _captioned_image(img: SyntaxTreeNode, api) -> Dict: - src = img.attrs.get("src", "") - if src.startswith("/"): - src = src[1:] - if api is not None and not src.startswith("http"): - try: - src = api.get_image(src).get("url") - except Exception: - pass - # markdown-it stores the image alt text as the node's content, not in attrs. - alt = img.content or img.attrs.get("alt") or None - # Standard markdown image title `![alt](src "caption")` maps to Substack's caption node. - title = img.attrs.get("title") or None - caption = [nodes.text(title)] if title else None - return nodes.captioned_image( - src, - alt=alt, - href=getattr(img, "_link_href", None), - caption=caption, - ) - - -def _render_block(node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]: - """Render a block-level node into zero or more Substack nodes.""" - t = node.type - - if t == "paragraph": - inline = node.children[0] - img = _only_image(inline) - if img is not None: - return [_captioned_image(img, api)] - return [nodes.paragraph(_render_inline(inline, [], ctx))] - - if t == "heading": - level = int(node.tag[1]) - return [nodes.heading(_render_inline(node.children[0], [], ctx), level=level)] - - if t == "hr": - return [nodes.horizontal_rule()] - - if t in ("fence", "code_block"): - return [ - nodes.code_block( - node.content.rstrip("\n"), language=node.info.strip() or None - ) - ] - - if t == "blockquote": - paras: List[Dict] = [] - for child in node.children: - paras.extend(_render_block(child, api, ctx)) - return [nodes.blockquote(paras)] - - if t == "bullet_list": - return [nodes.bullet_list(_render_list_items(node, api, ctx))] - - if t == "ordered_list": - return [nodes.ordered_list(_render_list_items(node, api, ctx))] - - # "$$...$$ (label)" tokenizes as math_block_label; Substack has no equation - # labels, so it renders like an unlabeled block. - if t in ("math_block", "math_block_label"): - return [nodes.latex_block(node.content.strip())] - - if t == "container_pullquote": - return [nodes.pullquote(_render_container_body(node, api, ctx))] - - if t == "container_callout": - return [nodes.callout_block(_render_container_body(node, api, ctx))] - - # footnote_block is handled separately in markdown_to_doc; ignore it here. - return [] - - -def _render_container_body(node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]: - body: List[Dict] = [] - for child in node.children: - body.extend(_render_block(child, api, ctx)) - return body - - -def _render_list_items(list_node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]: - items = [] - for li in list_node.children: - content: List[Dict] = [] - for child in li.children: - content.extend(_render_block(child, api, ctx)) - items.append({"type": NodeType.LIST_ITEM, "content": content}) - return items - - -def _footnote_definitions(tree: SyntaxTreeNode, api) -> Dict[int, List[Dict]]: - """Map each footnote id to its rendered block content.""" - definitions: Dict[int, List[Dict]] = {} - for node in tree.children: - if node.type != "footnote_block": - continue - for fn in node.children: - # A footnote's own content should not register anchors of its own. - local_ctx = {"order": []} - content: List[Dict] = [] - for child in fn.children: - content.extend(_render_block(child, api, local_ctx)) - definitions[fn.meta["id"]] = content - return definitions - - -def markdown_to_doc(markdown_content: str, api=None) -> List[Dict]: - """Convert Markdown into a list of Substack ProseMirror block nodes.""" - tree = SyntaxTreeNode(_make_parser().parse(markdown_content)) - - definitions = _footnote_definitions(tree, api) - - ctx: Dict = {"order": []} - out: List[Dict] = [] - for node in tree.children: - if node.type == "footnote_block": - continue - out.extend(_render_block(node, api, ctx)) - - # Emit one footnote block per reference, in anchor order, numbered to match. - for number, footnote_id in enumerate(ctx["order"], start=1): - content = copy.deepcopy(definitions.get(footnote_id, [])) - out.append(nodes.footnote(number, content)) - - return out +"""Markdown -> Substack ProseMirror via markdown-it-py. + +Implements Post.from_markdown() using a real CommonMark parser (markdown-it-py) +plus the standard footnote plugin, with a small renderer that walks the syntax +tree into Substack's node schema. + +Node construction goes through ``substack.nodes`` so the (undocumented) schema +lives in exactly one place. + +Footnotes: Substack numbers footnote anchors by their position in the document +and pairs them one-to-one, in order, with the footnote blocks at the end (it +ignores any explicit number and does not support one block serving several +anchors). So each reference is emitted as its own sequentially-numbered anchor, +and a matching footnote block is appended for each -- a definition referenced +more than once is duplicated, which mirrors how Substack's own editor behaves. +""" + +from __future__ import annotations + +import base64 +import copy +import json +import re +from typing import Dict, List, Optional + +from markdown_it import MarkdownIt +from markdown_it.tree import SyntaxTreeNode +from mdit_py_plugins.container import container_plugin +from mdit_py_plugins.dollarmath import dollarmath_plugin +from mdit_py_plugins.footnote import footnote_plugin +from mdit_py_plugins.subscript import sub_plugin +from mdit_py_plugins.superscript import superscript_plugin + +from substack import nodes +from substack.nodes import MarkType, NodeType + +_MARK_FOR = { + "strong": {"type": MarkType.STRONG}, + "em": {"type": MarkType.EM}, + "s": {"type": MarkType.STRIKETHROUGH}, + "sup": {"type": MarkType.SUPERSCRIPT}, + "sub": {"type": MarkType.SUBSCRIPT}, +} + + +def parse_node_marker(comment_content: str) -> dict | None: + """ + Parse a python-substack-node:v1 comment marker and return the parsed JSON dictionary. + + If it is not a python-substack-node:v1 marker, returns None. + If it is an attempted marker but is corrupt/malformed, raises ValueError. + """ + clean = comment_content.strip() + match = re.match(r"^$", clean) + if not match: + if "python-substack-node:v1" in clean: + raise ValueError("Corrupt marker format") + return None + + encoded = match.group(1) + + try: + padding = len(encoded) % 4 + if padding: + encoded += "=" * (4 - padding) + decoded_bytes = base64.urlsafe_b64decode(encoded.encode("ascii")) + except Exception as exc: + raise ValueError("Invalid URL-safe base64 in marker") from exc + + try: + decoded_str = decoded_bytes.decode("utf-8") + except Exception as exc: + raise ValueError("Invalid UTF-8 in marker") from exc + + try: + data = json.loads(decoded_str) + except Exception as exc: + raise ValueError("Invalid JSON in marker") from exc + + if not isinstance(data, dict): + raise ValueError("Marker payload is not a top-level JSON object") + + node_type = data.get("type") + if not isinstance(node_type, str) or not node_type: + raise ValueError("Marker payload has missing or empty 'type'") + + return data + + +def _make_parser() -> MarkdownIt: + return ( + MarkdownIt("commonmark") + .use(footnote_plugin) + # Pandoc-style delimiters: no whitespace just inside the dollars and no + # digit just outside them, so paired currency amounts ("$5 ... $10") + # stay plain text instead of becoming math. + .use(dollarmath_plugin, allow_space=False, allow_digits=False) + .use(sub_plugin) + .use(superscript_plugin) + .use(container_plugin, name="pullquote") + .use(container_plugin, name="callout") + .enable("strikethrough") + ) + + +def _coalesce(out_nodes: List[Dict]) -> List[Dict]: + """Merge adjacent text nodes that carry identical marks (e.g. softbreaks).""" + merged: List[Dict] = [] + for node in out_nodes: + if ( + merged + and node.get("type") == NodeType.TEXT + and merged[-1].get("type") == NodeType.TEXT + and node.get("marks") == merged[-1].get("marks") + ): + merged[-1]["text"] += node["text"] + else: + merged.append(node) + return merged + + +def _render_inline(node: SyntaxTreeNode, marks: List[Dict], ctx: Dict) -> List[Dict]: + """Render an inline subtree into a flat list of text / anchor nodes.""" + out: List[Dict] = [] + for child in node.children: + t = child.type + if t == "text": + if child.content: + out.append(nodes.text(child.content, marks)) + elif t == "code_inline": + out.append(nodes.text(child.content, marks + [nodes.code_mark()])) + elif t == "math_inline": + out.append(nodes.latex_inline(child.content.strip())) + elif t in _MARK_FOR: + out.extend(_render_inline(child, marks + [_MARK_FOR[t]], ctx)) + elif t == "link": + href = child.attrs.get("href", "") + out.extend(_render_inline(child, marks + [nodes.link_mark(href)], ctx)) + elif t in ("softbreak", "hardbreak"): + out.append(nodes.text(" ", marks)) + elif t == "footnote_ref": + # Number anchors by document position and record which definition each + # one points to, so matching blocks can be emitted 1:1 afterwards. + ctx["order"].append(child.meta["id"]) + out.append(nodes.footnote_anchor(len(ctx["order"]))) + elif t == "image": + # Inline images are rare in this schema; fall back to alt text. + alt = child.attrs.get("alt") or "".join( + c.content for c in child.children if c.type == "text" + ) + if alt: + out.append(nodes.text(alt, marks)) + elif t == "html_inline": + marker_data = parse_node_marker(child.content) + if marker_data is not None: + out.append(marker_data) + return _coalesce(out) + + +def _only_image(inline: SyntaxTreeNode) -> Optional[SyntaxTreeNode]: + """If an inline node is just an image (optionally wrapped in a link), return it.""" + kids = [c for c in inline.children if c.type != "softbreak"] + if len(kids) == 1 and kids[0].type == "image": + return kids[0] + if len(kids) == 1 and kids[0].type == "link": + inner = [c for c in kids[0].children if c.type != "softbreak"] + if len(inner) == 1 and inner[0].type == "image": + img = inner[0] + img._link_href = kids[0].attrs.get("href") # type: ignore[attr-defined] + return img + return None + + +def _captioned_image(img: SyntaxTreeNode, api) -> Dict: + src = img.attrs.get("src", "") + if src.startswith("/"): + src = src[1:] + if api is not None and not src.startswith("http"): + try: + src = api.get_image(src).get("url") + except Exception: + pass + # markdown-it stores the image alt text as the node's content, not in attrs. + alt = img.content or img.attrs.get("alt") or None + # Standard markdown image title `![alt](src "caption")` maps to Substack's caption node. + title = img.attrs.get("title") or None + caption = [nodes.text(title)] if title else None + return nodes.captioned_image( + src, + alt=alt, + href=getattr(img, "_link_href", None), + caption=caption, + ) + + +def _render_block(node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]: + """Render a block-level node into zero or more Substack nodes.""" + t = node.type + + if t == "html_block": + marker_data = parse_node_marker(node.content) + if marker_data is not None: + return [marker_data] + return [] + + if t == "paragraph": + inline = node.children[0] + img = _only_image(inline) + if img is not None: + return [_captioned_image(img, api)] + return [nodes.paragraph(_render_inline(inline, [], ctx))] + + if t == "heading": + level = int(node.tag[1]) + return [nodes.heading(_render_inline(node.children[0], [], ctx), level=level)] + + if t == "hr": + return [nodes.horizontal_rule()] + + if t in ("fence", "code_block"): + return [ + nodes.code_block( + node.content.rstrip("\n"), language=node.info.strip() or None + ) + ] + + if t == "blockquote": + paras: List[Dict] = [] + for child in node.children: + paras.extend(_render_block(child, api, ctx)) + return [nodes.blockquote(paras)] + + if t == "bullet_list": + return [nodes.bullet_list(_render_list_items(node, api, ctx))] + + if t == "ordered_list": + return [nodes.ordered_list(_render_list_items(node, api, ctx))] + + # "$$...$$ (label)" tokenizes as math_block_label; Substack has no equation + # labels, so it renders like an unlabeled block. + if t in ("math_block", "math_block_label"): + return [nodes.latex_block(node.content.strip())] + + if t == "container_pullquote": + return [nodes.pullquote(_render_container_body(node, api, ctx))] + + if t == "container_callout": + return [nodes.callout_block(_render_container_body(node, api, ctx))] + + # footnote_block is handled separately in markdown_to_doc; ignore it here. + return [] + + +def _render_container_body(node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]: + body: List[Dict] = [] + for child in node.children: + body.extend(_render_block(child, api, ctx)) + return body + + +def _render_list_items(list_node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]: + items = [] + for li in list_node.children: + content: List[Dict] = [] + for child in li.children: + content.extend(_render_block(child, api, ctx)) + items.append({"type": NodeType.LIST_ITEM, "content": content}) + return items + + +def _footnote_definitions(tree: SyntaxTreeNode, api) -> Dict[int, List[Dict]]: + """Map each footnote id to its rendered block content.""" + definitions: Dict[int, List[Dict]] = {} + for node in tree.children: + if node.type != "footnote_block": + continue + for fn in node.children: + # A footnote's own content should not register anchors of its own. + local_ctx = {"order": []} + content: List[Dict] = [] + for child in fn.children: + content.extend(_render_block(child, api, local_ctx)) + definitions[fn.meta["id"]] = content + return definitions + + +def markdown_to_doc(markdown_content: str, api=None) -> List[Dict]: + """Convert Markdown into a list of Substack ProseMirror block nodes.""" + tree = SyntaxTreeNode(_make_parser().parse(markdown_content)) + + definitions = _footnote_definitions(tree, api) + + ctx: Dict = {"order": []} + out: List[Dict] = [] + for node in tree.children: + if node.type == "footnote_block": + continue + out.extend(_render_block(node, api, ctx)) + + # Emit one footnote block per reference, in anchor order, numbered to match. + for number, footnote_id in enumerate(ctx["order"], start=1): + content = copy.deepcopy(definitions.get(footnote_id, [])) + out.append(nodes.footnote(number, content)) + + return out diff --git a/tests/substack/test_widget_preservation.py b/tests/substack/test_widget_preservation.py new file mode 100644 index 0000000..02349c3 --- /dev/null +++ b/tests/substack/test_widget_preservation.py @@ -0,0 +1,159 @@ +import base64 +import json +from unittest.mock import Mock, patch + +import pytest + +from substack import Api, cli +from substack.mdrender import parse_node_marker, markdown_to_doc + + +def test_parse_node_marker_valid(): + node = {"type": "button", "attrs": {"text": "Click me", "url": "https://example.com"}} + payload = json.dumps(node, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + encoded = base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + comment = f"" + + parsed = parse_node_marker(comment) + assert parsed == node + + +def test_parse_node_marker_corrupt_format(): + with pytest.raises(ValueError, match="Corrupt marker format"): + parse_node_marker("") + + +def test_parse_node_marker_invalid_base64(): + # Attempted but bad character + with pytest.raises(ValueError, match="Corrupt marker format"): + parse_node_marker("") + + +def test_parse_node_marker_invalid_json(): + # Valid base64 but invalid JSON (not a complete object) + encoded = base64.urlsafe_b64encode(b"{not valid json").decode("ascii").rstrip("=") + with pytest.raises(ValueError, match="Invalid JSON"): + parse_node_marker(f"") + + +def test_parse_node_marker_not_object(): + # Valid JSON but a list instead of a dict + encoded = base64.urlsafe_b64encode(b"[]").decode("ascii").rstrip("=") + with pytest.raises(ValueError, match="not a top-level JSON object"): + parse_node_marker(f"") + + +def test_parse_node_marker_missing_type(): + # Valid object but missing type + encoded = base64.urlsafe_b64encode(b'{"attrs": {}}').decode("ascii").rstrip("=") + with pytest.raises(ValueError, match="missing or empty 'type'"): + parse_node_marker(f"") + + +def test_parse_node_marker_empty_type(): + # Valid object but empty type + encoded = base64.urlsafe_b64encode(b'{"type": ""}').decode("ascii").rstrip("=") + with pytest.raises(ValueError, match="missing or empty 'type'"): + parse_node_marker(f"") + + +def test_parse_node_marker_ordinary_comment(): + assert parse_node_marker("") is None + + +def test_markdown_to_doc_preserves_block_and_inline_markers(): + # Build a button node + btn = {"type": "button", "attrs": {"text": "Click me", "url": "https://example.com"}} + btn_payload = json.dumps(btn, separators=(",", ":"), sort_keys=True).encode("utf-8") + btn_encoded = base64.urlsafe_b64encode(btn_payload).decode("ascii").rstrip("=") + btn_comment = f"" + + # Build an inline recipe node + recipe = {"type": "recipe", "attrs": {"id": 123}} + recipe_payload = json.dumps(recipe, separators=(",", ":"), sort_keys=True).encode("utf-8") + recipe_encoded = base64.urlsafe_b64encode(recipe_payload).decode("ascii").rstrip("=") + recipe_comment = f"" + + markdown = f"""# Heading 1 + +Some text and inline {recipe_comment} recipe. + +{btn_comment} + + +""" + doc = markdown_to_doc(markdown) + + # doc should have heading, paragraph (with text, recipe inline node, text), and the button block node + assert len(doc) == 3 + assert doc[0]["type"] == "heading" + + p = doc[1] + assert p["type"] == "paragraph" + inline_content = p["content"] + assert len(inline_content) == 3 + assert inline_content[0]["text"] == "Some text and inline " + assert inline_content[1] == recipe + assert inline_content[2]["text"] == " recipe." + + assert doc[2] == btn + + +def test_update_draft_from_markdown_preservation(monkeypatch): + api = Api.__new__(Api) + api.publication_url = "https://test.substack.com" + monkeypatch.setattr(api, "get_user_id", lambda: 1) + + # Remote draft contains an unsupported "button" node + remote_btn = {"type": "button", "attrs": {"text": "Click", "url": "https://example.com"}} + remote_body = { + "type": "doc", + "content": [ + {"type": "heading", "attrs": {"level": 1}, "content": [{"type": "text", "text": "Hello"}]}, + remote_btn + ] + } + mock_get_draft = Mock(return_value={"id": 42, "draft_body": json.dumps(remote_body)}) + monkeypatch.setattr(api, "get_draft", mock_get_draft) + + mock_put_draft = Mock(return_value={"id": 42}) + monkeypatch.setattr(api, "put_draft", mock_put_draft) + + # 1. Update with correct marker matches and succeeds + btn_payload = json.dumps(remote_btn, separators=(",", ":"), sort_keys=True).encode("utf-8") + btn_encoded = base64.urlsafe_b64encode(btn_payload).decode("ascii").rstrip("=") + submitted_markdown = f"# New Title\n\n\n" + + res = api.update_draft_from_markdown(42, submitted_markdown) + assert res["action"] == "update" + assert res["dry_run"] is False + assert mock_put_draft.called + + # 2. Update without marker fails when allow_unsupported_change is False + mock_put_draft.reset_mock() + with pytest.raises(ValueError, match="remote unsupported nodes are missing"): + api.update_draft_from_markdown(42, "# New Title without button") + assert not mock_put_draft.called + + # 3. Update without marker succeeds when allow_unsupported_change is True + mock_put_draft.reset_mock() + res = api.update_draft_from_markdown(42, "# New Title without button", allow_unsupported_change=True) + assert res["action"] == "update" + assert mock_put_draft.called + + +def test_cli_update_requires_yes_for_allow_unsupported_change(tmp_path, monkeypatch, capsys): + # Mocking UpdateOperationsApi + class MockApi: + def update_draft_from_markdown(self, *args, **kwargs): + return {"action": "update"} + + monkeypatch.setattr(cli, "_api_from_env", lambda **kw: MockApi()) + + md_file = tmp_path / "test.md" + md_file.write_text("# Test", encoding="utf-8") + + # If passing --allow-unsupported-change without --yes, it must raise CLIUsageError + assert cli.main(["drafts", "update", "42", str(md_file), "--allow-unsupported-change"]) == 2 + err = capsys.readouterr().err + assert "--yes is required when using --allow-unsupported-change" in err