diff --git a/docs/source/conf.py b/docs/source/conf.py index d137489..4a56d18 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,8 +17,10 @@ sys.path.insert(0, os.path.abspath(".")) from filters import generate_filters_docs +from question_format import generate_question_format_tables generate_filters_docs() +generate_question_format_tables() sys.path.insert(0, os.path.abspath("../..")) @@ -60,7 +62,13 @@ graphviz_output_format = "svg" templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = [ + "_build", + "Thumbs.db", + ".DS_Store", + # Tables that question-format.md includes: pages of their own otherwise. + "_autosummary/question-format/*.md", +] source_suffix = [".rst", ".md"] copybutton_prompt_text = "$ " diff --git a/docs/source/question-format.md b/docs/source/question-format.md index c27ddf3..7d29937 100644 --- a/docs/source/question-format.md +++ b/docs/source/question-format.md @@ -122,68 +122,26 @@ Files are written on a single line. ### Set -| key | set by | meaning | -|---|---|---| -| `name` | `Set.set_name` | names the folder, the zip and the file | -| `description` | `Set.set_description` | shown with the set | -| `finalAnswerVisibility` | `Set._finalAnswerVisibility` | `OPEN`, `HIDE` or `OPEN_WITH_WARNINGS` | -| `workedSolutionVisibility` | `Set._workedSolutionVisibility` | as above | -| `structuredTutorialVisibility` | `Set._structuredTutorialVisibility` | as above | -| `manuallyHidden` | fixed by template (`true`) | | -| `chatbotVisibility` | fixed by template (`"HIDE"`) | | +```{include} _autosummary/question-format/set.md +``` Each visibility is a {class}`~in2lambda.api.visibility_status.VisibilityController`, changed with `to_open()`, `to_hide()` or `to_open_with_warnings()`. ### Question -| key | set by | meaning | -|---|---|---| -| `orderNumber` | position in `Set.questions`, from 0 | matches the filename | -| `title` | `Question.title` | shown to students; an empty title becomes `Question N` | -| `masterContent` | `Question.main_text` | markdown shared by every part: setup, data, figure | -| `skill` | `Question.skill` | difficulty; exports use 1/3 and 2/3. Omitted if unset | -| `guidance` | `Question.guidance` | a sentence to students about the question's purpose. Omitted if unset | -| `durationLowerBound`, `durationUpperBound` | `Question.duration_lower_bound`, `.duration_upper_bound` | expected minutes. Omitted if unset | -| `publish` | `Question.publish` | | -| `displayFinalAnswer` | `Question.display_final_answer` | | -| `displayWorkedSolution` | `Question.display_worked_solution` | | -| `displayStructuredTutorial` | `Question.display_structured_tutorial` | | -| `displayChatbot` | `Question.display_chatbot` | | -| `parts` | `Question.parts` | ordered from 0; students see (a), (b), ... | +```{include} _autosummary/question-format/question.md +``` ### Part -| key | set by | meaning | -|---|---|---| -| `orderNumber` | position in `Question.parts`, from 0 | | -| `content` | `Part.text` | markdown for this part | -| `answerContent` | `Part.answer` | the final answer shown to students, markdown | -| `responseAreas` | `Part.response_areas` | the answer boxes; may be empty | -| `workedSolution.content` | `Part.worked_solution` | markdown, split into tutorial steps on `---` | -| `workedSolution.children` | fixed by template (`[]`) | | +```{include} _autosummary/question-format/part.md +``` ### Response area -| key | set by | meaning | -|---|---|---| -| `orderNumber` | position in `Part.response_areas`, from 0 | | -| `preResponseText`, `postResponseText` | `pre_text`, `post_text` | labels either side of the box | -| `contentAfter` | `content_after` | markdown shown after the box, before the next one | -| `inputSymbols` | `input_symbols` | see below | -| `displayInputSymbols` | `display_input_symbols` | whether students are shown the palette | -| `evaluationFunctionName` | `evaluation_function` | how Lambda Feedback marks the response | -| `gradeParams` | `grade_params` | that function's settings; see below | -| `livePreview` | `live_preview` | render what is typed as the student types | -| `includeInPdf` | `include_in_pdf` | | -| `saveAllowed` | `save_allowed` | | -| `separateFeedback` | `separate_feedback` | | -| `commonFeedbackColor`, `correctFeedbackColor`, `correctFeedbackPrefix`, `incorrectFeedbackColor`, `incorrectFeedbackPrefix` | `common_feedback_color`, `correct_feedback_color`, `correct_feedback_prefix`, `incorrect_feedback_color`, `incorrect_feedback_prefix` | default to what Lambda Feedback fills in | -| `tests` | `tests` | see below | -| `cases` | `cases` | see below | -| `response.responseInput.responseType` | `response_type` | which box students get | -| `response.responseInput.answer` | `answer` | the correct answer | -| `response.responseInput.config` | `config` | the box's own settings; see below | +```{include} _autosummary/question-format/response_area.md +``` The three types in2lambda writes: diff --git a/docs/source/question_format.py b/docs/source/question_format.py new file mode 100644 index 0000000..4029c42 --- /dev/null +++ b/docs/source/question_format.py @@ -0,0 +1,234 @@ +"""Builds the key tables on the question format page from the real exports. + +Those tables used to be written by hand, so a change to Lambda Feedback's schema left +the page quietly describing the old one. They are now collected at docs build from the +exports in ``tests/fixtures/exports``: one table per kind of object, listing every key +those exports carry, its JSON type and an example value. What a key *means* cannot come +from the data, so it is kept in ``NOTES`` below, and a key with no note is an error +rather than a blank cell. +""" + +import json +from pathlib import Path +from typing import Any, Iterator, Sequence + +EXPORTS_DIR = Path(__file__).parents[2] / "tests" / "fixtures" / "exports" +"""Real Lambda Feedback exports, the same ones the test suite round-trips.""" + +EXPORTS = sorted(path for path in EXPORTS_DIR.iterdir() if path.is_dir()) +"""Every export folder, found rather than listed, as the test suite finds them.""" + +OUT_DIR = Path(__file__).parent / "_autosummary" / "question-format" +"""Where the tables are written for ``question-format.md`` to include.""" + +NOTES: dict[str, dict[str, str]] = { + "set": { + "name": "`Set.set_name` — names the folder, the zip and the file", + "description": "`Set.set_description` — shown with the set", + "isSurvey": "Neither read nor written by in2lambda; import fills it in", + "releasedAt": "Neither read nor written by in2lambda; import fills it in", + "manuallyHidden": "Fixed by the template (`true`)", + "finalAnswerVisibility": ( + "`Set._finalAnswerVisibility` — `OPEN`, `HIDE` or `OPEN_WITH_WARNINGS`" + ), + "workedSolutionVisibility": "`Set._workedSolutionVisibility` — as above", + "structuredTutorialVisibility": "`Set._structuredTutorialVisibility` — as above", + "chatbotVisibility": 'Fixed by the template (`"HIDE"`), whatever an export says', + }, + "question": { + "orderNumber": "Position in `Set.questions`, from 0; matches the filename", + "title": "`Question.title` — shown to students; an empty title becomes `Question N`", + "masterContent": ( + "`Question.main_text` — markdown shared by every part: setup, data, figure" + ), + "skill": "`Question.skill` — difficulty; exports use 1/3 and 2/3. Omitted if unset", + "guidance": ( + "`Question.guidance` — a sentence to students about the question's purpose." + " Omitted if unset" + ), + "durationLowerBound": ( + "`Question.duration_lower_bound` — expected minutes. Omitted if unset" + ), + "durationUpperBound": ( + "`Question.duration_upper_bound` — expected minutes. Omitted if unset" + ), + "publish": "`Question.publish` — whether the question is visible to students", + "displayFinalAnswer": "`Question.display_final_answer`", + "displayWorkedSolution": "`Question.display_worked_solution`", + "displayStructuredTutorial": "`Question.display_structured_tutorial`", + "displayChatbot": "`Question.display_chatbot`", + "parts": "`Question.parts` — ordered from 0; students see (a), (b), ...", + }, + "part": { + "orderNumber": "Position in `Question.parts`, from 0", + "content": "`Part.text` — markdown for this part", + "answerContent": "`Part.answer` — the final answer shown to students, markdown", + "responseAreas": "`Part.response_areas` — the answer boxes; may be empty", + "workedSolution.content": ( + "`Part.worked_solution` — markdown, split into tutorial steps on `---`" + ), + "workedSolution.children": "Fixed by the template (`[]`)", + }, + "response_area": { + "orderNumber": "Position in `Part.response_areas`, from 0", + "preResponseText": "`pre_text` — the label before the box", + "postResponseText": "`post_text` — the label after the box", + "contentAfter": "`content_after` — markdown shown after the box, before the next", + "inputSymbols": "`input_symbols` — the symbol palette; see below", + "displayInputSymbols": ( + "`display_input_symbols` — whether students are shown the palette" + ), + "evaluationFunctionName": ( + "`evaluation_function` — how Lambda Feedback marks the response" + ), + "gradeParams": "`grade_params` — that function's settings; see below", + "livePreview": "`live_preview` — render what is typed as the student types", + "includeInPdf": "`include_in_pdf` — whether the box appears in the printed question", + "saveAllowed": "`save_allowed` — whether a response can be saved unsubmitted", + "separateFeedback": "`separate_feedback` — show feedback apart from the mark", + "commonFeedbackColor": ( + "`common_feedback_color` — defaults to what Lambda Feedback fills in" + ), + "correctFeedbackColor": ( + "`correct_feedback_color` — defaults to what Lambda Feedback fills in" + ), + "correctFeedbackPrefix": ( + "`correct_feedback_prefix` — defaults to what Lambda Feedback fills in" + ), + "incorrectFeedbackColor": ( + "`incorrect_feedback_color` — defaults to what Lambda Feedback fills in" + ), + "incorrectFeedbackPrefix": ( + "`incorrect_feedback_prefix` — defaults to what Lambda Feedback fills in" + ), + "tests": "`tests` — the author's own checks of the marking; see below", + "cases": "`cases` — feedback for particular responses; see below", + "response.responseInput.responseType": "`response_type` — which box students get", + "response.responseInput.answer": "`answer` — the correct answer", + "response.responseInput.config": "`config` — the box's own settings; see below", + }, +} +"""What each key means, per kind of object, keyed by the key path in the table.""" + + +def _json_type(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list): + return "array" + return "object" + + +def _flatten( + obj: dict, notes: dict[str, str], path: str = "" +) -> Iterator[tuple[str, Any]]: + """Each key path in `obj` and its value. + + A key holding an object is descended into, so that `workedSolution` becomes + `workedSolution.content`, unless it has a note of its own — `gradeParams` and + `config` hold whatever the box type needs, and are described in prose instead. + An empty object is a row like any other: descending into it would yield nothing, + so a new key holding `{}` would go unnoticed rather than asking for a note. + """ + for key, value in obj.items(): + full_path = f"{path}{key}" + if isinstance(value, dict) and value and full_path not in notes: + yield from _flatten(value, notes, f"{full_path}.") + else: + yield full_path, value + + +def _keys(objects: list[dict], notes: dict[str, str]) -> dict[str, dict]: + """Every key path the objects carry, in order of first appearance.""" + keys: dict[str, dict] = {} + for obj in objects: + for path, value in _flatten(obj, notes): + key = keys.setdefault(path, {"types": set(), "example": value, "seen": 0}) + key["seen"] += 1 + key["types"].add(_json_type(value)) + # An empty string or list says nothing about the key, so keep looking. + if key["example"] in (None, "", [], {}): + key["example"] = value + for key in keys.values(): + key["optional"] = key["seen"] < len(objects) + return keys + + +def _cell(value: Any) -> str: + """One example value, short enough to sit in a table and not read as markdown.""" + example = json.dumps(value) + if len(example) > 40: + example = f"{example[:39]}…" + return f"`{example.replace('|', chr(92) + '|')}`" + + +def _table(keys: dict[str, dict], notes: dict[str, str]) -> str: + rows = ["| key | type | example | meaning |", "|---|---|---|---|"] + for path, key in keys.items(): + types = " or ".join(sorted(key["types"])) + if key["optional"]: + types += ", optional" + rows.append(f"| `{path}` | {types} | {_cell(key['example'])} | {notes[path]} |") + return "\n".join(rows) + "\n" + + +def _roots(export_dirs: Sequence[Path]) -> dict[str, list[dict]]: + """The objects each table is rooted on, gathered from every export.""" + roots: dict[str, list[dict]] = { + "set": [], + "question": [], + "part": [], + "response_area": [], + } + for export_dir in export_dirs: + for file in sorted(export_dir.glob("set_*.json")): + roots["set"].append(json.loads(file.read_text())) + for file in sorted(export_dir.glob("question_*.json")): + question = json.loads(file.read_text()) + roots["question"].append(question) + for part in question.get("parts", []): + roots["part"].append(part) + roots["response_area"] += part.get("responseAreas", []) + return roots + + +def generate_question_format_tables( + export_dirs: Sequence[Path] = EXPORTS, out_dir: Path = OUT_DIR +) -> None: + """Write a markdown table of keys per kind of object for the page to include. + + Args: + export_dirs: Lambda Feedback export folders to read the keys from. + out_dir: where `set.md`, `question.md`, `part.md` and `response_area.md` go. + + Raises: + ValueError: if an export carries a key with no note in `NOTES`, or a note + names a key no export carries. Either way the page and the schema have + drifted apart, and the error names every key that has to be dealt with. + """ + collected = { + kind: _keys(objects, NOTES[kind]) + for kind, objects in _roots(export_dirs).items() + } + + drifted = [] + for kind, keys in collected.items(): + notes = NOTES[kind] + drifted += [f"{kind}: no note for {path}" for path in keys if path not in notes] + drifted += [ + f"{kind}: note for {path}, which no export carries" + for path in notes + if path not in keys + ] + if drifted: + raise ValueError("; ".join(drifted)) + + out_dir.mkdir(parents=True, exist_ok=True) + for kind, keys in collected.items(): + (out_dir / f"{kind}.md").write_text(_table(keys, NOTES[kind])) diff --git a/tests/test_question_format_docs.py b/tests/test_question_format_docs.py new file mode 100644 index 0000000..d835a08 --- /dev/null +++ b/tests/test_question_format_docs.py @@ -0,0 +1,64 @@ +"""The question format page's key tables are generated from the real exports. + +``docs/source/question_format.py`` builds them at docs build. Running the same +generator here means a key Lambda Feedback adds to an export fails the test suite +until someone writes down what it means, rather than a docs build nobody watches. +""" + +import importlib.util +import json +import shutil +from pathlib import Path + +import pytest +from conftest import EXPORTS + +_module_path = Path(__file__).parents[1] / "docs" / "source" / "question_format.py" +_spec = importlib.util.spec_from_file_location("question_format", _module_path) +question_format = importlib.util.module_from_spec(_spec) +# Loaded by path because docs/source is not a package and holds Sphinx's conf.py. +_spec.loader.exec_module(question_format) + + +def _table_keys(table: Path) -> list[str]: + """The key column of a generated table, without its header rows.""" + return [ + line.split("|")[1].strip(" `") for line in table.read_text().splitlines()[2:] + ] + + +def _exported_keys(pattern: str) -> set[str]: + return { + key + for export_dir in EXPORTS + for file in export_dir.glob(pattern) + for key in json.loads(file.read_text()) + } + + +def test_tables_cover_the_exported_keys(tmp_path: Path) -> None: + """Each table lists the keys its exports carry, every one of them with a note.""" + question_format.generate_question_format_tables(EXPORTS, tmp_path) + + assert set(_table_keys(tmp_path / "set.md")) == _exported_keys("set_*.json") + assert set(_table_keys(tmp_path / "question.md")) == _exported_keys( + "question_*.json" + ) + # Objects a key holds are flattened onto their parent's table. + assert "workedSolution.content" in _table_keys(tmp_path / "part.md") + assert "response.responseInput.config" in _table_keys(tmp_path / "response_area.md") + + +# An empty object holds no keys to descend into, so it has to be a row in its own +# right for a new key to be named rather than silently dropped. +@pytest.mark.parametrize("value", [1, {}]) +def test_a_key_with_no_note_is_named(value: object, tmp_path: Path) -> None: + """A key added to the schema stops the build, saying which key needs a note.""" + export_dir = Path(shutil.copytree(EXPORTS[0], tmp_path / "export")) + file = sorted(export_dir.glob("question_*.json"))[0] + question = json.loads(file.read_text()) + question["madeUp"] = value + file.write_text(json.dumps(question)) + + with pytest.raises(ValueError, match="madeUp"): + question_format.generate_question_format_tables([export_dir], tmp_path / "out")