From 8d42b43427bcc9ee9be64beb930ebf15cb931231 Mon Sep 17 00:00:00 2001 From: Simon Halvorsen Date: Thu, 24 Sep 2026 17:40:48 +0200 Subject: [PATCH] Added yaml-promise-type Ticket: CFE-4574 Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Halvorsen --- cfbs.json | 11 + promise-types/yaml/README.md | 233 +++++++++ promise-types/yaml/enable.cf | 6 + promise-types/yaml/example.cf | 78 +++ promise-types/yaml/jq_filter.py | 286 +++++++++++ promise-types/yaml/tests/create.cf | 168 +++++++ promise-types/yaml/tests/edit.cf | 272 +++++++++++ promise-types/yaml/tests/errors.cf | 184 ++++++++ promise-types/yaml/yaml_lite.py | 599 ++++++++++++++++++++++++ promise-types/yaml/yaml_promise_type.py | 307 ++++++++++++ 10 files changed, 2144 insertions(+) create mode 100644 promise-types/yaml/README.md create mode 100644 promise-types/yaml/enable.cf create mode 100644 promise-types/yaml/example.cf create mode 100644 promise-types/yaml/jq_filter.py create mode 100644 promise-types/yaml/tests/create.cf create mode 100644 promise-types/yaml/tests/edit.cf create mode 100644 promise-types/yaml/tests/errors.cf create mode 100644 promise-types/yaml/yaml_lite.py create mode 100644 promise-types/yaml/yaml_promise_type.py diff --git a/cfbs.json b/cfbs.json index b4351c0..ea526b5 100644 --- a/cfbs.json +++ b/cfbs.json @@ -581,6 +581,17 @@ "append enable.cf services/init.cf" ] }, + "promise-type-yaml": { + "description": "Promise type to edit YAML files.", + "subdirectory": "promise-types/yaml", + "dependencies": ["library-for-promise-types-in-python"], + "steps": [ + "copy yaml_promise_type.py modules/promises/", + "copy yaml_lite.py modules/promises/", + "copy jq_filter.py modules/promises/", + "append enable.cf services/init.cf" + ] + }, "uninstall-packages": { "description": "Allows you to specify a list of packages you want uninstalled on your hosts.", "subdirectory": "security/uninstall-packages", diff --git a/promise-types/yaml/README.md b/promise-types/yaml/README.md new file mode 100644 index 0000000..94aebb4 --- /dev/null +++ b/promise-types/yaml/README.md @@ -0,0 +1,233 @@ +Promise type for editing values inside `yaml` files. + +Edits are applied as small text splices on the original lines, so comments, key order, quoting style and formatting of everything not touched by the promise are preserved. + +## Promiser + +Path to the YAML file to edit. + +A missing file is treated as an empty document: `present` creates it, `absent` leaves it missing. +Like with `files:` promises, missing parent directories are created. +New files get mode 0600 and new directories 0700, the same as `files:` promises create them with. To use other permissions, create the file with a `files:` promise and `perms` first. +An existing file keeps its permissions and owner. + +## Attributes + +| Name | Type | Required | Description | +| -------- | -------- | ------------------ | ----------------------------------------------------------------- | +| `target` | `string` | yes | jq-like path to the node(s), see [Target syntax](#target-syntax) | +| `state` | `string` | yes | `present` or `absent` | +| `value` | `string` | depends on `state` | A scalar value, see [States](#states) | + +### States + +Running a promise again once it is kept changes nothing. +What `state` means depends on whether the target ends in `[]`: + +```cfengine3 +target => ".server.port", state => "present", value => "8080"; # key has value +target => ".server.ports[]", state => "present", value => "22:22"; # list contains value +target => ".server.ports[]", state => "absent", value => "22:22"; # list doesn't contain value +target => ".server.debug", state => "absent"; # key removed +``` + +- `present` on a key: the key has `value`. Creates the key if it's missing, along with any missing parent keys, or replaces the current value, including nested content. + The quoting style of an existing value (plain, single or double quoted) is kept. + A key holding a sequence is not replaced; end the target with `[]` to add an item instead. +- `present` on `.list[]`: the sequence contains `value`. Appends it if it's missing, creating the sequence (and parent keys) if needed. +- `present` without `value`, on a target ending in `[] | select(.key == value)`: a sequence of mappings contains an item matching the select. + If none does, `- key: value` is appended, and other fields can then be set through the same `select()`: + + ```cfengine3 + "/etc/app/users.yaml" + target => '.users[] | select(.name == "carol")', + state => "present"; + + "/etc/app/users.yaml" + target => '.users[] | select(.name == "carol") | .shell', + state => "present", + value => "/bin/zsh"; + ``` +- `absent` without `value`: the key(s) or item(s) at `target` are removed, including anything nested under them. +- `absent` on `.list[]` with `value`: every item equal to `value` is removed; the rest of the sequence is kept. + +Parents are never removed, even if they end up empty. +Sequence items are matched by content, not by position: use `[]` with `value`, or `select()` for sequences of mappings. +`absent` on a plain index like `.hosts[1]` is rejected, since the next run would remove whatever moved into that position. + +`true`, `false`, `null` and numbers are written as-is, so `value => "false"` writes a boolean. +Values that would be misread (e.g. `no`, `on`, `22:22`, or anything containing `: `) are quoted. + +### Target syntax + +| Syntax | Meaning | +| ------------------------------ | --------------------------------------------------------- | +| `.a.b.c` | Key access | +| `.a["b.c"]` | Quoted key access (keys with dots, spaces, etc.) | +| `.a[1]`, `.a[-1]` | Sequence index (negative counts from the end) | +| `.a[]` | Every item of a sequence | +| `.a[] \| select(.b == "x")` | Items where `.b` equals a string, number, `true`, `false` or `null` | +| `... \| .c` | Continue from each selected item | + +A target can match several nodes (e.g. `.hosts[].port`); the state is applied to all of them. +A `select()` that matches nothing is kept. +Missing parents are only created for plain key paths like `.a.b.c`, not through `[]`, indexes or `select()`. + +## Example + +Given `/etc/netplan/01-netcfg.yaml`: + +```yaml +# Managed by cfengine +network: + version: 2 + ethernets: + eth0: + dhcp4: true + addresses: + - 192.168.1.10/24 + nameservers: + addresses: + - 8.8.8.8 + - 8.8.4.4 +``` + +the policy + +```cfengine3 +bundle agent main +{ + vars: + "netplan" string => "/etc/netplan/01-netcfg.yaml"; + "eth0" string => ".network.ethernets.eth0"; + + yaml: + "$(netplan)" + target => "$(eth0).dhcp4", + state => "present", + value => "false"; + + "$(netplan)" + target => "$(eth0).addresses[]", + state => "absent", + value => "192.168.1.10/24"; + + "$(netplan)" + target => "$(eth0).addresses[]", + state => "present", + value => "10.0.0.5/24"; + + "$(netplan)" + target => "$(eth0).nameservers.addresses[]", + state => "absent", + value => "8.8.4.4"; + + "$(netplan)" + target => "$(eth0).nameservers.addresses[]", + state => "present", + value => "1.1.1.1"; +} +``` + +results in: + +```yaml +# Managed by cfengine +network: + version: 2 + ethernets: + eth0: + dhcp4: false + addresses: + - 10.0.0.5/24 + nameservers: + addresses: + - 8.8.8.8 + - 1.1.1.1 +``` + +### More examples + +Set a field on the sequence item with `name: api`: + +```cfengine3 +"/etc/app/config.yaml" + target => '.backends[] | select(.name == "api") | .port', + state => "present", + value => "9100"; +``` + +Remove the sequence item with `name: legacy`: + +```cfengine3 +"/etc/app/config.yaml" + target => '.backends[] | select(.name == "legacy")', + state => "absent"; +``` + +Remove a key and everything under it: + +```cfengine3 +"/etc/app/config.yaml" + target => ".debug", + state => "absent"; +``` + +Set a key containing dots: + +```cfengine3 +"/etc/app/config.yaml" + target => '.labels["app.example.com/version"]', + state => "present", + value => "1.5.0"; +``` + +Set the same key on every item: + +```cfengine3 +"/etc/app/config.yaml" + target => ".backends[].timeout", + state => "present", + value => "10"; +``` + +## Limitations + +This module ships its own minimal YAML parser (`yaml_lite.py`) and it only understands a subset of YAML: + +- Block mappings, block sequences, plain/quoted scalars and comments are supported +- Block scalars (`|` / `>`), anchors, aliases and tags are not supported +- Flow collections are not supported. These are the compact, JSON-like way of writing sequences and mappings on one line: + + ```yaml + addresses: [10.0.0.5/24, 10.0.0.6/24] # flow sequence + tls: { enabled: true, port: 443 } # flow mapping + ``` + + instead of the block style this module understands: + + ```yaml + addresses: + - 10.0.0.5/24 + - 10.0.0.6/24 + tls: + enabled: true + port: 443 + ``` + + A flow collection is read as a single string: `present` on the key can replace it, but `.key[]` fails since it isn't a sequence, and keys inside it can not be reached by `target`. +- Only scalar values can be written; nested structures can not be created with `value` + +## Authors + +This software was created by the team at [Northern.tech](https://northern.tech), with many contributions from the community. +Thanks everyone! + +## Contribute + +Feel free to open pull requests to expand this documentation, add features, or fix problems. +You can also pick up an existing task or file an issue in [our bug tracker](https://northerntech.atlassian.net/). + +## License + +This software is licensed under the MIT License. See LICENSE in the root of the repository for the full license text. diff --git a/promise-types/yaml/enable.cf b/promise-types/yaml/enable.cf new file mode 100644 index 0000000..93a7300 --- /dev/null +++ b/promise-types/yaml/enable.cf @@ -0,0 +1,6 @@ +promise agent yaml +# @brief Define the 'yaml' promise type for editing YAML files +{ + path => "$(sys.workdir)/modules/promises/yaml_promise_type.py"; + interpreter => "/usr/bin/python3"; +} diff --git a/promise-types/yaml/example.cf b/promise-types/yaml/example.cf new file mode 100644 index 0000000..c8829a2 --- /dev/null +++ b/promise-types/yaml/example.cf @@ -0,0 +1,78 @@ +promise agent yaml +{ + path => "$(sys.workdir)/modules/promises/yaml_promise_type.py"; + interpreter => "/usr/bin/python3"; +} + +bundle agent main +{ + vars: + "yaml_path" string => "/tmp/promise-type-yaml-example.yaml"; + + "content" + string => "server: + host: '0.0.0.0' + port: 8080 + debug: true + allowed_networks: + - 10.0.0.0/8 + - 172.16.0.0/12 +backends: + - name: api + url: http://api.internal:9000 + - name: legacy + url: http://legacy.internal:8000 +"; + + files: + # Create the file with some content to edit + "$(yaml_path)" + create => "true", + content => "$(content)", + if => not(fileexists("$(yaml_path)")); + + yaml: + # Keeps the single quotes around the value + "$(yaml_path)" + target => ".server.host", + state => "present", + value => "127.0.0.1"; + + # Written as a boolean, not the string "false" + "$(yaml_path)" + target => ".server.debug", + state => "present", + value => "false"; + + # Creates the missing 'tls:' parent + "$(yaml_path)" + target => ".server.tls.cert", + state => "present", + value => "/etc/ssl/app.pem"; + + # List items are matched by value + "$(yaml_path)" + target => ".server.allowed_networks[]", + state => "absent", + value => "172.16.0.0/12"; + + "$(yaml_path)" + target => ".server.allowed_networks[]", + state => "present", + value => "192.168.0.0/16"; + + "$(yaml_path)" + target => '.backends[] | select(.name == "api") | .url', + state => "present", + value => "http://api.internal:9100"; + + "$(yaml_path)" + target => '.backends[] | select(.name == "legacy")', + state => "absent"; + + reports: + "Resulting YAML ($(yaml_path)): +$(with)" + with => readfile("$(yaml_path)", 0), + if => fileexists("$(yaml_path)"); +} diff --git a/promise-types/yaml/jq_filter.py b/promise-types/yaml/jq_filter.py new file mode 100644 index 0000000..819bbce --- /dev/null +++ b/promise-types/yaml/jq_filter.py @@ -0,0 +1,286 @@ +""" +A small jq-like path language used to locate nodes inside a parsed +yaml document. + +Supported syntax: + + .a.b.c key access + .a["b.c"] quoted key access + .a[1] .a[-1] numeric index (negative counts from the end) + .a[] every item of a sequence + .a[] | select(.b == "x") items where .b equals a string, number, + true, false or null +""" + +import re + +from yaml_lite import MappingNode, SequenceNode, ScalarNode, get_slot + + +class FilterError(Exception): + pass + + +# --------------------------------------------------------------------------- +# Tokenizer +# --------------------------------------------------------------------------- + +_TOKEN_SPEC = [ + ("DOT", r"\."), + ("PIPE", r"\|"), + ("LBRACKET", r"\["), + ("RBRACKET", r"\]"), + ("LPAREN", r"\("), + ("RPAREN", r"\)"), + ("EQ", r"=="), + ("STRING", r'"(?:[^"\\]|\\.)*"|\'(?:[^\'\\]|\\.)*\''), + ("NUMBER", r"-?\d+"), + ("IDENT", r"[A-Za-z_][A-Za-z0-9_-]*"), +] +_TOKEN_RE = re.compile("|".join("(?P<%s>%s)" % pair for pair in _TOKEN_SPEC)) +_WS_RE = re.compile(r"\s+") +_LITERALS = {"true": True, "false": False, "null": None} + + +class Token(object): + __slots__ = ("type", "value", "pos") + + def __init__(self, type_, value, pos): + self.type = type_ + self.value = value + self.pos = pos + + +def tokenize(text): + tokens = [] + pos = 0 + length = len(text) + while pos < length: + m = _WS_RE.match(text, pos) + if m: + pos = m.end() + continue + m = _TOKEN_RE.match(text, pos) + if not m: + raise FilterError( + "Unexpected character %r at position %d in filter %r" + % (text[pos], pos, text) + ) + kind = m.lastgroup or "" + tokens.append(Token(kind, m.group(kind), pos)) + pos = m.end() + return tokens + + +def _unquote(raw): + body = raw[1:-1] + out = [] + i = 0 + length = len(body) + while i < length: + char = body[i] + if char == "\\" and i + 1 < length: + nxt = body[i + 1] + if nxt == "n": + out.append("\n") + elif nxt == "t": + out.append("\t") + else: + out.append(nxt) + i += 2 + continue + out.append(char) + i += 1 + return "".join(out) + + +# --------------------------------------------------------------------------- +# Steps / AST +# --------------------------------------------------------------------------- + + +class KeyStep(object): + def __init__(self, name): + self.name = name + + +class IndexStep(object): + def __init__(self, index): + self.index = index + + +class IterStep(object): + pass + + +class SelectStep(object): + def __init__(self, path, literal): + self.path = path + self.literal = literal + + +# --------------------------------------------------------------------------- +# Parser +# --------------------------------------------------------------------------- + + +def _parse_steps(tokens, i, in_select): + steps = [] + length = len(tokens) + while i < length: + token = tokens[i] + if token.type in ("DOT", "PIPE"): + if token.type == "PIPE" and in_select: + raise FilterError("Unexpected '|' inside select()") + i += 1 + continue + if token.type == "EQ" and in_select: + return steps, i + if token.type == "LBRACKET": + i += 1 + if i < length and tokens[i].type == "RBRACKET": + if in_select: + raise FilterError("'[]' is not supported inside select()") + steps.append(IterStep()) + elif i < length and tokens[i].type == "NUMBER": + steps.append(IndexStep(int(tokens[i].value))) + i += 1 + elif i < length and tokens[i].type == "STRING": + steps.append(KeyStep(_unquote(tokens[i].value))) + i += 1 + else: + raise FilterError("Invalid index expression in filter") + if i >= length or tokens[i].type != "RBRACKET": + raise FilterError("Expected ']' in filter") + i += 1 + continue + if ( + token.type == "IDENT" + and token.value == "select" + and i + 1 < length + and tokens[i + 1].type == "LPAREN" + ): + if in_select: + raise FilterError("Nested select() is not supported") + path, i = _parse_steps(tokens, i + 2, True) + if i >= length or tokens[i].type != "EQ": + raise FilterError("Expected '==' in select()") + i += 1 + if i >= length: + raise FilterError("Expected a value after '==' in select()") + lit = tokens[i] + if lit.type == "STRING": + literal = _unquote(lit.value) + elif lit.type == "NUMBER": + literal = int(lit.value) + elif lit.type == "IDENT" and lit.value in _LITERALS: + literal = _LITERALS[lit.value] + else: + raise FilterError("Invalid value %r in select()" % lit.value) + i += 1 + if i >= length or tokens[i].type != "RPAREN": + raise FilterError("Expected ')' in select()") + steps.append(SelectStep(path, literal)) + i += 1 + continue + if token.type in ("IDENT", "STRING"): + name = token.value if token.type == "IDENT" else _unquote(token.value) + steps.append(KeyStep(name)) + i += 1 + continue + raise FilterError("Unexpected token %r in filter" % token.value) + if in_select: + raise FilterError("Unexpected end of filter inside select()") + return steps, i + + +def parse_filter(text): + steps, _ = _parse_steps(tokenize(text), 0, False) + if not steps: + raise FilterError("Filter must select a key or item, not the whole document") + return steps + + +# --------------------------------------------------------------------------- +# Evaluation +# --------------------------------------------------------------------------- + + +class Match(object): + """A location in the document. 'container' is the mapping/sequence that + holds the node (or the empty MappingEntry/SequenceItem slot to create it + under), 'key' its key or index and 'exists' whether it's there - a bare + 'key:' exists but its node is None. 'missing_parent' means 'key' is a + missing key the filter goes through, which can be created.""" + + __slots__ = ("container", "key", "node", "exists", "missing_parent") + + def __init__(self, container, key, node, exists, missing_parent=False): + self.container = container + self.key = key + self.node = node + self.exists = exists + self.missing_parent = missing_parent + + +def _step(match, step, rest): + if match.missing_parent: + return [match] + node = match.node + if isinstance(step, KeyStep): + if node is None and match.exists: + # Key under an empty 'parent:' + container, entry = get_slot(match.container, match.key), None + elif isinstance(node, MappingNode): + container, entry = node, node.get_entry(step.name) + else: + raise FilterError( + "Cannot look up key '%s' on a non-mapping value" % step.name + ) + if entry is not None: + return [Match(node, step.name, entry.value, True)] + if not rest: + return [Match(container, step.name, None, False)] + # Missing keys can only be created if the rest of the path is keys too + if all(isinstance(s, KeyStep) for s in rest): + return [Match(container, step.name, None, False, missing_parent=True)] + raise FilterError("Key '%s' not found" % step.name) + if isinstance(step, IndexStep): + if not isinstance(node, SequenceNode): + raise FilterError("Cannot index a non-sequence value") + idx = step.index if step.index >= 0 else step.index + len(node.items) + if not 0 <= idx < len(node.items): + raise FilterError("Index %d out of range" % step.index) + return [Match(node, idx, node.items[idx].value, True)] + if isinstance(step, IterStep): + if not isinstance(node, SequenceNode): + raise FilterError("Cannot iterate over a non-sequence value") + return [Match(node, i, item.value, True) for i, item in enumerate(node.items)] + if isinstance(step, SelectStep): + return [match] if selects(node, step) else [] + raise FilterError("Unknown filter step") + + +def selects(node, step): + """Whether select() step keeps node.""" + for sub in step.path: + if isinstance(sub, KeyStep) and isinstance(node, MappingNode): + entry = node.get_entry(sub.name) + node = entry.value if entry is not None else None + elif isinstance(sub, IndexStep) and isinstance(node, SequenceNode): + idx = sub.index if sub.index >= 0 else sub.index + len(node.items) + node = node.items[idx].value if 0 <= idx < len(node.items) else None + else: + return False + if node is None: + return step.literal is None + return isinstance(node, ScalarNode) and node.value == step.literal + + +def evaluate(steps, root): + """Return every Match the filter resolves to.""" + matches = [Match(None, None, root, True)] + for pos, step in enumerate(steps): + rest = steps[pos + 1 :] + matches = [m2 for m in matches for m2 in _step(m, step, rest)] + return matches diff --git a/promise-types/yaml/tests/create.cf b/promise-types/yaml/tests/create.cf new file mode 100644 index 0000000..fc089f4 --- /dev/null +++ b/promise-types/yaml/tests/create.cf @@ -0,0 +1,168 @@ +# Create a config file from nothing +# +# The file and its directory don't exist: present creates them, +# along with missing parent keys and sequences. Like other files CFEngine +# creates, the file gets mode 0600 and directories 0700. +# absent on something that isn't there is kept. +# +# The promises are applied twice: the first pass must repair the file, the +# second must only keep promises, and the result must match 'expected'. +body common control +{ + bundlesequence => { "init", "test", "check", "cleanup" }; +} + +promise agent yaml +{ + path => "$(sys.workdir)/modules/promises/yaml_promise_type.py"; + interpreter => "/usr/bin/python3"; +} + +body classes outcome(prefix) +{ + promise_kept => { "$(prefix)_kept" }; + promise_repaired => { "$(prefix)_repaired" }; + repair_failed => { "$(prefix)_failed" }; + scope => "namespace"; +} + +body delete remove_file +{ + dirlinks => "delete"; + rmdirs => "true"; +} + +bundle common g +{ + vars: + "dir" string => "/tmp/yaml-promise-type-test-create"; + "file" string => "$(dir)/config.cfg"; + + "expected" + string => "server: + port: 8080 + tls: + cert: /etc/ssl/app.pem +log_level: info +allowed_networks: + - 10.0.0.0/8 + - 192.168.0.0/16 +"; +} + +bundle agent init +{ + files: + "$(g.file)" delete => remove_file; + "$(g.dir)" delete => remove_file; +} + +bundle agent test +{ + methods: + "first pass" usebundle => apply("first"); + "second pass" usebundle => apply("second"); +} + +bundle agent apply(pass) +{ + yaml: + # Creates the file and the 'server:' parent + "$(g.file)" + target => ".server.port", + state => "present", + value => "8080", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".server.tls.cert", + state => "present", + value => "/etc/ssl/app.pem", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".log_level", + state => "present", + value => "info", + classes => outcome("$(pass)"); + + # Creates the sequence + "$(g.file)" + target => ".allowed_networks[]", + state => "present", + value => "10.0.0.0/8", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".allowed_networks[]", + state => "present", + value => "192.168.0.0/16", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".debug", + state => "absent", + classes => outcome("$(pass)"); +} + +bundle agent check +{ + vars: + "actual" + string => readfile("$(g.file)", 0), + if => fileexists("$(g.file)"); + + "mode" + string => filestat("$(g.file)", "modeoct"), + if => fileexists("$(g.file)"); + + "dir_mode" + string => filestat("$(g.dir)", "modeoct"), + if => fileexists("$(g.dir)"); + + classes: + "content_ok" expression => strcmp("$(actual)", "$(g.expected)"); + "mode_ok" expression => strcmp("$(mode)", "100600"); + "dir_mode_ok" expression => strcmp("$(dir_mode)", "40700"); + "first_ok" expression => "first_repaired.!first_failed"; + "second_ok" expression => "second_kept.!(second_repaired|second_failed)"; + + "ok" + and => { + "content_ok", "mode_ok", "dir_mode_ok", "first_ok", "second_ok" + }; + + reports: + ok:: + "$(this.promise_filename) Pass"; + + !ok:: + "$(this.promise_filename) FAIL"; + + !content_ok:: + "Expected: +$(g.expected)"; + "Actual: +$(actual)"; + + !mode_ok:: + "Expected mode 100600, got $(mode)"; + + !dir_mode_ok:: + "Expected directory mode 40700, got $(dir_mode)"; + + !first_ok:: + "First pass did not repair cleanly: $(with)" + with => join(", ", classesmatching("first_.*")); + + !second_ok:: + "Second pass was not a no-op: $(with)" + with => join(", ", classesmatching("second_.*")); +} + +bundle agent cleanup +{ + files: + "$(g.file)" delete => remove_file; + "$(g.dir)" delete => remove_file; +} diff --git a/promise-types/yaml/tests/edit.cf b/promise-types/yaml/tests/edit.cf new file mode 100644 index 0000000..10362d9 --- /dev/null +++ b/promise-types/yaml/tests/edit.cf @@ -0,0 +1,272 @@ +# Edit an existing file with present, absent, [] and select() +# +# The promises are applied twice: the first pass must repair the file, the +# second must only keep promises, and the result must match 'expected'. +body common control +{ + bundlesequence => { "init", "test", "check", "cleanup" }; +} + +promise agent yaml +{ + path => "$(sys.workdir)/modules/promises/yaml_promise_type.py"; + interpreter => "/usr/bin/python3"; +} + +body classes outcome(prefix) +{ + promise_kept => { "$(prefix)_kept" }; + promise_repaired => { "$(prefix)_repaired" }; + repair_failed => { "$(prefix)_failed" }; + scope => "namespace"; +} + +body perms mode(m) +{ + mode => "$(m)"; + rxdirs => "false"; +} + +body delete remove_file +{ + dirlinks => "delete"; +} + +bundle common g +{ + vars: + "file" string => "/tmp/yaml-promise-type-test-edit.yaml"; + + "start" + string => "# Managed by cfengine +server: + host: '0.0.0.0' + port: 8080 + tls: true + motd: 'It''s back at 9' # shown at login + allowed_networks: + - 10.0.0.0/8 + - 172.16.0.0/12 + trusted_proxies: +backends: + - name: api + url: \"http://api.internal:9000\" + timeout: 5 # seconds + - name: legacy + url: http://legacy.internal:8000 +labels: + app.example.com/name: shop + \"app.example.com/owner\": \"ops \\\"core\\\" team\""; + + "expected" + string => "# Managed by cfengine +server: + host: '127.0.0.1' + port: 8080 + tls: false + motd: 'It''s back at 10' # shown at login + allowed_networks: + - 10.0.0.0/8 + - 192.168.0.0/16 + trusted_proxies: + - 10.0.0.1 + workers: 4 + region: \"no\" + banner: \"Welcome: staff only\" + ports: + - \"22:22\" +backends: + - name: api + url: \"http://api.internal:9100\" + timeout: 10 # seconds + - name: cache + url: http://cache.internal:6379 +labels: + app.example.com/name: shop + \"app.example.com/owner\": \"platform\" + app.example.com/version: 1.5.0 +"; +} + +bundle agent init +{ + files: + "$(g.file)" + create => "true", + content => "$(g.start)", + perms => mode("640"); +} + +bundle agent test +{ + methods: + "first pass" usebundle => apply("first"); + "second pass" usebundle => apply("second"); +} + +bundle agent apply(pass) +{ + yaml: + # Single quotes are kept + "$(g.file)" + target => ".server.host", + state => "present", + value => "127.0.0.1", + classes => outcome("$(pass)"); + + # Written as a boolean, not the string "false" + "$(g.file)" + target => ".server.tls", + state => "present", + value => "false", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".server.allowed_networks[]", + state => "absent", + value => "172.16.0.0/12", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".server.allowed_networks[]", + state => "present", + value => "192.168.0.0/16", + classes => outcome("$(pass)"); + + # 'trusted_proxies:' has no value yet, the list is created under it + "$(g.file)" + target => ".server.trusted_proxies[]", + state => "present", + value => "10.0.0.1", + classes => outcome("$(pass)"); + + # New key goes after the last (nested) entry + "$(g.file)" + target => ".server.workers", + state => "present", + value => "4", + classes => outcome("$(pass)"); + + # Values that YAML would read as something else get quoted: "no" as a + # boolean, "22:22" as a number, and ": " as a nested mapping + "$(g.file)" + target => ".server.region", + state => "present", + value => "no", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".server.banner", + state => "present", + value => "Welcome: staff only", + classes => outcome("$(pass)"); + + "$(g.file)" + target => ".server.ports[]", + state => "present", + value => "22:22", + classes => outcome("$(pass)"); + + # Single quotes are kept, with ' written as '' + "$(g.file)" + target => ".server.motd", + state => "present", + value => "It's back at 10", + classes => outcome("$(pass)"); + + # The start content has no final newline, the new key must still go on + # its own line + "$(g.file)" + target => '.labels["app.example.com/version"]', + state => "present", + value => "1.5.0", + classes => outcome("$(pass)"); + + # A quoted key in the file, with a quoted value containing \" + "$(g.file)" + target => '.labels["app.example.com/owner"]', + state => "present", + value => "platform", + classes => outcome("$(pass)"); + + # Double quotes are kept + "$(g.file)" + target => '.backends[] | select(.name == "api") | .url', + state => "present", + value => "http://api.internal:9100", + classes => outcome("$(pass)"); + + # Trailing comment is kept + "$(g.file)" + target => '.backends[] | select(.name == "api") | .timeout', + state => "present", + value => "10", + classes => outcome("$(pass)"); + + # Adds '- name: cache' since no item matches, then sets a field on it + "$(g.file)" + target => '.backends[] | select(.name == "cache")', + state => "present", + classes => outcome("$(pass)"); + + "$(g.file)" + target => '.backends[] | select(.name == "cache") | .url', + state => "present", + value => "http://cache.internal:6379", + classes => outcome("$(pass)"); + + # Removes the whole item, found by name rather than position + "$(g.file)" + target => '.backends[] | select(.name == "legacy")', + state => "absent", + classes => outcome("$(pass)"); +} + +bundle agent check +{ + vars: + "actual" + string => readfile("$(g.file)", 0), + if => fileexists("$(g.file)"); + + "mode" + string => filestat("$(g.file)", "modeoct"), + if => fileexists("$(g.file)"); + + classes: + "content_ok" expression => strcmp("$(actual)", "$(g.expected)"); + "mode_ok" expression => strcmp("$(mode)", "100640"); + "first_ok" expression => "first_repaired.!first_failed"; + "second_ok" expression => "second_kept.!(second_repaired|second_failed)"; + "ok" and => { "content_ok", "mode_ok", "first_ok", "second_ok" }; + + reports: + ok:: + "$(this.promise_filename) Pass"; + + !ok:: + "$(this.promise_filename) FAIL"; + + !content_ok:: + "Expected: +$(g.expected)"; + "Actual: +$(actual)"; + + !mode_ok:: + "Expected mode 100640, got $(mode)"; + + !first_ok:: + "First pass did not repair cleanly: $(with)" + with => join(", ", classesmatching("first_.*")); + + !second_ok:: + "Second pass was not a no-op: $(with)" + with => join(", ", classesmatching("second_.*")); +} + +bundle agent cleanup +{ + files: + "$(g.file)" delete => remove_file; +} diff --git a/promise-types/yaml/tests/errors.cf b/promise-types/yaml/tests/errors.cf new file mode 100644 index 0000000..6e40504 --- /dev/null +++ b/promise-types/yaml/tests/errors.cf @@ -0,0 +1,184 @@ +# Promises that must not change the file +# +# Invalid targets must fail, and promises that are already satisfied must +# be kept. In both cases the file must be left untouched. +body common control +{ + bundlesequence => { "init", "test", "check", "cleanup" }; +} + +promise agent yaml +{ + path => "$(sys.workdir)/modules/promises/yaml_promise_type.py"; + interpreter => "/usr/bin/python3"; +} + +body classes outcome(prefix) +{ + promise_kept => { "$(prefix)_kept" }; + promise_repaired => { "$(prefix)_repaired" }; + repair_failed => { "$(prefix)_failed" }; + scope => "namespace"; +} + +body delete remove_file +{ + dirlinks => "delete"; +} + +bundle common g +{ + vars: + "file" string => "/tmp/yaml-promise-type-test-errors.yaml"; + "missing" string => "/tmp/yaml-promise-type-test-missing.yaml"; + + "start" + string => "server: + host: '0.0.0.0' + port: 8080 +users: + - name: alice + - name: bob +"; + + "must_fail" + slist => { + "through_scalar", + "missing_parent", + "present_on_mapping", + "value_on_sequence", + "index_out_of_range", + }; + + "must_keep" + slist => { + "same_value", + "same_quoted_value", + "absent_key", + "absent_no_match", + "absent_missing_file", + "set_existing_item", + }; +} + +bundle agent init +{ + files: + "$(g.file)" + create => "true", + content => "$(g.start)"; +} + +bundle agent test +{ + yaml: + # port is a plain value (8080), so it has no keys to look up + "$(g.file)" + target => ".server.port.number", + state => "present", + value => "1", + classes => outcome("through_scalar"); + + # Missing parents are only created for plain key paths + "$(g.file)" + target => '.backends[] | select(.name == "api") | .port', + state => "present", + value => "9000", + classes => outcome("missing_parent"); + + "$(g.file)" + target => ".server[]", + state => "present", + value => "x", + classes => outcome("present_on_mapping"); + + # Replacing a list with a scalar is refused, '.users[]' adds an item + "$(g.file)" + target => ".users", + state => "present", + value => "carol", + classes => outcome("value_on_sequence"); + + "$(g.file)" + target => ".users[5].name", + state => "present", + value => "carol", + classes => outcome("index_out_of_range"); + + "$(g.missing)" + target => ".a", + state => "absent", + classes => outcome("absent_missing_file"); + + "$(g.file)" + target => ".server.port", + state => "present", + value => "8080", + classes => outcome("same_value"); + + "$(g.file)" + target => ".server.host", + state => "present", + value => "0.0.0.0", + classes => outcome("same_quoted_value"); + + "$(g.file)" + target => ".server.tls", + state => "absent", + classes => outcome("absent_key"); + + "$(g.file)" + target => '.users[] | select(.name == "mallory")', + state => "absent", + classes => outcome("absent_no_match"); + + "$(g.file)" + target => ".users[0].name", + state => "present", + value => "alice", + classes => outcome("set_existing_item"); +} + +bundle agent check +{ + vars: + "actual" + string => readfile("$(g.file)", 0), + if => fileexists("$(g.file)"); + + classes: + "failed_$(g.must_fail)" + expression => "$(g.must_fail)_failed.!$(g.must_fail)_repaired"; + + "kept_$(g.must_keep)" + expression => "$(g.must_keep)_kept.!($(g.must_keep)_repaired|$(g.must_keep)_failed)"; + + "unchanged" expression => strcmp("$(actual)", "$(g.start)"); + "not_created" expression => not(fileexists("$(g.missing)")); + "fail" expression => "!failed_$(g.must_fail)"; + "fail" expression => "!kept_$(g.must_keep)"; + "fail" expression => "!(unchanged.not_created)"; + + reports: + "$(g.must_fail): expected the promise to fail" + unless => "failed_$(g.must_fail)"; + + "$(g.must_keep): expected the promise to be kept" + unless => "kept_$(g.must_keep)"; + + "File was modified: +$(actual)" unless => "unchanged"; + "$(g.missing) was created" unless => "not_created"; + + !fail:: + "$(this.promise_filename) Pass"; + + fail:: + "$(this.promise_filename) FAIL"; +} + +bundle agent cleanup +{ + files: + "$(g.file)" delete => remove_file; +} diff --git a/promise-types/yaml/yaml_lite.py b/promise-types/yaml/yaml_lite.py new file mode 100644 index 0000000..27fb929 --- /dev/null +++ b/promise-types/yaml/yaml_lite.py @@ -0,0 +1,599 @@ +""" +Minimal YAML editor. + +This is NOT a general-purpose YAML library. It only understands the subset of +YAML needed by the 'yaml' promise type: block mappings, block sequences, +plain/quoted scalars and comments. Block scalars (`|`/`>`) and flow +collections (`{...}` / `[...]`) are not supported. + +Every node keeps track of the source line(s) it came from, so that edits can +be applied as text splices on the original lines instead of a full +re-parse/re-dump of the document. This preserves comments, key order, +quoting style and formatting of everything the caller didn't touch. +""" + +import re + + +class YamlSyntaxError(Exception): + pass + + +# --------------------------------------------------------------------------- +# Nodes +# --------------------------------------------------------------------------- + + +class ScalarNode(object): + def __init__( + self, start_line, end_line, col_start, col_end, value, raw, style, trailing="" + ): + self.start_line = start_line + self.end_line = end_line + self.col_start = col_start + self.col_end = col_end + self.value = value + self.raw = raw + self.style = style # 'plain' | 'single' | 'double' + # Whitespace/comment text that followed the value on its source line + # (e.g. " # primary"), preserved across edits. + self.trailing = trailing + + +class MappingEntry(object): + def __init__( + self, + key, + key_line, + key_col, + colon_col, + value_start_col, + needs_space, + value, + end_line, + ): + self.key = key + self.key_line = key_line + self.key_col = key_col + self.colon_col = colon_col + self.value_start_col = value_start_col + self.needs_space = needs_space + self.value = value + self.end_line = end_line + + +class MappingNode(object): + def __init__(self, start_line, indent): + self.start_line = start_line + self.indent = indent + self.entries = [] + self.end_line = start_line + + def get_entry(self, key): + for entry in self.entries: + if entry.key == key: + return entry + return None + + +class SequenceItem(object): + def __init__(self, item_line, dash_col, value_start_col, value, needs_space): + self.item_line = item_line + self.dash_col = dash_col + self.value_start_col = value_start_col + self.value = value + self.needs_space = needs_space + self.end_line = value.end_line if value is not None else item_line + + +class SequenceNode(object): + def __init__(self, start_line, indent): + self.start_line = start_line + self.indent = indent + self.items = [] + self.end_line = start_line + + +# --------------------------------------------------------------------------- +# Low-level line helpers +# --------------------------------------------------------------------------- + + +def _strip_newline(line): + if line.endswith("\r\n"): + return line[:-2] + if line.endswith("\n"): + return line[:-1] + return line + + +def _leading_spaces(line): + return len(line) - len(line.lstrip(" ")) + + +def _is_noise(line): + s = line.strip() + return s == "" or s.startswith("#") or s in ("---", "...") + + +def _find_comment_start(text): + i = 1 + length = len(text) + while i < length: + if text[i] == "#" and text[i - 1] in " \t": + return i + i += 1 + return None + + +def _find_dquote_end(text): + i = 1 + length = len(text) + while i < length: + if text[i] == "\\": + i += 2 + continue + if text[i] == '"': + return i + i += 1 + return length - 1 + + +def _find_squote_end(text): + i = 1 + length = len(text) + while i < length: + if text[i] == "'": + if i + 1 < length and text[i + 1] == "'": + i += 2 + continue + return i + i += 1 + return length - 1 + + +def _find_key_colon(text): + """Return the index of the ':' that separates a mapping key from its + value on this (indent-stripped, newline-stripped) line, or -1 if the + line doesn't look like a mapping entry.""" + i = 0 + length = len(text) + if length == 0: + return -1 + if text[0] == "'": + i = _find_squote_end(text) + 1 + elif text[0] == '"': + i = _find_dquote_end(text) + 1 + while i < length: + char = text[i] + if char == ":": + if i + 1 == length or text[i + 1] in " \t": + return i + elif char == "#" and (i == 0 or text[i - 1] in " \t"): + return -1 + i += 1 + return -1 + + +def _decode_double_quoted(raw): + body = raw[1:-1] + out = [] + i = 0 + length = len(body) + while i < length: + char = body[i] + if char == "\\" and i + 1 < length: + nxt = body[i + 1] + if nxt == "n": + out.append("\n") + elif nxt == "t": + out.append("\t") + elif nxt == "r": + out.append("\r") + elif nxt in ('"', "\\"): + out.append(nxt) + else: + out.append(nxt) + i += 2 + continue + out.append(char) + i += 1 + return "".join(out) + + +def _decode_single_quoted(raw): + return raw[1:-1].replace("''", "'") + + +def _decode_scalar_text(text): + if text.startswith('"'): + end = _find_dquote_end(text) + return _decode_double_quoted(text[: end + 1]) + if text.startswith("'"): + end = _find_squote_end(text) + return _decode_single_quoted(text[: end + 1]) + return text.strip() + + +def decode_plain_scalar(raw): + if raw == "" or raw == "~" or raw.lower() == "null": + return None + low = raw.lower() + if low == "true": + return True + if low == "false": + return False + try: + return int(raw) + except ValueError: + pass + try: + return float(raw) + except ValueError: + pass + return raw + + +# --------------------------------------------------------------------------- +# Parser +# --------------------------------------------------------------------------- + + +class _Parser(object): + def __init__(self, lines): + self.lines = lines + self.line_count = len(lines) + + def _skip_noise(self, idx): + while idx < self.line_count and _is_noise(self.lines[idx]): + idx += 1 + return idx + + def parse_block(self, idx, min_indent): + idx = self._skip_noise(idx) + if idx >= self.line_count: + return None, idx + line = self.lines[idx] + indent = _leading_spaces(line) + if indent < min_indent: + return None, idx + content = _strip_newline(line)[indent:] + if content == "-" or content.startswith("- "): + return self._parse_sequence(idx, indent) + if _find_key_colon(content) != -1: + return self._parse_mapping(idx, indent) + node = self._parse_inline_scalar(idx, indent, content) + return node, idx + 1 + + # -- mappings --------------------------------------------------------- + + def _parse_mapping(self, idx, indent): + node = MappingNode(start_line=idx, indent=indent) + idx = self._parse_mapping_body(node, idx, indent) + node.end_line = node.entries[-1].end_line if node.entries else node.start_line + return node, idx + + def _parse_inline_mapping( + self, dash_line, key_col, key_line_content, indent_for_siblings + ): + node = MappingNode(start_line=dash_line, indent=indent_for_siblings) + entry, next_idx = self._make_entry_from_content( + dash_line, key_col, key_line_content + ) + node.entries.append(entry) + idx = self._parse_mapping_body(node, next_idx, indent_for_siblings) + node.end_line = node.entries[-1].end_line + return node, idx + + def _parse_mapping_body(self, node, idx, indent): + while True: + idx = self._skip_noise(idx) + if idx >= self.line_count: + break + line = self.lines[idx] + if _leading_spaces(line) != indent: + break + content = _strip_newline(line)[indent:] + if _find_key_colon(content) == -1: + break + entry, idx = self._make_entry_from_content(idx, indent, content) + node.entries.append(entry) + return idx + + def _make_entry_from_content(self, line_idx, key_col, content): + colon_idx = _find_key_colon(content) + key_text = content[:colon_idx] + key = _decode_scalar_text(key_text) + rest = content[colon_idx + 1 :] + rest_lstripped = rest.lstrip(" ") + leading_ws_len = len(rest) - len(rest_lstripped) + value_col = key_col + colon_idx + 1 + leading_ws_len + + if rest_lstripped == "" or rest_lstripped.startswith("#"): + nxt = self._skip_noise(line_idx + 1) + if nxt < self.line_count and _leading_spaces(self.lines[nxt]) > key_col: + value_node, next_idx = self.parse_block(line_idx + 1, key_col + 1) + else: + value_node, next_idx = None, line_idx + 1 + elif rest_lstripped == "-" or rest_lstripped.startswith("- "): + value_node, next_idx = self._parse_inline_sequence( + line_idx, value_col, rest_lstripped, value_col + ) + elif _find_key_colon(rest_lstripped) != -1: + value_node, next_idx = self._parse_inline_mapping( + line_idx, value_col, rest_lstripped, value_col + ) + else: + value_node = self._parse_inline_scalar(line_idx, value_col, rest_lstripped) + next_idx = line_idx + 1 + + entry = MappingEntry( + key=key, + key_line=line_idx, + key_col=key_col, + colon_col=key_col + colon_idx, + value_start_col=value_col, + needs_space=(leading_ws_len == 0), + value=value_node, + end_line=value_node.end_line if value_node is not None else line_idx, + ) + return entry, next_idx + + # -- sequences ---------------------------------------------------------- + + def _parse_sequence(self, idx, indent): + node = SequenceNode(start_line=idx, indent=indent) + idx = self._parse_sequence_body(node, idx, indent) + node.end_line = node.items[-1].end_line if node.items else node.start_line + return node, idx + + def _parse_inline_sequence(self, dash_line, col, content, indent_for_siblings): + node = SequenceNode(start_line=dash_line, indent=indent_for_siblings) + item, next_idx = self._make_item_from_content(dash_line, col, content) + node.items.append(item) + idx = self._parse_sequence_body(node, next_idx, indent_for_siblings) + node.end_line = node.items[-1].end_line + return node, idx + + def _parse_sequence_body(self, node, idx, indent): + while True: + idx = self._skip_noise(idx) + if idx >= self.line_count: + break + line = self.lines[idx] + if _leading_spaces(line) != indent: + break + content = _strip_newline(line)[indent:] + if content != "-" and not content.startswith("- "): + break + item, idx = self._make_item_from_content(idx, indent, content) + node.items.append(item) + return idx + + def _make_item_from_content(self, line_idx, dash_col, content): + if content == "-": + rest = "" + marker_width = 1 + else: + rest = content[2:] + marker_width = 2 + rest_lstripped = rest.lstrip(" ") + leading_ws_len = len(rest) - len(rest_lstripped) + value_col = dash_col + marker_width + leading_ws_len + + if rest_lstripped == "" or rest_lstripped.startswith("#"): + nxt = self._skip_noise(line_idx + 1) + if nxt < self.line_count and _leading_spaces(self.lines[nxt]) > dash_col: + value_node, next_idx = self.parse_block(line_idx + 1, dash_col + 1) + else: + value_node, next_idx = None, line_idx + 1 + elif rest_lstripped == "-" or rest_lstripped.startswith("- "): + value_node, next_idx = self._parse_inline_sequence( + line_idx, value_col, rest_lstripped, value_col + ) + elif _find_key_colon(rest_lstripped) != -1: + value_node, next_idx = self._parse_inline_mapping( + line_idx, value_col, rest_lstripped, value_col + ) + else: + value_node = self._parse_inline_scalar(line_idx, value_col, rest_lstripped) + next_idx = line_idx + 1 + + item = SequenceItem( + item_line=line_idx, + dash_col=dash_col, + value_start_col=value_col, + value=value_node, + needs_space=(content == "-"), + ) + return item, next_idx + + # -- scalars -------------------------------------------------------- + + def _parse_inline_scalar(self, line_idx, col, text): + if text.startswith('"'): + end = _find_dquote_end(text) + raw = text[: end + 1] + value = _decode_double_quoted(raw) + style = "double" + col_end = col + end + 1 + elif text.startswith("'"): + end = _find_squote_end(text) + raw = text[: end + 1] + value = _decode_single_quoted(raw) + style = "single" + col_end = col + end + 1 + else: + cut = _find_comment_start(text) + raw = text[:cut] if cut is not None else text + raw = raw.rstrip() + value = decode_plain_scalar(raw) + style = "plain" + col_end = col + len(raw) + return ScalarNode( + start_line=line_idx, + end_line=line_idx, + col_start=col, + col_end=col_end, + value=value, + raw=raw, + style=style, + trailing=text[len(raw) :], + ) + + +def parse_document(text): + lines = text.splitlines(True) if text else [] + parser = _Parser(lines) + idx = parser._skip_noise(0) + if idx >= parser.line_count: + return lines, None + root, _ = parser.parse_block(idx, 0) + return lines, root + + +# --------------------------------------------------------------------------- +# Scalar formatting (for writing new/changed values back out) +# --------------------------------------------------------------------------- + +# Plain scalars that YAML 1.1 parsers read as booleans. true/false/null are +# left alone, so that values can be written as booleans/null on purpose. +_RESERVED_PLAIN_WORDS = set(["yes", "no", "on", "off", "y", "n"]) +# YAML 1.1 reads e.g. 22:22 as a base 60 number +_SEXAGESIMAL_RE = re.compile(r"^[0-9][0-9_]*(:[0-5]?[0-9])+(\.[0-9_]*)?$") +_COLON_SPACE_RE = re.compile(r":(\s|$)") +_SPACE_HASH_RE = re.compile(r"\s#") + + +def _needs_quote(s): + if s == "": + return True + if s != s.strip(): + return True + if "\n" in s or "\r" in s: + return True + if s.lower() in _RESERVED_PLAIN_WORDS: + return True + if s[0] in "!&*?|>%@`\"'#,[]{}": + return True + if s[0] == "-" and (len(s) == 1 or s[1] == " "): + return True + if _COLON_SPACE_RE.search(s): + return True + if _SPACE_HASH_RE.search(s): + return True + if _SEXAGESIMAL_RE.match(s): + return True + return False + + +def _encode_double_quoted(s): + out = ['"'] + for ch in s: + if ch == "\\": + out.append("\\\\") + elif ch == '"': + out.append('\\"') + elif ch == "\n": + out.append("\\n") + elif ch == "\r": + out.append("\\r") + elif ch == "\t": + out.append("\\t") + else: + out.append(ch) + out.append('"') + return "".join(out) + + +def _encode_single_quoted(s): + return "'" + s.replace("'", "''") + "'" + + +def format_scalar(value, preferred_style=None): + value = "" if value is None else str(value) + if "\n" in value or "\r" in value: + return _encode_double_quoted(value) + if preferred_style == "single": + return _encode_single_quoted(value) + if preferred_style == "double": + return _encode_double_quoted(value) + if _needs_quote(value): + return _encode_double_quoted(value) + return value + + +# --------------------------------------------------------------------------- +# Mutation helpers (line splicing) +# --------------------------------------------------------------------------- + + +def get_slot(container, key): + if isinstance(container, MappingNode): + entry = container.get_entry(key) + assert entry is not None + return entry + return container.items[key] + + +def slot_line(slot): + return slot.key_line if isinstance(slot, MappingEntry) else slot.item_line + + +def set_slot_value(lines, slot, new_value_text): + def_line = slot_line(slot) + line = lines[def_line] + if line.endswith("\r\n"): + nl = "\r\n" + elif line.endswith("\n"): + nl = "\n" + else: + nl = "\n" + prefix = line[: slot.value_start_col] + if slot.needs_space: + prefix += " " + trailing = "" + if ( + isinstance(slot.value, ScalarNode) + and slot.value.start_line == def_line + and slot.value.end_line == def_line + ): + trailing = slot.value.trailing + new_line = prefix + new_value_text + trailing + nl + old_end = slot.value.end_line if slot.value is not None else def_line + lines[def_line : old_end + 1] = [new_line] + + +def delete_slot(lines, slot): + start = slot_line(slot) + end = slot.end_line + del lines[start : end + 1] + + +def _child_insert_point(container): + """Return (line index, indent) for a new last child of container: a + mapping/sequence node, or an empty 'key:' / '-' slot.""" + if isinstance(container, (MappingEntry, SequenceItem)): + col = ( + container.key_col + if isinstance(container, MappingEntry) + else container.dash_col + ) + return slot_line(container) + 1, col + 2 + children = ( + container.entries if isinstance(container, MappingNode) else container.items + ) + at = children[-1].end_line + 1 if children else container.start_line + 1 + return at, container.indent + + +def insert_mapping_key(lines, container, key, value_text): + at, indent = _child_insert_point(container) + # An empty value_text creates an empty 'key:' to add children under + value = ": " + value_text if value_text else ":" + lines.insert(at, " " * indent + format_scalar(key) + value + "\n") + + +def append_sequence_item(lines, container, value_text): + at, indent = _child_insert_point(container) + lines.insert(at, " " * indent + "- " + value_text + "\n") diff --git a/promise-types/yaml/yaml_promise_type.py b/promise-types/yaml/yaml_promise_type.py new file mode 100644 index 0000000..31cd627 --- /dev/null +++ b/promise-types/yaml/yaml_promise_type.py @@ -0,0 +1,307 @@ +import os +import stat +import tempfile + +from cfengine_module_library import PromiseModule, ValidationError, Result + +import yaml_lite +import jq_filter + +_STATES = ("present", "absent") + +# Each repair changes one match and re-parses the file, so this only guards +# against a promise that never converges +_MAX_REPAIRS = 1000 + + +class YamlPromiseTypeModule(PromiseModule): + def __init__(self): + super(YamlPromiseTypeModule, self).__init__("yaml_promise_module", "0.0.0") + + def validate_promise(self, promiser, attributes, metadata): + target = attributes.get("target") + if not target: + raise ValidationError("Attribute 'target' is required") + try: + steps = jq_filter.parse_filter(target) + except jq_filter.FilterError as e: + raise ValidationError("Invalid 'target': %s" % e) + + state = attributes.get("state") + if state not in _STATES: + raise ValidationError( + "Attribute 'state' must be one of: %s" % ", ".join(_STATES) + ) + + # Without a value, 'present' adds an item to a sequence of mappings, + # built from the select() condition + if state == "present" and "value" not in attributes: + if not _is_item_select(steps): + raise ValidationError( + "State 'present' requires 'value', or a target ending in " + "'[] | select(.key == value)'" + ) + + if state == "absent" and "value" in attributes: + if not isinstance(steps[-1], jq_filter.IterStep): + raise ValidationError( + "State 'absent' with 'value' removes sequence items, " + "use a target ending in '[]'" + ) + + # Removing by position is not convergent: the next run would remove + # whatever moved into that position + if ( + state == "absent" + and "value" not in attributes + and isinstance(steps[-1], jq_filter.IndexStep) + ): + raise ValidationError( + "State 'absent' can not remove a sequence item by index, " + "use '[]' with 'value', or select(), to match it by content" + ) + + def evaluate_promise(self, promiser, attributes, metadata): + state = attributes["state"] + target = attributes["target"] + value = attributes.get("value") + steps = jq_filter.parse_filter(target) + + # Like the json promise type, a missing file is treated as empty and + # created when something is added to it + if os.path.exists(promiser) and not os.path.isfile(promiser): + self.log_error("'%s' already exists and is not a regular file" % promiser) + return Result.NOT_KEPT + content = "" + if os.path.exists(promiser): + with open(promiser, "r", newline="") as f: + content = f.read() + # Edit with \n line endings and write back with the file's own + newline = "\r\n" if "\r\n" in content else "\n" + content = content.replace("\r\n", "\n") + # New lines are added after the last one, so it needs a line ending. + # Only written to the file if something else changes too. + if content and not content.endswith("\n"): + content += "\n" + + changed = False + for _ in range(_MAX_REPAIRS): + lines, root = yaml_lite.parse_document(content) + if root is None: + # Empty document (or only comments): new keys go at the end + root = yaml_lite.MappingNode(start_line=len(lines) - 1, indent=0) + try: + if state == "present" and value is None: + # The sequence(s) that '[] | select()' looks in + sequences = jq_filter.evaluate(steps[:-2], root) + repaired = _present_selected(lines, sequences, steps[-1]) + elif value is not None and isinstance(steps[-1], jq_filter.IterStep): + # '.list[]' with a value: the sequence contains it or not + sequences = jq_filter.evaluate(steps[:-1], root) + repaired = _repair_one(lines, sequences, state, value, True) + else: + matches = jq_filter.evaluate(steps, root) + repaired = _repair_one(lines, matches, state, value, False) + if not repaired: + break + except (jq_filter.FilterError, _OperationError) as e: + self.log_error("Target '%s' failed on '%s': %s" % (target, promiser, e)) + return Result.NOT_KEPT + content = "".join(lines) + changed = True + else: + self.log_error("'%s' did not converge for target '%s'" % (promiser, target)) + return Result.NOT_KEPT + + if not changed: + return Result.KEPT + + try: + self._write_atomically(promiser, content, newline) + except OSError as e: + self.log_error("Failed to write '%s': %s" % (promiser, e)) + return Result.NOT_KEPT + self.log_info( + "Updated '%s' (target '%s', state '%s')" % (promiser, target, state) + ) + return Result.REPAIRED + + def _write_atomically(self, path, content, newline): + """Write to a temporary file next to 'path' and rename it into place, + so readers never see a partially written file.""" + # Edit the file a symlink points to, not replace the symlink + path = os.path.realpath(path) + # Dirs/files are created with mode 0700 and 0600 respectively + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory, 0o700) + self.log_info("Created directory for '%s'" % path) + fd, tmp = tempfile.mkstemp( + prefix="." + os.path.basename(path) + ".", suffix=".tmp", dir=directory + ) + try: + with os.fdopen(fd, "w", newline=newline) as f: + # Keep the permissions and owner of an existing file + if os.path.exists(path): + st = os.stat(path) + os.fchmod(f.fileno(), stat.S_IMODE(st.st_mode)) + if (st.st_uid, st.st_gid) != (os.getuid(), os.getgid()): + os.fchown(f.fileno(), st.st_uid, st.st_gid) + f.write(content) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except BaseException: + os.unlink(tmp) + raise + dir_fd = os.open(directory, os.O_RDONLY) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + + +class _OperationError(Exception): + pass + + +def _repair_one(lines, matches, state, value, in_sequence): + """Fix the first match that isn't in the desired state. Returns False if + all matches are already in the desired state. With in_sequence, matches + are sequences and value an item of them.""" + for match in matches: + if match.missing_parent: + if state == "absent": + continue + # Create the missing key empty, the next repair adds to it + yaml_lite.insert_mapping_key(lines, match.container, match.key, "") + return True + if in_sequence: + if state == "present": + repaired = _present(lines, match, value) + else: + repaired = _absent_item(lines, match, value) + if repaired: + return True + elif state == "present": + if _set(lines, match, value): + return True + elif match.exists: + yaml_lite.delete_slot(lines, yaml_lite.get_slot(match.container, match.key)) + return True + return False + + +def _set(lines, match, value): + if not match.exists: + if isinstance(match.container, yaml_lite.SequenceNode): + raise _OperationError("Can not set a sequence item that does not exist") + yaml_lite.insert_mapping_key( + lines, match.container, match.key, yaml_lite.format_scalar(value) + ) + return True + node = match.node + if isinstance(node, yaml_lite.SequenceNode): + raise _OperationError( + "Value is a sequence, end the target with '[]' to add an item to it" + ) + style = node.style if isinstance(node, yaml_lite.ScalarNode) else None + new_text = yaml_lite.format_scalar(value, style) + if isinstance(node, yaml_lite.ScalarNode) and node.raw == new_text: + return False + yaml_lite.set_slot_value( + lines, yaml_lite.get_slot(match.container, match.key), new_text + ) + return True + + +def _sequence_or_slot(match): + """The sequence the filter points at, or the empty 'key:' slot to create + it under.""" + if isinstance(match.node, yaml_lite.SequenceNode): + return match.node + if match.exists and match.node is None: + return yaml_lite.get_slot(match.container, match.key) + raise _OperationError("Target ending in '[]' is not a sequence") + + +def _item_text(node): + if not isinstance(node, yaml_lite.ScalarNode): + return None + return node.raw if node.style == "plain" else node.value + + +def _present(lines, match, value): + if not match.exists: + if isinstance(match.container, yaml_lite.SequenceNode): + raise _OperationError("Target ending in '[]' is not a sequence") + yaml_lite.insert_mapping_key(lines, match.container, match.key, "") + return True + seq = _sequence_or_slot(match) + if isinstance(seq, yaml_lite.SequenceNode): + if any(_item_text(item.value) == value for item in seq.items): + return False + yaml_lite.append_sequence_item(lines, seq, yaml_lite.format_scalar(value)) + return True + + +def _is_item_select(steps): + return ( + len(steps) >= 2 + and isinstance(steps[-2], jq_filter.IterStep) + and isinstance(steps[-1], jq_filter.SelectStep) + and len(steps[-1].path) == 1 + and isinstance(steps[-1].path[0], jq_filter.KeyStep) + ) + + +def _literal_text(literal): + """YAML text for a select() literal that reads back as the same value.""" + if literal is None: + return "null" + if isinstance(literal, bool): + return "true" if literal else "false" + if isinstance(literal, int): + return str(literal) + text = yaml_lite.format_scalar(literal) + if text == literal and yaml_lite.decode_plain_scalar(text) != literal: + # e.g. the string "true" or "8080", which would be read as non-strings + return yaml_lite.format_scalar(literal, "double") + return text + + +def _present_selected(lines, matches, select): + """Append a '- key: value' item to every sequence where select() matches + no item. Returns False if all of them already have one.""" + key = select.path[0].name + for match in matches: + if match.missing_parent or not match.exists: + if isinstance(match.container, yaml_lite.SequenceNode): + raise _OperationError("Target ending in '[]' is not a sequence") + # Create the missing 'key:', the next repair adds the item + yaml_lite.insert_mapping_key(lines, match.container, match.key, "") + return True + seq = _sequence_or_slot(match) + if isinstance(seq, yaml_lite.SequenceNode) and any( + jq_filter.selects(item.value, select) for item in seq.items + ): + continue + item = yaml_lite.format_scalar(key) + ": " + _literal_text(select.literal) + yaml_lite.append_sequence_item(lines, seq, item) + return True + return False + + +def _absent_item(lines, match, value): + seq = _sequence_or_slot(match) + if not isinstance(seq, yaml_lite.SequenceNode): + return False + for item in seq.items: + if _item_text(item.value) == value: + yaml_lite.delete_slot(lines, item) + return True + return False + + +if __name__ == "__main__": + YamlPromiseTypeModule().start()