From 2ed57354b02b984b5610555c0af881b9ff6cbb2b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:16:08 +0200 Subject: [PATCH 1/4] fix(integrations): escape control characters in goose recipe YAML renderer YAML forbids C0 control characters (except tab and newline) and DEL in every scalar form, and a bare CR acts as a line break inside a block scalar. _render_yaml wrote the body verbatim into a |2 literal block scalar, so such bodies produced recipes the YAML parser rejects. Detect block-scalar-unsafe characters and fall back to an escaped double-quoted scalar via yaml.safe_dump, mirroring the TOML renderer's fallback strategy from #3341. Fixes #3382 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/integrations/base.py | 22 ++++++++++++++++++ .../test_integration_base_yaml.py | 23 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.py index bfbb81b85b..631b111306 100644 --- a/src/specify_cli/integrations/base.py +++ b/src/specify_cli/integrations/base.py @@ -1122,6 +1122,11 @@ def setup( # YamlIntegration — YAML-format agents (Goose) # --------------------------------------------------------------------------- +# Characters a YAML literal block scalar cannot carry: C0 controls other +# than tab/LF (a bare CR acts as a line break inside the scalar), and DEL. +_YAML_BLOCK_SCALAR_UNSAFE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") + + class YamlIntegration(IntegrationBase): """Concrete base for integrations that use YAML recipe format. @@ -1240,6 +1245,23 @@ def _render_yaml(cls, title: str, description: str, body: str, source_id: str) - default_flow_style=False, ).strip() + # YAML forbids C0 control characters (except tab and newline) and + # DEL in every scalar form, and a bare CR acts as a line break + # inside a block scalar. A literal block scalar emits such bytes + # verbatim, producing a recipe the YAML parser rejects, so fall + # back to an escaped double-quoted scalar for those bodies. + if _YAML_BLOCK_SCALAR_UNSAFE.search(body): + prompt_yaml = yaml.safe_dump( + {"prompt": body}, allow_unicode=True, default_style='"', width=float("inf") + ).strip() + lines = [ + header_yaml, + prompt_yaml, + "", + f"# Source: {source_id}", + ] + return "\n".join(lines) + "\n" + # Indent the body for YAML block scalar. Use an explicit indentation # indicator ("|2") rather than a bare "|": YAML infers a plain block # scalar's indentation from its first non-empty line, so a body whose diff --git a/tests/integrations/test_integration_base_yaml.py b/tests/integrations/test_integration_base_yaml.py index 56bed09eb2..f46ba89143 100644 --- a/tests/integrations/test_integration_base_yaml.py +++ b/tests/integrations/test_integration_base_yaml.py @@ -201,6 +201,29 @@ def test_yaml_prompt_with_indented_first_line_stays_valid(self): parsed = yaml.safe_load("\n".join(yaml_lines)) assert parsed["prompt"].rstrip("\n") == body + def test_yaml_prompt_with_control_characters_stays_valid(self): + """A body containing control characters must still produce parseable YAML. + + YAML forbids C0 control characters (except tab and newline) and DEL + in every scalar form; a literal block scalar emits them verbatim, + so the generated recipe failed to load. The renderer falls back to + an escaped double-quoted scalar for such bodies.""" + for ch in ("\x08", "\x0c", "\x1b", "\x7f"): + body = f"before{ch}after\nsecond line" + rendered = YamlIntegration._render_yaml("Title", "Desc", body, "src") + parsed = yaml.safe_load(rendered) + assert parsed["prompt"].rstrip("\n") == body, f"char {ch!r} round-trip" + + def test_yaml_prompt_with_bare_carriage_return_stays_valid(self): + """A bare CR (not part of CRLF) must not break the generated YAML. + + Inside a block scalar a lone \r acts as a line break, corrupting + the document structure.""" + body = "line1\rstill line1\nline2" + rendered = YamlIntegration._render_yaml("Title", "Desc", body, "src") + parsed = yaml.safe_load(rendered) + assert parsed["prompt"].rstrip("\n") == body + def test_plan_command_has_no_context_placeholder(self, tmp_path): """The generated plan command must not carry a context-file placeholder. From 664db97ded9d0d586428be95f653b756bcc8b24b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:50:42 +0200 Subject: [PATCH 2/4] fix(integrations): use sys.maxsize instead of float inf for yaml width Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/integrations/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.py index 631b111306..57b863b991 100644 --- a/src/specify_cli/integrations/base.py +++ b/src/specify_cli/integrations/base.py @@ -1252,7 +1252,7 @@ def _render_yaml(cls, title: str, description: str, body: str, source_id: str) - # back to an escaped double-quoted scalar for those bodies. if _YAML_BLOCK_SCALAR_UNSAFE.search(body): prompt_yaml = yaml.safe_dump( - {"prompt": body}, allow_unicode=True, default_style='"', width=float("inf") + {"prompt": body}, allow_unicode=True, default_style='"', width=sys.maxsize ).strip() lines = [ header_yaml, From e80814fb5cb2b86a79617fc6e644d3dcef42be5a Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:24:53 +0200 Subject: [PATCH 3/4] fix(integrations): extend block-scalar guard to C1 controls and Unicode line breaks YAML's printable set excludes C1 controls (U+0080-U+009F except NEL), and YAML 1.1 treats NEL/LS/PS as line breaks inside a literal block scalar, so bodies carrying any of these still produced unparseable recipes. Widen the fallback guard to the full class and cover it in the regression loop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/integrations/base.py | 7 +++++-- tests/integrations/test_integration_base_yaml.py | 15 ++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.py index 57b863b991..d23ab447e7 100644 --- a/src/specify_cli/integrations/base.py +++ b/src/specify_cli/integrations/base.py @@ -1123,8 +1123,11 @@ def setup( # --------------------------------------------------------------------------- # Characters a YAML literal block scalar cannot carry: C0 controls other -# than tab/LF (a bare CR acts as a line break inside the scalar), and DEL. -_YAML_BLOCK_SCALAR_UNSAFE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") +# than tab/LF (a bare CR acts as a line break inside the scalar), DEL, and +# the C1 range. NEL (U+0085) is YAML-printable but, like LS/PS +# (U+2028/U+2029), YAML 1.1 treats it as a line break, which corrupts the +# block scalar's structure just the same, so all three are included. +_YAML_BLOCK_SCALAR_UNSAFE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f\u2028\u2029]") class YamlIntegration(IntegrationBase): diff --git a/tests/integrations/test_integration_base_yaml.py b/tests/integrations/test_integration_base_yaml.py index f46ba89143..a5a35aada7 100644 --- a/tests/integrations/test_integration_base_yaml.py +++ b/tests/integrations/test_integration_base_yaml.py @@ -204,11 +204,16 @@ def test_yaml_prompt_with_indented_first_line_stays_valid(self): def test_yaml_prompt_with_control_characters_stays_valid(self): """A body containing control characters must still produce parseable YAML. - YAML forbids C0 control characters (except tab and newline) and DEL - in every scalar form; a literal block scalar emits them verbatim, - so the generated recipe failed to load. The renderer falls back to - an escaped double-quoted scalar for such bodies.""" - for ch in ("\x08", "\x0c", "\x1b", "\x7f"): + YAML forbids C0 control characters (except tab and newline), DEL, + and C1 controls in every scalar form, and YAML 1.1 treats NEL + (U+0085), LS (U+2028) and PS (U+2029) as line breaks that corrupt a + literal block scalar's structure. The renderer falls back to an + escaped double-quoted scalar for such bodies.""" + for ch in ( + "\x08", "\x0c", "\x1b", "\x7f", + "\x80", "\x84", "\x85", "\x86", "\x9f", + "\u2028", "\u2029", + ): body = f"before{ch}after\nsecond line" rendered = YamlIntegration._render_yaml("Title", "Desc", body, "src") parsed = yaml.safe_load(rendered) From 5be4acc081f929f51a37d0445e9f3abde7b0eae2 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:31:08 +0200 Subject: [PATCH 4/4] fix(integrations): also treat surrogates and U+FFFE/U+FFFF as block-scalar unsafe YAML's printable set also excludes lone UTF-16 surrogates and the non-characters U+FFFE/U+FFFF; bodies carrying them still hit the literal block path and produced unparseable recipes. Extend the guard and the regression loop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/integrations/base.py | 13 ++++++++----- tests/integrations/test_integration_base_yaml.py | 10 ++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.py index d23ab447e7..c5d72306a0 100644 --- a/src/specify_cli/integrations/base.py +++ b/src/specify_cli/integrations/base.py @@ -1123,11 +1123,14 @@ def setup( # --------------------------------------------------------------------------- # Characters a YAML literal block scalar cannot carry: C0 controls other -# than tab/LF (a bare CR acts as a line break inside the scalar), DEL, and -# the C1 range. NEL (U+0085) is YAML-printable but, like LS/PS -# (U+2028/U+2029), YAML 1.1 treats it as a line break, which corrupts the -# block scalar's structure just the same, so all three are included. -_YAML_BLOCK_SCALAR_UNSAFE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f\u2028\u2029]") +# than tab/LF (a bare CR acts as a line break inside the scalar), DEL, the +# C1 range, lone UTF-16 surrogates, and the non-characters U+FFFE/U+FFFF. +# NEL (U+0085) is YAML-printable but, like LS/PS (U+2028/U+2029), YAML 1.1 +# treats it as a line break, which corrupts the block scalar's structure +# just the same, so all three are included. +_YAML_BLOCK_SCALAR_UNSAFE = re.compile( + r"[\x00-\x08\x0b-\x1f\x7f-\x9f\u2028\u2029\ud800-\udfff\ufffe\uffff]" +) class YamlIntegration(IntegrationBase): diff --git a/tests/integrations/test_integration_base_yaml.py b/tests/integrations/test_integration_base_yaml.py index a5a35aada7..a3968384f2 100644 --- a/tests/integrations/test_integration_base_yaml.py +++ b/tests/integrations/test_integration_base_yaml.py @@ -205,14 +205,16 @@ def test_yaml_prompt_with_control_characters_stays_valid(self): """A body containing control characters must still produce parseable YAML. YAML forbids C0 control characters (except tab and newline), DEL, - and C1 controls in every scalar form, and YAML 1.1 treats NEL - (U+0085), LS (U+2028) and PS (U+2029) as line breaks that corrupt a - literal block scalar's structure. The renderer falls back to an - escaped double-quoted scalar for such bodies.""" + C1 controls, lone surrogates and U+FFFE/U+FFFF in every scalar form, + and YAML 1.1 treats NEL (U+0085), LS (U+2028) and PS (U+2029) as + line breaks that corrupt a literal block scalar's structure. The + renderer falls back to an escaped double-quoted scalar for such + bodies.""" for ch in ( "\x08", "\x0c", "\x1b", "\x7f", "\x80", "\x84", "\x85", "\x86", "\x9f", "\u2028", "\u2029", + "\ud800", "\udfff", "\ufffe", "\uffff", ): body = f"before{ch}after\nsecond line" rendered = YamlIntegration._render_yaml("Title", "Desc", body, "src")