From 5aca37c94bafeb7e85310fa9f0280b5b455392ba Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Sat, 17 Oct 2026 07:03:41 +0000 Subject: [PATCH 1/5] Implement bounded nonblocking HTTP and verified HTTPS downloads --- .claude/skills/dmod-boot/SKILL.md | 65 +++ .../skills/dmod-coding-conventions/SKILL.md | 90 +++ .claude/skills/dmod-ecosystem/SKILL.md | 118 ++++ .claude/skills/dmod-hardware-ports/SKILL.md | 45 ++ .claude/skills/dmod-module-structure/SKILL.md | 49 ++ .github/workflows/ci.yml | 33 ++ .github/workflows/release.yml | 246 ++++++++ .gitignore | 36 +- .vscode/settings.json | 8 + .vscode/tasks.json | 55 ++ CMakeLists.txt | 81 +++ README.md | 57 +- cmake/pointer-audit.cmake | 11 + dmhttp.dmr | 33 ++ docs/README.md | 4 + docs/api-reference.md | 3 + docs/api.md | 68 +++ docs/testing.md | 70 +++ include/dmhttp.h | 107 ++++ manifest.dmm | 5 + scripts/check_dmod_pointers.py | 67 +++ scripts/sync-claude.sh | 117 ++++ src/dmhttp.c | 535 ++++++++++++++++++ src/dmhttp_internal.h | 72 +++ src/dmhttp_parser.c | 211 +++++++ src/dmhttp_url.c | 239 ++++++++ tests/CMakeLists.txt | 47 ++ tests/dmhttp_test.c | 108 ++++ tests/host/CMakeLists.txt | 22 + tests/host/interoperability.py | 234 ++++++++ tests/host/platform.c | 113 ++++ tests/host/stubs/dmhttp_defs.h | 1 + tests/host/stubs/dmod.h | 20 + tests/host/stubs/dmosi.h | 11 + tests/host/stubs/dmtls_defs.h | 4 + tests/host/stubs/dmutc.h | 18 + tests/host/stubs/dmutc_defs.h | 4 + tests/probe/CMakeLists.txt | 14 + tests/probe/httpprobe.c | 272 +++++++++ tests/probe/httpprobe.dmr | 3 + tests/probe/make_fixtures.py | 29 + tests/probe/run_wire.py | 119 ++++ tests/probe/wireprobe.c | 228 ++++++++ 43 files changed, 3653 insertions(+), 19 deletions(-) create mode 100644 .claude/skills/dmod-boot/SKILL.md create mode 100644 .claude/skills/dmod-coding-conventions/SKILL.md create mode 100644 .claude/skills/dmod-ecosystem/SKILL.md create mode 100644 .claude/skills/dmod-hardware-ports/SKILL.md create mode 100644 .claude/skills/dmod-module-structure/SKILL.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json create mode 100644 CMakeLists.txt create mode 100644 cmake/pointer-audit.cmake create mode 100644 dmhttp.dmr create mode 100644 docs/README.md create mode 100644 docs/api-reference.md create mode 100644 docs/api.md create mode 100644 docs/testing.md create mode 100644 include/dmhttp.h create mode 100644 manifest.dmm create mode 100644 scripts/check_dmod_pointers.py create mode 100755 scripts/sync-claude.sh create mode 100644 src/dmhttp.c create mode 100644 src/dmhttp_internal.h create mode 100644 src/dmhttp_parser.c create mode 100644 src/dmhttp_url.c create mode 100644 tests/CMakeLists.txt create mode 100644 tests/dmhttp_test.c create mode 100644 tests/host/CMakeLists.txt create mode 100644 tests/host/interoperability.py create mode 100644 tests/host/platform.c create mode 100644 tests/host/stubs/dmhttp_defs.h create mode 100644 tests/host/stubs/dmod.h create mode 100644 tests/host/stubs/dmosi.h create mode 100644 tests/host/stubs/dmtls_defs.h create mode 100644 tests/host/stubs/dmutc.h create mode 100644 tests/host/stubs/dmutc_defs.h create mode 100644 tests/probe/CMakeLists.txt create mode 100644 tests/probe/httpprobe.c create mode 100644 tests/probe/httpprobe.dmr create mode 100644 tests/probe/make_fixtures.py create mode 100644 tests/probe/run_wire.py create mode 100644 tests/probe/wireprobe.c diff --git a/.claude/skills/dmod-boot/SKILL.md b/.claude/skills/dmod-boot/SKILL.md new file mode 100644 index 0000000..efe6bb1 --- /dev/null +++ b/.claude/skills/dmod-boot/SKILL.md @@ -0,0 +1,65 @@ +--- +name: dmod-boot +description: Explains dmod-boot, the firmware entry point that initializes hardware/FreeRTOS/heap/VFS/logging and boots into a loaded module package — how module packaging (modules.dmp) works, how to test a locally-built .dmf on real hardware without a release, and the flash/connect/monitor CMake targets for working with real boards. Load this when working in the dmod-boot repo, tracing how firmware boots into a usable system, or flashing/debugging a board. Companion to the dmod-ecosystem skill. +--- + +# `dmod-boot`: the entry point + +`dmod-boot` is the bootloader/firmware entry point for a real board. It +initializes hardware, FreeRTOS (via `dmosi-freertos`), heap (`dmheap`), VFS +(`dmvfs`, mounting `dmramfs` on `/`, `dmdevfs` on `/dev`), logging (`dmlog`), +then loads a package of modules embedded directly in ROM at link time and +starts a main module — typically the shell, `dmell`. If you're tracing "how +does firmware boot into a usable system", `dmod-boot/src/main.c` is the +starting point. + +**Module packaging.** During the build, `dmod-boot` bundles the modules it +finds in `/build/dmf/` into a single `modules.dmp` package, which +is then embedded in flash alongside the rest of the firmware (`modules/modules.dmd` +is the manifest that determines which modules get fetched/built into +`build/dmf/` in the first place). A module can be present there in two forms: +`.dmf` (plain) and `.dmfc` (the same module, fastlz-compressed). +`dmf-get` normally downloads both, and **the `.dmfc` takes precedence when the +module is loaded**. + +**Testing a locally-built module on real hardware, without a release:** drop +the `.dmf` binary straight into `/build/dmf/`, **delete that +module's `.dmfc` from the same directory**, then delete +`/build/modules.dmp` and `/build/__modules_dmp.o` so the +next build is forced to regenerate the package (an incremental build won't +notice a `.dmf` was added/changed in place otherwise). + +Skipping the `.dmfc` step is the classic trap: the build succeeds, the new +`.dmf` really is in `build/dmf/`, and the board still silently runs the old +released module. It usually surfaces as a *dependency* error rather than an +obvious "stale module" one — the released build was linked against older +versions of its dependencies, so you get +`Module version mismatch: != ` and +`API '@:/' is not connected`, followed by the module failing +to enable. Two things make this hard to see: `strings build/modules.dmp` will +*not* find the offending signature (it is compressed inside the `.dmfc`), and +comparing `.dmf`/`.dmfc` timestamps in `build/dmf/` is the quickest way to +spot it — every module should have a matching pair, so the one module whose +two files disagree is the culprit. + +**Beware of reconfiguring while testing a local module.** Re-running +`cmake -S . -B ` (as opposed to `cmake --build `) re-runs +`dmf-get`, which re-downloads *every* module into `build/dmf/` and overwrites +any `.dmf` you dropped there by hand — and restores the `.dmfc` you deleted. +After any reconfigure, redo the steps above. Note also that `dmf-get` resolves +each repo's dependencies independently, so a refresh can legitimately leave +the set internally inconsistent (e.g. a command module still built against an +older version of a library module it uses); such a module fails to bind on its +own, which is harmless to the rest of the boot, but it is worth telling apart +from the stale-`.dmfc` case above. + +**Flashing and debugging** (from `/build`): +- `cmake --build . --target install-firmware` — flash the firmware to the board. +- `cmake --build . --target connect` — attach OpenOCD to the target. +- `cmake --build . --target monitor` — attach a console to `dmell` over the + existing OpenOCD connection (run `connect` first). This gives you stdin/stdout + independent of any firmware-level driver — you get a working shell even + before a UART driver is loaded, since it doesn't go through `dmuart`/`dmtty` + at all. Log output for this relies on `dmlog`, the base/kernel-level logging + system used from very early boot (before most modules are up), which is why + `dmod-boot` depends on it directly rather than through a loaded module. diff --git a/.claude/skills/dmod-coding-conventions/SKILL.md b/.claude/skills/dmod-coding-conventions/SKILL.md new file mode 100644 index 0000000..bdda94b --- /dev/null +++ b/.claude/skills/dmod-coding-conventions/SKILL.md @@ -0,0 +1,90 @@ +--- +name: dmod-coding-conventions +description: Coding conventions for DMOD module C code — heap-allocate strings/lists instead of fixed-size buffers (Dmod_StrDup, dmlist), never statically link a dependency into a module (always dmod_link_modules), keep functions under ~50 lines, and the _create/_destroy opaque-context pattern with a magic-field guard. Load this when writing or reviewing module implementation code (.c/.h files) in a dmod-* repo. Companion to the dmod-ecosystem skill. +--- + +# DMOD Module Coding Conventions + +## Strings: prefer the heap over fixed-size buffers + +The target is embedded (STM32F4/F7, ESP32-S3) — RAM is scarce and every byte +of it matters, so default to heap-allocating rather than reserving fixed +static/stack space "just in case". This applies to strings in general, not +just paths: don't declare a fixed-size `char buf[N]` for a string whose +length isn't tightly bounded — heap-allocate it instead. Beyond the general +RAM-budget argument, fixed-size stack/static buffers silently truncate or +overflow once a value gets longer than whoever wrote `N` expected, which is a +real risk for anything composed at runtime (paths, module names, mount +points under `/`, `.dmr`/`.dmm` entries, log/error messages, etc.). A static +fixed-size array is only acceptable when you can genuinely guarantee the +string will never exceed ~20 bytes (e.g. a short fixed enum-like tag) — +paths and other composed/runtime strings do not qualify. + +When you need an owned copy of a string (not just a borrowed pointer), copy +it with `Dmod_StrDup` rather than `malloc` + `strcpy`/`memcpy`, and `free` it +when done. + +The same reasoning applies to lists: don't hand-roll a fixed-size static +array for a collection whose size grows/shrinks at runtime. Use the `dmlist` +module instead — pull it in with `dmod_link_modules(${DMOD_MODULE_NAME} +dmlist)` in `CMakeLists.txt` (see `dmtty/CMakeLists.txt` for a working +example) rather than reimplementing a dynamic list. + +## Never statically link libraries into a module + +Always link a module against another module's `_if` target dynamically +(`dmod_link_modules`, the standard Built-in API / MAL / DIF mechanisms) — +never statically link a library's object code directly into a module. DMOD +modules are loaded/unloaded independently at runtime; statically linking a +dependency into a module defeats that (bloats the `.dmf`, duplicates the +dependency's RAM/flash footprint across every module that links it, and +breaks the loader's ability to resolve/share a single instance of it). If +you find yourself reaching for `target_link_libraries` with anything other +than a module's own `_if` interface target, stop and use +`dmod_link_modules` instead. + +## Keep functions short and extract shared logic + +Functions should stay short — **max ~50 lines**. If a function is growing +past that, it's a signal to split it, not to keep going. Always look for the +common part between two similar code paths (two callers doing almost the +same thing, two branches with overlapping logic, ...) and pull it into its +own function rather than duplicating it inline. + +## Prefer the `_create`/`_destroy` context pattern + +The preferred way to manage a piece of stateful data is a context object: +a `_create` function allocates and initializes a context struct holding all +the data that logic needs, callers thread that context pointer through the +rest of the API, and a `_destroy` function frees it once it's no longer +needed. See `dm_sw_ring_create()`/`dm_sw_ring_destroy()` in +`dm_sw_ring/include/dm_sw_ring.h` for a worked example of this shape. + +**Context structs must be opaque/private.** Declare only the typedef in the +public/interface header, and define the actual struct body in the private +source file: + +```c +/* public header (e.g. include/.h) */ +typedef struct context context_t; +``` + +```c +/* private file (e.g. src/.c or a private header) */ +struct context +{ + ... +}; +``` + +This keeps callers from reaching into the struct directly or depending on +its layout — all access must go through the module's API functions. + +**Guard context structs with a magic field.** Give the struct a `magic` +field (`uint32_t`) set in `_create` to a fixed value derived from a short +form of the module's name (e.g. a 4-char tag packed into a `uint32_t`), and +check it at the top of every function that receives the context pointer, +before the context is dereferenced (or used) — the check should be present +in every entry point (`_destroy` included), not just a subset. This catches +use of a stale/freed, corrupted, or wrong-type pointer early instead of +leading to undefined behavior further down. diff --git a/.claude/skills/dmod-ecosystem/SKILL.md b/.claude/skills/dmod-ecosystem/SKILL.md new file mode 100644 index 0000000..b267107 --- /dev/null +++ b/.claude/skills/dmod-ecosystem/SKILL.md @@ -0,0 +1,118 @@ +--- +name: dmod-ecosystem +description: Entry-point map of the DMOD (Dynamic Modules) embedded ecosystem architecture — the core loader, the three inter-module communication mechanisms (Built-in API, MAL, DIF), and the shared CMake/Make build system. Load this first when working in any dmod-* repository (dmod, dmod-boot, dmuart, dmtty, dmfmc, dmell, dm_sw_ring, dmclk, dmgpio, dmini, dmheap, dmvfs, dmffs, dmdevfs, dmdrvi, dmhaman, dmosi*, or a new module created from this template) to understand how the repo fits into the larger system before making changes. Points to companion skills (dmod-module-structure, dmod-boot, dmod-hardware-ports, dmod-coding-conventions) for deeper topics. +--- + +# DMOD Ecosystem + +DMOD ("Dynamic Modules") is a framework for dynamically loading and running code +on embedded microcontrollers (STM32F4/F7, ESP32-S3 seen so far), similar in spirit +to shared libraries on a desktop OS but built for firmware. A "module" is a +self-contained unit of code (library or application) compiled to a `.dmf` +(DMOD File) — optionally compressed to `.dmfc` — that a loader can load, run, +and unload at runtime without recompiling or resetting the whole firmware. + +This ecosystem is spread across many small, independent git repositories under +one parent directory (each repo = one module, or the core framework). There is +no monorepo and no shared parent git tree — treat each repo you open as +self-contained, but expect it to `FetchContent` the `dmod` repo at configure time. + +**Naming convention: snake_case everywhere** — module names, file names, +functions, types (`dm_sw_ring`, `dm_sw_ring_create`, `dm_sw_ring_t`, +`dmfmc_configure_sdram`, ...). Don't introduce camelCase or PascalCase names +in module code (the handful of `Dmod_*`/`Pascal_Case` symbols you'll see, like +`Dmod_Printf`, are dmod's own SAL/core API, not a pattern to copy in module code). + +## The core repo: `dmod` + +`dmod` is the SDK and build system every other repo depends on. It provides: + +- **The loader/runtime** (`src/system/`, `src/module/`) — loads `.dmf`/`.dmfc`/`.dmp` + files, manages module contexts, resolves inter-module calls. +- **Two build modes**, selected via `DMOD_MODE`: + - `DMOD_SYSTEM` — builds the loader itself, embedded in firmware. + - `DMOD_MODULE` — builds a single loadable module (`.dmf`). +- **CMake functions** every module repo uses (see `dmod/docs/cmake-functions.md` + for the full reference): + - `dmod_add_library(name version sources...)` / `dmod_add_executable(...)` — + create a module. Reads `DMOD_MODULE_NAME`, `DMOD_MODULE_VERSION`, + `DMOD_AUTHOR_NAME`, `DMOD_STACK_SIZE`, optionally `DMOD_PRIORITY`, + `DMOD_MAL_IMPLS`, `DMOD_DIF_IMPLS`, `DMOD_DMR_PATH` (set before calling). + Also creates a `_if` INTERFACE target for consumers that only need headers. + - `dmod_link_modules(target [PRIVATE|PUBLIC|INTERFACE] module[@version]...)` — + downloads another module's headers via `dmf-get` at configure time and adds + them as include dirs. This is how modules depend on each other without + vendoring source. + - `dmod_setup_external_module()` — call after `project()` in a standalone repo + to wire up DMOD's CMake machinery. +- **The package manager**: `dmf-get` (installs/downloads modules), plus + `todmfc`/`todmd`/`todmm`/`todmp`/`mkdmrpkg`/`dmf-man` tools and three manifest + file formats, all documented under `dmod/docs/`: + - `.dmm` (manifest) — registry mapping module name → download URL. + - `.dmr` (resource file) — declares what files go where when a module is + installed/packaged (`docs/dmr-file-format.md`). + - `.dmd` (dependencies) — generated list of a module's dependency versions. + +## Three ways modules talk to each other + +1. **Built-in / Module API** — a module's own public functions, declared in + its header with the `dmod__api(version, ret, _suffix, (args))` + macro and defined in the `.c` file with the matching + `dmod__api_declaration(version, ret, _suffix, (args))` macro (both + auto-generated per-module into `_defs.h` by the dmod core build + from `dmod/scripts/api.h.in` — you only follow the naming convention, never + hand-write the macro itself). **This is not optional boilerplate**: a + plain C function prototype/definition in a module's public header will + compile but **fail to link** (or simply never be found), because the + loader resolves these calls dynamically rather than through normal static + linkage. Use this as the default way to expose any function another module + (or this module's own `tests/`) should be able to call — see + `dm_sw_ring/include/dm_sw_ring.h` + `dm_sw_ring/src/dm_sw_ring.c` for a + fully worked example, and the `dmod__port_api(...)` port variant + described below. + - A `dmod__global_api(...)` / `_global_api_declaration(...)` + variant also exists, declaring the function in the *global* namespace + instead of the module's own — reserved for exceptional cases (e.g. + defining a `printf`-style utility meant to be called unqualified from + anywhere). Default to the plain `_api`/`_api_declaration` pair unless you + have a specific reason to go global. +2. **MAL (Module Abstraction Layer)** — 1:1 inversion of control. One module + defines an interface; another module implements it and registers via + `DMOD_MAL_IMPLS` in its `CMakeLists.txt`/`Makefile`. Lets you swap an + implementation (e.g. which UART driver backs a generic "serial" interface) + without touching the caller. +3. **DIF (Dmod Interface)** — 1:N. Multiple modules implement the same + interface (e.g. several SPI/UART drivers) and are discovered dynamically at + runtime via `Dmod_GetNextDifModule()`/`Dmod_GetDifFunction()`, registered + through `DMOD_DIF_IMPLS`. + +Pick Built-in API for a hard compile-time dependency, MAL for "one pluggable +backend", DIF for "many interchangeable backends discovered at runtime". + +## Companion skills for deeper topics + +This skill is the map, not the territory — load these alongside it as the +task calls for them, rather than assuming everything is covered here: + +- **`dmod-module-structure`** — the standard module repo layout + (`CMakeLists.txt`, `manifest.dmm`, `docs/`, `tests/`), scaffolding a new + module with `dmod/scripts/new-module.sh`, and when to extract a reusable + library out of an application module. Load when creating or restructuring + a module repo. +- **`dmod-boot`** — the firmware entry point (`dmod-boot`): boot sequence, + module packaging (`modules.dmp`), testing a locally-built `.dmf` on real + hardware, and the flash/connect/monitor CMake targets. Load when working + in `dmod-boot` or tracing how firmware boots into a usable system. +- **`dmod-hardware-ports`** — the driver+port split used by hardware + modules (`dmuart`, `dmfmc`, ...), `DMOD_CPU_FAMILY`, and the + `dmod__port_api` macro. Load when writing or modifying a module + that talks to real hardware/peripherals. +- **`dmod-coding-conventions`** — module C code style: heap over + fixed-size buffers, never statically link a dependency into a module, + keep functions short, and the `_create`/`_destroy` opaque-context pattern + with a magic-field guard. Load when writing or reviewing module + implementation code. + +For a specific module's exact API, config keys, or behavior, read that +repo's `docs/api-reference.md` and `include/.h` directly rather than +guessing from any of these skills — module APIs evolve independently of them. diff --git a/.claude/skills/dmod-hardware-ports/SKILL.md b/.claude/skills/dmod-hardware-ports/SKILL.md new file mode 100644 index 0000000..9d9c9ef --- /dev/null +++ b/.claude/skills/dmod-hardware-ports/SKILL.md @@ -0,0 +1,45 @@ +--- +name: dmod-hardware-ports +description: Explains the driver+port pattern used by DMOD hardware modules (dmuart, dmfmc, dmgpio, dmclk, and any new peripheral driver) — splitting a module into a hardware-independent core and an architecture-specific "_port" module selected via DMOD_CPU_FAMILY, the dmod__port_api macro, and sharing logic across MCU families via src/port/_common/. Load this when writing or modifying a module that talks to real hardware/peripherals. Companion to the dmod-ecosystem skill. +--- + +# Driver + port pattern (hardware modules) + +Modules that touch real hardware (`dmuart`, `dmfmc`, and any new peripheral +driver) split into **two independent DMOD modules in one repo**: + +- `` — the core module: public API, config parsing (via `dmini`), + hardware-independent logic. Links against `_port_if`. +- `_port` — a second module, built from `src/port/CMakeLists.txt`, + containing the architecture-specific implementation. One subdirectory per + MCU family under `src/port//` (e.g. `src/port/stm32f7/`), each with a + `config.cmake` that sets `DMOD_TOOLS_NAME` (e.g. + `"arch/armv7/cortex-m7"`) and a `port.c` implementing the lifecycle + (`dmod_init`/`dmod_deinit`) plus any `DMOD_IRQ_HANDLER(...)`. + +The active architecture is selected via **`DMOD_CPU_FAMILY`** (a cache +variable already defined in `dmod/dmod-cfg.cmake`, consumed by `dmf-get +--cpu-family` for package resolution) — set it directly in the module's +`CMakeLists.txt` before `FetchContent_MakeAvailable(dmod)`: + +```cmake +set(DMOD_CPU_FAMILY "stm32f7" CACHE STRING "Target CPU family") +include(${CMAKE_CURRENT_SOURCE_DIR}/src/port/${DMOD_CPU_FAMILY}/config.cmake) +``` + +Do **not** invent a module-specific variable like `_MCU_SERIES` for +this — `DMOD_CPU_FAMILY` already exists and is understood by the rest of the +toolchain (`dmf-get`, package naming). `dmfmc`'s `DMFMC_MCU_SERIES` predates +this and is legacy, not a pattern to copy in new modules. + +The port's public API is declared with the `dmod__port_api(version, +return_type, _suffix, (args))` macro — this macro is generated automatically +by the dmod core build from `dmod/scripts/api.h.in` into +`_port_defs.h`; you only need to follow the naming convention, not +hand-write the macro. See `dmfmc/include/dmfmc_port.h` for a full example, or +`dmuart/include/dmuart_port.h` for a simpler one. + +If a peripheral IP block is identical across MCU families, keep the shared +logic in `src/port/_common/` and make each `src/port//port.c` a +thin wrapper (lifecycle + IRQ only) — see `dmfmc/docs/port-implementation.md` +for the exact recipe used there. diff --git a/.claude/skills/dmod-module-structure/SKILL.md b/.claude/skills/dmod-module-structure/SKILL.md new file mode 100644 index 0000000..77446b2 --- /dev/null +++ b/.claude/skills/dmod-module-structure/SKILL.md @@ -0,0 +1,49 @@ +--- +name: dmod-module-structure +description: Explains the standard DMOD module repo layout (CMakeLists.txt/Makefile skeleton, manifest.dmm, docs/, tests/), how to scaffold a new module with dmod/scripts/new-module.sh, and when to extract a reusable library out of an application module. Load this when creating a new dmod-* module repo, restructuring an existing one, or deciding whether functionality should become its own library module. Companion to the dmod-ecosystem skill. +--- + +# DMOD Module Structure + +Every real module repo (`dmuart`, `dmtty`, `dmfmc`, `dm_sw_ring`, `dmell`, ...) +follows the same shape: + +``` +/ +├── CMakeLists.txt # FetchContent's dmod, then dmod_add_library/_executable +├── Makefile # Make-based equivalent build +├── manifest.dmm # $include .../versions.dmm + this module's download URL +├── .dmr # install/package resource mapping +├── include/.h # public API +├── src/.c # implementation + dmod_init()/dmod_deinit() +├── docs/ +│ ├── README.md # doc index +│ ├── api-reference.md # full API reference +│ └── (examples.md, configuration.md, port-implementation.md as needed) +├── tests/ +│ ├── CMakeLists.txt # dmod_add_test(...) +│ └── _test.c # #include "dmod_test.h", DMOD_TEST_STEP(...) macros +└── README.md # includes a "Project Structure" tree section +``` + +Every `CMakeLists.txt` in a standalone module repo follows the same skeleton — +`FetchContent_Declare(dmod GIT_REPOSITORY .../dmod.git GIT_TAG develop)` + +`FetchContent_MakeAvailable(dmod)` + `include(${DMOD_DIR}/paths.cmake)` + +`dmod_setup_external_module()` — rather than assuming a local checkout path. +Look at `dm_sw_ring/CMakeLists.txt` or `dmtty/CMakeLists.txt` for a clean example. + +New modules should be scaffolded with `dmod/scripts/new-module.sh` (see +`--help` for `--type library|application`, `--port`, `--dif`, `--mal`), which +generates this exact layout. + +## Extracting a library out of an application module + +Sometimes an `application`-type module accumulates functionality that other +modules need to call — not just as an implementation detail, but as an API +other modules should link against. In that case, split the reusable part +out into its own `library`-type DMOD module rather than exposing the +application module's internals directly, and give that library module a +`lib` prefix. E.g. if the application module is `systemd`, the extracted +library becomes `libsystemd`: `systemd` itself becomes a consumer of +`libsystemd` (via `dmod_link_modules`) alongside every other module that +needs that API, rather than a special case other modules reach into. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bacc27a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI +on: + push: + branches: [main, master, develop] + pull_request: + branches: [main, master, develop] +jobs: + build: + name: Build and test (${{ matrix.tools }}) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + tools: [arch/x86_64, arch/armv7/cortex-m4, arch/armv7/cortex-m7] + container: + image: chocotechnologies/dmod:1.0.4 + steps: + - uses: actions/checkout@v4 + - name: Install test and audit tools + run: | + apt-get update + apt-get install -y --no-install-recommends python3 openssl + - name: Build + run: | + cmake -S . -B build -DDMOD_TOOLS_NAME=${{ matrix.tools }} + cmake --build build --parallel 4 + test -f build/dmf/dmhttp.dmf + test -f build/dmf/test_dmhttp.dmf + - name: Native tests + if: matrix.tools == 'arch/x86_64' + run: ctest --test-dir build --output-on-failure diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c9cbd1d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,246 @@ +name: Release + +on: + release: + types: [created] + +jobs: + build-release: + name: Build Release for ${{ matrix.arch_name }} + runs-on: ubuntu-latest + permissions: + contents: write + strategy: + matrix: + arch_name: [x86_64, armv7/cortex-m4, armv7/cortex-m7] + + container: + image: chocotechnologies/dmod:1.0.4 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install audit interpreter + run: | + apt-get update + apt-get install -y --no-install-recommends python3 + + - name: Extract version from tag + id: get_version + run: | + # Extract version from tag (e.g., v1.2 -> 1.2) + VERSION="${{ github.event.release.tag_name }}" + VERSION="${VERSION#v}" # Remove 'v' prefix if present + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + - name: Build dmhttp for ${{ matrix.arch_name }} + run: | + set -e + ARCH_DIR_NAME=$(echo "${{ matrix.arch_name }}" | sed 's|/|-|') + echo "ARCH_DIR_NAME=$ARCH_DIR_NAME" >> $GITHUB_ENV + echo "ARTIFACT_NAME=release-$ARCH_DIR_NAME" >> $GITHUB_ENV + mkdir -p build_$ARCH_DIR_NAME + cd build_$ARCH_DIR_NAME + + # DMOD_DMR_PATH is already set in CMakeLists.txt, so this build also + # assembles build_$ARCH_DIR_NAME/packages// for every module + # defined in this repo's CMake config - nothing left to do here but + # discover whatever got packaged, add release notes, and zip it. + cmake .. -DDMOD_TOOLS_NAME=arch/${{ matrix.arch_name }} -DDMOD_MODULE_VERSION="${{ steps.get_version.outputs.version }}" + cmake --build . + + echo "Build completed for ${{ matrix.arch_name }}" + ls -la packages/ + + - name: Add release notes and create release archives + run: | + set -e + BUILD_DIR="build_$ARCH_DIR_NAME" + TAG="${{ github.event.release.tag_name }}" + PACKAGES_DIR="$BUILD_DIR/packages" + + if [ ! -d "$PACKAGES_DIR" ] || [ -z "$(ls -A "$PACKAGES_DIR")" ]; then + echo "Error: no packages found in $PACKAGES_DIR" + exit 1 + fi + + # A repo may define more than one module (e.g. a core + port split, + # or several independent modules) - package every subdirectory + # found under packages/, not just dmhttp. + : > package-names.txt + for pkg_dir in "$PACKAGES_DIR"/*/; do + pkg=$(basename "$pkg_dir") + echo "$pkg" >> package-names.txt + + echo "${{ github.event.release.body }}" > "${pkg_dir}RELEASE_NOTES.txt" + (cd "$pkg_dir" && zip -r "${OLDPWD}/${pkg}-${TAG}-${ARCH_DIR_NAME}.zip" .) + done + + echo "Created archives:" + ls -lh *-"${TAG}"-"${ARCH_DIR_NAME}".zip + echo "Packages found:" + cat package-names.txt + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: | + *.zip + package-names.txt + retention-days: 1 + + generate-versions-manifest: + name: Generate versions.dmm + needs: [build-release] + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history to get all tags + + - name: Download release artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Generate versions.dmm + run: | + set -e + echo "# List of available versions for dmhttp modules" > versions.dmm + echo "# Generated automatically by CI" >> versions.dmm + echo "" >> versions.dmm + + # Get all version tags (starting with 'v') and extract version numbers + VERSIONS=$(git tag -l 'v*' | sed 's/^v//' | sort -V | tr '\n' ' ' | sed 's/ $//') + + # Remove vlatest from the list if present + VERSIONS=$(echo $VERSIONS | sed 's/\blatest\b//g' | xargs) + + if [ -z "$VERSIONS" ]; then + echo "Warning: No version tags found" + VERSIONS="${{ github.event.release.tag_name }}" + VERSIONS="${VERSIONS#v}" + fi + + echo "Found versions: $VERSIONS" + + # Union of package names discovered across every architecture build + # (the same set is expected everywhere, but union is safe even if + # some packages only exist for a subset of architectures). + PACKAGES=$(cat artifacts/release-*/package-names.txt | sort -u) + echo "Found packages: $PACKAGES" + + for pkg in $PACKAGES; do + echo "\$version-available $pkg $VERSIONS" >> versions.dmm + done + + echo "Generated versions.dmm:" + cat versions.dmm + + - name: Upload versions.dmm as artifact + uses: actions/upload-artifact@v4 + with: + name: versions-manifest + path: versions.dmm + retention-days: 1 + + upload-release-assets: + name: Upload Release Assets + needs: [build-release, generate-versions-manifest] + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Display artifact structure + run: | + echo "Downloaded artifacts:" + ls -lR artifacts/ + + - name: Upload release assets to versioned tag + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -e + shopt -s nullglob + zip_files=(artifacts/release-*/*.zip) + + if [ ${#zip_files[@]} -eq 0 ]; then + echo "Error: No artifacts found to upload" + exit 1 + fi + + for zip_file in "${zip_files[@]}"; do + echo "Uploading $zip_file to ${{ github.event.release.tag_name }}..." + gh release upload ${{ github.event.release.tag_name }} \ + "$zip_file" \ + --repo ${{ github.repository }} \ + --clobber + done + + if [ -f artifacts/versions-manifest/versions.dmm ]; then + echo "Uploading versions.dmm to ${{ github.event.release.tag_name }}..." + gh release upload ${{ github.event.release.tag_name }} \ + artifacts/versions-manifest/versions.dmm \ + --repo ${{ github.repository }} \ + --clobber + fi + + echo "Successfully uploaded ${#zip_files[@]} artifact(s) to ${{ github.event.release.tag_name }}" + + - name: Create or update latest release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -e + if gh release view vlatest --repo ${{ github.repository }} >/dev/null 2>&1; then + echo "Release vlatest exists, deleting it..." + gh release delete vlatest --repo ${{ github.repository }} --yes + fi + + echo "Creating vlatest release..." + gh release create vlatest \ + --repo ${{ github.repository }} \ + --title "Latest Release (based on ${{ github.event.release.tag_name }})" \ + --notes "This release always points to the latest stable version. Currently based on ${{ github.event.release.tag_name }}." + + - name: Upload release assets to latest tag + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -e + shopt -s nullglob + zip_files=(artifacts/release-*/*.zip) + + for zip_file in "${zip_files[@]}"; do + echo "Uploading $zip_file to vlatest..." + gh release upload vlatest \ + "$zip_file" \ + --repo ${{ github.repository }} \ + --clobber + done + + if [ -f artifacts/versions-manifest/versions.dmm ]; then + echo "Uploading versions.dmm to vlatest..." + gh release upload vlatest \ + artifacts/versions-manifest/versions.dmm \ + --repo ${{ github.repository }} \ + --clobber + fi + + echo "Successfully uploaded ${#zip_files[@]} artifact(s) to vlatest" diff --git a/.gitignore b/.gitignore index 1f99f9d..2eb0b42 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,19 @@ -CMakeLists.txt.user -CMakeCache.txt -CMakeFiles -CMakeScripts -Testing -Makefile -cmake_install.cmake -install_manifest.txt -compile_commands.json -CTestTestfile.cmake -_deps -CMakeUserPresets.json +# Build directories +build/ +*.dmf +*.dmfc -# CLion -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#cmake-build-* +# IDE +# .vscode/tasks.json and .vscode/settings.json are committed (see every real +# module repo) so the build tasks are shared with the team - don't ignore +# .vscode/ wholesale. +.idea/ + +# Temporary files +*.tmp +*.swp +*~ + +# Local diagnostics +__pycache__/ +*.pyc diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a65676b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "files.associations": { + "dmhttp.h": "c" + }, + "C_Cpp.default.compileCommands": "${workspaceFolder}/build/compile_commands.json", + "C_Cpp.intelliSenseEngine": "default", + "C_Cpp.errorSquiggles": "enabled" +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..2a66b75 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,55 @@ +{ + // See https://code.visualstudio.com/docs/editor/tasks for more information + "version": "2.0.0", + "tasks": [ + { + "label": "CMake: Configure", + "type": "shell", + "command": "cmake", + "args": [ + "-S", + "${workspaceFolder}", + "-B", + "${workspaceFolder}/build", + "-DDMOD_DIR=${input:dmodDir}" + ], + "group": "build", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "CMake: Build", + "type": "shell", + "command": "cmake", + "args": [ + "--build", + "${workspaceFolder}/build" + ], + "group": "build", + "problemMatcher": [] + }, + { + "label": "CMake: Clean", + "type": "shell", + "command": "cmake", + "args": [ + "--build", + "${workspaceFolder}/build", + "--target", + "clean" + ], + "group": "build", + "problemMatcher": [] + } + ], + "inputs": [ + { + "id": "dmodDir", + "description": "Path to a local dmod checkout (leave blank to fetch dmod's develop branch from GitHub instead)", + "default": "${workspaceFolder}/../dmod", + "type": "promptString" + } + ] +} diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..39f8624 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,81 @@ +cmake_minimum_required(VERSION 3.18) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +if(NOT DEFINED DMOD_MODULE_VERSION) + set(DMOD_MODULE_VERSION "0.1" CACHE STRING "DMOD module version") +endif() + +# If a local dmod checkout is provided, use it instead of fetching from git +if(DEFINED DMOD_DIR AND NOT "${DMOD_DIR}" STREQUAL "") + set(FETCHCONTENT_SOURCE_DIR_DMOD "${DMOD_DIR}" CACHE PATH "" FORCE) +endif() + +include(FetchContent) +FetchContent_Declare( + dmod + GIT_REPOSITORY https://github.com/choco-technologies/dmod.git + GIT_TAG develop +) + +set(DMOD_MODE "DMOD_MODULE" CACHE STRING "DMOD build mode") +set(DMOD_BUILD_TESTS OFF CACHE BOOL "Build tests") +set(DMOD_BUILD_EXAMPLES OFF CACHE BOOL "Build examples") +set(DMOD_BUILD_TOOLS OFF CACHE BOOL "Build tools") +set(DMOD_BUILD_TEMPLATES OFF CACHE BOOL "Build templates") + +FetchContent_MakeAvailable(dmod) + +project(dmhttp VERSION ${DMOD_MODULE_VERSION} + DESCRIPTION "dmhttp DMOD library module" + LANGUAGES C CXX) + +set(DMOD_DIR ${dmod_SOURCE_DIR} CACHE PATH "DMOD source directory") +set(DMOD_SCRIPTS_DIR ${DMOD_DIR}/scripts CACHE PATH "DMOD scripts directory") +include(${DMOD_DIR}/paths.cmake) +dmod_setup_external_module() +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/pointer-audit.cmake) + +# Name of the module +set(DMOD_MODULE_NAME dmhttp) + +# Author (should be string) +set(DMOD_AUTHOR_NAME "Patryk Kubiak") + +# Stack size for the module (should be integer) +set(DMOD_STACK_SIZE 8192) + +# Path to the .dmr file - enables release package creation in build/packages/ +set(DMOD_DMR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/dmhttp.dmr) + +# List of MAL interfaces implemented by this module (leave empty if none) +set(DMOD_MAL_IMPLS ) + +# List of DIF interfaces implemented by this module (leave empty if none) +set(DMOD_DIF_IMPLS ) + +# +# dmod_add_library - create a library module +# it has the same signature as add_library +# and can be used in the same way after the creation +# (for example, to link libraries) +# +dmod_add_library(${DMOD_MODULE_NAME} ${DMOD_MODULE_VERSION} + # List of source files - can include C and C++ files + src/dmhttp.c + src/dmhttp_url.c + src/dmhttp_parser.c +) + +target_include_directories(${DMOD_MODULE_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include +) + +target_include_directories(${DMOD_MODULE_NAME}_if INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/include +) + +dmhttp_audit_module(${DMOD_MODULE_NAME}) +dmod_link_modules(${DMOD_MODULE_NAME} dmosi dmtls) +enable_testing() +add_subdirectory(tests) diff --git a/README.md b/README.md index 4a84494..a8cfa8d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,57 @@ # dmhttp -http + +A bounded-memory HTTP/1.1 download client for DMOD. It supports GET and HEAD, +HTTP and verified HTTPS through dmtls, streamed response bodies, chunked framing, +redirects, finite transaction deadlines, cancellation and optional byte ranges. +There is no libcurl dependency, hardware port or driver layer. + +The application supplies a nonblocking connector (DNS/connect, send, receive, +close), initializes dmtls with strong entropy and UTC, and supplies a trust-store +configuration. dmhttp opens/closes each connector and creates/destroys its TLS +connection. It never owns the application adapter or TLS configuration. + +## Build + +```sh +cmake -S . -B build/native +cmake --build build/native --parallel +ctest --test-dir build/native --output-on-failure +``` + +A local SDK can be selected with `-DDMOD_DIR=/path/to/dmod`. Supported build +profiles are `arch/x86_64`, `arch/armv7/cortex-m4` and `arch/armv7/cortex-m7`; +select with `-DDMOD_TOOLS_NAME=...`. Module dependencies use the packager's +version selection. The existing SDK/loader and 240-byte DMF format are unchanged. + +## Use + +1. Fill `dmhttp_options_t` with a connector and a nonzero timeout. For HTTPS, + provide a dmtls configuration with explicit trust anchors. The TLS owner + supplies strong entropy and a working dmutc source before starting requests. +2. Call `dmhttp_create(url, &options, &request)`. +3. Advance `dmhttp_step()` until `DMHTTP_OK`: final response headers are ready. +4. Inspect `dmhttp_response()`, including status and, for resumed downloads, + `Content-Range`. Read the body with `dmhttp_read()` into your own buffers. +5. Commit a downloaded file only after `DMHTTP_DONE`; destroy the handle on all + paths. HTTP error statuses remain available to the application. + +On `DMHTTP_WANT_READ`/`DMHTTP_WANT_WRITE`, wait for the adapter's readiness event +or transaction deadline. On `DMHTTP_AGAIN`, schedule another bounded execution +step. The library never sleeps or polls the network. Caller backpressure is +natural: when it stops reading, dmhttp stops consuming transport bytes. + +See [API and contracts](docs/api.md) and [validation](docs/testing.md). + +## Scope + +Requests send `Connection: close` and `Accept-Encoding: identity`. Responses are +not transparently decompressed. Unsupported transfer/content encodings fail. +Redirects are bounded, resolved relative to the current URL, create a fresh +connection and repeat certificate/hostname validation. HTTPS-to-HTTP redirects +are rejected. No credentials, cookies or arbitrary request headers are forwarded. + +URL input supports ASCII DNS/IPv4 hosts, ports and escaped path/query components; +fragments are omitted. IPv6 literals, authentication, uploads, proxy tunnelling, +HTTP/2, connection pooling and caching are outside this interface. + +License: MIT. dmtls and its private crypto backend are separate dependencies. diff --git a/cmake/pointer-audit.cmake b/cmake/pointer-audit.cmake new file mode 100644 index 0000000..51ce5b7 --- /dev/null +++ b/cmake/pointer-audit.cmake @@ -0,0 +1,11 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter) +function(dmhttp_audit_module target) + # Metadata is checked at build time and excluded from the binary by objcopy. + target_link_options(${target} PRIVATE -Wl,--emit-relocs,--no-relax) + set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS + ${PROJECT_SOURCE_DIR}/scripts/check_dmod_pointers.py) + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/check_dmod_pointers.py + $ ${DMOD_DMF_DIR}/${target}.dmf + COMMENT "Checking compatibility with the existing DMOD GOT loader" VERBATIM) +endfunction() diff --git a/dmhttp.dmr b/dmhttp.dmr new file mode 100644 index 0000000..a2248af --- /dev/null +++ b/dmhttp.dmr @@ -0,0 +1,33 @@ +# DMOD Resource File for dmhttp +# This file specifies where resources should be installed + +# === Core Module === +# Main module file - always installed +dmf=./${module}.dmf => ${destination}/${module}.dmf [origin=${dmf_dir}/${module}.dmf] + +# Compressed module +dmfc=./${module}.dmfc => ${destination}/${module}.dmfc [origin=${build_dir}/dmfc/${module}.dmfc] + +# Dependencies file (if exists) +dmd=./${module}.dmd => ${destination}/${module}.dmd [origin=${dmf_dir}/${module}.dmd] + +# Version information +version=./${module}_version.txt => ${destination}/${module}_version.txt [origin=${dmf_dir}/${module}_version.txt] + +# === Documentation === +# Module documentation in markdown format for dmf-man tool +docs=./docs => ${destination}/${module}/docs [origin=${repo_dir}/docs] + +# README file +readme=./README.md => ${destination}/${module}/README.md [origin=${repo_dir}/README.md] + +# === Header Files === +# Include directory with all headers (public header + generated _defs.h) +inc=./include => ${destination}/${module}/include [origin=${repo_dir}/include] [origin=${build_dir}/${module}_defs.h] + +# === License === +# License file +license=./LICENSE => ${destination}/${module}/LICENSE [origin=${repo_dir}/LICENSE] + +# Adding itself +dmr=./${module}.dmr => ${destination}/${module}/${module}.dmr [origin=${repo_dir}/${module}.dmr] diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..d25aa0d --- /dev/null +++ b/docs/README.md @@ -0,0 +1,4 @@ +# dmhttp documentation + +- [API and contracts](api.md) +- [Build and validation](testing.md) diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 0000000..86ca4be --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,3 @@ +# API reference + +See [API and ownership contracts](api.md) and the public `include/dmhttp.h`. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..3bcf71d --- /dev/null +++ b/docs/api.md @@ -0,0 +1,68 @@ +# API and ownership + +Public declarations and exact return values live in `include/dmhttp.h`. + +A request owns a copied URL/options structure and fixed parser/I/O buffers. +It borrows adapter user data and the TLS trust-store configuration until destroy. +One handle uses one connector; simultaneous requests need distinct connector +instances. Serialize all operations on the same handle. Callbacks must not +reenter it; operation reentry is rejected. Destroy during a callback is ignored. + +`transport.open(user, hostname, port)` starts or advances nonblocking name +resolution and TCP connection establishment. It returns dmtls-style OK/WANT/error +results. A repeat after WANT uses the same endpoint. `send`/`receive` are the +same contract as dmtls: OK reports positive progress, receive OK/0 is EOF, +WANT reports zero bytes. `close` must synchronously cancel pending work, release +the stream and detach callbacks. It is called exactly once per attempted open, +including a failed/pending open. A redirect can then start a new open on that +same adapter. The connector supplies readiness notification to its owner. + +The application schedules the absolute transaction deadline as well as stream +readiness; it cannot wait forever for an absent peer event. `timeout_ms` is +nonzero, at most INT32_MAX, and starts at create. It includes all redirects, +handshakes, transfer and time spent paused by the application. There is no +library worker thread. Each parser call processes at most 1024 framing bytes +before yielding AGAIN. Body reads copy at most one 1024-byte input buffer. + +Headers are bounded to 8192 bytes per response hop, including informational +responses; individual lines to 2048 bytes. At most eight informational responses +and sixteen redirects are accepted. Trailers have a separate 8192-byte budget; +chunk extensions are line-bounded. Header parsing requires CRLF, valid field +names and values, and rejects folded fields, conflicting lengths, CL+TE, +unsupported transfer codings, upgrades and framing fields in trailers. +Identical duplicate Content-Length values are accepted; comma lists are not. + +Bodies use Content-Length, decoded chunked framing, or EOF framing when neither +is given. HEAD, 204 and 304 have no body. Known lengths and chunk sizes are +checked for overflow and optional body limits before consuming data. TCP EOF +before a declared body/chunk/trailer is complete is an error. For HTTPS, +close-delimited completion requires TLS close_notify; an abrupt TCP EOF fails. +A framed body completes when its HTTP framing is complete, independently of the +peer's subsequent connection closure. Before normal completion and redirects, dmhttp sends TLS close_notify, yielding +on WANT without blocking. The connector is then released; its close operation +should preserve queued bytes on an orderly TCP close. Cancellation, destruction +and errors abort immediately. Module deinitialization returns BUSY while any +request handle remains alive. + +Callers must check HTTP status explicitly. A completed 404 body is a successful +protocol transfer. Range requests send bytes=N-; applications must validate a +206 status and Content-Range before appending, and restart on a 200 response. +Metadata strings that exceed the public response buffers fail with LIMIT. + +URL parsing accepts http/https schemes case-insensitively and no userinfo. +DNS/IPv4 hosts are lowercased. Query/path percent escapes are validated but not +decoded. Relative redirect references use RFC 3986 path merging and dot-segment +removal; unsupported schemes and HTTPS downgrade fail. Since no application +credentials/custom headers are accepted, redirects cannot forward those secrets +across origins. An application that restricts network destinations must enforce +its policy in the connector before every open. + +`max_body_bytes=0` explicitly permits unlimited streaming; payload size does not +change the library's fixed per-request allocation. TLS and adapter queues have +separate memory budgets. The real TCP test adapter has a bounded receive ring +and fails on overflow; dmtcp currently has no consumer-controlled receive window. +An adapter must never silently discard acknowledged bytes when its queue fills. + +Protocol references: [HTTP semantics](https://www.rfc-editor.org/rfc/rfc9110), +[HTTP/1.1 framing](https://www.rfc-editor.org/rfc/rfc9112), +[URI resolution](https://www.rfc-editor.org/rfc/rfc3986). diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..16fa19b --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,70 @@ +# Build and validation + +Build profiles: x86-64, Cortex-M4 and Cortex-M7, with the existing SDK/loader. +A read-only final-ELF audit requires absolute pointers to use the existing +header/API/GOT sections; it does not rewrite binaries or add runtime fixups. +The normal DMF header remains 240 bytes. + +## Native + +```sh +cmake -S . -B build/native +cmake --build build/native --parallel +ctest --test-dir build/native --output-on-failure +``` + +`test_dmhttp` runs the actual module through dmod_loader (3 checks): URL/TLS +validation, fragmented chunk decoding, and cancellation/connector ownership. + +`http_interoperability` compiles production HTTP and real dmtls/crypto sources +with host SAL/OSI/time providers. Independent OpenSSL peers exercise HTTP, +TLS 1.2/1.3 HTTPS, close_notify, hostname/CA rejection, HTTPS redirects and +downgrade rejection. +Scripted streams split headers, chunk sizes/CRLF, trailers and body bytes, +interleave WANT results and inject malformed/truncated framing. Other checks +cover redirect resolution/limits (including informational responses), HEAD/204/304, +206 metadata, pending connectors, cancellation/deadlines, module lifetime, +header/body limits, overflow and a 2 MiB streamed binary body. All 29 tests pass. +Tests verify final allocation cleanup after TLS shutdown. + +Host tests fetch dmtls main solely for source. Production module dependency +versions remain unpinned. Standard FetchContent overrides select local sources; +DMOD_DMM_URL can select a local package manifest. CI requires no developer paths. + +## STM32F746G-DISCO + +```sh +cmake -S . -B build/cortex-m7 -DDMOD_TOOLS_NAME=arch/armv7/cortex-m7 \ + -DDMOD_CPU_FAMILY=stm32f7 -DDMHTTP_BUILD_BOARD_PROBE=ON +cmake --build build/cortex-m7 --parallel +python3 tests/probe/make_fixtures.py build/board-fixtures +``` + +Install the local dmhttp/test_dmhttp/httpprobe/httpwireprobe packages in test +firmware, with dmtls, RNG, RTC and the running UTC library-service unit. Upload +only ca.pem/other.pem to the board, not private keys. Keep raw logs and generated +keys under ignored build paths or outside the repository. + +`httpprobe URL SERVER_IPV4 CA_FILE` uses real dmtcp and a semaphore notified by +callbacks. The IP override separates transport tests from DNS provisioning. +A reachable HTTP/HTTPS server must listen on the URL's port. Results report +status, length, FNV-1a body hash, redirects and TLS verification flags. +The receive ring fails visibly on overflow; it never silently drops bytes. +This adapter is a diagnostic, not a public DNS/TCP interface. + +`httpwireprobe` uses UART byte queues against a PC OpenSSL peer: + +```sh +python3 tests/probe/run_wire.py --boot-dir /path/to/dmod-boot \ + --fixtures build/board-fixtures +``` + +This executes the real MCU HTTP/TLS/RNG/UTC engines and checks decoded body +length/hash and redirects. It does not cover Ethernet/dmtcp. The UART fixture +has a finite 90-second total deadline to include serial handshakes/redirects. +Native socket tests and the TCP diagnostic use shorter finite limits. + +Board module checks passed 3/3. Ethernet validation is pending a server address +reachable from the board; the isolated build container was unreachable from +its LAN. UART interoperability is tracked separately and does not substitute +for the missing network test. Renode is omitted as requested. diff --git a/include/dmhttp.h b/include/dmhttp.h new file mode 100644 index 0000000..16d02c9 --- /dev/null +++ b/include/dmhttp.h @@ -0,0 +1,107 @@ +#ifndef DMHTTP_H +#define DMHTTP_H +#include +#include +#include +#include "dmtls.h" +#include "dmhttp_defs.h" +#ifdef __cplusplus +extern "C" +{ +#endif +#define DMHTTP_URL_MAX 2048u +#define DMHTTP_HEADER_MAX 8192u +#define DMHTTP_HOST_MAX 253u + + typedef struct dmhttp *dmhttp_t; + typedef enum + { + DMHTTP_OK = 0, + DMHTTP_WANT_READ = 1, + DMHTTP_WANT_WRITE = 2, + DMHTTP_AGAIN = 3, + DMHTTP_DONE = 4, + DMHTTP_ERR_ARGUMENT = -1, + DMHTTP_ERR_MEMORY = -2, + DMHTTP_ERR_STATE = -3, + DMHTTP_ERR_URL = -4, + DMHTTP_ERR_TRANSPORT = -5, + DMHTTP_ERR_TLS = -6, + DMHTTP_ERR_PROTOCOL = -7, + DMHTTP_ERR_LIMIT = -8, + DMHTTP_ERR_TRUNCATED = -9, + DMHTTP_ERR_TIMEOUT = -10, + DMHTTP_ERR_CANCELLED = -11, + DMHTTP_ERR_REDIRECT = -12 + } dmhttp_result_t; + typedef enum + { + DMHTTP_GET = 0, + DMHTTP_HEAD = 1 + } dmhttp_method_t; + /* Each handle owns one connector. open starts or continues asynchronous + * DNS/connect; retry it with identical host/port after WANT_*. close cancels + * pending DNS/connect, releases the stream and detaches all callbacks before + * returning. It is called once after every attempted open, even on failure. + * send/receive obey dmtls's nonblocking byte-count contract (OK/0 = read EOF). + * The adapter must arrange external readiness notifications; no polling thread + * is created by dmhttp. Separate handles require separate adapter user data. */ + typedef dmtls_result_t (*dmhttp_open_fn)(void *, const char *, uint16_t); + typedef void (*dmhttp_close_fn)(void *); + typedef struct + { + dmhttp_open_fn open; + dmhttp_close_fn close; + dmtls_transport_t stream; + } dmhttp_transport_t; + typedef struct + { + dmhttp_transport_t transport; + dmtls_config_t tls_config; /* Borrowed until destroy. Required for HTTPS. */ + dmhttp_method_t method; + uint32_t timeout_ms; /* Entire transaction, including DNS and redirects. */ + uint32_t max_redirects; /* 0 disables following, at most 16. */ + uint64_t max_body_bytes; /* 0 explicitly permits unlimited streamed bytes. */ + bool range_enabled; + uint64_t range_start; /* GET: Range: bytes=N-; unchanged across redirects. */ + } dmhttp_options_t; + typedef struct + { + unsigned status_code; + unsigned redirects; + bool content_length_known; + uint64_t content_length; + uint64_t body_bytes; + /* A 206 response is never treated as proof that the requested range was + * accepted: the application must validate this field before appending. */ + char content_range[128]; + char content_type[128]; + dmhttp_result_t error; + dmtls_result_t tls_error; + uint32_t verification_flags; + } dmhttp_response_t; + /* URL/options are copied. Only http/https, ASCII DNS/IPv4 hosts and escaped + * origin-form targets are accepted (no credentials or IPv6 literals). + * Requests use Connection: close and Accept-Encoding: identity. No uploads, + * authentication, decompression, pooling or proxy tunnelling are performed. + * Calls on one handle must be externally serialized, including cancel/destroy. + * Callbacks must not reenter that handle. TLS must be initialized by its owner. */ + dmod_dmhttp_api(1.0, dmhttp_result_t, _create, + (const char *url, const dmhttp_options_t *options, dmhttp_t *output)); + /* Advance to final response headers. OK means headers are available. AGAIN + * means bounded local progress; WANT_* means wait for stream readiness. + * Also schedule the remaining transaction deadline: inactivity cannot disable + * timeouts. No call waits for network events. */ + dmod_dmhttp_api(1.0, dmhttp_result_t, _step, (dmhttp_t handle)); + /* Call after step returns OK. Streams decoded body bytes into caller memory. + * May return OK with bytes and report a framing error on the next call; commit + * downloaded files only after DONE. DONE is repeated on subsequent reads. */ + dmod_dmhttp_api(1.0, dmhttp_result_t, _read, + (dmhttp_t handle, void *buffer, size_t capacity, size_t *received)); + dmod_dmhttp_api(1.0, dmhttp_result_t, _response, (dmhttp_t handle, dmhttp_response_t *output)); + dmod_dmhttp_api(1.0, dmhttp_result_t, _cancel, (dmhttp_t handle)); + dmod_dmhttp_api(1.0, void, _destroy, (dmhttp_t handle)); +#ifdef __cplusplus +} +#endif +#endif diff --git a/manifest.dmm b/manifest.dmm new file mode 100644 index 0000000..f8b8dc7 --- /dev/null +++ b/manifest.dmm @@ -0,0 +1,5 @@ +# Include dynamically generated versions list from latest release +$include https://github.com/choco-technologies/dmhttp/releases/download/vlatest/versions.dmm + +# Module entry with version placeholder - expanded by $version-available +dmhttp https://github.com/choco-technologies/dmhttp/releases/download/v/dmhttp-v-.zip diff --git a/scripts/check_dmod_pointers.py b/scripts/check_dmod_pointers.py new file mode 100644 index 0000000..6a8603a --- /dev/null +++ b/scripts/check_dmod_pointers.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Reject absolute pointers outside legacy DMOD relocation sections. + +Read-only ELF audit: it never modifies ELF/DMF or emits runtime relocation data. +The fork must use DMOD_GLOBAL_POINTER or eliminate stored absolute pointers. +""" +import struct +import sys +from pathlib import Path + + +def check(elf, dmf): + if len(elf)<64 or elf[:4]!=b'\x7fELF' or elf[5]!=1: + raise ValueError('expected little-endian ELF') + wide=elf[4]==2 + machine=struct.unpack_from('=count: + raise ValueError('invalid ELF section table') + sections=[struct.unpack_from(fmt,elf,shoff+i*stride) for i in range(count)] + ns=sections[names_index];names=elf[ns[4]:ns[4]+ns[5]] + def name(section): + start=section[0];return names[start:names.index(b'\0',start)].decode('ascii') + allowed={'.header','.inputs','.outputs','.got'} + absolute=(1,10,11) if wide else (2,5,6,8,38,43,44,47,48) + relative=(0,2,4,9,24,25,26,27,28,29,41,42) if wide else (0,3,10,24,25,26,28,29,30,42,51,96,102,103) + records=0;got=0;failures=[] + for section in sections: + if section[1]!=(4 if wide else 9):continue + target=sections[section[7]] + if not target[2]&2:continue + symbol_table=sections[section[6]] + expected=24 if wide else 8 + if section[9]!=expected:raise ValueError('invalid relocation record size') + for pos in range(section[4],section[4]+section[5],expected): + offset,info=struct.unpack_from('>32) if wide else (info&255,info>>8) + records+=1 + if kind in absolute: + symbol_index=struct.unpack_from('&2 +} + +print_success() { + echo -e "${GREEN}$1${NC}" +} + +print_help() { + cat << EOF +Sync Claude Code skills from dmod + +Usage: + ./scripts/sync-claude.sh [--dmod-dir PATH] [--ref REF] + +Options: + --dmod-dir PATH Use a local dmod checkout instead of cloning from GitHub + --ref REF Git branch/tag to clone when no --dmod-dir is given (default: develop) + --help Show this help message +EOF +} + +while [[ $# -gt 0 ]]; do + case $1 in + --dmod-dir) + DMOD_DIR="$2" + shift 2 + ;; + --ref) + GIT_REF="$2" + shift 2 + ;; + --help) + print_help + exit 0 + ;; + *) + print_error "Unknown parameter: $1" + print_help + exit 1 + ;; + esac +done + +TMP_CLONE_DIR="" + +cleanup() { + if [[ -n "${TMP_CLONE_DIR}" && -d "${TMP_CLONE_DIR}" ]]; then + rm -rf "${TMP_CLONE_DIR}" + fi +} +trap cleanup EXIT + +if [[ -n "${DMOD_DIR}" ]]; then + if [[ ! -d "${DMOD_DIR}" ]]; then + print_error "--dmod-dir does not exist: ${DMOD_DIR}" + exit 1 + fi + SOURCE_SKILLS_DIR="${DMOD_DIR}/.claude/skills" + echo "Using local dmod checkout: ${DMOD_DIR}" +else + TMP_CLONE_DIR="$(mktemp -d)" + echo "Cloning dmod (branch: ${GIT_REF}) ..." + git clone --depth 1 --branch "${GIT_REF}" "${DMOD_REPO_URL}" "${TMP_CLONE_DIR}" > /dev/null 2>&1 + SOURCE_SKILLS_DIR="${TMP_CLONE_DIR}/.claude/skills" +fi + +if [[ ! -d "${SOURCE_SKILLS_DIR}" ]]; then + print_error "No .claude/skills directory found at ${SOURCE_SKILLS_DIR}" + exit 1 +fi + +mkdir -p .claude/skills +cp -r "${SOURCE_SKILLS_DIR}/." .claude/skills/ + +echo "" +print_success "Synced Claude Code skills:" +for skill_dir in .claude/skills/*/; do + [[ -d "${skill_dir}" ]] || continue + echo " - $(basename "${skill_dir}")" +done +echo "" +echo "Review the changes with 'git status'/'git diff' and commit them like any other file." diff --git a/src/dmhttp.c b/src/dmhttp.c new file mode 100644 index 0000000..bd6252d --- /dev/null +++ b/src/dmhttp.c @@ -0,0 +1,535 @@ +#define DMOD_ENABLE_REGISTRATION ON +#include "dmhttp_internal.h" +#include +#include +static atomic_uint live_requests; +#define HTTP_MAGIC 0x48545450u +static bool valid(dmhttp_t h) +{ + return h && h->magic == HTTP_MAGIC; +} + +void http_release(dmhttp_t h) +{ + if (h->tls) + { + dmtls_connection_destroy(h->tls); + h->tls = NULL; + } + if (h->opened) + { + h->options.transport.close(h->options.transport.stream.user); + h->opened = false; + } +} +dmhttp_result_t http_fail(dmhttp_t h, dmhttp_result_t rc) +{ + h->response.error = rc; + h->state = HTTP_FAILED; + http_release(h); + return rc; +} +static dmhttp_result_t tls_result(dmhttp_t h, dmtls_result_t rc) +{ + if (rc == DMTLS_OK) + return DMHTTP_OK; + if (rc == DMTLS_WANT_READ) + return DMHTTP_WANT_READ; + if (rc == DMTLS_WANT_WRITE) + return DMHTTP_WANT_WRITE; + h->response.tls_error = rc; + if (h->tls) + { + dmtls_status_t status; + if (dmtls_status(h->tls, &status) == DMTLS_OK) + h->response.verification_flags = status.verification_flags; + } + return rc == DMTLS_ERR_TIMEOUT ? DMHTTP_ERR_TIMEOUT : DMHTTP_ERR_TLS; +} +static dmhttp_result_t check(dmhttp_t h) +{ + if (h->state == HTTP_FAILED) + return h->response.error; + if (h->state == HTTP_FINISHED) + return DMHTTP_DONE; + if ((uint32_t)(dmosi_get_tick_count() - h->started) >= h->options.timeout_ms) + return http_fail(h, DMHTTP_ERR_TIMEOUT); + return DMHTTP_OK; +} +static dmhttp_result_t receive(dmhttp_t h) +{ + if (h->input_at < h->input_size) + return DMHTTP_OK; + h->input_at = h->input_size = 0; + size_t n = 0; + dmtls_result_t tr; + if (h->tls) + tr = dmtls_read(h->tls, h->input, sizeof(h->input), &n); + else + tr = h->options.transport.stream.receive(h->options.transport.stream.user, h->input, + sizeof(h->input), &n); + if (n > sizeof(h->input) || (tr != DMTLS_OK && n)) + return DMHTTP_ERR_TRANSPORT; + if (tr == DMTLS_WANT_READ) + return DMHTTP_WANT_READ; + if (tr == DMTLS_WANT_WRITE) + return DMHTTP_WANT_WRITE; + if (h->tls && tr == DMTLS_PEER_CLOSED) + return DMHTTP_DONE; + if (tr != DMTLS_OK) + return h->tls ? tls_result(h, tr) : DMHTTP_ERR_TRANSPORT; + if (!n) + return DMHTTP_DONE; + h->input_size = n; + return DMHTTP_OK; +} +/* Consumes at most one buffered byte. Strict CRLF and bounded line storage. */ +static dmhttp_result_t line_byte(dmhttp_t h, size_t *budget, size_t limit, bool *complete) +{ + *complete = false; + dmhttp_result_t rc = receive(h); + if (rc != DMHTTP_OK) + return rc == DMHTTP_DONE ? DMHTTP_ERR_TRUNCATED : rc; + unsigned char c = h->input[h->input_at++]; + if (++*budget > limit) + return DMHTTP_ERR_LIMIT; + if (c == '\n') + { + if (!h->line_size || h->line[h->line_size - 1] != '\r') + return DMHTTP_ERR_PROTOCOL; + h->line[--h->line_size] = 0; + *complete = true; + return DMHTTP_OK; + } + if (!c || (h->line_size && h->line[h->line_size - 1] == '\r')) + return DMHTTP_ERR_PROTOCOL; + if (h->line_size == HTTP_LINE_MAX) + return DMHTTP_ERR_LIMIT; + h->line[h->line_size++] = (char)c; + return DMHTTP_AGAIN; +} +static dmhttp_result_t connect_stream(dmhttp_t h) +{ + dmhttp_result_t rc; + + h->opened = true; + dmtls_result_t tr = + h->options.transport.open(h->options.transport.stream.user, h->url.host, h->url.port); + if (tr == DMTLS_WANT_READ) + return DMHTTP_WANT_READ; + if (tr == DMTLS_WANT_WRITE) + return DMHTTP_WANT_WRITE; + if (tr != DMTLS_OK) + return http_fail(h, DMHTTP_ERR_TRANSPORT); + if (h->url.secure) + { + rc = tls_result(h, dmtls_connection_create( + h->options.tls_config, h->url.host, &h->options.transport.stream, + h->options.timeout_ms, h->options.timeout_ms, &h->tls)); + if (rc != DMHTTP_OK) + return http_fail(h, rc); + h->state = HTTP_TLS; + } + else + h->state = HTTP_SEND; + return DMHTTP_AGAIN; +} +static dmhttp_result_t send_request(dmhttp_t h) +{ + + size_t n = 0, left = h->request_size - h->request_sent; + dmtls_result_t tr; + if (h->tls) + tr = dmtls_write(h->tls, h->request + h->request_sent, left, &n); + else + tr = h->options.transport.stream.send(h->options.transport.stream.user, + (const uint8_t *)h->request + h->request_sent, left, + &n); + if (n > left || (tr != DMTLS_OK && n)) + return http_fail(h, DMHTTP_ERR_TRANSPORT); + if (tr == DMTLS_WANT_READ) + return DMHTTP_WANT_READ; + if (tr == DMTLS_WANT_WRITE) + return DMHTTP_WANT_WRITE; + if (tr != DMTLS_OK) + return http_fail(h, h->tls ? tls_result(h, tr) : DMHTTP_ERR_TRANSPORT); + if (!n) + return http_fail(h, DMHTTP_ERR_TRANSPORT); + h->request_sent += n; + if (h->request_sent == h->request_size) + h->state = HTTP_HEADERS; + return DMHTTP_AGAIN; +} +static dmhttp_result_t read_headers(dmhttp_t h) +{ + dmhttp_result_t rc; + for (unsigned work = 0; work < HTTP_IO_SIZE; ++work) + { + bool complete; + rc = line_byte(h, &h->header_bytes, DMHTTP_HEADER_MAX, &complete); + if (rc < 0) + return http_fail(h, rc); + if (rc == DMHTTP_WANT_READ || rc == DMHTTP_WANT_WRITE) + return rc; + if (!complete) + continue; + if (h->line_size) + rc = http_header_line(h, false); + else + rc = http_headers_done(h); + h->line_size = 0; + if (rc < 0) + return http_fail(h, rc); + if (h->state != HTTP_HEADERS) + return rc; + } + return DMHTTP_AGAIN; +} +/* Send close_notify without waiting for a peer alert. The connector's close + * releases the stream after these bytes have been handed to its send queue. */ +static dmhttp_result_t close_tls(dmhttp_t h) +{ + if (!h->tls) + return DMHTTP_OK; + dmhttp_result_t rc = tls_result(h, dmtls_close(h->tls)); + return rc < 0 ? http_fail(h, rc) : rc; +} +static dmhttp_result_t step(dmhttp_t h) +{ + dmhttp_result_t rc = check(h); + if (rc != DMHTTP_OK) + return rc; + if (h->state == HTTP_BODY || h->state == HTTP_CLOSING) + return DMHTTP_OK; + if (h->state == HTTP_REDIRECT) + { + rc = close_tls(h); + if (rc != DMHTTP_OK) + return rc; + rc = http_redirect_restart(h); + return rc < 0 ? http_fail(h, rc) : rc; + } + if (h->state == HTTP_CONNECT) + return connect_stream(h); + if (h->state == HTTP_TLS) + { + rc = tls_result(h, dmtls_handshake(h->tls)); + if (rc < 0) + return http_fail(h, rc); + if (rc != DMHTTP_OK) + return rc; + h->state = HTTP_SEND; + return DMHTTP_AGAIN; + } + if (h->state == HTTP_SEND) + return send_request(h); + return read_headers(h); +} +static dmhttp_result_t finish(dmhttp_t h) +{ + h->state = HTTP_CLOSING; + dmhttp_result_t rc = close_tls(h); + if (rc != DMHTTP_OK) + return rc; + h->state = HTTP_FINISHED; + http_release(h); + return DMHTTP_DONE; +} +static dmhttp_result_t chunk_extensions(const char *p) +{ + /* Validate extension grammar, including quoted strings and escapes. */ + while (*p) + { + while (*p == ' ' || *p == '\t') + ++p; + if (*p++ != ';') + return DMHTTP_ERR_PROTOCOL; + while (*p == ' ' || *p == '\t') + ++p; + size_t tokens = 0; + while (http_token((unsigned char)*p)) + { + ++p; + ++tokens; + } + if (!tokens) + return DMHTTP_ERR_PROTOCOL; + while (*p == ' ' || *p == '\t') + ++p; + if (*p == '=') + { + ++p; + while (*p == ' ' || *p == '\t') + ++p; + if (*p == '"') + { + ++p; + while (*p && *p != '"') + { + unsigned char c = *p++; + if (c == '\\') + { + if (!*p) + return DMHTTP_ERR_PROTOCOL; + c = *p++; + } + if ((c < 32 && c != '\t') || c == 127) + return DMHTTP_ERR_PROTOCOL; + } + if (*p++ != '"') + return DMHTTP_ERR_PROTOCOL; + } + else + { + tokens = 0; + while (http_token((unsigned char)*p)) + { + ++p; + ++tokens; + } + if (!tokens) + return DMHTTP_ERR_PROTOCOL; + } + } + } + return DMHTTP_OK; +} +static dmhttp_result_t chunk_size(dmhttp_t h) +{ + const char *p = h->line; + uint64_t n = 0; + size_t digits = 0; + while (*p && *p != ';' && *p != ' ' && *p != '\t') + { + unsigned char c = *p++; + unsigned v; + if (c >= '0' && c <= '9') + v = c - '0'; + else if (c >= 'a' && c <= 'f') + v = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + v = c - 'A' + 10; + else + return DMHTTP_ERR_PROTOCOL; + if (n > (UINT64_MAX - v) / 16) + return DMHTTP_ERR_LIMIT; + n = n * 16 + v; + ++digits; + } + if (!digits) + return DMHTTP_ERR_PROTOCOL; + dmhttp_result_t rc = chunk_extensions(p); + if (rc != DMHTTP_OK) + return rc; + if (h->options.max_body_bytes && n > h->options.max_body_bytes - h->response.body_bytes) + return DMHTTP_ERR_LIMIT; + h->remaining = n; + h->framing = n ? HTTP_CHUNK_DATA : HTTP_TRAILERS; + return DMHTTP_OK; +} +static dmhttp_result_t read_chunk_line(dmhttp_t h) +{ + dmhttp_result_t rc; + + bool complete; + rc = line_byte(h, &h->trailer_bytes, DMHTTP_HEADER_MAX, &complete); + if (rc < 0) + return http_fail(h, rc); + if (rc == DMHTTP_WANT_READ || rc == DMHTTP_WANT_WRITE) + return rc; + if (!complete) + return DMHTTP_AGAIN; + if (h->framing == HTTP_TRAILERS) + { + if (!h->line_size) + return finish(h); + rc = http_header_line(h, true); + } + else + rc = chunk_size(h); + h->line_size = 0; + /* Chunk framing budget is per chunk; trailer budget is cumulative. */ + if (h->framing == HTTP_CHUNK_DATA) + h->trailer_bytes = 0; + if (rc < 0) + return http_fail(h, rc); + return DMHTTP_AGAIN; +} +static dmhttp_result_t read_chunk_crlf(dmhttp_t h) +{ + dmhttp_result_t rc; + + rc = receive(h); + if (rc == DMHTTP_DONE) + rc = DMHTTP_ERR_TRUNCATED; + if (rc < 0) + return http_fail(h, rc); + if (rc != DMHTTP_OK) + return rc; + if (h->input[h->input_at++] != (h->chunk_crlf == 0 ? '\r' : '\n')) + return http_fail(h, DMHTTP_ERR_PROTOCOL); + if (++h->chunk_crlf == 2) + { + h->chunk_crlf = 0; + h->framing = HTTP_CHUNK_SIZE; + } + return DMHTTP_AGAIN; +} +static dmhttp_result_t read_payload(dmhttp_t h, void *output, size_t capacity, size_t *received) +{ + dmhttp_result_t rc; + rc = receive(h); + if (rc == DMHTTP_DONE) + return h->framing == HTTP_CLOSE ? finish(h) : http_fail(h, DMHTTP_ERR_TRUNCATED); + if (rc < 0) + return http_fail(h, rc); + if (rc != DMHTTP_OK) + return rc; + size_t n = h->input_size - h->input_at; + if (n > capacity) + n = capacity; + if (h->framing != HTTP_CLOSE && n > h->remaining) + n = (size_t)h->remaining; + if (h->response.body_bytes > UINT64_MAX - n) + return http_fail(h, DMHTTP_ERR_LIMIT); + if (h->options.max_body_bytes && n > h->options.max_body_bytes - h->response.body_bytes) + return http_fail(h, DMHTTP_ERR_LIMIT); + memcpy(output, h->input + h->input_at, n); + h->input_at += n; + h->response.body_bytes += n; + *received = n; + if (h->framing != HTTP_CLOSE) + { + h->remaining -= n; + if (!h->remaining && h->framing == HTTP_CHUNK_DATA) + h->framing = HTTP_CHUNK_CRLF; + } + return DMHTTP_OK; +} +static dmhttp_result_t read_body(dmhttp_t h, void *output, size_t capacity, size_t *received) +{ + dmhttp_result_t rc = check(h); + if (rc != DMHTTP_OK) + return rc; + if (h->state == HTTP_CLOSING) + return finish(h); + if (h->state != HTTP_BODY) + return DMHTTP_ERR_STATE; + for (unsigned work = 0; work < HTTP_IO_SIZE; ++work) + { + if (h->framing == HTTP_LENGTH && !h->remaining) + return finish(h); + if (h->framing == HTTP_CHUNK_SIZE || h->framing == HTTP_TRAILERS) + rc = read_chunk_line(h); + else if (h->framing == HTTP_CHUNK_CRLF) + rc = read_chunk_crlf(h); + else + return read_payload(h, output, capacity, received); + if (rc != DMHTTP_AGAIN) + return rc; + } + return DMHTTP_AGAIN; +} +dmod_dmhttp_api_declaration(1.0, dmhttp_result_t, _create, + (const char *url, const dmhttp_options_t *options, dmhttp_t *output)) +{ + if (!output) + return DMHTTP_ERR_ARGUMENT; + *output = NULL; + if (!options || !options->transport.open || !options->transport.close || + !options->transport.stream.send || !options->transport.stream.receive || + !options->timeout_ms || options->timeout_ms > INT32_MAX || options->max_redirects > 16 || + (options->method != DMHTTP_GET && options->method != DMHTTP_HEAD) || + (options->method == DMHTTP_HEAD && options->range_enabled)) + return DMHTTP_ERR_ARGUMENT; + dmhttp_t h = Dmod_Malloc(sizeof(*h)); + if (!h) + return DMHTTP_ERR_MEMORY; + memset(h, 0, sizeof(*h)); + h->options = *options; + h->magic = HTTP_MAGIC; + dmhttp_result_t rc = http_url_parse(url, &h->url); + if (rc == DMHTTP_OK && h->url.secure && !options->tls_config) + rc = DMHTTP_ERR_TLS; + if (rc == DMHTTP_OK) + rc = http_request_build(h); + if (rc != DMHTTP_OK) + { + Dmod_Free(h); + return rc; + } + h->started = dmosi_get_tick_count(); + h->state = HTTP_CONNECT; + atomic_fetch_add_explicit(&live_requests, 1, memory_order_relaxed); + *output = h; + return DMHTTP_OK; +} +dmod_dmhttp_api_declaration(1.0, dmhttp_result_t, _step, (dmhttp_t h)) +{ + if (!valid(h)) + return DMHTTP_ERR_ARGUMENT; + if (h->busy) + return DMHTTP_ERR_STATE; + h->busy = true; + dmhttp_result_t rc = step(h); + h->busy = false; + return rc; +} +dmod_dmhttp_api_declaration(1.0, dmhttp_result_t, _read, + (dmhttp_t h, void *buffer, size_t capacity, size_t *received)) +{ + if (!received) + return DMHTTP_ERR_ARGUMENT; + *received = 0; + if (!valid(h) || !buffer || !capacity) + return DMHTTP_ERR_ARGUMENT; + if (h->busy) + return DMHTTP_ERR_STATE; + h->busy = true; + dmhttp_result_t rc = read_body(h, buffer, capacity, received); + h->busy = false; + return rc; +} +dmod_dmhttp_api_declaration(1.0, dmhttp_result_t, _response, + (dmhttp_t h, dmhttp_response_t *output)) +{ + if (!valid(h) || !output) + return DMHTTP_ERR_ARGUMENT; + *output = h->response; + return DMHTTP_OK; +} +dmod_dmhttp_api_declaration(1.0, dmhttp_result_t, _cancel, (dmhttp_t h)) +{ + if (!valid(h)) + return DMHTTP_ERR_ARGUMENT; + if (h->busy) + return DMHTTP_ERR_STATE; + if (h->state == HTTP_FINISHED) + return DMHTTP_DONE; + if (h->state == HTTP_FAILED) + return h->response.error; + h->busy = true; + dmhttp_result_t rc = http_fail(h, DMHTTP_ERR_CANCELLED); + h->busy = false; + return rc; +} +dmod_dmhttp_api_declaration(1.0, void, _destroy, (dmhttp_t h)) +{ + if (valid(h) && !h->busy) + { + h->busy = true; + http_release(h); + h->magic = 0; + Dmod_Free(h); + atomic_fetch_sub_explicit(&live_requests, 1, memory_order_relaxed); + } +} +int dmod_init(const Dmod_Config_t *config) +{ + (void)config; + return 0; +} +int dmod_deinit(void) +{ + return atomic_load_explicit(&live_requests, memory_order_relaxed) ? -EBUSY : 0; +} diff --git a/src/dmhttp_internal.h b/src/dmhttp_internal.h new file mode 100644 index 0000000..5119fef --- /dev/null +++ b/src/dmhttp_internal.h @@ -0,0 +1,72 @@ +#ifndef DMHTTP_INTERNAL_H +#define DMHTTP_INTERNAL_H +#include "dmod.h" +#include "dmhttp.h" +#include "dmosi.h" +#include +#include +#define HTTP_IO_SIZE 1024u +#define HTTP_LINE_MAX 2048u +struct http_url +{ + char host[DMHTTP_HOST_MAX + 1]; + char target[DMHTTP_URL_MAX + 1]; + uint16_t port; + bool secure; +}; +enum http_state +{ + HTTP_CONNECT, + HTTP_TLS, + HTTP_SEND, + HTTP_HEADERS, + HTTP_BODY, + HTTP_CLOSING, + HTTP_REDIRECT, + HTTP_FINISHED, + HTTP_FAILED +}; +enum http_framing +{ + HTTP_LENGTH, + HTTP_CLOSE, + HTTP_CHUNK_SIZE, + HTTP_CHUNK_DATA, + HTTP_CHUNK_CRLF, + HTTP_TRAILERS +}; +struct dmhttp +{ + uint32_t magic; + dmhttp_options_t options; + struct http_url url; + dmhttp_response_t response; + enum http_state state; + enum http_framing framing; + dmtls_connection_t tls; + uint32_t started; + bool opened; + bool busy; + char request[DMHTTP_URL_MAX + 512]; + size_t request_size, request_sent; + char line[HTTP_LINE_MAX + 1]; + size_t line_size, header_bytes, trailer_bytes; + bool status_seen, length_seen, transfer_seen, location_seen; + unsigned informational; + char location[DMHTTP_URL_MAX + 1]; + uint64_t remaining; + unsigned chunk_crlf; + uint8_t input[HTTP_IO_SIZE]; + size_t input_at, input_size; +}; +dmhttp_result_t http_url_parse(const char *, struct http_url *); +dmhttp_result_t http_url_resolve(const struct http_url *, const char *, struct http_url *); +dmhttp_result_t http_request_build(dmhttp_t); +dmhttp_result_t http_header_line(dmhttp_t, bool trailer); +dmhttp_result_t http_headers_done(dmhttp_t); +dmhttp_result_t http_redirect_restart(dmhttp_t); +bool http_equal(const char *, const char *); +bool http_token(unsigned char); +void http_release(dmhttp_t); +dmhttp_result_t http_fail(dmhttp_t, dmhttp_result_t); +#endif diff --git a/src/dmhttp_parser.c b/src/dmhttp_parser.c new file mode 100644 index 0000000..ed81657 --- /dev/null +++ b/src/dmhttp_parser.c @@ -0,0 +1,211 @@ +#include "dmhttp_internal.h" + +bool http_equal(const char *a, const char *b) +{ + while (*a && *b) + { + unsigned char x = *a++, y = *b++; + if (x >= 'A' && x <= 'Z') + x += 32; + if (y >= 'A' && y <= 'Z') + y += 32; + if (x != y) + return false; + } + return *a == *b; +} +bool http_token(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '!' || + c == '#' || c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' || c == '+' || + c == '-' || c == '.' || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; +} +static bool decimal(const char *text, uint64_t *out) +{ + uint64_t v = 0; + if (!*text) + return false; + while (*text) + { + unsigned c = (unsigned char)*text++; + if (c < '0' || c > '9' || v > (UINT64_MAX - (c - '0')) / 10) + return false; + v = v * 10 + c - '0'; + } + *out = v; + return true; +} +static dmhttp_result_t field_copy(char *out, size_t capacity, const char *value) +{ + size_t n = strlen(value); + if (n >= capacity) + return DMHTTP_ERR_LIMIT; + memcpy(out, value, n + 1); + return DMHTTP_OK; +} +static dmhttp_result_t status_line(dmhttp_t h) +{ + char *line = h->line; + + size_t n = strlen(line); + if (n < 13 || (memcmp(line, "HTTP/1.1 ", 9) != 0 && memcmp(line, "HTTP/1.0 ", 9) != 0) || + line[9] < '1' || line[9] > '5' || line[10] < '0' || line[10] > '9' || line[11] < '0' || + line[11] > '9' || line[12] != ' ') + return DMHTTP_ERR_PROTOCOL; + for (size_t i = 13; i < n; ++i) + if (((unsigned char)line[i] < 32 && line[i] != '\t') || (unsigned char)line[i] == 127) + return DMHTTP_ERR_PROTOCOL; + h->response.status_code = (line[9] - '0') * 100 + (line[10] - '0') * 10 + line[11] - '0'; + h->status_seen = true; + return DMHTTP_OK; +} +dmhttp_result_t http_header_line(dmhttp_t h, bool trailer) +{ + char *line = h->line; + if (!trailer && !h->status_seen) + return status_line(h); + char *colon = line; + while (*colon && *colon != ':') + { + if (!http_token((unsigned char)*colon)) + return DMHTTP_ERR_PROTOCOL; + ++colon; + } + if (colon == line || *colon != ':') + return DMHTTP_ERR_PROTOCOL; + *colon++ = 0; + while (*colon == ' ' || *colon == '\t') + ++colon; + size_t n = strlen(colon); + while (n && (colon[n - 1] == ' ' || colon[n - 1] == '\t')) + colon[--n] = 0; + for (size_t i = 0; i < n; ++i) + if (((unsigned char)colon[i] < 32 && colon[i] != '\t') || (unsigned char)colon[i] == 127) + return DMHTTP_ERR_PROTOCOL; + if (trailer) + { + if (http_equal(line, "Content-Length") || http_equal(line, "Transfer-Encoding") || + http_equal(line, "Host") || http_equal(line, "Content-Encoding") || + http_equal(line, "Content-Range") || http_equal(line, "Trailer")) + return DMHTTP_ERR_PROTOCOL; + return DMHTTP_OK; + } + if (http_equal(line, "Content-Length")) + { + uint64_t value; + if (!decimal(colon, &value)) + return DMHTTP_ERR_PROTOCOL; + if (h->length_seen && value != h->response.content_length) + return DMHTTP_ERR_PROTOCOL; + h->length_seen = true; + h->response.content_length_known = true; + h->response.content_length = value; + } + else if (http_equal(line, "Transfer-Encoding")) + { + if (h->transfer_seen || !http_equal(colon, "chunked")) + return DMHTTP_ERR_PROTOCOL; + h->transfer_seen = true; + } + else if (http_equal(line, "Content-Encoding")) + { + if (!http_equal(colon, "identity")) + return DMHTTP_ERR_PROTOCOL; + } + else if (http_equal(line, "Location")) + { + if (h->location_seen) + return DMHTTP_ERR_PROTOCOL; + h->location_seen = true; + return field_copy(h->location, sizeof(h->location), colon); + } + else if (http_equal(line, "Content-Type")) + return field_copy(h->response.content_type, sizeof(h->response.content_type), colon); + else if (http_equal(line, "Content-Range")) + { + if (h->response.content_range[0]) + return DMHTTP_ERR_PROTOCOL; + return field_copy(h->response.content_range, sizeof(h->response.content_range), colon); + } + return DMHTTP_OK; +} +static dmhttp_result_t redirect_request(dmhttp_t h) +{ + + if (h->response.redirects >= h->options.max_redirects) + return DMHTTP_ERR_REDIRECT; + struct http_url *next = Dmod_Malloc(sizeof(*next)); + if (!next) + return DMHTTP_ERR_MEMORY; + dmhttp_result_t rc = http_url_resolve(&h->url, h->location, next); + if (rc == DMHTTP_OK && h->url.secure && !next->secure) + rc = DMHTTP_ERR_REDIRECT; + if (rc == DMHTTP_OK && next->secure && !h->options.tls_config) + rc = DMHTTP_ERR_TLS; + if (rc == DMHTTP_OK) + h->url = *next; + Dmod_Free(next); + if (rc != DMHTTP_OK) + return rc; + h->state = HTTP_REDIRECT; + return DMHTTP_AGAIN; +} +dmhttp_result_t http_redirect_restart(dmhttp_t h) +{ + dmhttp_result_t rc; + unsigned redirects = h->response.redirects + 1; + http_release(h); + memset(&h->response, 0, sizeof(h->response)); + h->response.redirects = redirects; + h->header_bytes = 0; + h->status_seen = h->length_seen = h->transfer_seen = h->location_seen = false; + h->location[0] = 0; + h->input_at = h->input_size = 0; + h->informational = 0; + rc = http_request_build(h); + if (rc != DMHTTP_OK) + return rc; + h->state = HTTP_CONNECT; + return DMHTTP_AGAIN; +} +dmhttp_result_t http_headers_done(dmhttp_t h) +{ + if (!h->status_seen || (h->length_seen && h->transfer_seen)) + return DMHTTP_ERR_PROTOCOL; + unsigned code = h->response.status_code; + if (code < 200) + { + if (code == 101 || h->length_seen || h->transfer_seen || ++h->informational > 8) + return DMHTTP_ERR_PROTOCOL; + unsigned redirects = h->response.redirects; + memset(&h->response, 0, sizeof(h->response)); + h->response.redirects = redirects; + h->status_seen = false; + h->location_seen = false; + h->location[0] = 0; + return DMHTTP_AGAIN; + } + if ((code == 301 || code == 302 || code == 303 || code == 307 || code == 308) && + h->location_seen && h->options.max_redirects) + return redirect_request(h); + if (code == 204 && (h->length_seen || h->transfer_seen)) + return DMHTTP_ERR_PROTOCOL; + if (h->options.method == DMHTTP_HEAD || code == 204 || code == 304) + { + h->remaining = 0; + h->framing = HTTP_LENGTH; + } + else if (h->transfer_seen) + h->framing = HTTP_CHUNK_SIZE; + else if (h->length_seen) + { + if (h->options.max_body_bytes && h->response.content_length > h->options.max_body_bytes) + return DMHTTP_ERR_LIMIT; + h->remaining = h->response.content_length; + h->framing = HTTP_LENGTH; + } + else + h->framing = HTTP_CLOSE; + h->state = HTTP_BODY; + return DMHTTP_OK; +} diff --git a/src/dmhttp_url.c b/src/dmhttp_url.c new file mode 100644 index 0000000..2caf04b --- /dev/null +++ b/src/dmhttp_url.c @@ -0,0 +1,239 @@ +#include "dmhttp_internal.h" + +static bool starts(const char *text, const char *prefix) +{ + while (*prefix) + if (*text++ != *prefix++) + return false; + return true; +} +static bool scheme(const char *text, const char *prefix) +{ + while (*prefix) + { + unsigned char c = *text++; + if (c >= 'A' && c <= 'Z') + c += 32; + if (c != (unsigned char)*prefix++) + return false; + } + return true; +} +static bool hex(unsigned char c) +{ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} +static dmhttp_result_t target_copy(const char *p, char *target) +{ + size_t n = 0; + if (*p != '/' && *p != '?' && *p != '#' && *p) + return DMHTTP_ERR_URL; + if (*p != '/') + target[n++] = '/'; + while (*p && *p != '#') + { + unsigned char c = (unsigned char)*p; + if (c <= 32 || c >= 127 || c == '\\' || c == '"' || c == '<' || c == '>' || c == '[' || + c == ']' || c == '^' || c == '`' || c == '{' || c == '|' || c == '}') + return DMHTTP_ERR_URL; + if (c == '%' && (!p[1] || !p[2] || !hex(p[1]) || !hex(p[2]))) + return DMHTTP_ERR_URL; + if (n == DMHTTP_URL_MAX) + return DMHTTP_ERR_LIMIT; + target[n++] = *p++; + } + target[n] = 0; + return DMHTTP_OK; +} +dmhttp_result_t http_url_parse(const char *text, struct http_url *out) +{ + if (!text || !out) + return DMHTTP_ERR_ARGUMENT; + size_t total = 0; + while (text[total]) + if (++total > DMHTTP_URL_MAX) + return DMHTTP_ERR_LIMIT; + memset(out, 0, sizeof(*out)); + if (scheme(text, "https://")) + { + out->secure = true; + text += 8; + out->port = 443; + } + else if (scheme(text, "http://")) + { + text += 7; + out->port = 80; + } + else + return DMHTTP_ERR_URL; + size_t n = 0; + while (*text && *text != ':' && *text != '/' && *text != '?' && *text != '#') + { + unsigned char c = (unsigned char)*text++; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '.' || c == '-')) + return DMHTTP_ERR_URL; + if (n == DMHTTP_HOST_MAX) + return DMHTTP_ERR_LIMIT; + out->host[n++] = (c >= 'A' && c <= 'Z') ? c + 32 : c; + } + if (!n) + return DMHTTP_ERR_URL; + out->host[n] = 0; + if (*text == ':') + { + unsigned port = 0; + size_t digits = 0; + ++text; + while (*text >= '0' && *text <= '9') + { + if (++digits > 5) + return DMHTTP_ERR_URL; + port = port * 10 + (*text++ - '0'); + } + if (!digits || !port || port > 65535) + return DMHTTP_ERR_URL; + out->port = (uint16_t)port; + } + return target_copy(text, out->target); +} +/* RFC 3986 remove-dot-segments on the path only; preserve duplicate slashes, + * percent encoding and query bytes. Input/output may not alias. */ +static void normalize(const char *source, char *output) +{ + size_t used = 0; + while (*source && *source != '?') + { + if (starts(source, "/./")) + { + source += 2; + continue; + } + if (strcmp(source, "/.") == 0 || starts(source, "/.?")) + { + source += 2; + output[used++] = '/'; + break; + } + if (starts(source, "/../") || strcmp(source, "/..") == 0 || starts(source, "/..?")) + { + source += 3; + if (used) + { + while (used && output[used - 1] != '/') + --used; + if (used) + --used; + } + if (!*source || *source == '?') + { + output[used++] = '/'; + break; + } + continue; + } + output[used++] = *source++; + while (*source && *source != '/' && *source != '?') + output[used++] = *source++; + } + while (*source) + output[used++] = *source++; + if (!used) + output[used++] = '/'; + output[used] = 0; +} +dmhttp_result_t http_url_resolve(const struct http_url *base, const char *ref, struct http_url *out) +{ + if (scheme(ref, "http://") || scheme(ref, "https://")) + { + dmhttp_result_t rc = http_url_parse(ref, out); + if (rc != DMHTTP_OK) + return rc; + char clean[DMHTTP_URL_MAX + 1]; + normalize(out->target, clean); + memcpy(out->target, clean, strlen(clean) + 1); + return DMHTTP_OK; + } + if (starts(ref, "//")) + { + char full[DMHTTP_URL_MAX + 1]; + int n = Dmod_SnPrintf(full, sizeof(full), "%s:%s", base->secure ? "https" : "http", ref); + if (n < 0 || (size_t)n >= sizeof(full)) + return DMHTTP_ERR_LIMIT; + dmhttp_result_t rc = http_url_parse(full, out); + if (rc != DMHTTP_OK) + return rc; + char clean[DMHTTP_URL_MAX + 1]; + normalize(out->target, clean); + memcpy(out->target, clean, strlen(clean) + 1); + return DMHTTP_OK; + } + *out = *base; + char joined[DMHTTP_URL_MAX + 1], clean[DMHTTP_URL_MAX + 1]; + if (!*ref || *ref == '#') + return DMHTTP_OK; + if (*ref == '/') + { + dmhttp_result_t rc = target_copy(ref, clean); + if (rc == DMHTTP_OK) + normalize(clean, out->target); + return rc; + } + size_t prefix = 0; + while (base->target[prefix] && base->target[prefix] != '?') + ++prefix; + if (*ref != '?') + while (prefix && base->target[prefix - 1] != '/') + --prefix; + /* A colon in the first relative segment denotes an unsupported scheme. */ + for (size_t i = 0; ref[i] && ref[i] != '/' && ref[i] != '?' && ref[i] != '#'; ++i) + if (ref[i] == ':') + return DMHTTP_ERR_URL; + size_t length = strlen(ref); + if (prefix + length > DMHTTP_URL_MAX) + return DMHTTP_ERR_LIMIT; + memcpy(joined, base->target, prefix); + memcpy(joined + prefix, ref, length + 1); + dmhttp_result_t rc = target_copy(joined, clean); + if (rc != DMHTTP_OK) + return rc; + normalize(clean, out->target); + return DMHTTP_OK; +} +dmhttp_result_t http_request_build(dmhttp_t h) +{ + char authority[DMHTTP_HOST_MAX + 8], range[64]; + range[0] = 0; + int n; + if (h->url.port == (h->url.secure ? 443 : 80)) + n = Dmod_SnPrintf(authority, sizeof(authority), "%s", h->url.host); + else + n = Dmod_SnPrintf(authority, sizeof(authority), "%s:%u", h->url.host, + (unsigned)h->url.port); + if (n < 0 || (size_t)n >= sizeof(authority)) + return DMHTTP_ERR_LIMIT; + if (h->options.range_enabled) + { + /* Avoid requiring 64-bit printf support in embedded SAL. */ + char digits[21]; + size_t at = sizeof(digits) - 1; + uint64_t v = h->options.range_start; + digits[at] = 0; + do + { + digits[--at] = '0' + v % 10; + v /= 10; + } while (v); + Dmod_SnPrintf(range, sizeof(range), "Range: bytes=%s-\r\n", digits + at); + } + n = Dmod_SnPrintf( + h->request, sizeof(h->request), + "%s %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nAccept-Encoding: identity\r\n%s\r\n", + h->options.method == DMHTTP_HEAD ? "HEAD" : "GET", h->url.target, authority, range); + if (n < 0 || (size_t)n >= sizeof(h->request)) + return DMHTTP_ERR_LIMIT; + h->request_size = (size_t)n; + h->request_sent = 0; + return DMHTTP_OK; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..055febe --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,47 @@ +set(DMOD_MODULE_NAME test_dmhttp) +set(DMOD_MODULE_VERSION ${PROJECT_VERSION}) +set(DMOD_AUTHOR_NAME "Patryk Kubiak") +set(DMOD_STACK_SIZE 16384) +set(DMOD_PRIORITY 0) + +dmod_add_test(${DMOD_MODULE_NAME} ${DMOD_MODULE_VERSION} + dmhttp_test.c +) + +target_include_directories(${DMOD_MODULE_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../include +) + +target_link_libraries(${DMOD_MODULE_NAME} dmhttp_if) +dmhttp_audit_module(${DMOD_MODULE_NAME}) +dmod_link_modules(${DMOD_MODULE_NAME} dmtls) + +# Register the test module with ctest - this replays the same two steps CI +# runs (see .github/templates/ci.yml.template): install the test module's +# dependencies with dmf-get, then execute it through dmod_loader. The test's +# exit code equals its number of failed steps (see dmod_add_test). +find_program(DMF_GET_EXECUTABLE dmf-get HINTS ${DMOD_TOOLS_BIN_DIR} PATHS ENV PATH) +find_program(DMOD_LOADER_EXECUTABLE dmod_loader HINTS ${DMOD_TOOLS_BIN_DIR} PATHS ENV PATH) + +set(test_manifest_env "") +if(DEFINED DMOD_DMM_URL AND NOT "${DMOD_DMM_URL}" STREQUAL "") + list(APPEND test_manifest_env "DMOD_MANIFEST=${DMOD_DMM_URL}") +endif() +if(DMF_GET_EXECUTABLE AND DMOD_LOADER_EXECUTABLE) + add_test( + NAME ${DMOD_MODULE_NAME} + COMMAND ${CMAKE_COMMAND} -E env DMOD_DMF_DIR=${DMOD_DMF_DIR} + ${test_manifest_env} + bash -c "${DMF_GET_EXECUTABLE} install -d ${DMOD_DMF_DIR}/${DMOD_MODULE_NAME}-local.dmd -y && ${DMOD_LOADER_EXECUTABLE} ${DMOD_DMF_DIR}/${DMOD_MODULE_NAME}.dmf" + ) +else() + message(WARNING "dmf-get/dmod_loader not found - 'ctest' will not run ${DMOD_MODULE_NAME}. Build the tools first with -DDMOD_BUILD_TOOLS=ON") +endif() + +if(NOT CMAKE_CROSSCOMPILING AND NOT DMOD_ARCH MATCHES "arm") + add_subdirectory(host) +endif() +option(DMHTTP_BUILD_BOARD_PROBE "Build real TCP hardware diagnostic" OFF) +if(DMHTTP_BUILD_BOARD_PROBE) + add_subdirectory(probe) +endif() diff --git a/tests/dmhttp_test.c b/tests/dmhttp_test.c new file mode 100644 index 0000000..af0535f --- /dev/null +++ b/tests/dmhttp_test.c @@ -0,0 +1,108 @@ +#define DMOD_ENABLE_REGISTRATION ON +#include "dmod_test.h" +#include "dmhttp.h" +#include +struct fixture +{ + size_t at; + unsigned closes; +}; +static dmtls_result_t open_stream(void *p, const char *host, uint16_t port) +{ + (void)p; + return strcmp(host, "example.test") == 0 && port == 80 ? DMTLS_OK : DMTLS_ERR_TRANSPORT; +} +static void close_stream(void *p) +{ + ((struct fixture *)p)->closes++; +} +static dmtls_result_t send_stream(void *p, const uint8_t *bytes, size_t n, size_t *sent) +{ + (void)p; + (void)bytes; + *sent = n; + return DMTLS_OK; +} +static dmtls_result_t recv_stream(void *p, uint8_t *out, size_t cap, size_t *n) +{ + struct fixture *f = p; + static const char reply[] = + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"; + *n = 0; + if (f->at == sizeof(reply) - 1) + return DMTLS_OK; + if (cap) + { + out[0] = reply[f->at++]; + *n = 1; + } + return DMTLS_OK; +} +static dmhttp_open_fn const volatile open_cb DMOD_GLOBAL_POINTER = open_stream; +static dmhttp_close_fn const volatile close_cb DMOD_GLOBAL_POINTER = close_stream; +static dmtls_send_fn const volatile send_cb DMOD_GLOBAL_POINTER = send_stream; +static dmtls_recv_fn const volatile recv_cb DMOD_GLOBAL_POINTER = recv_stream; +static void options(dmhttp_options_t *o, struct fixture *f) +{ + memset(o, 0, sizeof(*o)); + o->timeout_ms = 5000; + o->transport.open = open_cb; + o->transport.close = close_cb; + o->transport.stream.send = send_cb; + o->transport.stream.receive = recv_cb; + o->transport.stream.user = f; +} +DMOD_TEST_STEP(http_validates_urls_and_tls_requirement) +{ + struct fixture f = {0}; + dmhttp_options_t o; + options(&o, &f); + dmhttp_t h = NULL; + DMOD_TEST_EXPECT_EQ(dmhttp_create("http://user:pass@example.test/", &o, &h), DMHTTP_ERR_URL); + DMOD_TEST_EXPECT_NULL(h); + DMOD_TEST_EXPECT_EQ(dmhttp_create("https://example.test/", &o, &h), DMHTTP_ERR_TLS); + DMOD_TEST_EXPECT_NULL(h); +} +DMOD_TEST_STEP(http_decodes_fragmented_chunked_response) +{ + struct fixture f = {0}; + dmhttp_options_t o; + options(&o, &f); + dmhttp_t h = NULL; + DMOD_TEST_EXPECT_EQ(dmhttp_create("http://example.test/a", &o, &h), DMHTTP_OK); + dmhttp_result_t rc; + unsigned guard = 0; + do + { + rc = dmhttp_step(h); + } while (rc == DMHTTP_AGAIN && ++guard < 1000); + DMOD_TEST_EXPECT_EQ(rc, DMHTTP_OK); + char body[16] = {0}; + size_t size = 0, n = 0; + do + { + rc = dmhttp_read(h, body + size, sizeof(body) - 1 - size, &n); + size += n; + } while ((rc == DMHTTP_OK || rc == DMHTTP_AGAIN) && ++guard < 1000); + DMOD_TEST_EXPECT_EQ(rc, DMHTTP_DONE); + DMOD_TEST_EXPECT_EQ(size, 5); + DMOD_TEST_EXPECT_EQ(strcmp(body, "hello"), 0); + dmhttp_response_t r; + DMOD_TEST_EXPECT_EQ(dmhttp_response(h, &r), DMHTTP_OK); + DMOD_TEST_EXPECT_EQ(r.status_code, 200); + dmhttp_destroy(h); + DMOD_TEST_EXPECT_EQ(f.closes, 1); +} +DMOD_TEST_STEP(http_cancel_is_terminal_and_releases_connector) +{ + struct fixture f = {0}; + dmhttp_options_t o; + options(&o, &f); + dmhttp_t h = NULL; + DMOD_TEST_EXPECT_EQ(dmhttp_create("http://example.test/", &o, &h), DMHTTP_OK); + DMOD_TEST_EXPECT_EQ(dmhttp_step(h), DMHTTP_AGAIN); + DMOD_TEST_EXPECT_EQ(dmhttp_cancel(h), DMHTTP_ERR_CANCELLED); + DMOD_TEST_EXPECT_EQ(dmhttp_step(h), DMHTTP_ERR_CANCELLED); + dmhttp_destroy(h); + DMOD_TEST_EXPECT_EQ(f.closes, 1); +} diff --git a/tests/host/CMakeLists.txt b/tests/host/CMakeLists.txt new file mode 100644 index 0000000..1d64473 --- /dev/null +++ b/tests/host/CMakeLists.txt @@ -0,0 +1,22 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter) +find_package(Threads REQUIRED) +include(FetchContent) +FetchContent_Declare(dmhttp_tls GIT_REPOSITORY https://github.com/choco-technologies/dmtls.git GIT_TAG main GIT_SHALLOW TRUE) +FetchContent_GetProperties(dmhttp_tls) +if(NOT dmhttp_tls_POPULATED) + FetchContent_Populate(dmhttp_tls) +endif() +include(${dmhttp_tls_SOURCE_DIR}/cmake/backend.cmake) +set(tls_sources ${dmhttp_tls_SOURCE_DIR}/src/dmtls.c ${dmhttp_tls_SOURCE_DIR}/src/dmtls_platform.c + ${dmhttp_tls_SOURCE_DIR}/src/dmtls_config.c ${dmhttp_tls_SOURCE_DIR}/src/dmtls_connection.c ${dmhttp_tls_SOURCE_DIR}/src/dmtls_io.c) +set_source_files_properties(${dmhttp_tls_SOURCE_DIR}/src/dmtls.c PROPERTIES COMPILE_DEFINITIONS "dmod_init=dmtls_test_module_init;dmod_deinit=dmtls_test_module_deinit") +add_library(dmhttp_host_engine SHARED ../../src/dmhttp.c ../../src/dmhttp_url.c ../../src/dmhttp_parser.c + platform.c ${tls_sources} ${DMTLS_BACKEND_SOURCES}) +target_include_directories(dmhttp_host_engine PRIVATE stubs ../../include ../../src + ${dmhttp_tls_SOURCE_DIR}/include ${dmhttp_tls_SOURCE_DIR}/src ${dmhttp_tls_SOURCE_DIR}/config + ${DMOD_DMF_DIR}/inc/dmtls/include ${DMTLS_BACKEND_INCLUDES}) +target_compile_definitions(dmhttp_host_engine PRIVATE strstr=dmtls_backend_strstr + MBEDTLS_CONFIG_FILE="dmtls_mbedtls_config.h" TF_PSA_CRYPTO_CONFIG_FILE="dmtls_psa_config.h") +target_link_libraries(dmhttp_host_engine PRIVATE Threads::Threads) +add_test(NAME http_interoperability COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/interoperability.py $) +set_tests_properties(http_interoperability PROPERTIES TIMEOUT 180) diff --git a/tests/host/interoperability.py b/tests/host/interoperability.py new file mode 100644 index 0000000..952bc4b --- /dev/null +++ b/tests/host/interoperability.py @@ -0,0 +1,234 @@ +"""Production HTTP + dmtls engines: fragmented framing and independent peers.""" +import ctypes as C +import os +from pathlib import Path +import select +import socket +import ssl +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +lib=C.CDLL(sys.argv.pop(1)) +IO=C.CFUNCTYPE(C.c_int,C.c_void_p,C.c_void_p,C.c_size_t,C.POINTER(C.c_size_t)) +Open=C.CFUNCTYPE(C.c_int,C.c_void_p,C.c_char_p,C.c_uint16) +Close=C.CFUNCTYPE(None,C.c_void_p) +Entropy=C.CFUNCTYPE(C.c_int,C.c_void_p,C.c_void_p,C.c_size_t) +class Stream(C.Structure):_fields_=[('send',IO),('receive',IO),('user',C.c_void_p)] +class Transport(C.Structure):_fields_=[('open',Open),('close',Close),('stream',Stream)] +class Options(C.Structure): + _fields_=[('transport',Transport),('tls_config',C.c_void_p),('method',C.c_int),('timeout_ms',C.c_uint32),('max_redirects',C.c_uint32),('max_body_bytes',C.c_uint64),('range_enabled',C.c_bool),('range_start',C.c_uint64)] +class Response(C.Structure): + _fields_=[('status_code',C.c_uint),('redirects',C.c_uint),('content_length_known',C.c_bool),('content_length',C.c_uint64),('body_bytes',C.c_uint64),('content_range',C.c_char*128),('content_type',C.c_char*128),('error',C.c_int),('tls_error',C.c_int),('verification_flags',C.c_uint32)] +class Platform(C.Structure):_fields_=[('entropy',Entropy),('user',C.c_void_p),('heap_limit',C.c_size_t),('max_connections',C.c_uint32)] +for name,args in {'create':[C.c_char_p,C.POINTER(Options),C.POINTER(C.c_void_p)],'step':[C.c_void_p], + 'read':[C.c_void_p,C.c_void_p,C.c_size_t,C.POINTER(C.c_size_t)],'response':[C.c_void_p,C.POINTER(Response)],'cancel':[C.c_void_p]}.items(): + f=getattr(lib,'dmhttp_'+name);f.argtypes=args;f.restype=C.c_int +lib.dmhttp_destroy.argtypes=[C.c_void_p] +lib.dmtls_initialize.argtypes=[C.POINTER(Platform)] +lib.dmtls_config_create.argtypes=[C.c_void_p,C.c_size_t,C.POINTER(C.c_void_p)] +lib.dmtls_config_destroy.argtypes=[C.c_void_p] +lib.test_live_allocations.restype=C.c_size_t +lib.test_ticks_advance.argtypes=[C.c_uint32] +assert lib.dmtls_test_module_init(None)==0 +@Entropy +def entropy(user,out,n):C.memmove(out,os.urandom(n),n);return 0 +assert lib.dmtls_initialize(C.byref(Platform(entropy,None,4*1024*1024,4)))==0 + +class Script: + def __init__(self,*replies,fragment=1,wait=True): + self.replies=replies;self.fragment=fragment;self.wait=wait;self.calls=0;self.closes=0;self.hosts=[];self.requests=[];self.data=b'' + self.callbacks=(Open(self.open),Close(self.close),IO(self.send),IO(self.recv)) + def open(self,user,host,port): + self.hosts.append((host,port));self.data=self.replies[min(len(self.hosts)-1,len(self.replies)-1)];self.requests.append(bytearray());return 0 + def close(self,user):self.closes+=1 + def send(self,user,data,n,out): + out[0]=0;self.calls+=1 + if self.wait and self.calls%3==0:return 2 + count=min(n,13);self.requests[-1].extend(C.string_at(data,count));out[0]=count;return 0 + def recv(self,user,data,n,out): + out[0]=0;self.calls+=1 + if self.wait and self.calls%3==0:return 1 + count=min(n,len(self.data),self.fragment);C.memmove(data,self.data[:count],count);self.data=self.data[count:];out[0]=count;return 0 + def transport(self):return Transport(self.callbacks[0],self.callbacks[1],Stream(self.callbacks[2],self.callbacks[3],None)) + def wait_ready(self,rc):pass + +class Network(Script): + def __init__(self,context,reply): + super().__init__(fragment=71);self.context=context;self.reply=reply;self.errors=[];self.notifications=0 + def open(self,user,host,port): + self.hosts.append((host,port));self.requests.append(bytearray());self.socket,server=socket.socketpair();self.socket.setblocking(False) + def peer(): + try: + with server: + server.settimeout(5) + s=self.context.wrap_socket(server,server_side=True) if self.context else server + with s: + request=b'' + while b'\r\n\r\n' not in request:request+=s.recv(4096) + self.requests[-1].extend(request) + for at in range(0,len(self.reply),29):s.sendall(self.reply[at:at+29]) + if self.context: + try:s.unwrap();self.notifications+=1 + except (ssl.SSLError,TimeoutError,ConnectionError):pass + except (ssl.SSLError,ConnectionError,OSError) as e:self.errors.append(e) + self.thread=threading.Thread(target=peer,daemon=True);self.thread.start();return 0 + def close(self,user): + self.closes+=1;self.socket.close();self.thread.join(6) + def send(self,user,data,n,out): + out[0]=0 + try:out[0]=self.socket.send(C.string_at(data,min(n,97)));return 0 + except BlockingIOError:return 2 + except OSError:return -8 + def recv(self,user,data,n,out): + out[0]=0 + try:chunk=self.socket.recv(min(n,71));C.memmove(data,chunk,len(chunk));out[0]=len(chunk);return 0 + except BlockingIOError:return 1 + except OSError:return -8 + def wait_ready(self,rc):select.select([self.socket] if rc==1 else [],[self.socket] if rc==2 else [],[],.01) + +class Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name) + cls.configs=[] + for name in ('server','other'): + subprocess.run(['openssl','req','-x509','-newkey','ec','-pkeyopt','ec_paramgen_curve:P-256','-nodes','-days','2','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost','-keyout',str(cls.root/(name+'.key')),'-out',str(cls.root/(name+'.pem'))],check=True,capture_output=True) + ca=(cls.root/(name+'.pem')).read_bytes();config=C.c_void_p();assert lib.dmtls_config_create(ca,len(ca),C.byref(config))==0;cls.configs.append(config) + @classmethod + def tearDownClass(cls): + for config in cls.configs:assert lib.dmtls_config_destroy(config)==0 + assert lib.dmtls_shutdown()==0 + assert lib.dmod_deinit()==0 + assert lib.test_live_allocations()==0 + cls.temp.cleanup() + def run_http(self,adapter,url=b'http://example.test/a/b',**overrides): + options=Options();options.transport=adapter.transport();options.timeout_ms=15000;options.max_redirects=3 + for k,v in overrides.items():setattr(options,k,v) + h=C.c_void_p();rc=lib.dmhttp_create(url,C.byref(options),C.byref(h));body=bytearray();response=Response() + try: + if rc!=0:return rc,body,response + end=time.monotonic()+20 + while True: + rc=lib.dmhttp_step(h) + if rc<=0 or rc==4:break + adapter.wait_ready(rc) + if time.monotonic()>end:raise TimeoutError('headers') + if rc==0: + while True: + out=C.create_string_buffer(113);n=C.c_size_t();rc=lib.dmhttp_read(h,out,len(out),C.byref(n));body.extend(out.raw[:n.value]) + if rc<0 or rc==4:break + adapter.wait_ready(rc) + if time.monotonic()>end:raise TimeoutError('body') + lib.dmhttp_response(h,C.byref(response));return rc,body,response + finally:lib.dmhttp_destroy(h) + def test_fixed_length_and_request(self): + s=Script(b'HTTP/1.1 200 OK\r\nContent-Length: 5\r\nContent-Type: audio/mpeg\r\n\r\nhello') + rc,body,r=self.run_http(s);self.assertEqual((rc,body,r.status_code),(4,b'hello',200));self.assertEqual(r.content_type,b'audio/mpeg');self.assertEqual(s.closes,1) + self.assertIn(b'GET /a/b HTTP/1.1\r\nHost: example.test\r\n',s.requests[0]);self.assertIn(b'Accept-Encoding: identity',s.requests[0]) + def test_chunked_extensions_trailers(self): + s=Script(b'HTTP/1.1 200 OK\r\nTransfer-Encoding: cHuNkEd\r\n\r\n2;foo="a\\\"b"\r\nhe\r\n3\r\nllo\r\n0\r\nX-Checksum: yes\r\n\r\n') + rc,body,r=self.run_http(s);self.assertEqual((rc,body),(4,b'hello')) + def test_close_delimited(self):self.assertEqual(self.run_http(Script(b'HTTP/1.0 200 OK\r\n\r\nhello'))[:2],(4,b'hello')) + def test_empty_head_and_304(self): + for status,extra,method in [(200,b'Content-Length: 9000\r\n',1),(204,b'',0),(304,b'Content-Length: 12\r\n',0)]: + with self.subTest(status=status):self.assertEqual(self.run_http(Script(b'HTTP/1.1 '+str(status).encode()+b' OK\r\n'+extra+b'\r\n'),method=method)[:2],(4,b'')) + def test_informational_response(self):self.assertEqual(self.run_http(Script(b'HTTP/1.1 103 Early Hints\r\nLink: \r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok'))[:2],(4,b'ok')) + def test_http_error_status_is_available(self): + rc,body,r=self.run_http(Script(b'HTTP/1.1 404 Missing\r\nContent-Length: 2\r\n\r\nno'));self.assertEqual((rc,body,r.status_code),(4,b'no',404)) + def test_relative_redirect(self): + s=Script(b'HTTP/1.1 302 Found\r\nLocation: ../c?q=1#fragment\r\n\r\n',b'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok') + rc,body,r=self.run_http(s);self.assertEqual((rc,body,r.redirects,s.closes),(4,b'ok',1,2));self.assertIn(b'GET /c?q=1 HTTP/1.1',s.requests[1]) + def test_redirect_references(self): + for ref,target,host in [(b'/x/../y',b'/y',b'example.test'),(b'?q=2',b'/a/b?q=2',b'example.test'),(b'//other.test:90/a/../z',b'/z',b'other.test'),(b'http://other.test/a/../z',b'/z',b'other.test')]: + with self.subTest(ref=ref): + s=Script(b'HTTP/1.1 307 Temporary\r\nLocation: '+ref+b'\r\n\r\n',b'HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n');self.assertEqual(self.run_http(s)[0],4);self.assertIn(b'GET '+target+b' HTTP',s.requests[1]);self.assertEqual(s.hosts[1][0],host) + def test_redirect_limit(self):self.assertEqual(self.run_http(Script(b'HTTP/1.1 302 Loop\r\nLocation: /a\r\n\r\n'),max_redirects=2)[0],-12) + def test_informational_cannot_reset_redirect_limit(self): + s=Script(b'HTTP/1.1 103 Hint\r\n\r\nHTTP/1.1 302 Loop\r\nLocation: /a\r\n\r\n') + self.assertEqual(self.run_http(s,max_redirects=2)[0],-12) + def test_redirect_disabled(self): + rc,_,r=self.run_http(Script(b'HTTP/1.1 302 Here\r\nLocation: /a\r\nContent-Length: 0\r\n\r\n'),max_redirects=0);self.assertEqual((rc,r.status_code),(4,302)) + def test_range(self): + s=Script(b'HTTP/1.1 206 Partial\r\nContent-Length: 2\r\nContent-Range: bytes 5-6/7\r\n\r\nok');rc,_,r=self.run_http(s,range_enabled=True,range_start=5);self.assertEqual(rc,4);self.assertEqual(r.content_range,b'bytes 5-6/7');self.assertIn(b'Range: bytes=5-',s.requests[0]) + def test_large_stream(self): + data=bytes(range(256))*8192;s=Script(b'HTTP/1.1 200 OK\r\nContent-Length: '+str(len(data)).encode()+b'\r\n\r\n'+data,fragment=997,wait=False) + rc,body,r=self.run_http(s);self.assertEqual(rc,4);self.assertEqual(body,data);self.assertEqual(r.body_bytes,len(data)) + def test_framing_errors(self): + cases=[(b'Content-Length: 2\r\nContent-Length: 3',b'abc',-7),(b'Content-Length: -1',b'',-7),(b'Content-Length: 18446744073709551616',b'',-7), + (b'Transfer-Encoding: chunked\r\nContent-Length: 3',b'abc',-7),(b'Transfer-Encoding: gzip, chunked',b'',-7), + (b'Content-Encoding: gzip',b'',-7),(b'Bad : value',b'',-7),(b' folded: x',b'',-7), + (b'Content-Length: 5',b'abc',-9),(b'Transfer-Encoding: chunked',b'3\r\nab',-9), + (b'Transfer-Encoding: chunked',b'1\r\nxXX',-7),(b'Transfer-Encoding: chunked',b'0\r\nContent-Length: 0\r\n\r\n',-7), + (b'Transfer-Encoding: chunked',b'Z\r\n',-7),(b'Transfer-Encoding: chunked',b'1;=bad\r\n',-7), + (b'Transfer-Encoding: chunked',b'fffffffffffffffff\r\n',-8), (b'X: '+b'x'*2050,b'',-8), + (b'Transfer-Encoding: chunked',b'0\r\n',-9)] + for fields,body,error in cases: + with self.subTest(fields=fields[:80],body=body[:30]):self.assertEqual(self.run_http(Script(b'HTTP/1.1 200 OK\r\n'+fields+b'\r\n\r\n'+body))[0],error) + def test_status_and_line_errors(self): + for data in [b'HTTP/2 200 OK\r\n\r\n',b'HTTP/1.1 200 OK\n\n',b'HTTP/1.1 101 Switch\r\n\r\n',b'HTTP/1.1 204 None\r\nContent-Length: 1\r\n\r\nx',b'HTTP/1.1 200 OK\r\nX: a\x00b\r\n\r\n']: + with self.subTest(data=data):self.assertEqual(self.run_http(Script(data))[0],-7) + def test_body_limits(self): + for data in [b'Content-Length: 5\r\n\r\nhello',b'Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n',b'\r\nhello']: + with self.subTest(data=data):self.assertEqual(self.run_http(Script(b'HTTP/1.1 200 OK\r\n'+data),max_body_bytes=3)[0],-8) + def test_invalid_urls(self): + for url in [b'ftp://a/b',b'http://user@a/',b'http://a:0/',b'http://a:99999/',b'http://[::1]/',b'http://a/x\r\ny',b'http://a/a b',b'http://a/%xx',b'http://a/\\b']: + with self.subTest(url=url):self.assertEqual(self.run_http(Script(b''),url=url)[0],-4) + def test_pending_open_cancel_and_transport_contract(self): + class Pending(Script): + def open(self,user,host,port):return 1 + s=Pending(b'');o=Options();o.transport=s.transport();o.timeout_ms=1000;h=C.c_void_p() + self.assertEqual(lib.dmhttp_create(b'http://a/',C.byref(o),C.byref(h)),0) + self.assertEqual(lib.dmhttp_step(h),1);self.assertEqual(lib.dmhttp_cancel(h),-11) + lib.dmhttp_destroy(h);self.assertEqual(s.closes,1) + class Bad(Script): + def send(self,user,data,n,out):out[0]=1;return 2 + self.assertEqual(self.run_http(Bad(b''))[0],-5) + def test_header_budget(self): + fields=b'X-Test: '+b'x'*100+b'\r\n' + self.assertEqual(self.run_http(Script(b'HTTP/1.1 200 OK\r\n'+fields*80+b'\r\n',fragment=997,wait=False))[0],-8) + def test_case_insensitive_scheme(self): + self.assertEqual(self.run_http(Script(b'HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n'),url=b'HTTP://EXAMPLE.TEST/')[0],4) + def test_cancel_and_timeout(self): + s=Script(b'');o=Options();o.transport=s.transport();o.timeout_ms=100;h=C.c_void_p() + for timeout in (False,True): + self.assertEqual(lib.dmhttp_create(b'http://a/',C.byref(o),C.byref(h)),0);self.assertEqual(lib.dmhttp_step(h),3) + if timeout:lib.test_ticks_advance(101);self.assertEqual(lib.dmhttp_step(h),-10) + else:self.assertEqual(lib.dmhttp_cancel(h),-11);self.assertEqual(lib.dmhttp_step(h),-11) + lib.dmhttp_destroy(h) + self.assertEqual(s.closes,2) + def test_module_lifecycle_busy(self): + s=Script(b'');o=Options();o.transport=s.transport();o.timeout_ms=1000;h=C.c_void_p() + self.assertEqual(lib.dmhttp_create(b'http://a/',C.byref(o),C.byref(h)),0) + self.assertLess(lib.dmod_deinit(),0);lib.dmhttp_destroy(h);self.assertEqual(lib.dmod_deinit(),0) + def context(self,version): + ctx=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER);ctx.minimum_version=ctx.maximum_version=version;ctx.load_cert_chain(self.root/'server.pem',self.root/'server.key');return ctx + def test_http_socket(self):self.assertEqual(self.run_http(Network(None,b'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello'))[:2],(4,b'hello')) + def test_https_tls12_tls13(self): + for version in (ssl.TLSVersion.TLSv1_2,ssl.TLSVersion.TLSv1_3): + with self.subTest(version=version): + s=Network(self.context(version),b'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n') + self.assertEqual(self.run_http(s,url=b'https://localhost/',tls_config=self.configs[0])[:2],(4,b'hello'));self.assertEqual(s.notifications,len(s.hosts)) + def test_https_redirect(self): + class Redirect(Network): + def open(self,user,host,port): + self.reply=(b'HTTP/1.1 302 Found\r\nLocation: /next\r\nContent-Length: 0\r\n\r\n' if not self.hosts else b'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello') + return super().open(user,host,port) + s=Redirect(self.context(ssl.TLSVersion.TLSv1_3),b'') + self.assertEqual(self.run_http(s,url=b'https://localhost/',tls_config=self.configs[0])[:2],(4,b'hello'));self.assertEqual(s.notifications,len(s.hosts)) + def test_https_close_delimited(self): + self.assertEqual(self.run_http(Network(self.context(ssl.TLSVersion.TLSv1_3),b'HTTP/1.1 200 OK\r\n\r\nhello'),url=b'https://localhost/',tls_config=self.configs[0])[:2],(4,b'hello')) + def test_https_untrusted_and_hostname(self): + for host,config,flag in [(b'localhost',self.configs[1],1),(b'wrong.test',self.configs[0],2)]: + with self.subTest(host=host): + rc,_,r=self.run_http(Network(self.context(ssl.TLSVersion.TLSv1_3),b''),url=b'https://'+host+b'/',tls_config=config) + self.assertEqual((rc,r.tls_error),(-6,-6));self.assertTrue(r.verification_flags&flag) + def test_https_downgrade(self): + s=Network(self.context(ssl.TLSVersion.TLSv1_3),b'HTTP/1.1 302 Found\r\nLocation: http://localhost/plain\r\n\r\n') + self.assertEqual(self.run_http(s,url=b'https://localhost/',tls_config=self.configs[0])[0],-12) + def test_https_missing_tls_config(self):self.assertEqual(self.run_http(Script(b''),url=b'https://localhost/')[0],-6) + +if __name__=='__main__':unittest.main(verbosity=2) diff --git a/tests/host/platform.c b/tests/host/platform.c new file mode 100644 index 0000000..05447df --- /dev/null +++ b/tests/host/platform.c @@ -0,0 +1,113 @@ +#define _POSIX_C_SOURCE 200809L +#include "dmod.h" +#include "dmosi.h" +#include "dmutc.h" +#include +#include +#include +#include +#include +#include +static _Atomic uint32_t ticks_offset; +static _Atomic int64_t utc_offset; +static _Atomic bool utc_available = true; +static _Atomic size_t allocations; +void test_ticks_advance(uint32_t ms) +{ + ticks_offset += ms; +} +void test_utc_offset(int64_t seconds) +{ + utc_offset = seconds; +} +void test_utc_available(int available) +{ + utc_available = available != 0; +} +size_t test_live_allocations(void) +{ + return allocations; +} +void *Dmod_Malloc(size_t size) +{ + void *p = malloc(size); + if (p != NULL) + ++allocations; + return p; +} +void Dmod_Free(void *p) +{ + if (p != NULL) + --allocations; + free(p); +} +int Dmod_VSnPrintf(char *out, size_t n, const char *fmt, va_list args) +{ + return vsnprintf(out, n, fmt, args); +} +struct test_mutex +{ + pthread_mutex_t native; +}; +dmosi_mutex_t dmosi_mutex_create(bool recursive) +{ + dmosi_mutex_t p = malloc(sizeof(*p)); + if (p == NULL) + return NULL; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + if (recursive) + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + int rc = pthread_mutex_init(&p->native, &attr); + pthread_mutexattr_destroy(&attr); + if (rc != 0) + { + free(p); + return NULL; + } + return p; +} +void dmosi_mutex_destroy(dmosi_mutex_t p) +{ + pthread_mutex_destroy(&p->native); + free(p); +} +int dmosi_mutex_lock(dmosi_mutex_t p) +{ + return pthread_mutex_lock(&p->native); +} +int dmosi_mutex_unlock(dmosi_mutex_t p) +{ + return pthread_mutex_unlock(&p->native); +} +uint32_t dmosi_get_tick_count(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint32_t)((uint64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000) + ticks_offset; +} +int dmutc_get(dmutc_time_t *out) +{ + if (!utc_available) + return -ENODATA; + *out = (dmutc_time_t){time(NULL) + utc_offset, 0, 0}; + return 0; +} +int dmutc_to_calendar(const dmutc_time_t *value, dmutc_calendar_t *out) +{ + time_t t = (time_t)value->seconds; + struct tm c; + if (gmtime_r(&t, &c) == NULL) + return -ERANGE; + *out = (dmutc_calendar_t){ + c.tm_year + 1900, c.tm_mon + 1, c.tm_mday, c.tm_hour, c.tm_min, c.tm_sec, 0, 0, 0}; + return 0; +} +int Dmod_SnPrintf(char *out, size_t n, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + int rc = vsnprintf(out, n, fmt, args); + va_end(args); + return rc; +} diff --git a/tests/host/stubs/dmhttp_defs.h b/tests/host/stubs/dmhttp_defs.h new file mode 100644 index 0000000..8552c55 --- /dev/null +++ b/tests/host/stubs/dmhttp_defs.h @@ -0,0 +1 @@ +#define dmod_dmhttp_api(v, r, n, a) r dmhttp##n a diff --git a/tests/host/stubs/dmod.h b/tests/host/stubs/dmod.h new file mode 100644 index 0000000..f1ed300 --- /dev/null +++ b/tests/host/stubs/dmod.h @@ -0,0 +1,20 @@ +#ifndef TEST_DMOD_H +#define TEST_DMOD_H +#define DMOD_GLOBAL_POINTER __attribute__((section(".got"))) +#define DMOD_MOD_DEFS_H_dmutc +#define dmod_dmutc_api(v, r, n, a) r dmutc##n a +#include +#include +#include +#include +typedef struct +{ + int unused; +} Dmod_Config_t; +void *Dmod_Malloc(size_t size); +void Dmod_Free(void *p); +int Dmod_VSnPrintf(char *, size_t, const char *, va_list); +#define dmod_dmtls_api_declaration(v, r, n, a) r dmtls##n a +#endif +#define dmod_dmhttp_api_declaration(v, r, n, a) r dmhttp##n a +int Dmod_SnPrintf(char *, size_t, const char *, ...); diff --git a/tests/host/stubs/dmosi.h b/tests/host/stubs/dmosi.h new file mode 100644 index 0000000..645de01 --- /dev/null +++ b/tests/host/stubs/dmosi.h @@ -0,0 +1,11 @@ +#ifndef TEST_DMOSI_H +#define TEST_DMOSI_H +#include +#include +typedef struct test_mutex *dmosi_mutex_t; +dmosi_mutex_t dmosi_mutex_create(bool recursive); +void dmosi_mutex_destroy(dmosi_mutex_t mutex); +int dmosi_mutex_lock(dmosi_mutex_t mutex); +int dmosi_mutex_unlock(dmosi_mutex_t mutex); +uint32_t dmosi_get_tick_count(void); +#endif diff --git a/tests/host/stubs/dmtls_defs.h b/tests/host/stubs/dmtls_defs.h new file mode 100644 index 0000000..d542e6d --- /dev/null +++ b/tests/host/stubs/dmtls_defs.h @@ -0,0 +1,4 @@ +#ifndef TEST_DMTLS_DEFS_H +#define TEST_DMTLS_DEFS_H +#define dmod_dmtls_api(v, r, n, a) r dmtls##n a +#endif diff --git a/tests/host/stubs/dmutc.h b/tests/host/stubs/dmutc.h new file mode 100644 index 0000000..f0a3961 --- /dev/null +++ b/tests/host/stubs/dmutc.h @@ -0,0 +1,18 @@ +#ifndef TEST_DMUTC_H +#define TEST_DMUTC_H +#include +typedef struct +{ + int64_t seconds; + uint32_t nanosecond, reserved; +} dmutc_time_t; +typedef struct +{ + int32_t year; + uint8_t month, day, hour, minute, second, weekday; + uint16_t reserved; + uint32_t nanosecond; +} dmutc_calendar_t; +int dmutc_get(dmutc_time_t *); +int dmutc_to_calendar(const dmutc_time_t *, dmutc_calendar_t *); +#endif diff --git a/tests/host/stubs/dmutc_defs.h b/tests/host/stubs/dmutc_defs.h new file mode 100644 index 0000000..dd3befe --- /dev/null +++ b/tests/host/stubs/dmutc_defs.h @@ -0,0 +1,4 @@ +#ifndef TEST_DMUTC_DEFS_H +#define TEST_DMUTC_DEFS_H +#define dmod_dmutc_api(v, r, n, a) r dmutc##n a +#endif diff --git a/tests/probe/CMakeLists.txt b/tests/probe/CMakeLists.txt new file mode 100644 index 0000000..3a255f1 --- /dev/null +++ b/tests/probe/CMakeLists.txt @@ -0,0 +1,14 @@ +set(DMOD_MODULE_NAME httpprobe) +set(DMOD_STACK_SIZE 16384) +set(DMOD_DMR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/httpprobe.dmr) +dmod_add_executable(${DMOD_MODULE_NAME} ${PROJECT_VERSION} httpprobe.c) +target_link_libraries(${DMOD_MODULE_NAME} dmhttp_if) +dmhttp_audit_module(${DMOD_MODULE_NAME}) +dmod_link_modules(${DMOD_MODULE_NAME} dmtls dmtcp dmdns dmip dmroute dmnetif dmrng dmosi) + +set(DMOD_MODULE_NAME httpwireprobe) +set(DMOD_DMR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/httpprobe.dmr) +dmod_add_executable(${DMOD_MODULE_NAME} ${PROJECT_VERSION} wireprobe.c) +target_link_libraries(${DMOD_MODULE_NAME} dmhttp_if) +dmhttp_audit_module(${DMOD_MODULE_NAME}) +dmod_link_modules(${DMOD_MODULE_NAME} dmtls dmrng dmosi dmtty) diff --git a/tests/probe/httpprobe.c b/tests/probe/httpprobe.c new file mode 100644 index 0000000..74d8621 --- /dev/null +++ b/tests/probe/httpprobe.c @@ -0,0 +1,272 @@ +/* Real dmtcp diagnostic. IP override isolates HTTP/TLS from DNS provisioning. + * TCP callbacks wake the waiting application; no network polling loop. */ +#include "dmod.h" +#include "dmhttp.h" +#include "dmtcp.h" +#include "dmdns.h" +#include "dmrng.h" +#include "dmosi.h" +#include +#define RX_CAPACITY 32768u +struct adapter +{ + dmip_addr_t ip; + dmtcp_conn_t conn; + dmosi_mutex_t lock; + dmosi_semaphore_t ready; + uint8_t rx[RX_CAPACITY]; + size_t head, count; + bool started, connected, eof, failed; +}; +static void wake(struct adapter *a) +{ + dmosi_semaphore_post(a->ready, 1); +} +static void established(dmtcp_conn_t conn, void *user) +{ + (void)conn; + struct adapter *a = user; + dmosi_mutex_lock(a->lock); + a->connected = true; + wake(a); + dmosi_mutex_unlock(a->lock); +} +static void data(dmtcp_conn_t conn, const uint8_t *bytes, size_t n, void *user) +{ + (void)conn; + struct adapter *a = user; + dmosi_mutex_lock(a->lock); + if (!n) + a->eof = true; + else if (n > RX_CAPACITY - a->count) + a->failed = true; + else + for (size_t i = 0; i < n; ++i) + { + a->rx[(a->head + a->count) % RX_CAPACITY] = bytes[i]; + ++a->count; + } + wake(a); + dmosi_mutex_unlock(a->lock); +} +static void writable(dmtcp_conn_t conn, size_t n, void *user) +{ + (void)conn; + (void)n; + wake(user); +} +static void closed(dmtcp_conn_t conn, void *user) +{ + (void)conn; + struct adapter *a = user; + dmosi_mutex_lock(a->lock); + a->conn = NULL; + a->eof = true; + wake(a); + dmosi_mutex_unlock(a->lock); +} +static void reset(dmtcp_conn_t conn, void *user) +{ + (void)conn; + struct adapter *a = user; + dmosi_mutex_lock(a->lock); + a->conn = NULL; + a->failed = true; + wake(a); + dmosi_mutex_unlock(a->lock); +} +static void error(dmtcp_conn_t conn, int rc, void *user) +{ + (void)rc; + reset(conn, user); +} +static dmtcp_established_handler_t const volatile established_cb DMOD_GLOBAL_POINTER = established; +static dmtcp_data_handler_t const volatile data_cb DMOD_GLOBAL_POINTER = data; +static dmtcp_writable_handler_t const volatile writable_cb DMOD_GLOBAL_POINTER = writable; +static dmtcp_closed_handler_t const volatile closed_cb DMOD_GLOBAL_POINTER = closed; +static dmtcp_reset_handler_t const volatile reset_cb DMOD_GLOBAL_POINTER = reset; +static dmtcp_error_handler_t const volatile error_cb DMOD_GLOBAL_POINTER = error; +static dmtls_result_t open_stream(void *user, const char *host, uint16_t port) +{ + (void)host; + struct adapter *a = user; + dmosi_mutex_lock(a->lock); + if (!a->started) + { + a->started = true; + dmtcp_conn_callbacks_t cb = {0}; + cb.on_established = established_cb; + cb.on_data = data_cb; + cb.on_writable = writable_cb; + cb.on_closed = closed_cb; + cb.on_reset = reset_cb; + cb.on_error = error_cb; + if (dmtcp_connect(&a->ip, port, 0, &cb, a, &a->conn) != 0) + a->failed = true; + } + dmtls_result_t rc = a->failed ? DMTLS_ERR_TRANSPORT : a->connected ? DMTLS_OK : DMTLS_WANT_READ; + dmosi_mutex_unlock(a->lock); + return rc; +} +static void close_stream(void *user) +{ + struct adapter *a = user; + dmosi_mutex_lock(a->lock); + if (a->conn) + { + dmtcp_conn_set_callbacks(a->conn, NULL, NULL); + dmtcp_abort(a->conn); + a->conn = NULL; + } + a->started = a->connected = a->eof = a->failed = false; + a->head = a->count = 0; + dmosi_mutex_unlock(a->lock); +} +static dmtls_result_t send_stream(void *user, const uint8_t *bytes, size_t n, size_t *sent) +{ + struct adapter *a = user; + *sent = 0; + dmosi_mutex_lock(a->lock); + int count = a->failed || !a->conn ? -1 : dmtcp_send(a->conn, bytes, n); + dmosi_mutex_unlock(a->lock); + if (count < 0) + return DMTLS_ERR_TRANSPORT; + if (!count) + return DMTLS_WANT_WRITE; + *sent = (size_t)count; + return DMTLS_OK; +} +static dmtls_result_t recv_stream(void *user, uint8_t *bytes, size_t n, size_t *received) +{ + struct adapter *a = user; + *received = 0; + dmosi_mutex_lock(a->lock); + dmtls_result_t rc = DMTLS_OK; + if (a->failed) + rc = DMTLS_ERR_TRANSPORT; + else if (a->count) + { + if (n > a->count) + n = a->count; + for (size_t i = 0; i < n; ++i) + bytes[i] = a->rx[(a->head + i) % RX_CAPACITY]; + a->head = (a->head + n) % RX_CAPACITY; + a->count -= n; + *received = n; + } + else if (!a->eof) + rc = DMTLS_WANT_READ; + dmosi_mutex_unlock(a->lock); + return rc; +} +static int entropy(void *user, uint8_t *bytes, size_t n) +{ + (void)user; + return dmrng_fill(bytes, n); +} +static dmhttp_open_fn const volatile open_cb DMOD_GLOBAL_POINTER = open_stream; +static dmhttp_close_fn const volatile close_cb DMOD_GLOBAL_POINTER = close_stream; +static dmtls_send_fn const volatile send_cb DMOD_GLOBAL_POINTER = send_stream; +static dmtls_recv_fn const volatile recv_cb DMOD_GLOBAL_POINTER = recv_stream; +static dmtls_entropy_fn const volatile entropy_cb DMOD_GLOBAL_POINTER = entropy; +int main(int argc, char *argv[]) +{ + if (argc != 4) + { + Dmod_Printf("Usage: httpprobe URL SERVER_IPV4 CA_FILE\n"); + return 1; + } + struct adapter *a = Dmod_Malloc(sizeof(*a)); + if (!a) + return 1; + memset(a, 0, sizeof(*a)); + a->lock = dmosi_mutex_create(true); + a->ready = dmosi_semaphore_create(0, 1); + int rc = DMHTTP_ERR_ARGUMENT; + dmtls_config_t config = NULL; + dmhttp_t request = NULL; + bool initialized = false; + if (!a->lock || !a->ready || dmdns_parse_address(argv[2], &a->ip) != 0) + goto cleanup; + if (strncmp(argv[1], "https://", 8) == 0) + { + void *file = Dmod_FileOpen(argv[3], "rb"); + if (!file) + goto cleanup; + uint8_t ca[4096]; + size_t n = Dmod_FileRead(ca, 1, sizeof(ca), file); + Dmod_FileClose(file); + dmtls_platform_t platform = {0}; + platform.entropy = entropy_cb; + platform.heap_limit = 512 * 1024; + platform.max_connections = 1; + rc = dmtls_initialize(&platform); + if (rc != DMTLS_OK) + goto cleanup; + initialized = true; + rc = dmtls_config_create(ca, n, &config); + if (rc != DMTLS_OK) + goto cleanup; + } + dmhttp_options_t options = {0}; + options.transport.open = open_cb; + options.transport.close = close_cb; + options.transport.stream.send = send_cb; + options.transport.stream.receive = recv_cb; + options.transport.stream.user = a; + options.tls_config = config; + options.timeout_ms = 20000; + options.max_redirects = 3; + options.max_body_bytes = 8 * 1024 * 1024; + rc = dmhttp_create(argv[1], &options, &request); + if (rc != DMHTTP_OK) + goto cleanup; + bool headers = false; + uint32_t hash = 2166136261u, started = dmosi_get_tick_count(); + for (;;) + { + if (!headers) + { + rc = dmhttp_step(request); + if (rc == DMHTTP_OK) + { + headers = true; + continue; + } + } + else + { + uint8_t block[1024]; + size_t n = 0; + rc = dmhttp_read(request, block, sizeof(block), &n); + for (size_t i = 0; i < n; ++i) + hash = (hash ^ block[i]) * 16777619u; + } + if (rc < 0 || rc == DMHTTP_DONE) + break; + if (rc == DMHTTP_WANT_READ || rc == DMHTTP_WANT_WRITE) + { + uint32_t elapsed = dmosi_get_tick_count() - started; + if (elapsed < options.timeout_ms) + dmosi_semaphore_wait(a->ready, 1, (int32_t)(options.timeout_ms - elapsed)); + } + } + dmhttp_response_t response; + dmhttp_response(request, &response); + Dmod_Printf("HTTP_RESULT result=%d status=%u bytes=%u hash=%u redirects=%u tls=%d verify=%u\n", + rc, response.status_code, (unsigned)response.body_bytes, hash, response.redirects, + response.tls_error, response.verification_flags); +cleanup: + if (request) + dmhttp_destroy(request); + if (config) + dmtls_config_destroy(config); + if (initialized) + dmtls_shutdown(); + if (a->ready) + dmosi_semaphore_destroy(a->ready); + if (a->lock) + dmosi_mutex_destroy(a->lock); + Dmod_Free(a); + return rc == DMHTTP_DONE ? 0 : 1; +} diff --git a/tests/probe/httpprobe.dmr b/tests/probe/httpprobe.dmr new file mode 100644 index 0000000..29bc853 --- /dev/null +++ b/tests/probe/httpprobe.dmr @@ -0,0 +1,3 @@ +dmf=./${module}.dmf => ${destination}/${module}.dmf [origin=${dmf_dir}/${module}.dmf] +dmfc=./${module}.dmfc => ${destination}/${module}.dmfc [origin=${build_dir}/dmfc/${module}.dmfc] +dmd=./${module}.dmd => ${destination}/${module}.dmd [origin=${dmf_dir}/${module}.dmd] diff --git a/tests/probe/make_fixtures.py b/tests/probe/make_fixtures.py new file mode 100644 index 0000000..910c396 --- /dev/null +++ b/tests/probe/make_fixtures.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Create disposable hardware TLS fixtures. Keep the output under ignored build/.""" +import argparse +from pathlib import Path +import subprocess + +parser=argparse.ArgumentParser(description=__doc__) +parser.add_argument('directory',type=Path) +args=parser.parse_args() +root=args.directory +root.mkdir(parents=True,exist_ok=True) +if any(root.iterdir()):parser.error('output directory must be empty') + +def openssl(*args): + subprocess.run(['openssl',*map(str,args)],check=True,capture_output=True) + +for stem in ('ca','other'): + openssl('req','-x509','-newkey','ec','-pkeyopt','ec_paramgen_curve:P-256', + '-nodes','-keyout',root/(stem+'.key'),'-out',root/(stem+'.pem'), + '-days','365','-subj','/CN=dmtls disposable '+stem, + '-addext','basicConstraints=critical,CA:TRUE', + '-addext','keyUsage=critical,keyCertSign,cRLSign') +openssl('req','-new','-newkey','ec','-pkeyopt','ec_paramgen_curve:P-256', + '-nodes','-keyout',root/'server.key','-out',root/'server.csr','-subj','/CN=tls.test') +(root/'extensions.cnf').write_text('basicConstraints=critical,CA:FALSE\n' + 'keyUsage=critical,digitalSignature\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:tls.test\n') +openssl('x509','-req','-in',root/'server.csr','-CA',root/'ca.pem','-CAkey',root/'ca.key', + '-CAcreateserial','-out',root/'server.pem','-days','2','-extfile',root/'extensions.cnf') +print('Created disposable fixtures. Only ca.pem and other.pem belong on the board.') diff --git a/tests/probe/run_wire.py b/tests/probe/run_wire.py new file mode 100644 index 0000000..47e66e4 --- /dev/null +++ b/tests/probe/run_wire.py @@ -0,0 +1,119 @@ +"""STM32 HTTP + TLS/RNG/UTC engines against PC OpenSSL over UART byte queues. + +This diagnostic does not claim to exercise Ethernet/dmtcp. Use httpprobe for +that path when the PC server is reachable from the board's network. +""" +import argparse +import datetime +from pathlib import Path +import re +import select +import ssl +import sys +import time +import os + +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--boot-dir',type=Path,required=True) +p.add_argument('--fixtures',type=Path,required=True) +p.add_argument('--port',default='/dev/ttyACM0') +p.add_argument('--case',action='append',default=[]) +a=p.parse_args() +sys.path.insert(0,str(a.boot_dir/'scripts')) +from ymodem_send import Link +link=Link(a.port,921600) +completed=[] +ansi=re.compile(rb'\x1b\[[0-9;]*[A-Za-z]') + +def command(text): + marker=b'HTTPWIRE_CMD_DONE_219';link.write(text.encode()+b'\necho '+marker+b'\n');out=b'';end=time.monotonic()+110 + while time.monotonic() +#define CAPACITY 32768u +struct transport +{ + unsigned char rx[CAPACITY]; + size_t rx_pos, rx_size; + unsigned char tx[CAPACITY]; + size_t tx_size; +}; +static int entropy(void *user, uint8_t *out, size_t size) +{ + (void)user; + return dmrng_fill(out, size); +} +static dmtls_entropy_fn const volatile entropy_source DMOD_GLOBAL_POINTER = entropy; +static dmtls_result_t send_bytes(void *user, const uint8_t *in, size_t size, size_t *sent) +{ + struct transport *t = user; + *sent = 0; + size_t n = CAPACITY - t->tx_size; + if (n == 0) + return DMTLS_WANT_WRITE; + if (n > size) + n = size; + if (n > 257) + n = 257; + memcpy(t->tx + t->tx_size, in, n); + t->tx_size += n; + *sent = n; + return DMTLS_OK; +} +static dmtls_result_t receive_bytes(void *user, uint8_t *out, size_t size, size_t *received) +{ + struct transport *t = user; + *received = 0; + size_t n = t->rx_size - t->rx_pos; + if (n == 0) + return DMTLS_WANT_READ; + if (n > size) + n = size; + if (n > 113) + n = 113; + memcpy(out, t->rx + t->rx_pos, n); + t->rx_pos += n; + *received = n; + return DMTLS_OK; +} +static void flush(struct transport *t) +{ + if (t->tx_size == 0) + return; + static const char digits[] = "0123456789abcdef"; + Dmod_Printf("HTTPWIRE_OUT "); + char line[128]; + for (size_t pos = 0; pos < t->tx_size;) + { + size_t count = 0; + while (pos < t->tx_size && count < sizeof(line)) + { + line[count++] = digits[t->tx[pos] >> 4]; + line[count++] = digits[t->tx[pos] & 15]; + ++pos; + } + Dmod_FileWrite(line, 1, count, DMOD_STDOUT); + } + Dmod_Printf("\n"); + t->tx_size = 0; +} +static int unhex(int c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + return -1; +} +static bool exchange(struct transport *t) +{ + flush(t); + Dmod_Printf("HTTPWIRE_INPUT\n"); + if (t->rx_pos != 0) + { + memmove(t->rx, t->rx + t->rx_pos, t->rx_size - t->rx_pos); + t->rx_size -= t->rx_pos; + t->rx_pos = 0; + } + int ch; + do + { + ch = Dmod_Getc(); + } while (ch == '\r' || ch == '\n'); + if (ch != 'I') + return false; + int high = -1; + while ((ch = Dmod_Getc()) != '\n' && ch != '\r' && ch != DMOD_EOF) + { + if (ch == ' ') + continue; + int nibble = unhex(ch); + if (nibble < 0) + return false; + if (high < 0) + high = nibble; + else + { + if (t->rx_size == CAPACITY) + return false; + t->rx[t->rx_size++] = (unsigned char)((high << 4) | nibble); + high = -1; + } + } + return high < 0 && ch != DMOD_EOF; +} +static dmtls_result_t open_stream(void *user, const char *host, uint16_t port) +{ + struct transport *t = user; + t->rx_pos = t->rx_size = t->tx_size = 0; + Dmod_Printf("HTTPWIRE_OPEN %s %u\n", host, (unsigned)port); + return DMTLS_OK; +} +static void close_stream(void *user) +{ + (void)user; +} +static dmhttp_open_fn const volatile open_cb DMOD_GLOBAL_POINTER = open_stream; +static dmhttp_close_fn const volatile close_cb DMOD_GLOBAL_POINTER = close_stream; +static dmtls_send_fn const volatile send_cb DMOD_GLOBAL_POINTER = send_bytes; +static dmtls_recv_fn const volatile recv_cb DMOD_GLOBAL_POINTER = receive_bytes; +int main(int argc, char *argv[]) +{ + if (argc != 3) + { + Dmod_Printf("Usage: httpwireprobe URL CA_FILE\n"); + return 1; + } + struct transport *t = Dmod_Malloc(sizeof(*t)); + if (!t) + return 1; + memset(t, 0, sizeof(*t)); + void *file = Dmod_FileOpen(argv[2], "rb"); + if (!file) + { + Dmod_Free(t); + return 1; + } + unsigned char ca[4096]; + size_t size = Dmod_FileRead(ca, 1, sizeof(ca), file); + Dmod_FileClose(file); + dmtls_platform_t platform = {0}; + platform.entropy = entropy_source; + platform.heap_limit = 512 * 1024; + platform.max_connections = 1; + dmtls_config_t config = NULL; + dmhttp_t request = NULL; + int rc = dmtls_initialize(&platform); + bool initialized = rc == DMTLS_OK; + if (rc == DMTLS_OK) + rc = dmtls_config_create(ca, size, &config); + uint32_t flags = Dmod_Stdin_GetFlags(); + void *input = Dmod_LockStdio(DMOD_STDIN); + if (input == NULL || Dmod_Ioctl(input, dmtty_ioctl_cmd_claim_foreground, NULL) != 0) + rc = DMHTTP_ERR_TRANSPORT; + Dmod_UnlockStdio(DMOD_STDIN); + Dmod_Stdin_SetFlags(0); + dmhttp_options_t options = {0}; + options.transport.open = open_cb; + options.transport.close = close_cb; + options.transport.stream.send = send_cb; + options.transport.stream.receive = recv_cb; + options.transport.stream.user = t; + options.tls_config = config; + options.timeout_ms = 90000; + options.max_redirects = 3; + if (rc == DMTLS_OK) + rc = dmhttp_create(argv[1], &options, &request); + bool headers = false; + uint32_t hash = 2166136261u; + while (rc >= 0 && rc != DMHTTP_DONE) + { + if (!headers) + { + rc = dmhttp_step(request); + if (rc == DMHTTP_OK) + { + headers = true; + continue; + } + } + else + { + unsigned char block[257]; + size_t n = 0; + rc = dmhttp_read(request, block, sizeof(block), &n); + for (size_t i = 0; i < n; ++i) + hash = (hash ^ block[i]) * 16777619u; + } + if (rc == DMHTTP_WANT_READ || rc == DMHTTP_WANT_WRITE) + if (!exchange(t)) + { + rc = DMHTTP_ERR_TRANSPORT; + break; + } + } + dmhttp_response_t response = {0}; + if (request) + dmhttp_response(request, &response); + if (request) + dmhttp_destroy(request); + if (config) + dmtls_config_destroy(config); + if (initialized) + dmtls_shutdown(); + Dmod_Stdin_SetFlags(flags); + Dmod_Printf( + "HTTPWIRE_DONE result=%d status=%u bytes=%u hash=%u redirects=%u tls=%d verify=%u\n", rc, + response.status_code, (unsigned)response.body_bytes, hash, response.redirects, + response.tls_error, response.verification_flags); + Dmod_Free(t); + return rc == DMHTTP_DONE ? 0 : 1; +} From eb5db685d3219e9da18e48cd9c599dea94a9acda Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Sat, 17 Oct 2026 12:19:49 +0000 Subject: [PATCH 2/5] Validate MCU network transfers against an independent HTTPS client --- README.md | 3 +- docs/api.md | 4 ++ docs/testing.md | 44 +++++++++++++++++++-- tests/probe/CMakeLists.txt | 2 + tests/probe/httpprobe.c | 23 ++++++++++- tests/probe/run_tcp.py | 81 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 tests/probe/run_tcp.py diff --git a/README.md b/README.md index a8cfa8d..2efe469 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ version selection. The existing SDK/loader and 240-byte DMF format are unchanged paths. HTTP error statuses remain available to the application. On `DMHTTP_WANT_READ`/`DMHTTP_WANT_WRITE`, wait for the adapter's readiness event -or transaction deadline. On `DMHTTP_AGAIN`, schedule another bounded execution +or transaction deadline. Drain already-buffered adapter data before waiting for +a new edge notification. On `DMHTTP_AGAIN`, schedule another bounded execution step. The library never sleeps or polls the network. Caller backpressure is natural: when it stops reading, dmhttp stops consuming transport bytes. diff --git a/docs/api.md b/docs/api.md index 3bcf71d..9d5692a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -17,6 +17,10 @@ the stream and detach callbacks. It is called exactly once per attempted open, including a failed/pending open. A redirect can then start a new open on that same adapter. The connector supplies readiness notification to its owner. +The diagnostic applications reserve 16 KiB of stack; budget caller stack for +HTTP parsing and TLS arithmetic, rather than invoking them on a small network +callback stack. + The application schedules the absolute transaction deadline as well as stream readiness; it cannot wait forever for an absent peer event. `timeout_ms` is nonzero, at most INT32_MAX, and starts at create. It includes all redirects, diff --git a/docs/testing.md b/docs/testing.md index 16fa19b..dd1644e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -64,7 +64,43 @@ length/hash and redirects. It does not cover Ethernet/dmtcp. The UART fixture has a finite 90-second total deadline to include serial handshakes/redirects. Native socket tests and the TCP diagnostic use shorter finite limits. -Board module checks passed 3/3. Ethernet validation is pending a server address -reachable from the board; the isolated build container was unreachable from -its LAN. UART interoperability is tracked separately and does not substitute -for the missing network test. Renode is omitted as requested. +Recorded on STM32F746G-DISCO with the existing loader: + +- Actual module checks: 3/3. +- UART/OpenSSL engine scenarios: 9/9 (HTTP framing/redirects, HTTPS TLS 1.2/1.3, + HTTPS reconnect/redirect, hostname/CA rejection and downgrade rejection). +- Real Ethernet/dmtcp HTTP: three complete 200 responses from example.com. +- Real Ethernet/dmtcp HTTPS: 3/3 complete verified downloads from httpbin.org; + status, body length and FNV-1a hash match Python's independent HTTPS client. +- A 65536-byte HTTPS binary download also matched the independent reference; + one recorded full-firmware run took 50.25 seconds including the handshake. +- After network TLS: UTC 6/6, RTC 12/12 and RNG 6/6. PC UTC was restored and the + UTC library-service unit remained running. + +The TCP diagnostic uses priority 1, checks buffered readiness before waiting on +coalesced semaphore notifications, and has a finite 60-second transaction limit. +Applications must make the same level-versus-edge distinction when integrating +an event-based byte queue. TLS arithmetic consumes CPU even though I/O is +nonblocking; schedule the calling worker appropriately for the target firmware. + +A reproducible independent-reference network harness is included: + +```sh +python3 tests/probe/run_tcp.py --boot-dir /path/to/dmod-boot \ + --url https://server.example/file --server-ip SERVER_IPV4 \ + --ca-board /ca.pem --ca-local build/board-fixtures/ca.pem --regressions +``` + +For public servers, omit --ca-local to use the PC system trust store and upload +an appropriate public root CA to the board. Test fixtures/private keys remain +local. The PC's isolated container was not reachable from the board's LAN, so +network validation used public endpoints instead of claiming a local-server run. +Renode is omitted as requested. + +Long-chain latency remains a separate observed TLS limitation: some example.com +HTTPS attempts completed, while others ended in TLS truncation after a slow +handshake. Native execution and controlled OpenSSL peers passed. Track this in +[dmtls issue 3](https://github.com/choco-technologies/dmtls/issues/3); the three +passing HTTPS network repetitions above used httpbin.org. The upstream arithmetic +configuration update is [dmtls PR 2](https://github.com/choco-technologies/dmtls/pull/2). +No certificate or hostname verification is bypassed. diff --git a/tests/probe/CMakeLists.txt b/tests/probe/CMakeLists.txt index 3a255f1..9f57f1a 100644 --- a/tests/probe/CMakeLists.txt +++ b/tests/probe/CMakeLists.txt @@ -1,4 +1,6 @@ set(DMOD_MODULE_NAME httpprobe) +# Diagnostics do crypto work: keep them above the FreeRTOS idle priority. +set(DMOD_PRIORITY 1) set(DMOD_STACK_SIZE 16384) set(DMOD_DMR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/httpprobe.dmr) dmod_add_executable(${DMOD_MODULE_NAME} ${PROJECT_VERSION} httpprobe.c) diff --git a/tests/probe/httpprobe.c b/tests/probe/httpprobe.c index 74d8621..87833b2 100644 --- a/tests/probe/httpprobe.c +++ b/tests/probe/httpprobe.c @@ -16,6 +16,7 @@ struct adapter dmosi_semaphore_t ready; uint8_t rx[RX_CAPACITY]; size_t head, count; + size_t sent_bytes, received_bytes; bool started, connected, eof, failed; }; static void wake(struct adapter *a) @@ -36,6 +37,7 @@ static void data(dmtcp_conn_t conn, const uint8_t *bytes, size_t n, void *user) (void)conn; struct adapter *a = user; dmosi_mutex_lock(a->lock); + a->received_bytes += n; if (!n) a->eof = true; else if (n > RX_CAPACITY - a->count) @@ -133,6 +135,7 @@ static dmtls_result_t send_stream(void *user, const uint8_t *bytes, size_t n, si return DMTLS_ERR_TRANSPORT; if (!count) return DMTLS_WANT_WRITE; + a->sent_bytes += (size_t)count; *sent = (size_t)count; return DMTLS_OK; } @@ -169,6 +172,20 @@ static dmhttp_close_fn const volatile close_cb DMOD_GLOBAL_POINTER = close_strea static dmtls_send_fn const volatile send_cb DMOD_GLOBAL_POINTER = send_stream; static dmtls_recv_fn const volatile recv_cb DMOD_GLOBAL_POINTER = recv_stream; static dmtls_entropy_fn const volatile entropy_cb DMOD_GLOBAL_POINTER = entropy; +/* A coalesced callback is an edge, not a count of readable records. Drain + * already buffered bytes before waiting for the next notification. */ +static void wait_for_io(struct adapter *a, int result, uint32_t remaining) +{ + dmosi_mutex_lock(a->lock); + bool ready = a->failed || a->eof; + if (result == DMHTTP_WANT_READ) + ready = ready || a->count != 0; + else + ready = ready || (a->conn && dmtcp_send_space(a->conn) > 0); + dmosi_mutex_unlock(a->lock); + if (!ready && remaining) + dmosi_semaphore_wait(a->ready, 1, (int32_t)remaining); +} int main(int argc, char *argv[]) { if (argc != 4) @@ -215,7 +232,7 @@ int main(int argc, char *argv[]) options.transport.stream.receive = recv_cb; options.transport.stream.user = a; options.tls_config = config; - options.timeout_ms = 20000; + options.timeout_ms = 60000; options.max_redirects = 3; options.max_body_bytes = 8 * 1024 * 1024; rc = dmhttp_create(argv[1], &options, &request); @@ -248,7 +265,7 @@ int main(int argc, char *argv[]) { uint32_t elapsed = dmosi_get_tick_count() - started; if (elapsed < options.timeout_ms) - dmosi_semaphore_wait(a->ready, 1, (int32_t)(options.timeout_ms - elapsed)); + wait_for_io(a, rc, options.timeout_ms - elapsed); } } dmhttp_response_t response; @@ -257,6 +274,8 @@ int main(int argc, char *argv[]) rc, response.status_code, (unsigned)response.body_bytes, hash, response.redirects, response.tls_error, response.verification_flags); cleanup: + Dmod_Printf("HTTP_TCP_BYTES sent=%u received=%u queued=%u\n", (unsigned)a->sent_bytes, + (unsigned)a->received_bytes, (unsigned)a->count); if (request) dmhttp_destroy(request); if (config) diff --git a/tests/probe/run_tcp.py b/tests/probe/run_tcp.py new file mode 100644 index 0000000..bc3f723 --- /dev/null +++ b/tests/probe/run_tcp.py @@ -0,0 +1,81 @@ +"""Compare MCU Ethernet/dmtcp downloads with Python's independent HTTP client. + +The board must have a running UTC service and a public/fixture CA file. No TLS +verification is bypassed. The server must be reachable from both PC and board. +""" +import argparse +import datetime +import http.client +from pathlib import Path +import re +import shlex +import socket +import ssl +import sys +import time +from urllib.parse import urlsplit + +parser=argparse.ArgumentParser(description=__doc__) +parser.add_argument('--boot-dir',type=Path,required=True) +parser.add_argument('--url',action='append',required=True) +parser.add_argument('--server-ip',help='Override DNS for both reference and board') +parser.add_argument('--ca-board',required=True) +parser.add_argument('--ca-local',type=Path,help='Fixture CA; otherwise use the PC system trust store') +parser.add_argument('--module',default='httpprobe') +parser.add_argument('--repeat',type=int,default=1) +parser.add_argument('--regressions',action='store_true') +parser.add_argument('--port',default='/dev/ttyACM0') +args=parser.parse_args() +sys.path.insert(0,str(args.boot_dir/'scripts')) +from ymodem_send import Link +link=Link(args.port,921600) +ansi=re.compile(rb'\x1b\[[0-9;]*[A-Za-z]') + +def command(text,timeout=75): + marker=b'HTTP_TCP_TEST_END_812';link.write(text.encode()+b'\necho '+marker+b'\n');out=b'';end=time.monotonic()+timeout + while time.monotonic() Date: Sat, 17 Oct 2026 12:24:24 +0000 Subject: [PATCH 3/5] Document GOT-safe application callback registration --- docs/api.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/api.md b/docs/api.md index 9d5692a..ba99908 100644 --- a/docs/api.md +++ b/docs/api.md @@ -17,6 +17,22 @@ the stream and detach callbacks. It is called exactly once per attempted open, including a failed/pending open. A redirect can then start a new open on that same adapter. The connector supplies readiness notification to its owner. +For DMOD applications, callback addresses must use the existing GOT mechanism. +Keep each persistent function pointer in a typed `const volatile` variable +annotated with `DMOD_GLOBAL_POINTER`, then assign it into a zero-initialized +options structure at runtime. For example: + +```c +static dmhttp_open_fn const volatile open_callback DMOD_GLOBAL_POINTER = app_open; +/* Use the same pattern for close/send/receive. */ +dmhttp_options_t options = {0}; +options.transport.open = open_callback; +``` + +The complete pattern is in `tests/probe/httpprobe.c`. A mixed static aggregate +containing callbacks is not covered by the original loader's GOT relocation. + + The diagnostic applications reserve 16 KiB of stack; budget caller stack for HTTP parsing and TLS arithmetic, rather than invoking them on a small network callback stack. From 87230fdec0f6f9d7ccb6f96b33ac6c45c559b405 Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Sat, 17 Oct 2026 13:00:37 +0000 Subject: [PATCH 4/5] Restore workflows generated by new-module.sh --- .github/workflows/ci.yml | 49 +++++++++++++++++++++++------------ .github/workflows/release.yml | 46 +++++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bacc27a..729de89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,33 +1,50 @@ name: CI + on: push: - branches: [main, master, develop] + branches: [ main, master, develop ] pull_request: - branches: [main, master, develop] + branches: [ main, master, develop ] + jobs: - build: - name: Build and test (${{ matrix.tools }}) + build-and-test: + name: Build and Test runs-on: ubuntu-latest permissions: contents: read - strategy: - fail-fast: false - matrix: - tools: [arch/x86_64, arch/armv7/cortex-m4, arch/armv7/cortex-m7] container: image: chocotechnologies/dmod:1.0.4 + steps: - - uses: actions/checkout@v4 - - name: Install test and audit tools + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install audit and test dependencies run: | apt-get update apt-get install -y --no-install-recommends python3 openssl - - name: Build + + - name: Build dmhttp run: | - cmake -S . -B build -DDMOD_TOOLS_NAME=${{ matrix.tools }} - cmake --build build --parallel 4 + mkdir -p build + cd build + cmake .. -DDMOD_MODE=DMOD_MODULE + cmake --build . + + - name: Verify module files + run: | + echo "Checking for module files..." + ls -lh build/dmf/ test -f build/dmf/dmhttp.dmf + test -f build/dmf/dmhttp_version.txt test -f build/dmf/test_dmhttp.dmf - - name: Native tests - if: matrix.tools == 'arch/x86_64' - run: ctest --test-dir build --output-on-failure + echo "Module files present" + + - name: Run tests with dmod_loader + run: | + export DMOD_DMF_DIR=$(pwd)/build/dmf + dmf-get install -d ${DMOD_DMF_DIR}/test_dmhttp-local.dmd -y + dmod_loader build/dmf/test_dmhttp.dmf + + - name: Run HTTP interoperability tests + run: ctest --test-dir build --output-on-failure -R "^http_interoperability$" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9cbd1d..817193b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,14 +5,52 @@ on: types: [created] jobs: + discover-architectures: + name: Discover Architectures + runs-on: ubuntu-latest + permissions: + contents: read + container: + image: chocotechnologies/dmod:1.0.4 + outputs: + architectures: ${{ steps.list-archs.outputs.architectures }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Fetch dmod to discover architectures + run: | + # A plain `git clone` here (rather than configuring dmhttp's + # own CMakeLists.txt via FetchContent, as before) so this step only + # ever depends on dmod itself - not on dmhttp's + # dmod_link_modules() fetches of other modules, which this step has + # no business needing just to list dmod's supported architectures, + # and which previously made this discovery job (and therefore the + # whole release) fail outright whenever any of those unrelated + # external fetches hiccuped. + git clone --depth 1 https://github.com/choco-technologies/dmod.git dmod_discovery + + - name: List available architectures + id: list-archs + run: | + # create JSON array + ARCHS=$(find dmod_discovery/configs/arch -name "tools-cfg.cmake" | \ + sed "s|dmod_discovery/configs/arch/||g" | \ + sed 's|/tools-cfg.cmake||g' | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + + echo "Found architectures: $ARCHS" + echo "architectures=$ARCHS" >> $GITHUB_OUTPUT + build-release: name: Build Release for ${{ matrix.arch_name }} + needs: discover-architectures runs-on: ubuntu-latest permissions: contents: write strategy: matrix: - arch_name: [x86_64, armv7/cortex-m4, armv7/cortex-m7] + arch_name: ${{ fromJson(needs.discover-architectures.outputs.architectures) }} container: image: chocotechnologies/dmod:1.0.4 @@ -21,10 +59,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Install audit interpreter + - name: Install audit and test dependencies run: | apt-get update - apt-get install -y --no-install-recommends python3 + apt-get install -y --no-install-recommends python3 openssl - name: Extract version from tag id: get_version @@ -94,7 +132,7 @@ jobs: generate-versions-manifest: name: Generate versions.dmm - needs: [build-release] + needs: [discover-architectures, build-release] runs-on: ubuntu-latest permissions: contents: read From 2cdeb0af5aca0dd588492ed1afd4c1eab0343df6 Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Sat, 17 Oct 2026 13:17:57 +0000 Subject: [PATCH 5/5] Address review: remove pointer audit and document HTTP client usage --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- CMakeLists.txt | 2 - README.md | 4 +- cmake/pointer-audit.cmake | 11 --- docs/README.md | 153 ++++++++++++++++++++++++++++++++- docs/api-reference.md | 3 - docs/api.md | 60 +++++++++++++ docs/testing.md | 13 +-- include/dmhttp.h | 74 +++++++++++++++- scripts/check_dmod_pointers.py | 67 --------------- tests/CMakeLists.txt | 1 - tests/dmhttp_test.c | 46 ++++++++-- tests/probe/CMakeLists.txt | 2 - 14 files changed, 335 insertions(+), 105 deletions(-) delete mode 100644 cmake/pointer-audit.cmake delete mode 100644 docs/api-reference.md delete mode 100644 scripts/check_dmod_pointers.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 729de89..9fa60dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Install audit and test dependencies + - name: Install test dependencies run: | apt-get update apt-get install -y --no-install-recommends python3 openssl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 817193b..87390c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,7 +59,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Install audit and test dependencies + - name: Install test dependencies run: | apt-get update apt-get install -y --no-install-recommends python3 openssl diff --git a/CMakeLists.txt b/CMakeLists.txt index 39f8624..59224fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,6 @@ set(DMOD_DIR ${dmod_SOURCE_DIR} CACHE PATH "DMOD source directory") set(DMOD_SCRIPTS_DIR ${DMOD_DIR}/scripts CACHE PATH "DMOD scripts directory") include(${DMOD_DIR}/paths.cmake) dmod_setup_external_module() -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/pointer-audit.cmake) # Name of the module set(DMOD_MODULE_NAME dmhttp) @@ -75,7 +74,6 @@ target_include_directories(${DMOD_MODULE_NAME}_if INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include ) -dmhttp_audit_module(${DMOD_MODULE_NAME}) dmod_link_modules(${DMOD_MODULE_NAME} dmosi dmtls) enable_testing() add_subdirectory(tests) diff --git a/README.md b/README.md index 2efe469..47f8515 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ cmake --build build/native --parallel ctest --test-dir build/native --output-on-failure ``` -A local SDK can be selected with `-DDMOD_DIR=/path/to/dmod`. Supported build -profiles are `arch/x86_64`, `arch/armv7/cortex-m4` and `arch/armv7/cortex-m7`; +A local SDK can be selected with `-DDMOD_DIR=/path/to/dmod`. Locally validated build +profiles include `arch/x86_64`, `arch/armv7/cortex-m4` and `arch/armv7/cortex-m7`; select with `-DDMOD_TOOLS_NAME=...`. Module dependencies use the packager's version selection. The existing SDK/loader and 240-byte DMF format are unchanged. diff --git a/cmake/pointer-audit.cmake b/cmake/pointer-audit.cmake deleted file mode 100644 index 51ce5b7..0000000 --- a/cmake/pointer-audit.cmake +++ /dev/null @@ -1,11 +0,0 @@ -find_package(Python3 REQUIRED COMPONENTS Interpreter) -function(dmhttp_audit_module target) - # Metadata is checked at build time and excluded from the binary by objcopy. - target_link_options(${target} PRIVATE -Wl,--emit-relocs,--no-relax) - set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS - ${PROJECT_SOURCE_DIR}/scripts/check_dmod_pointers.py) - add_custom_command(TARGET ${target} POST_BUILD - COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/check_dmod_pointers.py - $ ${DMOD_DMF_DIR}/${target}.dmf - COMMENT "Checking compatibility with the existing DMOD GOT loader" VERBATIM) -endfunction() diff --git a/docs/README.md b/docs/README.md index d25aa0d..9a5c3e2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,151 @@ -# dmhttp documentation +# dmhttp — HTTP and HTTPS download library -- [API and contracts](api.md) -- [Build and validation](testing.md) +## Purpose + +dmhttp is a DMOD library for fetching resources with HTTP/1.1 GET and HEAD. +It streams response bytes into application buffers without keeping the whole +file in RAM. HTTPS uses dmtls with certificate and hostname verification. +There is no command-line download program in this module. + +The application provides a nonblocking network connector and schedules request +progress when the connector becomes ready. dmhttp handles request formatting, +response headers, chunk decoding, redirects, deadlines and connection cleanup. +It creates no thread and performs no network polling or sleeps. + +## Requirements + +Link your module with `dmod_link_modules(your_module dmhttp)`, then include +`dmhttp.h`. Install the module and documentation using the normal package tools: + +```sh +dmf-get install dmhttp +dmf-get docs dmhttp +dmf-man dmhttp +dmf-man dmhttp api +dmf-man dmhttp testing +``` + +Runtime dependencies are dmosi and dmtls. The application must implement the +connector's DNS/connect, send, receive and close callbacks. Each concurrent +request requires its own connector state. The diagnostic TCP adapter shipped +with the source is an integration example, not a DNS service supplied by dmhttp. + +For HTTPS, initialize dmtls with a strong entropy source and a working dmutc UTC +source. Create a dmtls configuration containing the appropriate trusted CA +certificates and pass it as `options.tls_config`. Keep the configuration alive +until all requests borrowing it have been destroyed. Plain HTTP does not require +TLS initialization; using an HTTPS URL without a TLS configuration fails. + +## Configure a request + +Zero-initialize `dmhttp_options_t`, then assign the connector callbacks and its +`transport.stream.user` state. Persistent callback pointers in DMOD applications +use the SDK `DMOD_GLOBAL_POINTER` macro; see `dmf-man dmhttp api` for the pattern. +Set these options before calling `dmhttp_create`: + +| Option | Meaning | +|--------|---------| +| method | DMHTTP_GET (zero/default) or DMHTTP_HEAD | +| timeout_ms | Required total deadline, 1 through INT32_MAX milliseconds | +| max_redirects | 0 disables following; 1 through 16 limits followed redirects | +| max_body_bytes | Maximum decoded body size; 0 permits unlimited streaming | +| tls_config | Borrowed dmtls trust configuration, required for HTTPS | +| range_enabled | Send a byte-range request; GET only | +| range_start | First requested byte when range_enabled is true | + +The URL and options are copied. Connector state and the TLS configuration remain +owned by the application. The deadline starts at create and includes DNS, +connection establishment, TLS, redirects, transfer and time paused by the caller. + +## Request lifecycle + +1. Call `dmhttp_create(url, &options, &request)`. Continue only on DMHTTP_OK. +2. Call `dmhttp_step(request)` until it returns DMHTTP_OK. Final response headers + are now available. Handle readiness and scheduling results as described below. +3. Call `dmhttp_response(request, &response)` and check response.status_code. + Decide whether to accept this status before writing to the destination file. +4. Call `dmhttp_read(request, buffer, capacity, &received)` to stream the decoded + body. The buffer and capacity must be valid and nonzero. Consume exactly + received bytes, which may contain binary data and are not NUL-terminated. +5. Complete the download only on DMHTTP_DONE. A later framing error can follow + earlier body bytes, so write to a temporary file until the transfer completes. +6. Call `dmhttp_destroy(request)` on success, failure and cancellation paths. + +Use `dmhttp_cancel(request)` to stop a pending request. It releases the connector +and returns DMHTTP_ERR_CANCELLED. Destroy the handle afterwards. Serialize calls +on a handle and do not call back into it from a transport callback. + +## Operation results and HTTP status + +The result returned by a library operation describes local progress: + +| Result | Caller action | +|--------|---------------| +| DMHTTP_OK | Operation made progress; step has headers, read may have bytes | +| DMHTTP_WANT_READ | Resume when the connector is readable or the deadline fires | +| DMHTTP_WANT_WRITE | Resume when the connector is writable or the deadline fires | +| DMHTTP_AGAIN | Schedule another bounded call without waiting for a network event | +| DMHTTP_DONE | The body and normal connection cleanup are complete | +| DMHTTP_ERR_* | Transfer failed; inspect response diagnostics and destroy | + +Always schedule the transaction deadline as well as readiness notifications. +Check data already buffered by the adapter before waiting for another event. +A lost or coalesced edge notification must not leave a readable request asleep. +TLS cryptography still consumes CPU during calls; use an appropriate worker +priority and stack budget rather than executing it on a network callback stack. + +HTTP response codes are standard numbers in `response.status_code`, with named +constants in `dmhttp_status_t`. For example, DMHTTP_STATUS_OK is 200, +DMHTTP_STATUS_PARTIAL_CONTENT is 206 and DMHTTP_STATUS_NOT_FOUND is 404. +A complete 404 response can finish with DMHTTP_DONE: the exchange succeeded and +the server reported that the resource was missing. A negative DMHTTP_ERR_TLS or +DMHTTP_ERR_TRANSPORT instead means the exchange failed locally. + +```c +dmhttp_response_t response; +if (dmhttp_response(request, &response) == DMHTTP_OK) { + if (response.status_code == DMHTTP_STATUS_NOT_FOUND) { + /* Report the missing resource; cancel or read its error body. */ + } else if (response.status_code == DMHTTP_STATUS_OK) { + /* Read the body into a new temporary destination file. */ + } +} +``` + +Inspect the status after step returns DMHTTP_OK; during connection/redirect +progress it may be zero or refer to an intermediate response. The numeric field +also preserves extension status codes without requiring a new library version. + +## Resuming downloads and redirects + +To request bytes from offset N, set range_enabled and range_start=N. Append only +after confirming status 206 and a Content-Range matching the desired offset and +resource. A 200 response requires starting a new file. A 416 response is a server +range error, not proof that an existing local file is complete. + +Redirects are followed only up to max_redirects. Each hop opens a fresh +connection; HTTPS repeats hostname and certificate validation. HTTPS-to-HTTP +redirects are rejected. With following disabled, inspect the 3xx status normally. + +## Limits and troubleshooting + +URLs support HTTP/HTTPS, ASCII DNS or IPv4 hosts, explicit ports and escaped +paths/queries. IPv6 literals and URL credentials are unsupported. Requests use +Connection: close and Accept-Encoding: identity. There are no uploads, cookies, +authentication, custom request headers, gzip decoding, proxy tunnelling or HTTP/2. + +Header storage is bounded to 8192 bytes per response hop and 2048 bytes per line; +trailers have a separate 8192-byte budget. Body framing supports Content-Length, +chunked and connection-close delimiters. Premature EOF, conflicting lengths and +unsupported encodings fail. HTTPS close-delimited bodies require close_notify. + +On failure, query dmhttp_response before destroy: error holds the local failure; +tls_error and verification_flags give TLS details. Check the CA store, hostname, +UTC and entropy source for certificate/TLS failures. DMHTTP_ERR_TIMEOUT covers +the entire transaction; DMHTTP_ERR_LIMIT indicates a configured or parser bound. +Use a transport adapter that reports overflow rather than dropping payload bytes. + +Current STM32 validation found slow TLS handshakes with long certificate chains +and intermittent truncation against some public servers. This is tracked in +dmtls issue 3. See `dmf-man dmhttp testing` for measured coverage and limitations, +and `dmf-man dmhttp api` for callback contracts and error definitions. diff --git a/docs/api-reference.md b/docs/api-reference.md deleted file mode 100644 index 86ca4be..0000000 --- a/docs/api-reference.md +++ /dev/null @@ -1,3 +0,0 @@ -# API reference - -See [API and ownership contracts](api.md) and the public `include/dmhttp.h`. diff --git a/docs/api.md b/docs/api.md index ba99908..1d672cd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -86,3 +86,63 @@ An adapter must never silently discard acknowledged bytes when its queue fills. Protocol references: [HTTP semantics](https://www.rfc-editor.org/rfc/rfc9110), [HTTP/1.1 framing](https://www.rfc-editor.org/rfc/rfc9112), [URI resolution](https://www.rfc-editor.org/rfc/rfc3986). + +## Public functions + +| Function | Contract | +|----------|----------| +| dmhttp_create(url, options, output) | Copies URL/options; returns OK with a new handle, otherwise sets output to NULL | +| dmhttp_step(handle) | Advances connection, redirects and final headers; OK means inspect response and start reading | +| dmhttp_read(handle, buffer, capacity, received) | Streams body after headers; requires capacity greater than zero; DONE means completion | +| dmhttp_response(handle, output) | Copies the current metadata and diagnostics; it does not advance the request | +| dmhttp_cancel(handle) | Aborts pending work; returns ERR_CANCELLED, or preserves an existing terminal result | +| dmhttp_destroy(handle) | Releases the handle and any pending connection; accepts NULL | + +Destroy every successfully created handle. Module unload is BUSY while handles +remain alive. Both TLS configuration and adapter user data must outlive the +handle. For HTTPS initialization and trust configuration, see `dmf-man dmtls`. + +## Local results versus HTTP statuses + +`dmhttp_result_t` is the local operation result, not an HTTP status code. +OK=0, WANT_READ=1, WANT_WRITE=2, AGAIN=3 and DONE=4 describe request progress. +Check WANT on both step and read. AGAIN needs another scheduled call; WANT +needs connector readiness or a deadline wakeup. DONE is a complete transfer, +including normal connection shutdown, and is repeated by subsequent reads. + +`dmhttp_status_t` supplies named HTTP codes with standard wire values, including +STATUS_OK=200, STATUS_PARTIAL_CONTENT=206 and STATUS_NOT_FOUND=404 (all prefixed +DMHTTP_). The authoritative registry is +https://www.iana.org/assignments/http-status-codes/ . Reserved, obsolete and +temporary assignments are not named constants. Naming a status does not add +support for its feature: e.g. protocol upgrades still fail. The unsigned +response.status_code preserves extension values and remains 0 until a status +line is received. Inspect it after step returns OK for the final response. + +HTTP 4xx and 5xx responses remain readable, including their error bodies. +For example, receiving all of a 404 response returns DONE with status_code=404 +and error=OK. A malformed or truncated 404 body instead fails with the appropriate +local error while retaining the HTTP status. Do not interpret DONE as HTTP 200. + +## Local errors + +All names below have the DMHTTP_ERR_ prefix. + +| Name | Value | Meaning | +|------|-------|---------| +| ARGUMENT | -1 | Missing required argument, invalid method/option, zero read capacity | +| MEMORY | -2 | Allocation failed | +| STATE | -3 | Wrong request phase or reentrant operation | +| URL | -4 | Invalid URL or unsupported URL form | +| TRANSPORT | -5 | Connector failure or violated callback byte-count contract | +| TLS | -6 | TLS setup, handshake, verification or record failure; inspect tls_error | +| PROTOCOL | -7 | Invalid HTTP syntax, unsupported framing or encoding | +| LIMIT | -8 | Header/line/metadata/body bound or numeric overflow | +| TRUNCATED | -9 | HTTP body/framing ended prematurely | +| TIMEOUT | -10 | Total transaction deadline or TLS timeout elapsed | +| CANCELLED | -11 | Request was cancelled | +| REDIRECT | -12 | Redirect limit or HTTPS downgrade rejection | + +A lower-layer TLS truncation is reported as ERR_TLS with the corresponding +DMTLS_ERR_TRUNCATED in response.tls_error. The response also exposes stable +DMTLS_VERIFY_* flags for trust, hostname and certificate-time failures. diff --git a/docs/testing.md b/docs/testing.md index dd1644e..c3705d5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,9 +1,9 @@ # Build and validation Build profiles: x86-64, Cortex-M4 and Cortex-M7, with the existing SDK/loader. -A read-only final-ELF audit requires absolute pointers to use the existing -header/API/GOT sections; it does not rewrite binaries or add runtime fixups. -The normal DMF header remains 240 bytes. +No additional linker flags, binary rewriting or runtime fixups are required. +The library stores callbacks in request instances; diagnostic applications use +the SDK DMOD_GLOBAL_POINTER macro for their persistent callback addresses. ## Native @@ -13,8 +13,9 @@ cmake --build build/native --parallel ctest --test-dir build/native --output-on-failure ``` -`test_dmhttp` runs the actual module through dmod_loader (3 checks): URL/TLS -validation, fragmented chunk decoding, and cancellation/connector ownership. +`test_dmhttp` runs the actual module through dmod_loader (4 checks): URL/TLS +validation, fragmented chunk decoding, HTTP 404 status/body handling, and +cancellation/connector ownership. `http_interoperability` compiles production HTTP and real dmtls/crypto sources with host SAL/OSI/time providers. Independent OpenSSL peers exercise HTTP, @@ -66,7 +67,7 @@ Native socket tests and the TCP diagnostic use shorter finite limits. Recorded on STM32F746G-DISCO with the existing loader: -- Actual module checks: 3/3. +- Actual module checks before the review additions: 3/3. - UART/OpenSSL engine scenarios: 9/9 (HTTP framing/redirects, HTTPS TLS 1.2/1.3, HTTPS reconnect/redirect, hostname/CA rejection and downgrade rejection). - Real Ethernet/dmtcp HTTP: three complete 200 responses from example.com. diff --git a/include/dmhttp.h b/include/dmhttp.h index 16d02c9..6d2586f 100644 --- a/include/dmhttp.h +++ b/include/dmhttp.h @@ -14,6 +14,8 @@ extern "C" #define DMHTTP_HOST_MAX 253u typedef struct dmhttp *dmhttp_t; + /* Local operation results, not HTTP response status codes. + * Read response.status_code for the server response (e.g. 404). */ typedef enum { DMHTTP_OK = 0, @@ -34,6 +36,72 @@ extern "C" DMHTTP_ERR_CANCELLED = -11, DMHTTP_ERR_REDIRECT = -12 } dmhttp_result_t; + /* HTTP response statuses use their standard wire values. This lists the + * permanent assigned statuses; status_code also preserves extension codes. + * https://www.iana.org/assignments/http-status-codes/ */ + typedef enum + { + DMHTTP_STATUS_CONTINUE = 100, + DMHTTP_STATUS_SWITCHING_PROTOCOLS = 101, + DMHTTP_STATUS_PROCESSING = 102, + DMHTTP_STATUS_EARLY_HINTS = 103, + DMHTTP_STATUS_OK = 200, + DMHTTP_STATUS_CREATED = 201, + DMHTTP_STATUS_ACCEPTED = 202, + DMHTTP_STATUS_NON_AUTHORITATIVE_INFORMATION = 203, + DMHTTP_STATUS_NO_CONTENT = 204, + DMHTTP_STATUS_RESET_CONTENT = 205, + DMHTTP_STATUS_PARTIAL_CONTENT = 206, + DMHTTP_STATUS_MULTI_STATUS = 207, + DMHTTP_STATUS_ALREADY_REPORTED = 208, + DMHTTP_STATUS_IM_USED = 226, + DMHTTP_STATUS_MULTIPLE_CHOICES = 300, + DMHTTP_STATUS_MOVED_PERMANENTLY = 301, + DMHTTP_STATUS_FOUND = 302, + DMHTTP_STATUS_SEE_OTHER = 303, + DMHTTP_STATUS_NOT_MODIFIED = 304, + DMHTTP_STATUS_USE_PROXY = 305, + DMHTTP_STATUS_TEMPORARY_REDIRECT = 307, + DMHTTP_STATUS_PERMANENT_REDIRECT = 308, + DMHTTP_STATUS_BAD_REQUEST = 400, + DMHTTP_STATUS_UNAUTHORIZED = 401, + DMHTTP_STATUS_PAYMENT_REQUIRED = 402, + DMHTTP_STATUS_FORBIDDEN = 403, + DMHTTP_STATUS_NOT_FOUND = 404, + DMHTTP_STATUS_METHOD_NOT_ALLOWED = 405, + DMHTTP_STATUS_NOT_ACCEPTABLE = 406, + DMHTTP_STATUS_PROXY_AUTHENTICATION_REQUIRED = 407, + DMHTTP_STATUS_REQUEST_TIMEOUT = 408, + DMHTTP_STATUS_CONFLICT = 409, + DMHTTP_STATUS_GONE = 410, + DMHTTP_STATUS_LENGTH_REQUIRED = 411, + DMHTTP_STATUS_PRECONDITION_FAILED = 412, + DMHTTP_STATUS_CONTENT_TOO_LARGE = 413, + DMHTTP_STATUS_URI_TOO_LONG = 414, + DMHTTP_STATUS_UNSUPPORTED_MEDIA_TYPE = 415, + DMHTTP_STATUS_RANGE_NOT_SATISFIABLE = 416, + DMHTTP_STATUS_EXPECTATION_FAILED = 417, + DMHTTP_STATUS_MISDIRECTED_REQUEST = 421, + DMHTTP_STATUS_UNPROCESSABLE_CONTENT = 422, + DMHTTP_STATUS_LOCKED = 423, + DMHTTP_STATUS_FAILED_DEPENDENCY = 424, + DMHTTP_STATUS_TOO_EARLY = 425, + DMHTTP_STATUS_UPGRADE_REQUIRED = 426, + DMHTTP_STATUS_PRECONDITION_REQUIRED = 428, + DMHTTP_STATUS_TOO_MANY_REQUESTS = 429, + DMHTTP_STATUS_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, + DMHTTP_STATUS_UNAVAILABLE_FOR_LEGAL_REASONS = 451, + DMHTTP_STATUS_INTERNAL_SERVER_ERROR = 500, + DMHTTP_STATUS_NOT_IMPLEMENTED = 501, + DMHTTP_STATUS_BAD_GATEWAY = 502, + DMHTTP_STATUS_SERVICE_UNAVAILABLE = 503, + DMHTTP_STATUS_GATEWAY_TIMEOUT = 504, + DMHTTP_STATUS_HTTP_VERSION_NOT_SUPPORTED = 505, + DMHTTP_STATUS_VARIANT_ALSO_NEGOTIATES = 506, + DMHTTP_STATUS_INSUFFICIENT_STORAGE = 507, + DMHTTP_STATUS_LOOP_DETECTED = 508, + DMHTTP_STATUS_NETWORK_AUTHENTICATION_REQUIRED = 511 + } dmhttp_status_t; typedef enum { DMHTTP_GET = 0, @@ -67,6 +135,9 @@ extern "C" } dmhttp_options_t; typedef struct { + /* Standard HTTP wire value; 0 until a status line is received. + * Inspect after step returns OK for the final response headers. + * HTTP 4xx/5xx do not become DMHTTP_ERR_* operation failures. */ unsigned status_code; unsigned redirects; bool content_length_known; @@ -85,7 +156,8 @@ extern "C" * Requests use Connection: close and Accept-Encoding: identity. No uploads, * authentication, decompression, pooling or proxy tunnelling are performed. * Calls on one handle must be externally serialized, including cancel/destroy. - * Callbacks must not reenter that handle. TLS must be initialized by its owner. */ + * Callbacks must not reenter that handle. TLS must be initialized by its owner. + */ dmod_dmhttp_api(1.0, dmhttp_result_t, _create, (const char *url, const dmhttp_options_t *options, dmhttp_t *output)); /* Advance to final response headers. OK means headers are available. AGAIN diff --git a/scripts/check_dmod_pointers.py b/scripts/check_dmod_pointers.py deleted file mode 100644 index 6a8603a..0000000 --- a/scripts/check_dmod_pointers.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -"""Reject absolute pointers outside legacy DMOD relocation sections. - -Read-only ELF audit: it never modifies ELF/DMF or emits runtime relocation data. -The fork must use DMOD_GLOBAL_POINTER or eliminate stored absolute pointers. -""" -import struct -import sys -from pathlib import Path - - -def check(elf, dmf): - if len(elf)<64 or elf[:4]!=b'\x7fELF' or elf[5]!=1: - raise ValueError('expected little-endian ELF') - wide=elf[4]==2 - machine=struct.unpack_from('=count: - raise ValueError('invalid ELF section table') - sections=[struct.unpack_from(fmt,elf,shoff+i*stride) for i in range(count)] - ns=sections[names_index];names=elf[ns[4]:ns[4]+ns[5]] - def name(section): - start=section[0];return names[start:names.index(b'\0',start)].decode('ascii') - allowed={'.header','.inputs','.outputs','.got'} - absolute=(1,10,11) if wide else (2,5,6,8,38,43,44,47,48) - relative=(0,2,4,9,24,25,26,27,28,29,41,42) if wide else (0,3,10,24,25,26,28,29,30,42,51,96,102,103) - records=0;got=0;failures=[] - for section in sections: - if section[1]!=(4 if wide else 9):continue - target=sections[section[7]] - if not target[2]&2:continue - symbol_table=sections[section[6]] - expected=24 if wide else 8 - if section[9]!=expected:raise ValueError('invalid relocation record size') - for pos in range(section[4],section[4]+section[5],expected): - offset,info=struct.unpack_from('>32) if wide else (info&255,info>>8) - records+=1 - if kind in absolute: - symbol_index=struct.unpack_from(' struct fixture { size_t at; unsigned closes; + bool not_found; }; static dmtls_result_t open_stream(void *p, const char *host, uint16_t port) { @@ -26,14 +27,16 @@ static dmtls_result_t send_stream(void *p, const uint8_t *bytes, size_t n, size_ static dmtls_result_t recv_stream(void *p, uint8_t *out, size_t cap, size_t *n) { struct fixture *f = p; - static const char reply[] = - "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"; + static const char reply[] = "HTTP/1.1 200 OK\r\nTransfer-Encoding: " + "chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"; + static const char missing[] = "HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n\r\nno"; + const char *response = f->not_found ? missing : reply; *n = 0; - if (f->at == sizeof(reply) - 1) + if (f->at == strlen(response)) return DMTLS_OK; if (cap) { - out[0] = reply[f->at++]; + out[0] = response[f->at++]; *n = 1; } return DMTLS_OK; @@ -106,3 +109,36 @@ DMOD_TEST_STEP(http_cancel_is_terminal_and_releases_connector) dmhttp_destroy(h); DMOD_TEST_EXPECT_EQ(f.closes, 1); } + +DMOD_TEST_STEP(http_preserves_not_found_status_and_body) +{ + struct fixture f = {0}; + f.not_found = true; + dmhttp_options_t o; + options(&o, &f); + dmhttp_t h = NULL; + DMOD_TEST_EXPECT_EQ(dmhttp_create("http://example.test/missing", &o, &h), DMHTTP_OK); + dmhttp_result_t rc; + unsigned guard = 0; + do + { + rc = dmhttp_step(h); + } while (rc == DMHTTP_AGAIN && ++guard < 1000); + DMOD_TEST_EXPECT_EQ(rc, DMHTTP_OK); + dmhttp_response_t response; + DMOD_TEST_EXPECT_EQ(dmhttp_response(h, &response), DMHTTP_OK); + DMOD_TEST_EXPECT_EQ(response.status_code, DMHTTP_STATUS_NOT_FOUND); + DMOD_TEST_EXPECT_EQ(response.error, DMHTTP_OK); + char body[4] = {0}; + size_t used = 0, received = 0; + do + { + rc = dmhttp_read(h, body + used, sizeof(body) - 1 - used, &received); + used += received; + } while ((rc == DMHTTP_OK || rc == DMHTTP_AGAIN) && ++guard < 1000); + DMOD_TEST_EXPECT_EQ(rc, DMHTTP_DONE); + DMOD_TEST_EXPECT_EQ(used, 2); + DMOD_TEST_EXPECT_EQ(strcmp(body, "no"), 0); + dmhttp_destroy(h); + DMOD_TEST_EXPECT_EQ(f.closes, 1); +} diff --git a/tests/probe/CMakeLists.txt b/tests/probe/CMakeLists.txt index 9f57f1a..67faf2e 100644 --- a/tests/probe/CMakeLists.txt +++ b/tests/probe/CMakeLists.txt @@ -5,12 +5,10 @@ set(DMOD_STACK_SIZE 16384) set(DMOD_DMR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/httpprobe.dmr) dmod_add_executable(${DMOD_MODULE_NAME} ${PROJECT_VERSION} httpprobe.c) target_link_libraries(${DMOD_MODULE_NAME} dmhttp_if) -dmhttp_audit_module(${DMOD_MODULE_NAME}) dmod_link_modules(${DMOD_MODULE_NAME} dmtls dmtcp dmdns dmip dmroute dmnetif dmrng dmosi) set(DMOD_MODULE_NAME httpwireprobe) set(DMOD_DMR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/httpprobe.dmr) dmod_add_executable(${DMOD_MODULE_NAME} ${PROJECT_VERSION} wireprobe.c) target_link_libraries(${DMOD_MODULE_NAME} dmhttp_if) -dmhttp_audit_module(${DMOD_MODULE_NAME}) dmod_link_modules(${DMOD_MODULE_NAME} dmtls dmrng dmosi dmtty)