Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CONFORMANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,13 @@ Pages avoid full-log/category/sales-history copies, but sparse filters can scan
the remaining catalogue. Existing unpaged reads remain unbounded; in-memory,
single-writer and witnessing limitations remain unchanged. See
`docs/marketplace-browse.md` for query syntax and compatibility limits.

The consumer API applies configurable deployment byte limits before all ten JSON
write handlers. `internal/consumerapi/request_limits.go` bounds the complete
encoded body before signature validation or any store mutation, while preserving
accepted signed bytes and the existing commons/Locker separation. The HTTP and
signed-artifact regressions in `request_limits_test.go` and
`request_limits_boundary_test.go`, `request_limits_http_test.go` and
`request_limits_io_test.go` bind this resource boundary. It is not a wire,
settlement or covenant rule; public reads and `/v0/` relay traffic retain their
existing contracts. See `docs/consumer-request-limits.md` for defaults and scope.
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,3 +165,9 @@ independent witnessing.
AGPL-3.0-or-later.

*Network Theory Applied Research Institute, Inc. — 501(c)(3) — EIN 92-3047136 — info@ntari.org*

## Consumer API deployment limits

The member-facing JSON write routes enforce configurable body limits before
decoding or mutation. See [consumer request limits](docs/consumer-request-limits.md)
for defaults, daemon flags and compatibility boundaries.
5 changes: 4 additions & 1 deletion cmd/cloudyd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ import (
func main() {
addr := flag.String("addr", ":8088", "listen address for the consumer JSON API")
platform := flag.String("platform", "cloudy", "platform identity (scopes member ids and every artifact)")
limits := consumerapi.DefaultRequestLimits()
flag.Int64Var(&limits.StructuredBytes, "request-bytes", limits.StructuredBytes, "maximum encoded JSON body bytes for structural consumer API writes")
flag.Int64Var(&limits.ClaimBytes, "claim-request-bytes", limits.ClaimBytes, "maximum encoded JSON body bytes for claims including member-local narrative")
var rc relayConfig
flag.StringVar(&rc.Orchestrator, "orchestrator", "", "orchestrator base URL; when set, relay members' nodes to it (decision 0001)")
flag.StringVar(&rc.OperatorID, "operator-id", "", "operator id this frontend enrolled under (cloudy-enroll -slug)")
Expand All @@ -42,7 +45,7 @@ func main() {
flag.DurationVar(&rc.Poll, "relay-poll", 0, "poll the orchestrator at most once per node per this interval (default 30s)")
flag.Parse()

srv, err := consumerapi.NewServer(*platform)
srv, err := consumerapi.NewServerWithLimits(*platform, limits)
if err != nil {
log.Fatalf("cloudyd: constructing server: %v", err)
}
Expand Down
34 changes: 34 additions & 0 deletions cmd/cloudyd/request_limits_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package main

import (
"bytes"
"context"
"os/exec"
"testing"
"time"
)

func TestRequestLimitFlagsAreAvailable(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
output, err := exec.CommandContext(ctx, "go", "run", ".", "-help").CombinedOutput()
if err != nil {
t.Fatalf("help failed: %v\n%s", err, output)
}
for _, flag := range []string{"-request-bytes", "-claim-request-bytes"} {
if !bytes.Contains(output, []byte(flag)) {
t.Errorf("help does not expose %s\n%s", flag, output)
}
}
}

func TestRequestLimitFlagsRejectInvalidConfiguration(t *testing.T) {
for _, args := range [][]string{{"-request-bytes", "0"}, {"-claim-request-bytes", "-1"}} {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
output, err := exec.CommandContext(ctx, "go", append([]string{"run", "."}, args...)...).CombinedOutput()
cancel()
if err == nil || !bytes.Contains(output, []byte("request byte limits must be positive")) {
t.Errorf("invalid limits %v: err=%v output=%s", args, err, output)
}
}
}
60 changes: 60 additions & 0 deletions docs/consumer-request-limits.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Consumer API request limits

The member-facing JSON write routes enforce a complete encoded-body byte limit
before JSON decoding, signature checks or store mutation. An oversized body
receives HTTP 413 with a JSON error. This applies with `Content-Length` and with
chunked or otherwise unknown-length bodies, including excess whitespace after a
valid signed object. The boundary is inclusive: a body exactly at the limit
continues to the existing validation and signature checks.

| POST routes | Default encoded bytes | Daemon flag |
| --- | ---: | --- |
| `/api/v1/claims` | 1,048,576 (1 MiB) | `-claim-request-bytes` |
| `/api/v1/members`, `/api/v1/references`, `/api/v1/market/listings`, `/api/v1/drops`, `/api/v1/credit/spends`, `/api/v1/assessments`, `/api/v1/assessments/{id}/answers`, `/api/v1/disputes`, `/api/v1/disputes/{id}/withdraw` | 65,536 (64 KiB) | `-request-bytes` |

Structural requests consist of keys, hashes, signatures, timestamps and labels.
The existing signed request fixtures fit well below the structural budget. Claims
also transport three arbitrary member-local narrative strings, so their default
budget is larger. Neither class has an existing domain-wide maximum: these are
adjustable frontend deployment budgets, not consensus rules, new enrollment
conditions, fees or limits on the canonical signed artifacts.

For a deployment that needs a larger narrative envelope:

```sh
go run ./cmd/cloudyd -request-bytes 65536 -claim-request-bytes 2097152
```

Both settings must be positive; zero or a negative value fails daemon startup.
Callers embedding the API may pass `consumerapi.RequestLimits` to
`consumerapi.NewServerWithLimits`. Existing `NewServer` callers use
`consumerapi.DefaultRequestLimits` without a source change. Configure a server
before exposing its handler; the settings are immutable during requests.

The count includes the whole encoded JSON envelope, escapes, ignored fields and
trailing bytes, not merely decoded narrative lengths. HTTP chunk framing is not
part of the body count. No request decompression is added. The guard reads at
most the configured limit plus one byte to detect an unknown-length overflow;
a declared oversized body is rejected without an application read. The original
body is closed on success, overflow and read failure. A body read or close failure returns
HTTP 400 without dispatching the write handler.

Accepted bytes are replayed unchanged to the existing decoder. Existing JSON
field handling and signature verification remain intact; this change adds no
end-of-document rule for an in-bounds body. Signatures, public structural reads,
member-local narrative separation and the escrow policy keep their existing
meaning. These controls apply only to the ten consumer JSON write routes, not
node relay `/v0/` traffic, unrelated witness daemons or response sizes.

Memory use is bounded per request, not across simultaneous requests. HTTP timeouts,
request concurrency limits and reverse-proxy policies remain separate deployment
concerns. Stores and operator keys retain their existing in-memory/ephemeral
limitations; request bounds do not make this a durable deployment.

Verification lives in `internal/consumerapi/request_limits_test.go`,
`internal/consumerapi/request_limits_boundary_test.go`,
`internal/consumerapi/request_limits_http_test.go`,
`internal/consumerapi/request_limits_io_test.go` and
`cmd/cloudyd/request_limits_test.go`. Tests cover every mounted JSON write route,
known/unknown lengths, real chunked HTTP, exact boundaries, signed registration
and narrative claims, unchanged stores on rejection, read bounds and body cleanup.
64 changes: 64 additions & 0 deletions internal/consumerapi/request_limits.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package consumerapi

import (
"bytes"
"errors"
"io"
"net/http"
)

// RequestLimits bounds encoded JSON bytes at this frontend's HTTP ingress.
// These are deployment controls, not limits on signed artifacts in the commons.
type RequestLimits struct {
StructuredBytes int64
ClaimBytes int64
}

// DefaultRequestLimits allows larger claim bodies for member-local narrative.
// Structural requests carry keys, hashes, signatures and short labels instead.
func DefaultRequestLimits() RequestLimits {
return RequestLimits{StructuredBytes: 64 << 10, ClaimBytes: 1 << 20}
}

// NewServerWithLimits constructs the same member-facing composition as NewServer
// with explicit positive HTTP body limits. It does not alter signed data.
func NewServerWithLimits(platform string, limits RequestLimits) (*Server, error) {
if limits.StructuredBytes <= 0 || limits.ClaimBytes <= 0 {
return nil, errors.New("consumerapi: request byte limits must be positive")
}
s, err := NewServer(platform)
if err != nil {
return nil, err
}
s.requestLimits = limits
return s, nil
}

// limitRequest reads the entire bounded body before a handler can mutate state.
// Merely bounding Decode would miss excess bytes after its first JSON value.
func limitRequest(limit int64, next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
body, err := readRequestBody(w, r, limit)
if err != nil {
var oversized *http.MaxBytesError
if errors.As(err, &oversized) {
writeErr(w, http.StatusRequestEntityTooLarge, "request body exceeds configured byte limit")
} else {
writeErr(w, http.StatusBadRequest, "invalid JSON body")
}
return
}
r.Body = io.NopCloser(bytes.NewReader(body))
next(w, r)
}
}

// readRequestBody closes the original body before any write handler runs.
func readRequestBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) {
bounded := http.MaxBytesReader(w, r.Body, limit)
if r.ContentLength > limit {
return nil, errors.Join(&http.MaxBytesError{Limit: limit}, bounded.Close())
}
body, err := io.ReadAll(bounded)
return body, errors.Join(err, bounded.Close())
}
120 changes: 120 additions & 0 deletions internal/consumerapi/request_limits_boundary_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package consumerapi

import (
"bytes"
"net/http"
"net/http/httptest"
"reflect"
"testing"
)

func configuredRequestServer(t *testing.T) *Server {
t.Helper()
s, err := NewServerWithLimits(platform, RequestLimits{StructuredBytes: 1024, ClaimBytes: 2048})
if err != nil {
t.Fatal(err)
}
return s
}

func TestRequestLimitsConfiguredBoundaries(t *testing.T) {
for _, path := range jsonWritePaths {
for _, extra := range []int{-1, 0, 1} {
for _, declared := range []bool{true, false} {
checkConfiguredBoundary(t, path, extra, declared)
}
}
}
}

func checkConfiguredBoundary(t *testing.T, path string, extra int, declared bool) {
t.Helper()
limit := 1024
if path == "/api/v1/claims" {
limit = 2048
}
s := configuredRequestServer(t)
body := append([]byte("{}"), bytes.Repeat([]byte(" "), limit+extra-2)...)
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(body))
if !declared {
req.ContentLength = -1
}
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, req)
want := http.StatusBadRequest
if extra > 0 {
want = http.StatusRequestEntityTooLarge
}
if rec.Code != want {
t.Errorf("%s size=%d declared=%v: got %d, want %d", path, len(body), declared, rec.Code, want)
}
}

func TestRequestLimitsSignedRegistrationBoundary(t *testing.T) {
for _, extra := range []int{-1, 0, 1} {
for _, declared := range []bool{true, false} {
checkSignedRegistrationBoundary(t, extra, declared)
}
}
}

func checkSignedRegistrationBoundary(t *testing.T, extra int, declared bool) {
t.Helper()
s := configuredRequestServer(t)
before := ingressState(t, s)
body := signedRegistration(t, s)
body = append(body, bytes.Repeat([]byte(" "), 1024+extra-len(body))...)
req := httptest.NewRequest(http.MethodPost, memberWritePath, bytes.NewReader(body))
if !declared {
req.ContentLength = -1
}
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, req)
want := http.StatusOK
if extra > 0 {
want = http.StatusRequestEntityTooLarge
}
if rec.Code != want {
t.Errorf("extra=%d declared=%v got %d, want %d", extra, declared, rec.Code, want)
}
if extra > 0 && !reflect.DeepEqual(before, ingressState(t, s)) {
t.Error("overflow mutated members")
}
}

func TestRequestLimitsSignedClaimBoundary(t *testing.T) {
for _, extra := range []int{-1, 0, 1} {
s, err := NewServerWithLimits(platform, RequestLimits{1024, 2048})
if err != nil {
t.Fatal(err)
}
h := s.Handler()
pub, priv := key(94)
registerMember(t, h, pub, priv)
narrative := "member-local narrative with escaped characters: \n\t and café"
c, body := signedClaim(t, pub, priv, narrative)
body = append(body, bytes.Repeat([]byte(" "), 2048+extra-len(body))...)
before := ingressState(t, s)
req := httptest.NewRequest(http.MethodPost, "/api/v1/claims", bytes.NewReader(body))
req.ContentLength = -1
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if extra > 0 {
if rec.Code != http.StatusRequestEntityTooLarge || !reflect.DeepEqual(before, ingressState(t, s)) {
t.Fatalf("overflow changed state or got %d", rec.Code)
}
continue
}
if rec.Code != http.StatusCreated {
t.Fatalf("got %d: %s", rec.Code, rec.Body.String())
}
id := c.ID()
get, view := do(t, h, http.MethodGet, "/api/v1/claims/"+hx(id[:]), nil)
if get.Code != http.StatusOK {
t.Fatalf("stored signed claim unreadable: %d", get.Code)
}
if view["narrative"].(map[string]any)["inputs"] != narrative {
t.Error("signed narrative changed")
}
}
}
65 changes: 65 additions & 0 deletions internal/consumerapi/request_limits_http_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package consumerapi

import (
"bytes"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
)

func TestRequestLimitsRealHTTP(t *testing.T) {
for _, extra := range []int{0, 1} {
for _, chunked := range []bool{false, true} {
checkRealRequestBoundary(t, extra, chunked)
}
}
}

func checkRealRequestBoundary(t *testing.T, extra int, chunked bool) {
t.Helper()
s := configuredRequestServer(t)
server := httptest.NewServer(s.Handler())
defer server.Close()
body := signedRegistration(t, s)
body = append(body, bytes.Repeat([]byte(" "), 1024+extra-len(body))...)
req, err := http.NewRequest(http.MethodPost, server.URL+memberWritePath, bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
if chunked {
req.ContentLength = -1
req.TransferEncoding = []string{"chunked"}
}
response, err := server.Client().Do(req)
if err != nil {
t.Fatal(err)
}
data, err := io.ReadAll(response.Body)
closeErr := response.Body.Close()
if err != nil {
t.Fatal(err)
}
if closeErr != nil {
t.Fatal(closeErr)
}
want := http.StatusOK
if extra > 0 {
want = http.StatusRequestEntityTooLarge
}
if response.StatusCode != want || !json.Valid(data) {
t.Errorf("extra=%d chunked=%v: status=%d body=%s", extra, chunked, response.StatusCode, data)
}
}

func TestRequestLimitsInvalidConfiguration(t *testing.T) {
for _, limits := range []RequestLimits{{0, 1024}, {1024, 0}, {-1, 1024}, {1024, -1}} {
if _, err := NewServerWithLimits(platform, limits); err == nil {
t.Errorf("accepted invalid limits %+v", limits)
}
}
if _, err := NewServerWithLimits("", RequestLimits{1024, 2048}); err == nil {
t.Error("accepted empty platform")
}
}
Loading