Skip to content

feat(db): path-in-URL streaming endpoint /api/db/resource/{path} (closes #35) #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
joewiz wants to merge 2 commits into from
Closed

feat(db): path-in-URL streaming endpoint /api/db/resource/{path} (closes #35) #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7b89741
fix(db): auto-detect mime from extension when mime-type is omitted
joewiz Jun 4, 2026
211b016
feat(db): path-in-URL streaming endpoint /api/db/resource/{path}
joewiz Jun 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions controller.xq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,32 @@ else if (matches($exist:path, "^/+modules/.*\.json$")) then
(: serve OpenAPI spec files directly :)
<dispatch xmlns="http://exist.sourceforge.net/NS/exist"/>

else if (matches($exist:path, "^/+api/db/resource/.+")) then
(: Path-in-URL variant: raw bytes streaming via eXist's REST servlet.
:
: Use this for binary uploads/downloads (PDFs, images, fonts, zips)
: and for any text upload where the JSON envelope's buffering would
: be wasteful and the metadata it bundles isn't needed. The bare
: /api/db/resource endpoint (path-in-JSON-body) stays the canonical
: metadata-bundled path for editors that want signature + mime +
: content in a single roundtrip.
:
: Forwarding via controller.xq is an internal servlet dispatch —
: the original request's headers, auth context, and body all flow
: through unchanged. Crucially, /exist/rest's RESTServer streams
: the body straight to broker.storeDocument(InputSource) without
: materializing it in JVM memory, so this path is suitable for
: large binaries that would OOM Roaster's parse-the-body pipeline. :)
let $db-path := replace($exist:path, "^/+api/db/resource", "")
return
<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
<forward url="/rest{$db-path}" absolute="yes">
<set-header name="Access-Control-Allow-Origin" value="*"/>
<set-header name="Access-Control-Allow-Methods" value="GET, PUT, DELETE, HEAD, OPTIONS"/>
<set-header name="Access-Control-Allow-Headers" value="Content-Type, Authorization"/>
</forward>
</dispatch>

else if (starts-with($exist:path, "/api")) then
<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
<forward url="{$exist:controller}/modules/api.xq">
Expand Down
73 changes: 68 additions & 5 deletions modules/api.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1296,7 +1296,7 @@
},
"content": {
"type": "string",
"description": "Resource content (text or base64-encoded for binary)"
"description": "Resource content. UTF-8 text for text-class resources; for binary resources the JSON envelope is lossy — use the sibling endpoint `GET /api/db/resource/{path}` to fetch raw bytes."
},
"mime-type": {
"type": "string",
Expand Down Expand Up @@ -1326,9 +1326,9 @@
}
},
"put": {
"summary": "Store resource",
"summary": "Store resource (JSON envelope, text content)",
"operationId": "db:store-resource",
"description": "Stores or updates a resource in the database. The parent collection must exist. MIME type is auto-detected from the file extension if not specified.",
"description": "Stores or updates a *text* resource in the database via a JSON envelope. The parent collection must exist. When `mime-type` is omitted the server looks up the appropriate MIME type for the path's filename extension via eXist's `mime-types.xml` (e.g. `.xq` → `application/xquery`, `.svg` → `image/svg+xml`, `.html` → `text/html`). Send `mime-type` only to override that mapping (e.g. force an `.html` file to be stored as `application/xhtml+xml`).\n\nFor **binary uploads** (PDFs, images, fonts, zips, …) and large text uploads, use the sibling path-in-URL endpoint `PUT /api/db/resource/{path}` with the raw bytes as the request body. That route streams via eXist's REST servlet without buffering the body in JVM memory.",
"tags": [
"Database"
],
Expand All @@ -1349,11 +1349,11 @@
},
"content": {
"type": "string",
"description": "Resource content to store"
"description": "Resource content to store (UTF-8 text)."
},
"mime-type": {
"type": "string",
"description": "MIME type (auto-detected if omitted)"
"description": "MIME type to store the resource as. When omitted the server looks up the path's filename extension in `mime-types.xml`. Supply this only to override that mapping."
}
}
}
Expand Down Expand Up @@ -1779,6 +1779,69 @@
}
}
},
"/api/db/resource/{path}": {
"description": "Raw-bytes (path-in-URL) variant of /api/db/resource. The DB path is part of the URL rather than a JSON field. The request/response body is the resource's raw bytes — suitable for binary uploads/downloads (PDFs, images, fonts, zips) and large text uploads where the JSON envelope's buffering would be wasteful or hit JVM memory limits. Internally these requests are forwarded by controller.xq to eXist's REST servlet (`/exist/rest/db/...`), which streams the request body directly into the broker without materializing it in JVM memory. Use this endpoint when streaming matters; use `/api/db/resource` (path-in-JSON-body) when you want the metadata-bundled JSON envelope back.",
"parameters": [
{
"name": "path",
"in": "path",
"required": true,
"schema": { "type": "string" },
"description": "Target DB path, e.g. `/db/apps/myapp/icon.png`. The leading `/db/` is part of the URL.",
"example": "/db/apps/myapp/icon.png"
}
],
"put": {
"summary": "Store resource (raw bytes, streaming)",
"description": "Stores or updates a resource with the request body taken as the raw bytes. The `Content-Type` request header sets the MIME type; clients should send the appropriate media type (e.g., `image/png`, `application/pdf`, `application/xquery`) so the stored resource is correctly labeled. Streams through eXist's REST servlet — suitable for large binary uploads.",
"tags": ["Database"],
"requestBody": {
"required": true,
"description": "Raw resource bytes. The `Content-Type` header selects the MIME type.",
"content": {
"application/octet-stream": {
"schema": { "type": "string", "format": "binary" }
}
}
},
"responses": {
"201": { "description": "Resource created" },
"200": { "description": "Resource updated" },
"400": { "description": "Bad request" },
"401": { "description": "Authentication required" },
"403": { "description": "Permission denied" }
}
},
"get": {
"summary": "Fetch resource (raw bytes, streaming)",
"description": "Returns the resource's raw bytes. The response's `Content-Type` header reflects the stored MIME type. Suitable for downloading binaries and for any client that wants to stream the body directly to disk.",
"tags": ["Database"],
"responses": {
"200": {
"description": "Resource bytes",
"content": {
"application/octet-stream": {
"schema": { "type": "string", "format": "binary" }
}
}
},
"404": { "description": "Resource not found" },
"401": { "description": "Authentication required" },
"403": { "description": "Permission denied" }
}
},
"delete": {
"summary": "Remove resource",
"description": "Deletes the resource at the given DB path.",
"tags": ["Database"],
"responses": {
"200": { "description": "Resource removed" },
"404": { "description": "Resource not found" },
"401": { "description": "Authentication required" },
"403": { "description": "Permission denied" }
}
}
},
"/api/db/properties": {
"get": {
"summary": "Get resource/collection properties",
Expand Down
44 changes: 26 additions & 18 deletions modules/db.xqm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,21 @@ declare function db:store-resource($request as map(*)) {
let $body := $request?body
let $path := $body?path
let $content := $body?content
let $mime-type := ($body?mime-type, "application/xml")[1]
(: Explicit mime from the client wins. When omitted we pass through to
: the 3-arg xmldb:store, which calls MimeTable.getContentTypeFor() on
: the resource name internally (see XMLDBStore.java lines 152-154 in
: eXist core) — so the server's mime-types.xml is the single source
: of truth. The pre-fix code defaulted to "application/xml", which
: routed .xq / .xqm / image content into the XML parser and either
: failed (XPST0003 / "Content is not allowed in prolog") or stored
: with the wrong content type.
:
: For binary uploads — large or otherwise — clients should use the
: path-in-URL sibling endpoint `PUT /api/db/resource/{path}` with the
: raw bytes as the request body and the mime in the Content-Type
: header. That route streams via eXist's REST servlet, bypassing
: Roaster's buffered request-body pipeline. :)
let $mime-type := $body?mime-type
return
if (empty($path) or empty($content))
then roaster:response(400, map { "error": "Missing required fields: path, content" })
Expand All @@ -258,7 +272,7 @@ declare function db:store-resource($request as map(*)) {
let $stored :=
if (util:binary-doc-available($path))
then xmldb:store-as-binary($collection, $resource, $content)
else if ($mime-type)
else if (exists($mime-type))
then xmldb:store($collection, $resource, $content, $mime-type)
else xmldb:store($collection, $resource, $content)
let $_ := if ($is-new) then db:fix-permissions($stored) else ()
Expand All @@ -270,22 +284,16 @@ declare function db:store-resource($request as map(*)) {
}
)
} catch * {
(: Fall back to binary store for HTML that isn't well-formed :)
if ($mime-type = "text/html")
then
let $stored := xmldb:store-as-binary($collection, $resource, $content)
let $_ := if ($is-new) then db:fix-permissions($stored) else ()
return roaster:response(
if ($is-new) then 201 else 200,
map {
"stored": $stored,
"runPath": db:get-run-path($stored)
}
)
else
roaster:response(400, map {
"error": replace(replace($err:description, "^.*XMLDBException:", ""), "\[at.*\]$", "")
})
(: The store failed — most commonly because the content
: wasn't well-formed XML and the target mime was an
: XML-class type (application/xml, text/html,
: application/xhtml+xml, …). Surface the parse error
: as a 400 with the eXist message; clients who want
: the raw bytes preserved should use the path-in-URL
: sibling endpoint `PUT /api/db/resource/{path}`. :)
roaster:response(400, map {
"error": replace(replace($err:description, "^.*XMLDBException:", ""), "\[at.*\]$", "")
})
}
};

Expand Down
Loading