diff --git a/controller.xq b/controller.xq index 54fe98b..35808ff 100644 --- a/controller.xq +++ b/controller.xq @@ -31,6 +31,32 @@ else if (matches($exist:path, "^/+modules/.*\.json$")) then (: serve OpenAPI spec files directly :) +else if (matches($exist:path, "^/+api/db/resource/.+")) then + (: Path-in-URL variant: raw bytes streaming via eXist's REST servlet. + : + : Use this for binary uploads/downloads (PDFs, images, fonts, zips) + : and for any text upload where the JSON envelope's buffering would + : be wasteful and the metadata it bundles isn't needed. The bare + : /api/db/resource endpoint (path-in-JSON-body) stays the canonical + : metadata-bundled path for editors that want signature + mime + + : content in a single roundtrip. + : + : Forwarding via controller.xq is an internal servlet dispatch — + : the original request's headers, auth context, and body all flow + : through unchanged. Crucially, /exist/rest's RESTServer streams + : the body straight to broker.storeDocument(InputSource) without + : materializing it in JVM memory, so this path is suitable for + : large binaries that would OOM Roaster's parse-the-body pipeline. :) + let $db-path := replace($exist:path, "^/+api/db/resource", "") + return + + + + + + + + else if (starts-with($exist:path, "/api")) then diff --git a/modules/api.json b/modules/api.json index c9f932e..3f16e8e 100644 --- a/modules/api.json +++ b/modules/api.json @@ -1296,7 +1296,7 @@ }, "content": { "type": "string", - "description": "Resource content (text or base64-encoded for binary)" + "description": "Resource content. UTF-8 text for text-class resources; for binary resources the JSON envelope is lossy — use the sibling endpoint `GET /api/db/resource/{path}` to fetch raw bytes." }, "mime-type": { "type": "string", @@ -1326,9 +1326,9 @@ } }, "put": { - "summary": "Store resource", + "summary": "Store resource (JSON envelope, text content)", "operationId": "db:store-resource", - "description": "Stores or updates a resource in the database. The parent collection must exist. MIME type is auto-detected from the file extension if not specified.", + "description": "Stores or updates a *text* resource in the database via a JSON envelope. The parent collection must exist. When `mime-type` is omitted the server looks up the appropriate MIME type for the path's filename extension via eXist's `mime-types.xml` (e.g. `.xq` → `application/xquery`, `.svg` → `image/svg+xml`, `.html` → `text/html`). Send `mime-type` only to override that mapping (e.g. force an `.html` file to be stored as `application/xhtml+xml`).\n\nFor **binary uploads** (PDFs, images, fonts, zips, …) and large text uploads, use the sibling path-in-URL endpoint `PUT /api/db/resource/{path}` with the raw bytes as the request body. That route streams via eXist's REST servlet without buffering the body in JVM memory.", "tags": [ "Database" ], @@ -1349,11 +1349,11 @@ }, "content": { "type": "string", - "description": "Resource content to store" + "description": "Resource content to store (UTF-8 text)." }, "mime-type": { "type": "string", - "description": "MIME type (auto-detected if omitted)" + "description": "MIME type to store the resource as. When omitted the server looks up the path's filename extension in `mime-types.xml`. Supply this only to override that mapping." } } } @@ -1779,6 +1779,69 @@ } } }, + "/api/db/resource/{path}": { + "description": "Raw-bytes (path-in-URL) variant of /api/db/resource. The DB path is part of the URL rather than a JSON field. The request/response body is the resource's raw bytes — suitable for binary uploads/downloads (PDFs, images, fonts, zips) and large text uploads where the JSON envelope's buffering would be wasteful or hit JVM memory limits. Internally these requests are forwarded by controller.xq to eXist's REST servlet (`/exist/rest/db/...`), which streams the request body directly into the broker without materializing it in JVM memory. Use this endpoint when streaming matters; use `/api/db/resource` (path-in-JSON-body) when you want the metadata-bundled JSON envelope back.", + "parameters": [ + { + "name": "path", + "in": "path", + "required": true, + "schema": { "type": "string" }, + "description": "Target DB path, e.g. `/db/apps/myapp/icon.png`. The leading `/db/` is part of the URL.", + "example": "/db/apps/myapp/icon.png" + } + ], + "put": { + "summary": "Store resource (raw bytes, streaming)", + "description": "Stores or updates a resource with the request body taken as the raw bytes. The `Content-Type` request header sets the MIME type; clients should send the appropriate media type (e.g., `image/png`, `application/pdf`, `application/xquery`) so the stored resource is correctly labeled. Streams through eXist's REST servlet — suitable for large binary uploads.", + "tags": ["Database"], + "requestBody": { + "required": true, + "description": "Raw resource bytes. The `Content-Type` header selects the MIME type.", + "content": { + "application/octet-stream": { + "schema": { "type": "string", "format": "binary" } + } + } + }, + "responses": { + "201": { "description": "Resource created" }, + "200": { "description": "Resource updated" }, + "400": { "description": "Bad request" }, + "401": { "description": "Authentication required" }, + "403": { "description": "Permission denied" } + } + }, + "get": { + "summary": "Fetch resource (raw bytes, streaming)", + "description": "Returns the resource's raw bytes. The response's `Content-Type` header reflects the stored MIME type. Suitable for downloading binaries and for any client that wants to stream the body directly to disk.", + "tags": ["Database"], + "responses": { + "200": { + "description": "Resource bytes", + "content": { + "application/octet-stream": { + "schema": { "type": "string", "format": "binary" } + } + } + }, + "404": { "description": "Resource not found" }, + "401": { "description": "Authentication required" }, + "403": { "description": "Permission denied" } + } + }, + "delete": { + "summary": "Remove resource", + "description": "Deletes the resource at the given DB path.", + "tags": ["Database"], + "responses": { + "200": { "description": "Resource removed" }, + "404": { "description": "Resource not found" }, + "401": { "description": "Authentication required" }, + "403": { "description": "Permission denied" } + } + } + }, "/api/db/properties": { "get": { "summary": "Get resource/collection properties", diff --git a/modules/db.xqm b/modules/db.xqm index a5842a9..5dad521 100644 --- a/modules/db.xqm +++ b/modules/db.xqm @@ -245,7 +245,21 @@ declare function db:store-resource($request as map(*)) { let $body := $request?body let $path := $body?path let $content := $body?content - let $mime-type := ($body?mime-type, "application/xml")[1] + (: Explicit mime from the client wins. When omitted we pass through to + : the 3-arg xmldb:store, which calls MimeTable.getContentTypeFor() on + : the resource name internally (see XMLDBStore.java lines 152-154 in + : eXist core) — so the server's mime-types.xml is the single source + : of truth. The pre-fix code defaulted to "application/xml", which + : routed .xq / .xqm / image content into the XML parser and either + : failed (XPST0003 / "Content is not allowed in prolog") or stored + : with the wrong content type. + : + : For binary uploads — large or otherwise — clients should use the + : path-in-URL sibling endpoint `PUT /api/db/resource/{path}` with the + : raw bytes as the request body and the mime in the Content-Type + : header. That route streams via eXist's REST servlet, bypassing + : Roaster's buffered request-body pipeline. :) + let $mime-type := $body?mime-type return if (empty($path) or empty($content)) then roaster:response(400, map { "error": "Missing required fields: path, content" }) @@ -258,7 +272,7 @@ declare function db:store-resource($request as map(*)) { let $stored := if (util:binary-doc-available($path)) then xmldb:store-as-binary($collection, $resource, $content) - else if ($mime-type) + else if (exists($mime-type)) then xmldb:store($collection, $resource, $content, $mime-type) else xmldb:store($collection, $resource, $content) let $_ := if ($is-new) then db:fix-permissions($stored) else () @@ -270,22 +284,16 @@ declare function db:store-resource($request as map(*)) { } ) } catch * { - (: Fall back to binary store for HTML that isn't well-formed :) - if ($mime-type = "text/html") - then - let $stored := xmldb:store-as-binary($collection, $resource, $content) - let $_ := if ($is-new) then db:fix-permissions($stored) else () - return roaster:response( - if ($is-new) then 201 else 200, - map { - "stored": $stored, - "runPath": db:get-run-path($stored) - } - ) - else - roaster:response(400, map { - "error": replace(replace($err:description, "^.*XMLDBException:", ""), "\[at.*\]$", "") - }) + (: The store failed — most commonly because the content + : wasn't well-formed XML and the target mime was an + : XML-class type (application/xml, text/html, + : application/xhtml+xml, …). Surface the parse error + : as a 400 with the eXist message; clients who want + : the raw bytes preserved should use the path-in-URL + : sibling endpoint `PUT /api/db/resource/{path}`. :) + roaster:response(400, map { + "error": replace(replace($err:description, "^.*XMLDBException:", ""), "\[at.*\]$", "") + }) } }; diff --git a/src/test/cypress/e2e/db.cy.js b/src/test/cypress/e2e/db.cy.js index 4213bdf..bc19130 100644 --- a/src/test/cypress/e2e/db.cy.js +++ b/src/test/cypress/e2e/db.cy.js @@ -85,7 +85,7 @@ describe('/api/db', () => { }); describe('PUT /api/db/resource — store', () => { - it('stores an XML resource', () => { + it('stores an XML resource with explicit mime-type', () => { cy.request({ url: '/api/db/resource', method: 'PUT', @@ -99,6 +99,264 @@ describe('/api/db', () => { expect(response.body).to.have.property('stored'); }); }); + + // Regression: prior to this fix the handler defaulted `mime-type` to + // "application/xml" when the client omitted it, so .xq / .xqm / .svg + // / .json content either failed the XML parse outright (XPST0003, + // "Content is not allowed in prolog") or stored with the wrong MIME. + // The fix drops the default and lets the 3-arg xmldb:store consult + // eXist's MimeTable to pick the right MIME from the extension. + it('auto-detects application/xquery from .xq extension when mime-type omitted', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { path: `${testCollection}/auto.xq`, content: 'xquery version "3.1"; 1+1' } + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + expect(response.body).to.have.property('stored'); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/auto.xq`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('application/xquery'); + }); + }); + + it('auto-detects application/xquery from .xqm extension when mime-type omitted', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { + path: `${testCollection}/auto.xqm`, + content: 'xquery version "3.1"; module namespace t = "http://example.com/t";' + } + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/auto.xqm`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('application/xquery'); + }); + }); + + it('auto-detects image/svg+xml from .svg extension when mime-type omitted', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { + path: `${testCollection}/auto.svg`, + content: '' + } + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/auto.svg`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('image/svg+xml'); + }); + }); + + it('auto-detects application/json from .json extension when mime-type omitted', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { path: `${testCollection}/auto.json`, content: '{"hello":"world"}' } + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/auto.json`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('application/json'); + }); + }); + + it('stores well-formed .html as text/html via auto-detection', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { + path: `${testCollection}/wellformed.html`, + content: 't

x

' + } + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + expect(response.body).to.have.property('stored'); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/wellformed.html`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('text/html'); + }); + }); + + it('returns 400 with a parse-error message for unparseable XML content', () => { + // Unclosed — not well-formed XML. eXist's text/html is + // an XML-class mime, so the parser runs and rejects. Clients + // who need to preserve raw bytes should send + // mime-type=application/octet-stream explicitly. + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { + path: `${testCollection}/bad.html`, + content: '' + }, + failOnStatusCode: false + }).then(response => { + expect(response.status).to.equal(400); + expect(response.body.error).to.match(/parser|XML|img/i); + }); + }); + + it('preserves raw bytes for unparseable content when mime-type=application/octet-stream', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { + path: `${testCollection}/raw.html`, + content: '', + 'mime-type': 'application/octet-stream' + } + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/raw.html`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('application/octet-stream'); + }); + }); + + it('still honors explicit mime-type override (e.g. xhtml+xml for an .html path)', () => { + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { + path: `${testCollection}/explicit.html`, + content: 't

x

', + 'mime-type': 'application/xhtml+xml' + } + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/explicit.html`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('application/xhtml+xml'); + }); + }); + }); + + // --------------------------------------------------------------------- + // Path-in-URL variant — raw bytes streaming via controller.xq forward + // to /exist/rest. Used for binary uploads/downloads (PDFs, images, + // fonts, zips, …) and any large text upload. The JSON envelope at + // /api/db/resource is unchanged and remains the metadata-bundled path + // for editors. + // Closes #35. + // --------------------------------------------------------------------- + describe('/api/db/resource/{path} — streaming via controller.xq forward to /exist/rest', () => { + // These tests exercise the path-in-URL endpoint that forwards to + // /exist/rest via controller.xq (so the request body streams through + // to broker.storeDocument without Roaster's body:parse buffering it + // first). Cypress's cy.request can't cleanly send raw binary bytes + // (the Buffer ↔ JSON serialization mangles them), so the binary + // round-trip is verified via curl in the PR description. The text- + // based tests below exercise the same forward routing — the only + // type-specific concern (mime auto-detection from the Content-Type + // header) is covered by the second test. + + it('PUT text via path-in-URL stores at the URL path', () => { + cy.request({ + url: `/api/db/resource${testCollection}/streamed.txt`, + method: 'PUT', + auth, + headers: { 'Content-Type': 'text/plain' }, + body: 'hello from the streaming endpoint' + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/streamed.txt`, + auth + }).then(response => { + // /exist/rest honors the Content-Type header on PUT + expect(response.body['mime-type']).to.equal('text/plain'); + expect(response.body.type).to.equal('resource'); + }); + }); + + it('PUT with application/xquery Content-Type stores as XQuery (no XML parse)', () => { + cy.request({ + url: `/api/db/resource${testCollection}/streamed.xq`, + method: 'PUT', + auth, + headers: { 'Content-Type': 'application/xquery' }, + body: 'xquery version "3.1"; declare namespace t = "http://t"; 1+1' + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/streamed.xq`, + auth + }).then(response => { + expect(response.body['mime-type']).to.equal('application/xquery'); + }); + }); + + it('GET on the path-in-URL endpoint returns raw bytes with the stored Content-Type', () => { + cy.request({ + url: `/api/db/resource${testCollection}/streamed.txt`, + auth + }).then(response => { + expect(response.status).to.equal(200); + expect(response.headers['content-type']).to.match(/^text\/plain/); + expect(response.body).to.equal('hello from the streaming endpoint'); + }); + }); + + it('DELETE via path-in-URL removes the resource', () => { + cy.request({ + url: `/api/db/resource${testCollection}/streamed.txt`, + method: 'DELETE', + auth + }).then(response => { + expect(response.status).to.be.oneOf([200, 204]); + }); + cy.request({ + url: `/api/db/properties?path=${testCollection}/streamed.txt`, + auth, + failOnStatusCode: false + }).then(response => { + expect(response.status).to.equal(404); + }); + }); + + it('JSON-envelope endpoint (path-in-body) still works for text — sibling, not replacement', () => { + // Sanity check that the path-in-URL forward doesn't shadow the + // bare /api/db/resource endpoint that takes the JSON envelope. + cy.request({ + url: '/api/db/resource', + method: 'PUT', + auth, + body: { path: `${testCollection}/json-sibling.xml`, content: '', 'mime-type': 'application/xml' } + }).then(response => { + expect(response.status).to.be.oneOf([200, 201]); + expect(response.body).to.have.property('stored'); + }); + }); }); describe('GET /api/db/resource — read', () => {