WIP: s3 request file upload

web/package.json

@@ -23,6 +23,7 @@
     "dependencies": {
         "@aws-sdk/client-s3": "^3.828.0",
         "@aws-sdk/lib-storage": "^3.828.0",
+        "@aws-sdk/s3-presigned-post": "3.828.0",
         "@aws-sdk/s3-request-presigner": "^3.828.0",
         "@aws-sdk/client-sts": "^3.907.0",
         "@babel/runtime": "7.26.0",

web/scripts/unyamlify-env-local.ts

@@ -65,8 +65,7 @@ if (!fs.existsSync(envLocalYamlFilePath)) {
                 `          sts: {`,
                 `            durationSeconds: 604800,`,
                 `            oidcConfiguration: {`,
-                `              issuerURI: "https://auth.lab.sspcloud.fr/auth/realms/sspcloud",`,
-                `              clientID: "onyxia"`,
+                `              clientID: "onyxia_minio"`,
                 `            },`,
                 `            role: {`,
                 `              profileName: "default",`,

web/src/core/adapters/s3Client/s3Client.ts

@@ -169,9 +169,15 @@ export function createS3Client(
                 import("@aws-sdk/client-s3").S3Client
             >();
 
-            async function getAwsS3Client() {
+            type Token = NonNullable<
+                Awaited<ReturnType<typeof getNewlyRequestedOrCachedToken>>
+            >;
+
+            async function getAwsS3Client(options?: { token: Token }) {
                 const [tokens, AwsS3Client] = await Promise.all([
-                    getNewlyRequestedOrCachedToken(),
+                    options === undefined
+                        ? getNewlyRequestedOrCachedToken()
+                        : Promise.resolve(options.token),
                     import("@aws-sdk/client-s3").then(({ S3Client }) => S3Client)
                 ] as const);
 
@@ -550,6 +556,71 @@ export function createS3Client(
 
             return downloadUrl;
         },
+        createPresignedPost: async ({
+            s3Uri,
+            validityDurationSecond,
+            maxObjectSizeInBytes
+        }) => {
+            assert(
+                !isAnonymousProfile,
+                "Trying to generate a presigned POST with a public client"
+            );
+
+            const { getAwsS3Client, getNewlyRequestedOrCachedToken } = await prApi;
+
+            // This is the only recoverable boundary in this operation: obtaining
+            // temporary credentials can fail when the identity or STS service is
+            // unreachable. Signing the POST below is otherwise a local operation.
+            const tokenResult = await getNewlyRequestedOrCachedToken().then(
+                token => ({ isSuccess: true as const, token }),
+                error => ({
+                    isSuccess: false as const,
+                    errorMessage: error instanceof Error ? error.message : String(error)
+                })
+            );
+
+            if (!tokenResult.isSuccess) {
+                return tokenResult;
+            }
+
+            const { token } = tokenResult;
+
+            assert(token !== undefined);
+
+            const { awsS3Client } = await getAwsS3Client({ token });
+
+            const now = Date.now();
+            const requestedExpirationTime = now + validityDurationSecond * 1_000;
+            const expirationTime = Math.min(
+                requestedExpirationTime,
+                token.expirationTime ?? requestedExpirationTime
+            );
+            const expiresInSecond = Math.max(
+                1,
+                Math.floor((expirationTime - now) / 1_000)
+            );
+
+            const { url, fields } = await (
+                await import("@aws-sdk/s3-presigned-post")
+            ).createPresignedPost(awsS3Client, {
+                Bucket: s3Uri.bucket,
+                Key: `${getS3UriKey(s3Uri)}\${filename}`,
+                Expires: expiresInSecond,
+                Conditions:
+                    maxObjectSizeInBytes === undefined
+                        ? []
+                        : [["content-length-range", 0, maxObjectSizeInBytes]]
+            });
+
+            return {
+                isSuccess: true,
+                presignedPost: {
+                    url,
+                    fields,
+                    expirationTime: now + expiresInSecond * 1_000
+                }
+            };
+        },
 
         getObjectContent: async ({ s3Uri, range }) => {
             const { getAwsS3Client } = await prApi;

web/src/core/ports/S3Client.ts

@@ -37,6 +37,24 @@ export type S3Client = {
         isForDirectDownload: boolean;
     }) => Promise<string>;
 
+    /**
+     * Creates a form that can be used without S3 credentials to upload objects
+     * whose keys start with the key of `s3Uri`.
+     *
+     * `maxObjectSizeInBytes` applies to each object independently. Enforcing a
+     * maximum size across all objects uploaded with the form requires a stateful
+     * service in front of S3.
+     *
+     * A failure value represents an expected failure to acquire temporary S3
+     * credentials. Invalid state or unexpected signing errors are not converted
+     * into this result and still throw.
+     */
+    createPresignedPost: (params: {
+        s3Uri: S3Uri.TerminatedByDelimiter;
+        validityDurationSecond: number;
+        maxObjectSizeInBytes: number | undefined;
+    }) => Promise<S3Client.CreatePresignedPostReturn>;
+
     getUnsignedObjectHttpUrl: (params: {
         s3Uri: S3Uri.NonTerminatedByDelimiter;
         isForDirectDownload: boolean;
@@ -77,6 +95,22 @@ export type S3Client = {
 export namespace S3Client {
     export type BucketPolicies = Record<string, unknown>;
 
+    export type PresignedPost = {
+        url: string;
+        fields: Record<string, string>;
+        expirationTime: number;
+    };
+
+    export type CreatePresignedPostReturn =
+        | {
+              isSuccess: true;
+              presignedPost: PresignedPost;
+          }
+        | {
+              isSuccess: false;
+              errorMessage: string;
+          };
+
     export type ListObjectsReturn = ListObjectsReturn.Error | ListObjectsReturn.Success;
 
     export namespace ListObjectsReturn {

web/src/core/usecases/index.ts

@@ -27,6 +27,8 @@ import * as s3ProfilesManagement from "./s3ProfilesManagement";
 import * as s3ShareObjectUiController from "./s3ShareObjectUiController";
 import * as s3ProfilesCreationUiController from "./s3ProfilesCreationUiController";
 import * as s3ExplorerUiController from "./s3ExplorerUiController";
+import * as s3FileRequestUiController from "./s3FileRequestUiController";
+import * as s3FileRequestCreationUiController from "./s3FileRequestCreationUiController";
 
 export const usecases = {
     aiProvidersManagements,
@@ -57,5 +59,7 @@ export const usecases = {
     s3ProfilesManagement,
     s3ShareObjectUiController,
     s3ProfilesCreationUiController,
-    s3ExplorerUiController
+    s3ExplorerUiController,
+    s3FileRequestUiController,
+    s3FileRequestCreationUiController
 };

web/src/core/usecases/s3ExplorerUiController/decoupledLogic/bucketPolicies.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import { assert } from "tsafe/assert";
 import { parseS3Uri, type S3Uri } from "core/tools/S3Uri";
 import {
+    getHasBucketPolicies,
     getHasPrefixBeMadePublic,
     getIsWithinPrefixThatHasBeenMadePublic,
     makePrefixPublic,
@@ -34,22 +35,45 @@ function getBucketPoliciesByBucket(bucketPolicies: BucketPolicies) {
 }
 
 describe("bucketPolicies", () => {
-    it("assumes private when the bucket policy is unavailable", () => {
+    it("reports unavailable bucket policies and rejects operations that require them", () => {
         const s3Uri = parsePrefix("s3://mybucket/public/");
 
         expect(
-            getHasPrefixBeMadePublic({
+            getHasBucketPolicies({
                 s3Uri,
                 bucketPoliciesByBucket: {}
             })
         ).toBe(false);
 
         expect(
+            getHasBucketPolicies({
+                s3Uri,
+                bucketPoliciesByBucket: {
+                    mybucket: { bucketPolicies: undefined }
+                }
+            })
+        ).toBe(false);
+
+        expect(
+            getHasBucketPolicies({
+                s3Uri,
+                bucketPoliciesByBucket: getBucketPoliciesByBucket({})
+            })
+        ).toBe(true);
+
+        expect(() =>
+            getHasPrefixBeMadePublic({
+                s3Uri,
+                bucketPoliciesByBucket: {}
+            })
+        ).toThrow();
+
+        expect(() =>
             getIsWithinPrefixThatHasBeenMadePublic({
                 s3Uri,
                 bucketPoliciesByBucket: {}
             })
-        ).toStrictEqual({ isWithinPrefixThatHasBeenMadePublic: false });
+        ).toThrow();
 
         expect(() =>
             makePrefixPublic({
@@ -173,6 +197,60 @@ describe("bucketPolicies", () => {
         ).toStrictEqual({ isWithinPrefixThatHasBeenMadePublic: false });
     });
 
+    it("supports making the bucket root public", () => {
+        const bucketRoot = parsePrefix("s3://mybucket/");
+
+        const { updatedBucketPolicies } = makePrefixPublic({
+            s3Uri: bucketRoot,
+            bucketPoliciesByBucket: getBucketPoliciesByBucket({
+                Version: "2012-10-17",
+                Statement: []
+            })
+        });
+
+        const statements = updatedBucketPolicies.Statement;
+
+        assert(Array.isArray(statements));
+        expect(statements).toHaveLength(2);
+        expect(statements[0]).toMatchObject({
+            Sid: "OnyxiaMakePrefixPublicGetObject",
+            Resource: ["arn:aws:s3:::mybucket/*"]
+        });
+        expect(statements[1]).toMatchObject({
+            Sid: "OnyxiaMakePrefixPublicListBucket",
+            Condition: {
+                StringLike: {
+                    "s3:prefix": ["*"]
+                }
+            }
+        });
+
+        const bucketPoliciesByBucket = getBucketPoliciesByBucket(updatedBucketPolicies);
+
+        expect(
+            getHasPrefixBeMadePublic({
+                s3Uri: bucketRoot,
+                bucketPoliciesByBucket
+            })
+        ).toBe(true);
+        expect(
+            getIsWithinPrefixThatHasBeenMadePublic({
+                s3Uri: parseObject("s3://mybucket/nested/file.csv"),
+                bucketPoliciesByBucket
+            })
+        ).toStrictEqual({
+            isWithinPrefixThatHasBeenMadePublic: true,
+            s3Uri_publicPrefix: bucketRoot
+        });
+
+        expect(
+            undoMakePrefixPublic({
+                s3Uri: bucketRoot,
+                bucketPoliciesByBucket
+            }).updatedBucketPolicies.Statement
+        ).toStrictEqual([]);
+    });
+
     it("returns the public prefix that contains an object", () => {
         const updatedBucketPolicies = makePrefixPublic({
             s3Uri: parsePrefix("s3://mybucket/foo/"),

web/src/core/usecases/s3ExplorerUiController/decoupledLogic/bucketPolicies.ts

@@ -8,7 +8,7 @@ export type BucketPolicies = Record<string, unknown>;
 
 assert<Equals<S3Client.BucketPolicies, BucketPolicies>>;
 
-type BucketPoliciesByBucket = Record<
+export type BucketPoliciesByBucket = Record<
     string,
     { bucketPolicies: BucketPolicies | undefined } | undefined
 >;
@@ -40,6 +40,17 @@ type ManagedPrefixKeysByKind = {
     listBucket: string[];
 };
 
+export function getHasBucketPolicies(params: {
+    s3Uri: S3Uri;
+    bucketPoliciesByBucket: BucketPoliciesByBucket;
+}): boolean {
+    const { s3Uri, bucketPoliciesByBucket } = params;
+
+    const bucketPolicies = getBucketPolicies({ s3Uri, bucketPoliciesByBucket });
+
+    return bucketPolicies !== undefined;
+}
+
 export function getHasPrefixBeMadePublic(params: {
     s3Uri: S3Uri.TerminatedByDelimiter;
     bucketPoliciesByBucket: BucketPoliciesByBucket;
@@ -48,9 +59,7 @@ export function getHasPrefixBeMadePublic(params: {
 
     const bucketPolicies = getBucketPolicies({ s3Uri, bucketPoliciesByBucket });
 
-    if (bucketPolicies === undefined) {
-        return false;
-    }
+    assert(bucketPolicies !== undefined);
 
     const prefixKey = getS3UriKey(s3Uri);
 
@@ -76,9 +85,7 @@ export function getIsWithinPrefixThatHasBeenMadePublic(params: {
 
     const bucketPolicies = getBucketPolicies({ s3Uri, bucketPoliciesByBucket });
 
-    if (bucketPolicies === undefined) {
-        return { isWithinPrefixThatHasBeenMadePublic: false };
-    }
+    assert(bucketPolicies !== undefined);
 
     const s3UriKey = getS3UriKey(s3Uri);
 

web/src/core/usecases/s3ExplorerUiController/decoupledLogic/computeUploadStatusAtPrefix.ts

@@ -72,8 +72,9 @@ export function computeUploadStatusAtPrefix(params: {
                 displayName,
                 s3Uri: s3Uri_newItem,
                 isDeleting: false,
-                policy: { isPublic: false, canBeMadePublic: false },
-                profileNameForSharing: undefined,
+                publicAccessAction: undefined,
+                shouldShowShareAction: false,
+                shouldShowRequestFilesAction: false,
                 uploadProgressPercent: NaN
             });
         }