diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index a80c16515..e90ec8492 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -63,6 +63,14 @@ jobs: language: java version: "21" architecture: "arm64" + - platform: aws + language: dotnet + version: "10" + architecture: "x64" + - platform: aws + language: dotnet + version: "10" + architecture: "arm64" - platform: gcp language: python version: "3.11" diff --git a/CHANGELOG.md b/CHANGELOG.md index 406922e01..176e002a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Features +* Object storage for Java and .NET on AWS: `Storage.java` (AWS SDK v2) and `Storage.cs` (AWSSDK.S3) implement the interface of the Python wrapper's `storage` class. Both are added only to benchmarks declaring the `storage` module, through the new `module_files` and `module_packages` entries in `configs/systems.json`, so benchmarks without storage keep their package size and cold start. + * Cold start enforcement for the local deployment: SeBS replaces the function container, and the local function servers report the first invocation of a container as cold. Requires the updated local run images. * [RustFS](https://github.com/rustfs/rustfs) is the new default S3-compatible object storage for local and OpenWhisk deployments. Minio is deprecated and kept for reproducibility (`configs/storage-minio.json`). The self-hosted object storage implementation is now shared between backends. diff --git a/benchmarks/000.microbenchmarks/010.sleep/config.json b/benchmarks/000.microbenchmarks/010.sleep/config.json index 86b4cc64c..fe134a5a7 100644 --- a/benchmarks/000.microbenchmarks/010.sleep/config.json +++ b/benchmarks/000.microbenchmarks/010.sleep/config.json @@ -1,7 +1,7 @@ { "timeout": 120, "memory": 128, - "languages": ["python", "nodejs", "java", "cpp"], + "languages": ["python", "nodejs", "java", "cpp", "dotnet"], "modules": [], "cpp_dependencies": [] } diff --git a/benchmarks/000.microbenchmarks/010.sleep/dotnet/Function.cs b/benchmarks/000.microbenchmarks/010.sleep/dotnet/Function.cs new file mode 100644 index 000000000..22263d8da --- /dev/null +++ b/benchmarks/000.microbenchmarks/010.sleep/dotnet/Function.cs @@ -0,0 +1,20 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +using System; +using System.Diagnostics; +using System.Text.Json.Nodes; +using System.Threading; + +namespace Benchmark; + +public class Function +{ + public JsonObject Handler(JsonObject input) + { + double seconds = input["sleep"]?.GetValue() + ?? throw new ArgumentException("expected a number in 'sleep'"); + + var stopwatch = Stopwatch.StartNew(); + Thread.Sleep(TimeSpan.FromSeconds(seconds)); + return new JsonObject { ["result"] = stopwatch.Elapsed.TotalSeconds }; + } +} diff --git a/benchmarks/000.microbenchmarks/010.sleep/dotnet/function.csproj b/benchmarks/000.microbenchmarks/010.sleep/dotnet/function.csproj new file mode 100644 index 000000000..d71f1b279 --- /dev/null +++ b/benchmarks/000.microbenchmarks/010.sleep/dotnet/function.csproj @@ -0,0 +1,17 @@ + + + + + net$(DOTNET_VERSION).0 + + function + enable + + true + + + + + + + diff --git a/benchmarks/wrappers/aws/dotnet/Handler.cs b/benchmarks/wrappers/aws/dotnet/Handler.cs new file mode 100644 index 000000000..cddd8e557 --- /dev/null +++ b/benchmarks/wrappers/aws/dotnet/Handler.cs @@ -0,0 +1,90 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +// The benchmark must define Benchmark.Function with JsonObject Handler(JsonObject input). + +using System; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using Amazon.Lambda.Core; + +namespace ServerlessBench; + +public class Handler +{ + private const string ColdStartMarker = "/tmp/cold_run"; + + // Skip HTML-safe escaping, matching the other wrappers' output. + private static readonly JsonSerializerOptions JsonOptions = + new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; + + public Stream HandleRequest(Stream input, ILambdaContext context) + { + JsonObject evt = ParseEvent(input); + + DateTimeOffset begin = DateTimeOffset.UtcNow; + JsonObject result = new Benchmark.Function().Handler(evt); + DateTimeOffset end = DateTimeOffset.UtcNow; + + (bool isCold, string containerId) = CheckColdStart(); + + var body = new JsonObject + { + ["begin"] = FormatTimestamp(begin), + ["end"] = FormatTimestamp(end), + ["results_time"] = 0, + ["is_cold"] = isCold, + ["result"] = result, + ["request_id"] = context.AwsRequestId, + ["cold_start_var"] = Environment.GetEnvironmentVariable("cold_start") ?? "", + ["container_id"] = containerId, + }; + var response = new JsonObject + { + ["statusCode"] = 200, + ["body"] = body.ToJsonString(JsonOptions), + }; + return new MemoryStream(Encoding.UTF8.GetBytes(response.ToJsonString(JsonOptions))); + } + + // HTTP triggers wrap the input in a JSON string under "body". + private static JsonObject ParseEvent(Stream input) + { + JsonObject evt = JsonNode.Parse(input) as JsonObject ?? new JsonObject(); + if (evt["body"] is JsonValue body && body.TryGetValue(out string? text) + && JsonNode.Parse(text) is JsonObject parsed) + { + return parsed; + } + return evt; + } + + private static (bool, string) CheckColdStart() + { + if (File.Exists(ColdStartMarker)) + { + return (false, File.ReadAllText(ColdStartMarker)); + } + string containerId = Guid.NewGuid().ToString("N").Substring(0, 8); + try + { + File.WriteAllText(ColdStartMarker, containerId); + } + catch (IOException) + { + // Best effort. + } + return (true, containerId); + } + + // seconds.microseconds, like the other wrappers. + private static string FormatTimestamp(DateTimeOffset ts) + { + long micros = (ts.UtcTicks - DateTimeOffset.UnixEpoch.UtcTicks) / 10; + return string.Format( + CultureInfo.InvariantCulture, "{0}.{1:D6}", micros / 1_000_000, micros % 1_000_000); + } +} diff --git a/benchmarks/wrappers/aws/dotnet/Storage.cs b/benchmarks/wrappers/aws/dotnet/Storage.cs new file mode 100644 index 000000000..e361c2d3a --- /dev/null +++ b/benchmarks/wrappers/aws/dotnet/Storage.cs @@ -0,0 +1,111 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +// Object storage, the counterpart of the Python wrapper's `storage` class. +// Added to the deployment package only for benchmarks declaring the `storage` module, +// because the SDK dominates the package size and the cold start. + +using System; +using System.Collections.Generic; +using System.IO; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; + +namespace ServerlessBench; + +public sealed class Storage +{ + private static Storage? instance; + + private readonly IAmazonS3 client; + private readonly TransferUtility transfer; + + private Storage() + { + this.client = new AmazonS3Client(); + this.transfer = new TransferUtility(this.client); + } + + // Created lazily, like the Python wrapper, so that the SDK's start-up cost falls into the + // first invocation rather than the platform's initialization phase. + public static Storage GetInstance() => instance ??= new Storage(); + + // Insert a random suffix before the file extension, as the other wrappers do. + public static string UniqueName(string name) + { + int separator = name.LastIndexOf('/'); + int dot = name.LastIndexOf('.'); + string file = name; + string extension = ""; + if (dot > separator) + { + file = name.Substring(0, dot); + extension = name.Substring(dot); + } + string random = Guid.NewGuid().ToString("N").Substring(0, 8); + return $"{file}.{random}{extension}"; + } + + // Uploads a local file under a randomized key and returns that key. + public string Upload(string bucket, string file, string filepath) + { + string keyName = UniqueName(file); + this.transfer.UploadAsync(filepath, bucket, keyName).GetAwaiter().GetResult(); + return keyName; + } + + // Downloads an object to a local file, creating parent directories as needed. + public void Download(string bucket, string file, string filepath) + { + string? directory = Path.GetDirectoryName(filepath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + this.transfer.DownloadAsync(filepath, bucket, file).GetAwaiter().GetResult(); + } + + // Downloads every object with the given prefix, recreating the key structure under a path. + public void DownloadDirectory(string bucket, string prefix, string path) + { + var request = new ListObjectsV2Request { BucketName = bucket, Prefix = prefix }; + bool found = false; + do + { + ListObjectsV2Response objects = + this.client.ListObjectsV2Async(request).GetAwaiter().GetResult(); + foreach (S3Object entry in objects.S3Objects ?? new List()) + { + found = true; + Download(bucket, entry.Key, Path.Combine(path, entry.Key)); + } + request.ContinuationToken = objects.NextContinuationToken; + } + while (request.ContinuationToken != null); + + if (!found) + { + throw new IOException( + $"No objects found in bucket '{bucket}' with prefix '{prefix}'"); + } + } + + // Uploads in-memory data under a randomized key and returns that key. + public string UploadStream(string bucket, string file, byte[] data) + { + string keyName = UniqueName(file); + using var stream = new MemoryStream(data); + this.transfer.UploadAsync(stream, bucket, keyName).GetAwaiter().GetResult(); + return keyName; + } + + // Downloads an object into memory. + public byte[] DownloadStream(string bucket, string file) + { + using Stream stream = + this.transfer.OpenStreamAsync(bucket, file).GetAwaiter().GetResult(); + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + return buffer.ToArray(); + } +} diff --git a/benchmarks/wrappers/aws/java/modules/storage/src/main/java/org/serverlessbench/Storage.java b/benchmarks/wrappers/aws/java/modules/storage/src/main/java/org/serverlessbench/Storage.java new file mode 100644 index 000000000..8d8e80358 --- /dev/null +++ b/benchmarks/wrappers/aws/java/modules/storage/src/main/java/org/serverlessbench/Storage.java @@ -0,0 +1,167 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +package org.serverlessbench; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +/** + * Object storage, the counterpart of the Python wrapper's {@code storage} class. + * + *

Added to the deployment package only for benchmarks declaring the {@code storage} module, + * because the SDK dominates the package size and the cold start. + */ +public final class Storage { + + private static Storage instance = null; + + private final S3Client client; + + private Storage() { + // The URL-connection client keeps the jar small; the default client pulls in Netty. + this.client = S3Client.builder().httpClient(UrlConnectionHttpClient.create()).build(); + } + + /** + * Returns the shared instance, creating it on first use. + * + *

Created lazily, like the Python wrapper, so that the SDK's start-up cost falls into the + * first invocation rather than the platform's initialization phase. + * + * @return the storage instance for this container + */ + public static synchronized Storage getInstance() { + if (instance == null) { + instance = new Storage(); + } + return instance; + } + + /** + * Inserts a random suffix before the file extension, as the other wrappers do. + * + * @param name key or file name to make unique + * @return the name with a random suffix, e.g. {@code archive.1a2b3c4d.zip} + */ + public static String uniqueName(String name) { + int separator = name.lastIndexOf('/'); + int dot = name.lastIndexOf('.'); + String base = name; + String extension = ""; + if (dot > separator) { + base = name.substring(0, dot); + extension = name.substring(dot); + } + String random = UUID.randomUUID().toString().split("-")[0]; + return base + "." + random + extension; + } + + /** + * Uploads a local file under a randomized key. + * + * @param bucket bucket to upload to + * @param file key to derive the randomized key from + * @param filepath local file to upload + * @return the key the object was stored under + */ + public String upload(String bucket, String file, String filepath) { + String keyName = uniqueName(file); + this.client.putObject( + PutObjectRequest.builder().bucket(bucket).key(keyName).build(), + RequestBody.fromFile(Paths.get(filepath))); + return keyName; + } + + /** + * Downloads an object to a local file, creating parent directories as needed. + * + * @param bucket bucket to download from + * @param file key of the object + * @param filepath local destination path + * @throws IOException when the destination cannot be prepared + */ + public void download(String bucket, String file, String filepath) throws IOException { + Path destination = Paths.get(filepath); + Path parent = destination.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + // The SDK refuses to overwrite an existing file. + Files.deleteIfExists(destination); + this.client.getObject( + GetObjectRequest.builder().bucket(bucket).key(file).build(), destination); + } + + /** + * Downloads every object with the given prefix, recreating the key structure under a path. + * + * @param bucket bucket to download from + * @param prefix key prefix selecting the objects + * @param path local directory to download into + * @throws IOException when a destination cannot be prepared + */ + public void downloadDirectory(String bucket, String prefix, String path) throws IOException { + ListObjectsV2Request.Builder request = + ListObjectsV2Request.builder().bucket(bucket).prefix(prefix); + String continuationToken = null; + boolean found = false; + do { + ListObjectsV2Response objects = + this.client.listObjectsV2(request.continuationToken(continuationToken).build()); + for (S3Object object : objects.contents()) { + found = true; + download(bucket, object.key(), Paths.get(path, object.key()).toString()); + } + continuationToken = objects.nextContinuationToken(); + } while (continuationToken != null); + + if (!found) { + throw new IOException( + String.format( + "No objects found in bucket '%s' with prefix '%s'", bucket, prefix)); + } + } + + /** + * Uploads in-memory data under a randomized key. + * + * @param bucket bucket to upload to + * @param file key to derive the randomized key from + * @param data bytes to upload + * @return the key the object was stored under + */ + public String uploadStream(String bucket, String file, byte[] data) { + String keyName = uniqueName(file); + this.client.putObject( + PutObjectRequest.builder().bucket(bucket).key(keyName).build(), + RequestBody.fromBytes(data)); + return keyName; + } + + /** + * Downloads an object into memory. + * + * @param bucket bucket to download from + * @param file key of the object + * @return the object's bytes + */ + public byte[] downloadStream(String bucket, String file) { + ResponseBytes response = + this.client.getObjectAsBytes( + GetObjectRequest.builder().bucket(bucket).key(file).build()); + return response.asByteArray(); + } +} diff --git a/configs/dotnet.json b/configs/dotnet.json new file mode 100644 index 000000000..bb2dceac7 --- /dev/null +++ b/configs/dotnet.json @@ -0,0 +1,108 @@ +{ + "experiments": { + "deployment": "aws", + "update_code": false, + "update_storage": false, + "download_results": false, + "architecture": "x64", + "system_variant": "package", + "runtime": { + "language": "dotnet", + "version": "10" + }, + "type": "invocation-overhead", + "perf-cost": { + "benchmark": "110.dynamic-html", + "experiments": ["cold", "warm", "burst", "sequential"], + "input-size": "test", + "repetitions": 50, + "concurrent-invocations": 50, + "memory-sizes": [128, 256] + }, + "network-ping-pong": { + "invocations": 50, + "repetitions": 1000, + "threads": 1 + }, + "invocation-overhead": { + "repetitions": 5, + "N": 20, + "type": "payload", + "payload_begin": 1024, + "payload_end": 6251000, + "payload_points": 20, + "code_begin": 1048576, + "code_end": 261619712, + "code_points": 20 + }, + "eviction-model": { + "invocations": 1, + "function_copy_idx": 0, + "repetitions": 5, + "sleep": 1 + } + }, + "deployment": { + "name": "aws", + "aws": { + "region": "us-east-1", + "lambda-role": "", + "resources": { + "use-function-url": true, + "function-url-auth-type": "NONE" + } + }, + "azure": { + "region": "westeurope" + }, + "gcp": { + "region": "europe-west1", + "project_name": "", + "credentials": "", + "configuration": { + "function-gen1": { + "min-instances": 0, + "max-instances": 20 + }, + "function-gen2": { + "vcpus": 1, + "gcp-concurrency": 80, + "worker-concurrency": 1, + "worker-threads": 8, + "min-instances": 0, + "max-instances": 20, + "cpu-boost": false, + "cpu-throttle": true + }, + "container": { + "environment": "gen2", + "vcpus": 1, + "gcp-concurrency": 80, + "worker-concurrency": 1, + "worker-threads": 8, + "min-instances": 0, + "max-instances": 20, + "cpu-boost": false, + "cpu-throttle": true + } + } + }, + "local": {}, + "openwhisk": { + "shutdownStorage": false, + "removeCluster": false, + "wskBypassSecurity": "true", + "wskExec": "wsk", + "experimentalManifest": false, + "removeStaleContainers": true, + "kubectlExec": "kubectl", + "kubernetesNamespace": "openwhisk", + "dockerhubRepository": null, + "docker_registry": { + "registry": "", + "username": "", + "password": "" + } + } + } +} diff --git a/configs/systems.json b/configs/systems.json index a3640c1f5..c63278457 100644 --- a/configs/systems.json +++ b/configs/systems.json @@ -170,7 +170,24 @@ "com.amazonaws:aws-lambda-java-core": "1.2.3", "com.fasterxml.jackson.core:jackson-databind": "2.17.1" }, - "module_packages": {} + "module_packages": { + "storage": { + "software.amazon.awssdk:s3": { + "version": "2.46.7", + "exclusions": [ + "software.amazon.awssdk:netty-nio-client", + "software.amazon.awssdk:apache-client", + "software.amazon.awssdk:apache5-client" + ] + }, + "software.amazon.awssdk:url-connection-client": "2.46.7" + } + }, + "module_files": { + "storage": [ + "modules/storage/src" + ] + } } }, "cpp": { @@ -192,6 +209,37 @@ "deployment": { "files": [ "handler.cpp", "key-value.cpp", "key-value.hpp", "storage.cpp", "storage.hpp", "redis.hpp", "redis.cpp", "utils.cpp", "utils.hpp"] } + }, + "dotnet": { + "base_images": { + "x64": { + "10": "amazon/aws-lambda-dotnet:10.2026.11.11.03-x86_64" + }, + "arm64": { + "10": "amazon/aws-lambda-dotnet:10.2026.11.11.03-arm64" + } + }, + "images": [ + "build" + ], + "deployment": { + "files": [ + "Handler.cs" + ], + "packages": { + "Amazon.Lambda.Core": "3.4.0" + }, + "module_packages": { + "storage": { + "AWSSDK.S3": "4.1.1" + } + }, + "module_files": { + "storage": [ + "Storage.cs" + ] + } + } } }, "architecture": [ diff --git a/dockerfiles/aws/dotnet/Dockerfile.build b/dockerfiles/aws/dotnet/Dockerfile.build new file mode 100644 index 000000000..5a140aef8 --- /dev/null +++ b/dockerfiles/aws/dotnet/Dockerfile.build @@ -0,0 +1,54 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +ARG VERSION +# Read by function.csproj. Overrides the base image's informational runtime version. +ENV DOTNET_VERSION=${VERSION} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# shadow-utils provides useradd and groupmod for the entrypoint. +RUN dnf install -y shadow-utils tar gzip && dnf clean all + +# The base image has only the runtime. SDK releases and checksums: +# https://builds.dotnet.microsoft.com/dotnet/release-metadata/.0/releases.json +RUN case "${VERSION}" in \ + 10) SDK_RELEASE=10.0.404; \ + SDK_SHA512_X64=577d0eb17fec33d029104c5c63a1ab0accc3e28776274abef2218dbac5af18d2109b5406d26b31d76f5c245215e3ba0926a94c20246c672c41847c9e1eb1dc50; \ + SDK_SHA512_ARM64=3533a3869da219cac45c567aadac3d8e96cdf1c6a04fe1c1d5fdeefea5d45af8424488f0144730d11b4a751f7ad546fb11e4c2b2725cd8401efee726711f8f11 ;; \ + *) echo "No pinned .NET SDK for version ${VERSION}" >&2; exit 1 ;; \ + esac && \ + case "$(uname -m)" in \ + x86_64) SDK_ARCH=x64; SDK_SHA512=${SDK_SHA512_X64} ;; \ + aarch64) SDK_ARCH=arm64; SDK_SHA512=${SDK_SHA512_ARM64} ;; \ + *) echo "Unsupported build architecture $(uname -m)" >&2; exit 1 ;; \ + esac && \ + curl -fsSL -o /tmp/sdk.tar.gz \ + "https://builds.dotnet.microsoft.com/dotnet/Sdk/${SDK_RELEASE}/dotnet-sdk-${SDK_RELEASE}-linux-${SDK_ARCH}.tar.gz" && \ + echo "${SDK_SHA512} /tmp/sdk.tar.gz" | sha512sum -c - && \ + mkdir -p /opt/dotnet && \ + tar -C /opt/dotnet -xzf /tmp/sdk.tar.gz && \ + rm /tmp/sdk.tar.gz +ENV DOTNET_ROOT=/opt/dotnet +ENV PATH=/opt/dotnet:$PATH +# Silences a first-run warning; benchmarks use no workloads. +ENV DOTNET_SKIP_WORKLOAD_INTEGRITY_CHECK=1 + +ENV GOSU_VERSION=1.14 +# Architecture from uname: single-platform builds leave TARGETARCH empty. +RUN case "$(uname -m)" in \ + x86_64) GOSU_ARCH=amd64 ;; \ + aarch64) GOSU_ARCH=arm64 ;; \ + esac && \ + curl -fsSL -o /usr/local/bin/gosu \ + "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${GOSU_ARCH}" && \ + chmod +x /usr/local/bin/gosu +RUN mkdir -p /sebs/ +COPY dockerfiles/dotnet_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh /sebs/installer.sh + +# useradd and groupmod are installed in /usr/sbin, which is not in PATH +ENV PATH=/usr/sbin:$PATH +ENV SCRIPT_FILE=/mnt/function/package.sh +CMD ["/bin/bash", "/sebs/installer.sh"] +ENTRYPOINT ["/sebs/entrypoint.sh"] diff --git a/dockerfiles/aws/dotnet/Dockerfile.function b/dockerfiles/aws/dotnet/Dockerfile.function new file mode 100644 index 000000000..83ba3bceb --- /dev/null +++ b/dockerfiles/aws/dotnet/Dockerfile.function @@ -0,0 +1,11 @@ +ARG BASE_IMAGE +ARG BASE_IMAGE_BUILDER + +FROM ${BASE_IMAGE_BUILDER} AS builder +ARG TARGET_ARCHITECTURE +COPY . /mnt/function +RUN TARGET_ARCHITECTURE=${TARGET_ARCHITECTURE} /bin/bash /sebs/installer.sh + +FROM ${BASE_IMAGE} +COPY --from=builder /mnt/function/publish/ ${LAMBDA_TASK_ROOT}/ +CMD ["function::ServerlessBench.Handler::HandleRequest"] diff --git a/dockerfiles/dotnet_installer.sh b/dockerfiles/dotnet_installer.sh new file mode 100644 index 000000000..9edc1fb9b --- /dev/null +++ b/dockerfiles/dotnet_installer.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Builds a .NET benchmark into publish/. +set -euo pipefail + +cd /mnt/function + +case "${TARGET_ARCHITECTURE:-x64}" in + x64) RID=linux-x64 ;; + arm64) RID=linux-arm64 ;; + *) + echo "Unsupported target architecture ${TARGET_ARCHITECTURE}" >&2 + exit 1 + ;; +esac + +# MSBuild would read SeBS's PLATFORM variable as the build platform. +unset PLATFORM + +dotnet --version +# Framework-dependent and without ReadyToRun, like Java's default build. +dotnet publish function.csproj -c Release -r "${RID}" --self-contained false -o publish +echo ".NET package size $(du -sb publish | cut -f1) bytes" + +if [[ -f "${SCRIPT_FILE:-}" ]]; then + /bin/bash "${SCRIPT_FILE}" . +fi diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9e5486285..aad296cc8 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -2,7 +2,7 @@ | Type | Benchmark | Languages | Architecture | Description | | :--- | :---: | :---: | :---: | :---: | -| Webapps | 010.sleep | Python, Node.js, C++, Java | x64, arm64 | Customizable sleep microbenchmark. | +| Webapps | 010.sleep | Python, Node.js, C++, Java, .NET | x64, arm64 | Customizable sleep microbenchmark. | | Webapps | 110.dynamic-html | Python, Node.js, Java | x64, arm64 | Generate dynamic HTML from a template. | | Webapps | 120.uploader | Python, Node.js | x64, arm64 | Uploader file from provided URL to cloud storage. | | Webapps | 130.crud-api | Python, Node.js | x64, arm64 | Simple CRUD application using NoSQL to store application data. | diff --git a/docs/build.md b/docs/build.md index c6c7f4bab..1f92b9be0 100644 --- a/docs/build.md +++ b/docs/build.md @@ -67,7 +67,7 @@ pushing images to a custom registry. For example, a local registry deployment ca ### AWS Lambda -We support deploying functions as Docker images for Python, Node.js, and C++ functions. We also support building arm64 images for this platform, except for C++ functions (extension to ARM functions is planned in the future). +We support deploying functions as Docker images for Python, Node.js, C++, and .NET functions. We also support building arm64 images for this platform, except for C++ functions (extension to ARM functions is planned in the future). ### OpenWhisk diff --git a/docs/modularity.md b/docs/modularity.md index 7a1e44784..444b85239 100644 --- a/docs/modularity.md +++ b/docs/modularity.md @@ -85,7 +85,7 @@ def validate_output( Args: input_config: The input configuration dict used to invoke the benchmark output: The output dict returned by the benchmark function handler - language: Programming language used (e.g., 'python', 'nodejs', 'cpp', 'java') + language: Programming language used (e.g., 'python', 'nodejs', 'cpp', 'java', 'dotnet') architecture: CPU architecture (e.g., 'x64', 'arm64') storage: Optional PersistentStorage client for downloading/verifying files @@ -133,7 +133,7 @@ For each benchmark, the `input.py` file provides a generation of inputs and uplo ## New Languages -At the moment, SeBS implements benchmarks in Python and Node.js. However, it is very easy to add new languages, +At the moment, SeBS implements benchmarks in Python, Node.js, Java, C++, and .NET. However, it is very easy to add new languages, and it requires only minor modifications. ### Language Name @@ -240,6 +240,27 @@ and `storage` which hides the cloud API of persistent storage. For example, examine the existing implementations in [`benchmarks/wrappers/`](/benchmarks/wrappers/). +Wrappers named in `deployment.files` are added to every benchmark. A wrapper whose SDK +is large enough to affect the package size and the cold start belongs in `deployment.module_files` +instead, keyed by the benchmark module that needs it; it is added only to benchmarks declaring +that module in their `config.json`. Its dependencies belong in `deployment.module_packages`, +keyed the same way. Languages naming dependencies as `"package": version` pairs, such as Java +and .NET, may replace the version with an object to drop transitive dependencies: + +```json +"module_packages": { + "storage": { + "software.amazon.awssdk:s3": { + "version": "2.46.7", + "exclusions": ["software.amazon.awssdk:netty-nio-client"] + } + } +}, +"module_files": { + "storage": ["modules/storage/src"] +} +``` + ### Adapt Platform Mode The final step is adjusting the SeBS code whenever we make a language-specific decision. diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index dd7077d18..b1a42646d 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -13,6 +13,7 @@ import shutil import time import uuid +import zipfile from typing import cast, Dict, List, Optional, Tuple, Type, Union # noqa import boto3 @@ -274,6 +275,25 @@ def package_code( mbytes = bytes_size / 1024.0 / 1024.0 self.logging.info("Zip archive size {:2f} MB".format(mbytes)) + elif language == Language.DOTNET: + # Assemblies go at the archive root. + publish_dir = os.path.join(directory, "publish") + if not os.path.isdir(publish_dir): + raise RuntimeError( + f".NET output {publish_dir} missing. Ensure the .NET build produced it." + ) + benchmark_archive = os.path.join(directory, "{}.zip".format(benchmark)) + with zipfile.ZipFile(benchmark_archive, "w", zipfile.ZIP_DEFLATED) as archive: + for root, _, files in os.walk(publish_dir): + for file in files: + path = os.path.join(root, file) + archive.write(path, os.path.relpath(path, publish_dir)) + self.logging.info("Created {} archive".format(benchmark_archive)) + + bytes_size = os.path.getsize(benchmark_archive) + mbytes = bytes_size / 1024.0 / 1024.0 + self.logging.info("Zip archive size {:2f} MB".format(mbytes)) + else: raise NotImplementedError() @@ -313,6 +333,8 @@ def cloud_runtime(self, language: Language, language_version: str): return "provided.al2023" elif language == Language.JAVA: return f"{language}{language_version}" + elif language == Language.DOTNET: + return f"{language}{language_version}" elif language in [Language.PYTHON]: return f"{language}{language_version}" else: @@ -414,6 +436,10 @@ def create_function( create_function_params["Runtime"] = self.cloud_runtime(language, language_runtime) if language == Language.JAVA: create_function_params["Handler"] = "org.serverlessbench.Handler::handleRequest" + elif language == Language.DOTNET: + # Assembly::Namespace.Type::Method + handler = "function::ServerlessBench.Handler::HandleRequest" + create_function_params["Handler"] = handler else: create_function_params["Handler"] = "handler.handler" diff --git a/sebs/benchmark.py b/sebs/benchmark.py index dff775f1c..966c249b3 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -686,12 +686,16 @@ def hash_directory( Language.NODEJS: ["*.js", "package.json"], Language.JAVA: ["*.java", "pom.xml"], Language.CPP: ["*.cpp", "*.hpp", "dependencies.json"], + Language.DOTNET: ["*.cs", "*.csproj"], } + # "modules" holds the wrappers added only for benchmarks using a module; they are + # hashed for every benchmark, which can invalidate a package that did not need them. WRAPPERS = { Language.PYTHON: ["*.py"], Language.NODEJS: ["*.js"], - Language.JAVA: ["src"], + Language.JAVA: ["src", "modules"], Language.CPP: ["*.cpp", "*.hpp"], + Language.DOTNET: ["*.cs"], } NON_LANG_FILES = ["*.sh", "*.json"] selected_files = FILES[language] + NON_LANG_FILES @@ -788,6 +792,7 @@ def copy_code(self, output_dir: str) -> None: Language.NODEJS: ["*.js", "package.json"], Language.JAVA: [], Language.CPP: ["*.cpp", "*.hpp", "dependencies.json"], + Language.DOTNET: ["*.cs", "*.csproj"], } path = os.path.join(self.benchmark_path, self.language_name) if self.language == Language.JAVA: @@ -910,6 +915,17 @@ def add_deployment_files(self, output_dir: str) -> None: ) ] + # Wrappers for modules the benchmark uses, e.g. object storage. They are kept out of + # the unconditional list because their dependencies increase the package size. + module_files = self._system_config.deployment_module_files( + self._deployment_name, self.language_name + ) + for bench_module in self._benchmark_config.modules: + handlers.extend( + os.path.join(handlers_dir, file) + for file in module_files.get(bench_module.value, []) + ) + for file in handlers: destination = os.path.join(output_dir, os.path.basename(file)) if os.path.isdir(file): @@ -977,7 +993,45 @@ def add_deployment_package_nodejs(self, output_dir: str) -> None: with open(package_config, "w") as package_file: json.dump(package_json, package_file, indent=2) - def format_maven_dependency(self, group_artifact: str, version: str) -> str: + def _module_packages(self) -> Dict[str, Any]: + """Collect the packages required by the modules this benchmark uses. + + Used by the Java and .NET package builders, whose dependencies are named as + "package": version pairs. Benchmarks without modules get no extra packages, keeping + their deployment package as small as it was. + + Returns: + Dict[str, Any]: Package names mapped to their specification. + """ + module_packages = self._system_config.deployment_module_package_versions( + self._deployment_name, self.language_name + ) + packages: Dict[str, Any] = {} + for bench_module in self._benchmark_config.modules: + packages.update(module_packages.get(bench_module.value, {})) + return packages + + @staticmethod + def parse_package_spec(spec: Any) -> Tuple[str, List[str]]: + """Split a dependency specification from systems.json into version and exclusions. + + A dependency is named either by its version alone, or by an object carrying the + version and the transitive dependencies to drop. + + Args: + spec: version string, or a dict with "version" and optional "exclusions" + + Returns: + Tuple[str, List[str]]: the version, and the excluded dependencies as + "group:artifact" names + """ + if isinstance(spec, dict): + return spec["version"], spec.get("exclusions", []) + return spec, [] + + def format_maven_dependency( + self, group_artifact: str, version: str, exclusions: Optional[List[str]] = None + ) -> str: """Helper method to format Java system dependencies. Dependencies in system.json are in "group:artifact": version format; this function converts them to proper Maven blocks. @@ -985,16 +1039,30 @@ def format_maven_dependency(self, group_artifact: str, version: str) -> str: Args: group_artifact: name of library to add to benchmark version: library version + exclusions: transitive dependencies to drop, as "group:artifact" names Returns: XML-formatted block inserted into pom.xml """ group_id, artifact_id = group_artifact.split(":") + exclusion_block = "" + if exclusions: + excluded = "".join( + f""" + + {excluded_artifact.split(":")[0]} + {excluded_artifact.split(":")[1]} + """ + for excluded_artifact in exclusions + ) + exclusion_block = f""" + {excluded} + """ return f""" {group_id} {artifact_id} - {version} + {version}{exclusion_block} """ def add_deployment_package_java(self, output_dir: str): @@ -1011,17 +1079,20 @@ def add_deployment_package_java(self, output_dir: str): with open(pom_path, "r") as f: pom_content = f.read() - packages = self._system_config.deployment_packages( - self._deployment_name, self.language_name + packages: Dict[str, Any] = dict( + self._system_config.deployment_packages(self._deployment_name, self.language_name) ) + packages.update(self._module_packages()) dependency_blocks = "" if len(packages): for key, val in packages.items(): dependency_name = key.strip('"').strip("'") - dependency_version = val.strip('"').strip("'") + version, exclusions = Benchmark.parse_package_spec(val) + dependency_version = version.strip('"').strip("'") dependency_blocks += ( - self.format_maven_dependency(dependency_name, dependency_version) + "\n" + self.format_maven_dependency(dependency_name, dependency_version, exclusions) + + "\n" ) if "" not in pom_content: @@ -1036,6 +1107,57 @@ def add_deployment_package_java(self, output_dir: str): with open(pom_path, "w") as f: f.write(pom_content) + def format_nuget_dependency(self, package: str, version: str) -> str: + """Helper method to format .NET system dependencies. + Dependencies in systems.json are in "package": version format; + this function converts them to MSBuild items. + + Args: + package: name of NuGet package to add to benchmark + version: package version + + Returns: + XML-formatted item inserted into function.csproj + """ + return f' ' + + def add_deployment_package_dotnet(self, output_dir: str) -> None: + """Extend benchmark's function.csproj with system-specific packages. + All .NET dependencies for each platform are defined in systems.json. + + Args: + output_dir: benchmark directory containing function.csproj to modify + + Raises: + ValueError: when benchmark's function.csproj is missing placeholder + """ + csproj_path = os.path.join(output_dir, "function.csproj") + with open(csproj_path, "r") as f: + csproj_content = f.read() + + packages: Dict[str, Any] = dict( + self._system_config.deployment_packages(self._deployment_name, self.language_name) + ) + packages.update(self._module_packages()) + + package_items = "" + for name, spec in packages.items(): + # NuGet resolves a single implementation per package, so .NET declares no exclusions. + version, _ = Benchmark.parse_package_spec(spec) + package_items += self.format_nuget_dependency(name, version) + "\n" + + if "" not in csproj_content: + raise ValueError( + "function.csproj template is missing placeholder" + ) + + csproj_content = csproj_content.replace( + "", package_items.strip() + ) + + with open(csproj_path, "w") as f: + f.write(csproj_content) + def add_deployment_package_cpp(self, output_dir: str) -> None: """Generates CMakeLists.txt file for C++ benchmark. @@ -1127,6 +1249,8 @@ def add_deployment_package(self, output_dir: str) -> None: self.add_deployment_package_java(output_dir) elif self.language == Language.CPP: self.add_deployment_package_cpp(output_dir) + elif self.language == Language.DOTNET: + self.add_deployment_package_dotnet(output_dir) else: raise NotImplementedError @@ -1273,6 +1397,7 @@ def ensure_image(name: str) -> None: Language.NODEJS: "package.json", Language.CPP: "CMakeLists.txt", Language.JAVA: "pom.xml", + Language.DOTNET: "function.csproj", } file = os.path.join(output_dir, PACKAGE_FILES[self.language]) if os.path.exists(file): diff --git a/sebs/cli.py b/sebs/cli.py index 59d794387..2537340bd 100644 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -81,7 +81,7 @@ def simplified_common_params(func): @click.option( "--language", default=None, - type=click.Choice(["python", "nodejs", "java", "cpp"]), + type=click.Choice(["python", "nodejs", "java", "cpp", "dotnet"]), help="Benchmark language", ) @click.option("--language-version", default=None, type=str, help="Benchmark language version") @@ -1044,7 +1044,7 @@ def docker_cmd(): @click.option( "--language", default=None, - type=click.Choice(["python", "nodejs", "java", "cpp"]), + type=click.Choice(["python", "nodejs", "java", "cpp", "dotnet"]), help="Programming language", ) @click.option("--language-version", default=None, type=str, help="Language version") @@ -1141,7 +1141,7 @@ def _exit_on_failed_images(operation: str, failed: List[str]) -> None: @click.option( "--language", default=None, - type=click.Choice(["python", "nodejs", "java", "cpp"]), + type=click.Choice(["python", "nodejs", "java", "cpp", "dotnet"]), help="Programming language", ) @click.option("--language-version", default=None, type=str, help="Language version") diff --git a/sebs/config.py b/sebs/config.py index 6609953e1..8f55759f8 100644 --- a/sebs/config.py +++ b/sebs/config.py @@ -11,7 +11,7 @@ """ import json -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from sebs.utils import get_resource_path @@ -102,6 +102,29 @@ def deployment_module_packages( "module_packages" ] + def deployment_module_package_versions( + self, deployment_name: str, language_name: str + ) -> Dict[str, Dict[str, Any]]: + """Get deployment module packages for languages declaring versions separately. + + Java and .NET name their dependencies as "package": version pairs, like + :func:`deployment_packages`, whereas Python and Node.js use a single specification + string per package. This accessor returns the former shape. A package needing + transitive dependencies dropped names them alongside its version, as an object + with "version" and "exclusions". + + Args: + deployment_name (str): Name of the deployment platform (e.g., 'aws', 'azure'). + language_name (str): Programming language name (e.g., 'java', 'dotnet'). + + Returns: + Dict[str, Dict[str, Any]]: Dictionary mapping benchmark modules to the packages + added for them, each mapping a package name to its version or specification. + """ + return self._system_config[deployment_name]["languages"][language_name]["deployment"].get( + "module_packages", {} + ) + def deployment_files(self, deployment_name: str, language_name: str) -> List[str]: """Get deployment files list for a specific deployment and language. @@ -116,6 +139,27 @@ def deployment_files(self, deployment_name: str, language_name: str) -> List[str "files" ] + def deployment_module_files( + self, deployment_name: str, language_name: str + ) -> Dict[str, List[str]]: + """Get wrapper files added only for benchmarks using a given module. + + Wrappers listed in :func:`deployment_files` are added to every benchmark, while these + are added only when the benchmark declares the module. Languages whose module wrappers + carry no extra dependencies do not need the distinction and omit the key. + + Args: + deployment_name (str): Name of the deployment platform (e.g., 'aws', 'azure'). + language_name (str): Programming language name (e.g., 'java', 'dotnet'). + + Returns: + Dict[str, List[str]]: Dictionary mapping benchmark modules to the wrapper files + added for them, relative to the language's wrapper directory. + """ + return self._system_config[deployment_name]["languages"][language_name]["deployment"].get( + "module_files", {} + ) + def docker_image_types(self, deployment_name: str, language_name: str) -> List[str]: """Get available Docker image types for a deployment and language. diff --git a/sebs/docker_builder.py b/sebs/docker_builder.py index 9785bb661..5e071dda8 100644 --- a/sebs/docker_builder.py +++ b/sebs/docker_builder.py @@ -85,7 +85,7 @@ def _should_use_multiplatform( """Check if multi-platform build should be used for this image. Multi-platform builds (x64 + arm64) are enabled for: - - AWS build images of Python, Node.js, and Java; + - AWS build images of Python, Node.js, Java, and .NET; - local build and run images of Python and Node.js: local functions run on the host, so x86 and ARM machines pull the image of their architecture. @@ -98,7 +98,7 @@ def _should_use_multiplatform( True if multi-platform build should be used, False otherwise """ if system == "aws": - return image_type == "build" and language in ["python", "nodejs", "java"] + return image_type == "build" and language in ["python", "nodejs", "java", "dotnet"] if system == "local": return image_type in ["build", "run"] and language in ["python", "nodejs"] return False diff --git a/sebs/faas/function.py b/sebs/faas/function.py index 01abe06f5..ab65a5048 100644 --- a/sebs/faas/function.py +++ b/sebs/faas/function.py @@ -575,6 +575,13 @@ class Cpp(Enum): DEFAULT = "default" + class Dotnet(Enum): + """.NET runtime variants. + Currently only the managed runtime. + """ + + DEFAULT = "default" + @classmethod def for_language(cls, language: Language, val: str) -> Enum: """Deserialize a variant string for the given language.""" @@ -598,6 +605,7 @@ def default(cls, language: Language) -> Enum: Language.NODEJS: Variant.NodeJS, Language.JAVA: Variant.Java, Language.CPP: Variant.Cpp, + Language.DOTNET: Variant.Dotnet, } diff --git a/sebs/regression.py b/sebs/regression.py index 5c464d377..09a460b43 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -72,6 +72,8 @@ "503.graph-bfs", ] +benchmarks_dotnet = ["010.sleep"] + # AWS-specific configurations architectures_aws = ["x64", "arm64"] deployments_aws = ["package", "container"] @@ -584,6 +586,58 @@ def get_deployment(self, benchmark_name, architecture, deployment_type): return deployment_client +class AWSTestSequenceDotnet( + unittest.TestCase, + metaclass=TestSequenceMeta, + benchmarks=benchmarks_dotnet, + architectures=architectures_aws, + deployments=deployments_aws, + deployment_name="aws", + triggers=[Trigger.TriggerType.LIBRARY, Trigger.TriggerType.HTTP], +): + """Test suite for .NET benchmarks on AWS Lambda.""" + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Get an AWS deployment client for the specified configuration. + + Args: + benchmark_name: Name of the benchmark to deploy + architecture: Architecture to deploy on (x64, arm64) + deployment_type: Deployment type (package, container) + + Returns: + An initialized AWS deployment client + + Raises: + AssertionError: If cloud_config is not set + """ + deployment_name = "aws" + assert cloud_config, "Cloud configuration is required" + + # Create a copy of the config and set architecture and deployment type + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + configure_regression_deployment(config_copy, deployment_name, deployment_type) + + f = f"regression_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + with AWSTestSequenceDotnet.lock: + deployment_client.initialize(resource_prefix=RESOURCE_PREFIX, quiet=LOGGING_REDACTED) + if LOGGING_REDACTED: + LOGGING_REDACTOR.set_resource_id( + deployment_client.config.resources.resources_id, + deployment_client.config.credentials.account_id, + ) + LoggingBase.set_filtering_resource_id( + deployment_client.config.resources.resources_id, + deployment_client.config.credentials.account_id, + ) + return deployment_client + + class AzureTestSequencePython( unittest.TestCase, metaclass=TestSequenceMeta, @@ -1465,6 +1519,8 @@ def regression_suite( suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AWSTestSequenceJava)) elif language == "cpp": suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AWSTestSequenceCpp)) + elif language == "dotnet": + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AWSTestSequenceDotnet)) # Add GCP tests if requested if "gcp" in providers: assert ( diff --git a/sebs/sebs_types.py b/sebs/sebs_types.py index b5ea4894f..426f259f9 100644 --- a/sebs/sebs_types.py +++ b/sebs/sebs_types.py @@ -67,13 +67,14 @@ class Language(str, Enum): """ Enumeration of supported programming languages. - Currently supports Python, Node.js, and C++ for serverless functions. + Currently supports Python, Node.js, C++, Java, and .NET for serverless functions. """ PYTHON = "python" NODEJS = "nodejs" CPP = "cpp" JAVA = "java" + DOTNET = "dotnet" @staticmethod def deserialize(val: str) -> Language: