Initial Commit
This commit is contained in:
44
packages/persistence/storage/storage-s3/README.md
Normal file
44
packages/persistence/storage/storage-s3/README.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# @suaveplan/storage-s3 - S3-compatible object storage adapter for @suaveplan/storage: implements AbstractStorage backed by @aws-sdk/client-s3 with presigned URLs, multipart upload, retry integration, and OTel instrumentation.
|
||||
|
||||
## Summary
|
||||
|
||||
S3-compatible object storage adapter for `@suaveplan/storage`. Extends `AbstractStorage` backed by `@aws-sdk/client-s3` with presigned URL generation, multipart upload, retry integration, and OpenTelemetry instrumentation.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [registry](./src/registry/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [registry](./src/registry/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [utils](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [credential-validator](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
<!-- end:auto-docs --> - [registry](./src/registry/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [utils](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_- [credential-validator](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [utils](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_- [credential-validator](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [utils](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_- [credential-validator](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_ - [s3](./src/s3/s3.md) — <!-- begin:auto-docs --> _(no submodules)_- [utils](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_ - [credential-validator](./src/utils/credential-validator.md) — <!-- begin:auto-docs --> _(no submodules)_
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/storage-s3
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```ts
|
||||
import { createS3 } from "@suaveplan/storage-s3";
|
||||
|
||||
const storage = createS3({
|
||||
bucket: "my-bucket",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
});
|
||||
|
||||
await storage.set("hello.txt", new TextEncoder().encode("hello"));
|
||||
const data = await storage.get("hello.txt");
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- [`src/s3/s3/s3`](src/s3/s3/s3.md) — `S3` class and `createS3` factory
|
||||
- [`src/types/s3.types`](src/types/s3.types.md) — `S3Options`, `PresignOptions`, `MultipartUploadOptions`, `S3ClientFactory`
|
||||
- [`src/utils/credential-validator`](src/utils/credential-validator.md) — `isAwsS3`, `validateCredentials`
|
||||
- [`src/registry/registry/registry`](src/registry/registry/registry.md) — `registerS3Provider`
|
||||
101
packages/persistence/storage/storage-s3/package.json
Normal file
101
packages/persistence/storage/storage-s3/package.json
Normal file
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"name": "@suaveplan/storage-s3",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "S3-compatible object storage adapter for @suaveplan/storage: implements AbstractStorage backed by @aws-sdk/client-s3 with presigned URLs, multipart upload, retry integration, and OTel instrumentation.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./s3/s3": {
|
||||
"import": {
|
||||
"types": "./dist/s3/s3/s3.d.ts",
|
||||
"default": "./dist/s3/s3/s3.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/s3/s3/s3.d.ts",
|
||||
"default": "./dist/s3/s3/s3.js"
|
||||
}
|
||||
},
|
||||
"./types": {
|
||||
"import": {
|
||||
"types": "./dist/types/index.d.ts",
|
||||
"default": "./dist/types/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/types/index.d.ts",
|
||||
"default": "./dist/types/index.js"
|
||||
}
|
||||
},
|
||||
"./utils/credential-validator": {
|
||||
"import": {
|
||||
"types": "./dist/utils/credential-validator.d.ts",
|
||||
"default": "./dist/utils/credential-validator.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/utils/credential-validator.d.ts",
|
||||
"default": "./dist/utils/credential-validator.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"test:integration": "vitest run --config vitest.integration.config.ts --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "catalog:storage",
|
||||
"@aws-sdk/s3-request-presigner": "catalog:storage",
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/retry": "workspace:^",
|
||||
"@suaveplan/storage": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@suaveplan/testing-integration": "workspace:^",
|
||||
"@suaveplan/testing-integration-garage-s3": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"fast-check": "catalog:testing",
|
||||
"mitata": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/** Default multipart upload part size: 5 MiB (AWS minimum). */
|
||||
export const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
|
||||
|
||||
/** Default presigned URL expiry in seconds: 1 hour. */
|
||||
export const DEFAULT_PRESIGN_EXPIRES = 3600;
|
||||
|
||||
/** Maximum number of concurrent multipart upload parts. */
|
||||
export const DEFAULT_MULTIPART_CONCURRENCY = 4;
|
||||
|
||||
/** AWS credential AKIA prefix length. */
|
||||
export const AWS_ACCESS_KEY_PREFIX = "AKIA";
|
||||
|
||||
/** Expected length of AWS access key ID. */
|
||||
export const AWS_ACCESS_KEY_LENGTH = 20;
|
||||
|
||||
/** Minimum length of AWS secret access key. */
|
||||
export const AWS_SECRET_KEY_MIN_LENGTH = 40;
|
||||
46
packages/persistence/storage/storage-s3/src/error-codes.ts
Normal file
46
packages/persistence/storage/storage-s3/src/error-codes.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import {
|
||||
defineCodes,
|
||||
SuaveplanError,
|
||||
type SuaveplanErrorOptions,
|
||||
} from "@suaveplan/error";
|
||||
|
||||
export const S3_CODES = defineCodes("S3", [
|
||||
"CREDENTIALS_MISSING",
|
||||
"CREDENTIALS_INVALID",
|
||||
"OBJECT_NOT_FOUND",
|
||||
"BUCKET_NOT_FOUND",
|
||||
"UPLOAD_FAILED",
|
||||
"DELETE_FAILED",
|
||||
"LIST_FAILED",
|
||||
"PRESIGN_FAILED",
|
||||
"MULTIPART_FAILED",
|
||||
"REGION_MISMATCH",
|
||||
"ABORTED",
|
||||
] as const);
|
||||
|
||||
export type S3ErrorCode = (typeof S3_CODES)[keyof typeof S3_CODES];
|
||||
|
||||
/**
|
||||
* Structured error for S3 storage operations.
|
||||
*/
|
||||
export class S3StorageError extends SuaveplanError<S3ErrorCode> {
|
||||
public readonly key: string | undefined;
|
||||
|
||||
public constructor(
|
||||
code: S3ErrorCode,
|
||||
message: string,
|
||||
options?: SuaveplanErrorOptions & { key?: string }
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.key = options?.key;
|
||||
this.name = "S3StorageError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register S3 error codes (idempotent — safe to call multiple times).
|
||||
*/
|
||||
export function registerS3ErrorCodes(): void {
|
||||
// defineCodes already registers on first call; this is a no-op hook
|
||||
// for consumers who want an explicit init step.
|
||||
}
|
||||
13
packages/persistence/storage/storage-s3/src/index.ts
Normal file
13
packages/persistence/storage/storage-s3/src/index.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
export {
|
||||
registerS3ErrorCodes,
|
||||
S3_CODES,
|
||||
S3StorageError,
|
||||
} from "./error-codes.js";
|
||||
export { createS3, S3 } from "./s3/s3.js";
|
||||
export type {
|
||||
MultipartUploadOptions,
|
||||
PresignOptions,
|
||||
RetryOptions,
|
||||
S3ClientFactory,
|
||||
S3Options,
|
||||
} from "./types/index.js";
|
||||
@@ -0,0 +1 @@
|
||||
export { registerS3Provider } from "./registry.js";
|
||||
@@ -0,0 +1,56 @@
|
||||
# registry
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Registers the `S3` storage provider with `StorageFactory` from `@suaveplan/storage` under the provider key `"s3"`. This side-effect runs automatically when the module is imported, enabling consumers to use `createStorage("s3", opts)` without manually wiring the class. The registration is idempotent — importing the module multiple times does not register duplicate providers.
|
||||
|
||||
Exposing the registration via a named `registerS3Provider` function (in addition to the auto-registration side-effect) allows consumers who tree-shake aggressively to call it explicitly in their initialization code.
|
||||
|
||||
## Features
|
||||
|
||||
- Auto-registers `S3` with `StorageFactory` on module import
|
||||
- Named `registerS3Provider()` export for explicit initialization in tree-shaken builds
|
||||
- Compatible with `StorageFactory.register` module-augmentation pattern
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
// Import to trigger registration (side-effect)
|
||||
import "@suaveplan/storage-s3/registry/registry";
|
||||
import { createStorage } from "@suaveplan/storage";
|
||||
|
||||
const storage = createStorage("s3", {
|
||||
bucket: "my-bucket",
|
||||
region: "us-east-1",
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { registerS3Provider } from "@suaveplan/storage-s3/registry/registry";
|
||||
import { createStorage } from "@suaveplan/storage";
|
||||
|
||||
// Explicit registration (useful when tree-shaking might remove the import)
|
||||
registerS3Provider();
|
||||
|
||||
const storage = createStorage("s3", {
|
||||
bucket: "prod-bucket",
|
||||
region: "eu-west-1",
|
||||
endpoint: "https://garage.example.com",
|
||||
});
|
||||
await storage.set("config.json", new TextEncoder().encode(JSON.stringify({})));
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `registerS3Provider(): void`
|
||||
|
||||
Registers the `S3` class with `StorageFactory` under the key `"s3"`. Safe to call multiple times — subsequent calls are idempotent if `StorageFactory.register` deduplicates by key.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The `StorageFactory.register` call uses a type cast because `StorageFactory` expects a constructor signature matching `AbstractStorage`, while `S3` extends `AbstractStorage` but accepts `S3Options` rather than a generic storage config. The cast is safe because `createStorage("s3", opts)` users are expected to pass `S3Options` compatible objects.
|
||||
@@ -0,0 +1,18 @@
|
||||
import { registerStorageProvider } from "@suaveplan/storage/factory";
|
||||
import { S3 } from "../../s3/s3.js";
|
||||
|
||||
/**
|
||||
* Registers the S3 storage provider with the global StorageFactory
|
||||
* under the key "s3". This side-effect runs on package import.
|
||||
*/
|
||||
export function registerS3Provider(): void {
|
||||
registerStorageProvider(
|
||||
"s3",
|
||||
S3 as unknown as new (
|
||||
config?: unknown
|
||||
) => S3
|
||||
);
|
||||
}
|
||||
|
||||
// Auto-register on import
|
||||
registerS3Provider();
|
||||
4
packages/persistence/storage/storage-s3/src/s3/index.ts
Normal file
4
packages/persistence/storage/storage-s3/src/s3/index.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
createS3,
|
||||
S3,
|
||||
} from "./s3.js";
|
||||
@@ -0,0 +1,751 @@
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CopyObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3ServiceException,
|
||||
UploadPartCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
registerS3ErrorCodes,
|
||||
S3_CODES,
|
||||
S3StorageError,
|
||||
} from "../error-codes.js";
|
||||
import type { S3ClientFactory } from "../types/s3.types.js";
|
||||
import { S3 } from "./s3.js";
|
||||
|
||||
const BASE_OPTS = {
|
||||
bucket: "test-bucket",
|
||||
region: "us-east-1",
|
||||
} as const;
|
||||
|
||||
function fakeFactory(send: (cmd: any) => Promise<any>): S3ClientFactory {
|
||||
return { create: () => ({ send }) as unknown as S3Client };
|
||||
}
|
||||
|
||||
// ── error-codes coverage ───────────────────────────────────────────────────
|
||||
|
||||
it("registerS3ErrorCodes is callable (no-op hook)", { timeout: 5000 }, () => {
|
||||
expect(() => registerS3ErrorCodes()).not.toThrow();
|
||||
});
|
||||
|
||||
// ── registry coverage ──────────────────────────────────────────────────────
|
||||
|
||||
it("registry auto-registers S3 provider on import", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const { registerS3Provider } = await import(
|
||||
"../registry/registry/registry.js"
|
||||
);
|
||||
expect(typeof registerS3Provider).toBe("function");
|
||||
// registerS3Provider was already called on import (side-effect)
|
||||
// calling again is idempotent
|
||||
expect(() => registerS3Provider()).not.toThrow();
|
||||
});
|
||||
|
||||
// ── defaultClientFactory branches ─────────────────────────────────────────
|
||||
|
||||
it("defaultClientFactory: no endpoint, no credentials", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const s = new S3({ bucket: "b", region: "us-east-1" });
|
||||
expect(s).toBeInstanceOf(S3);
|
||||
});
|
||||
|
||||
it("defaultClientFactory: with endpoint", { timeout: 5000 }, () => {
|
||||
const s = new S3({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
endpoint: "http://localhost:9000",
|
||||
});
|
||||
expect(s).toBeInstanceOf(S3);
|
||||
});
|
||||
|
||||
it("defaultClientFactory: with credentials", { timeout: 5000 }, () => {
|
||||
const s = new S3({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE", // valid format
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
});
|
||||
expect(s).toBeInstanceOf(S3);
|
||||
});
|
||||
|
||||
// ── isRetryableS3Error branches ────────────────────────────────────────────
|
||||
|
||||
it("non-retryable plain Error is thrown immediately without delay", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let calls = 0;
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 3, baseDelayMs: 0 } },
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
calls++;
|
||||
throw new Error("plain error"); // non-S3ServiceException
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
await expect(s.get("k")).rejects.toThrow("plain error");
|
||||
expect(calls).toBe(1); // no retries for non-S3ServiceException
|
||||
});
|
||||
|
||||
it("isRetryableS3Error: throttling flag", { timeout: 30000 }, async () => {
|
||||
let calls = 0;
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 2, baseDelayMs: 1 } },
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
calls++;
|
||||
if (calls < 2) {
|
||||
const e = new S3ServiceException({
|
||||
name: "ThrottlingException",
|
||||
message: "throttled",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 429 },
|
||||
});
|
||||
Object.assign(e, { $retryable: { throttling: true } });
|
||||
throw e;
|
||||
}
|
||||
return {
|
||||
Body: {
|
||||
[Symbol.asyncIterator]() {
|
||||
let done = false;
|
||||
return {
|
||||
async next() {
|
||||
if (done)
|
||||
return { value: undefined, done: true };
|
||||
done = true;
|
||||
return {
|
||||
value: new Uint8Array([1]),
|
||||
done: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.get("k");
|
||||
expect(result).toEqual(new Uint8Array([1]));
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
// ── get() branches ─────────────────────────────────────────────────────────
|
||||
|
||||
it("get: passes AbortSignal to command", { timeout: 5000 }, async () => {
|
||||
const ctrl = new AbortController();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
return {
|
||||
Body: {
|
||||
[Symbol.asyncIterator]() {
|
||||
let done = false;
|
||||
return {
|
||||
async next() {
|
||||
if (done)
|
||||
return { value: undefined, done: true };
|
||||
done = true;
|
||||
return {
|
||||
value: new Uint8Array([1]),
|
||||
done: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.get("k", ctrl.signal);
|
||||
expect(result).toEqual(new Uint8Array([1]));
|
||||
});
|
||||
|
||||
it("get: returns undefined when Body is null", { timeout: 5000 }, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) return { Body: undefined };
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.get("k");
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
// ── set() branches ─────────────────────────────────────────────────────────
|
||||
|
||||
it("set: throws S3StorageError on PutObject failure", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof PutObjectCommand)
|
||||
throw new Error("upload failed");
|
||||
return {};
|
||||
})
|
||||
);
|
||||
await expect(s.set("k", new Uint8Array([1]))).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
await expect(s.set("k", new Uint8Array([1]))).rejects.toMatchObject({
|
||||
code: S3_CODES.UPLOAD_FAILED,
|
||||
});
|
||||
});
|
||||
|
||||
it("set: passes AbortSignal to command", { timeout: 5000 }, async () => {
|
||||
const ctrl = new AbortController();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async () => ({}))
|
||||
);
|
||||
await expect(
|
||||
s.set("k", new Uint8Array([1]), undefined, ctrl.signal)
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
// ── delete() branches ──────────────────────────────────────────────────────
|
||||
|
||||
it("delete: passes AbortSignal to HeadObject and DeleteObject", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ctrl = new AbortController();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof DeleteObjectCommand) return {};
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.delete("k", ctrl.signal);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it("delete: re-throws S3StorageError directly", { timeout: 5000 }, async () => {
|
||||
const originalErr = new S3StorageError(
|
||||
S3_CODES.DELETE_FAILED,
|
||||
"already wrapped"
|
||||
);
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof DeleteObjectCommand) throw originalErr;
|
||||
return {};
|
||||
})
|
||||
);
|
||||
await expect(s.delete("k")).rejects.toBe(originalErr);
|
||||
});
|
||||
|
||||
// ── list() branches ────────────────────────────────────────────────────────
|
||||
|
||||
it("list: passes AbortSignal to command", { timeout: 5000 }, async () => {
|
||||
const ctrl = new AbortController();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof ListObjectsV2Command)
|
||||
return { Contents: [{ Key: "a" }], IsTruncated: false };
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.list(undefined, ctrl.signal);
|
||||
expect(result.keys).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("list: handles Contents=null", { timeout: 5000 }, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async () => ({ Contents: null, IsTruncated: false }))
|
||||
);
|
||||
const result = await s.list();
|
||||
expect(result.keys).toEqual([]);
|
||||
});
|
||||
|
||||
it("list: skips objects with no Key", { timeout: 5000 }, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async () => ({
|
||||
Contents: [{ Key: "good" }, {}], // second has no Key
|
||||
IsTruncated: false,
|
||||
}))
|
||||
);
|
||||
const result = await s.list();
|
||||
expect(result.keys).toEqual(["good"]);
|
||||
});
|
||||
|
||||
it("list: stops after first page when options.continuationToken provided", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let pages = 0;
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async () => {
|
||||
pages++;
|
||||
return {
|
||||
Contents: [{ Key: `page${pages}` }],
|
||||
NextContinuationToken: "token123",
|
||||
IsTruncated: true,
|
||||
};
|
||||
})
|
||||
);
|
||||
// When continuationToken is set, do-while stops after first iteration
|
||||
const result = await s.list({ continuationToken: "token123" });
|
||||
expect(pages).toBe(1);
|
||||
expect(result.keys).toEqual(["page1"]);
|
||||
});
|
||||
|
||||
it("list: re-throws S3StorageError directly", { timeout: 5000 }, async () => {
|
||||
const originalErr = new S3StorageError(
|
||||
S3_CODES.LIST_FAILED,
|
||||
"already wrapped"
|
||||
);
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async () => {
|
||||
throw originalErr;
|
||||
})
|
||||
);
|
||||
await expect(s.list()).rejects.toBe(originalErr);
|
||||
});
|
||||
|
||||
it("list: strips keyPrefix from result keys", { timeout: 5000 }, async () => {
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, keyPrefix: "pfx" },
|
||||
fakeFactory(async () => ({
|
||||
Contents: [{ Key: "pfx/file.txt" }],
|
||||
IsTruncated: false,
|
||||
}))
|
||||
);
|
||||
const result = await s.list();
|
||||
expect(result.keys).toEqual(["file.txt"]);
|
||||
});
|
||||
|
||||
it("list: no keyPrefix returns full key from stripPrefix", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS, // no keyPrefix
|
||||
fakeFactory(async () => ({
|
||||
Contents: [{ Key: "file.txt" }],
|
||||
IsTruncated: false,
|
||||
}))
|
||||
);
|
||||
const result = await s.list();
|
||||
expect(result.keys).toEqual(["file.txt"]);
|
||||
});
|
||||
|
||||
// ── getMetadata() branches ─────────────────────────────────────────────────
|
||||
|
||||
it("getMetadata: uses Date.now() when LastModified absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const before = Date.now();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand) return { ContentLength: 5 }; // no LastModified
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const meta = await s.getMetadata("k");
|
||||
expect(meta).not.toBeNull();
|
||||
expect(meta!.createdAt).toBeGreaterThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("getMetadata: uses 0 when ContentLength absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { LastModified: new Date() }; // no ContentLength
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const meta = await s.getMetadata("k");
|
||||
expect(meta!.size).toBe(0);
|
||||
});
|
||||
|
||||
it("getMetadata: passes AbortSignal to command", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ctrl = new AbortController();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const meta = await s.getMetadata("k", ctrl.signal);
|
||||
expect(meta).not.toBeNull();
|
||||
});
|
||||
|
||||
// ── getSize() branches ─────────────────────────────────────────────────────
|
||||
|
||||
it("getSize: treats null metadata as 0 bytes", { timeout: 5000 }, async () => {
|
||||
let headCount = 0;
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof ListObjectsV2Command)
|
||||
return { Contents: [{ Key: "k" }], IsTruncated: false };
|
||||
if (cmd instanceof HeadObjectCommand) {
|
||||
headCount++;
|
||||
// First call returns null-ish (404 simulation)
|
||||
const e = new S3ServiceException({
|
||||
name: "NoSuchKey",
|
||||
message: "not found",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const size = await s.getSize();
|
||||
expect(size).toBe(0);
|
||||
expect(headCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// ── presign() branches ─────────────────────────────────────────────────────
|
||||
|
||||
it("presign: PUT method uses PutObjectCommand", { timeout: 5000 }, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async () => {
|
||||
throw new Error("presign uses getSignedUrl, not send");
|
||||
})
|
||||
);
|
||||
// getSignedUrl will fail without a real client, but the error path covers the branch
|
||||
await expect(s.presign("k", { method: "PUT" })).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
});
|
||||
|
||||
// ── setMultipart() branches ────────────────────────────────────────────────
|
||||
|
||||
it("setMultipart: passes signal through operations", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ctrl = new AbortController();
|
||||
let uploadId: string | undefined;
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof CreateMultipartUploadCommand) {
|
||||
uploadId = "up-sig";
|
||||
return { UploadId: uploadId };
|
||||
}
|
||||
if (cmd instanceof UploadPartCommand) return { ETag: "etag-1" };
|
||||
if (cmd instanceof CompleteMultipartUploadCommand) return {};
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl2) {
|
||||
ctrl2.enqueue(new Uint8Array([1, 2, 3]));
|
||||
ctrl2.close();
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
s.setMultipart("f.bin", stream, { signal: ctrl.signal })
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("setMultipart: handles null ETag in part", { timeout: 5000 }, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof CreateMultipartUploadCommand)
|
||||
return { UploadId: "up-1" };
|
||||
if (cmd instanceof UploadPartCommand) return { ETag: null }; // null ETag → covers ?? "" branch
|
||||
if (cmd instanceof CompleteMultipartUploadCommand) return {};
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array([1, 2, 3]));
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await expect(s.setMultipart("f.bin", stream)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("setMultipart: abort cleanup swallows AbortMultipartUpload error", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof CreateMultipartUploadCommand)
|
||||
return { UploadId: "up-err" };
|
||||
if (cmd instanceof UploadPartCommand)
|
||||
throw new Error("part failed");
|
||||
if (cmd instanceof AbortMultipartUploadCommand)
|
||||
throw new Error("abort also failed"); // .catch(() => void 0) covers this
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array(6 * 1024 * 1024));
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await expect(s.setMultipart("f.bin", stream)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
});
|
||||
|
||||
it("setMultipart: no final buffer flush when stream ends exactly on part boundary", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
|
||||
let partUploads = 0;
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof CreateMultipartUploadCommand)
|
||||
return { UploadId: "up-boundary" };
|
||||
if (cmd instanceof UploadPartCommand) {
|
||||
partUploads++;
|
||||
return { ETag: `etag-${partUploads}` };
|
||||
}
|
||||
if (cmd instanceof CompleteMultipartUploadCommand) return {};
|
||||
return {};
|
||||
})
|
||||
);
|
||||
// Exactly partSize bytes → buffer drains to 0, no final flush needed
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array(DEFAULT_PART_SIZE));
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await expect(s.setMultipart("f.bin", stream)).resolves.toBeUndefined();
|
||||
expect(partUploads).toBe(1); // exactly one part, no remainder
|
||||
});
|
||||
|
||||
// ── isRetryableS3Error remaining branches ─────────────────────────────────
|
||||
|
||||
it("isRetryableS3Error: null err.name uses empty string", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
let calls = 0;
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 2, baseDelayMs: 1 } },
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
calls++;
|
||||
if (calls < 2) {
|
||||
// name is null → code = "" → uses httpStatusCode >= 500 path
|
||||
const e = new S3ServiceException({
|
||||
name: "InternalServerError",
|
||||
message: "server error",
|
||||
$fault: "server",
|
||||
$metadata: { httpStatusCode: 500 },
|
||||
});
|
||||
// Override name to null to exercise ?? "" branch
|
||||
Object.defineProperty(e, "name", { value: null });
|
||||
throw e;
|
||||
}
|
||||
return {
|
||||
Body: {
|
||||
[Symbol.asyncIterator]() {
|
||||
let done = false;
|
||||
return {
|
||||
async next() {
|
||||
if (done)
|
||||
return { value: undefined, done: true };
|
||||
done = true;
|
||||
return {
|
||||
value: new Uint8Array([1]),
|
||||
done: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.get("k");
|
||||
expect(result).toEqual(new Uint8Array([1]));
|
||||
});
|
||||
|
||||
it("isRetryableS3Error: RequestTimeout code retries", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
let calls = 0;
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 2, baseDelayMs: 1 } },
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
calls++;
|
||||
if (calls < 2) {
|
||||
throw new S3ServiceException({
|
||||
name: "RequestTimeout",
|
||||
message: "timeout",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 408 },
|
||||
});
|
||||
}
|
||||
return {
|
||||
Body: {
|
||||
[Symbol.asyncIterator]() {
|
||||
let done = false;
|
||||
return {
|
||||
async next() {
|
||||
if (done)
|
||||
return { value: undefined, done: true };
|
||||
done = true;
|
||||
return {
|
||||
value: new Uint8Array([1]),
|
||||
done: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.get("k");
|
||||
expect(result).toEqual(new Uint8Array([1]));
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
it("isRetryableS3Error: null httpStatusCode metadata → not retryable", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// maxAttempts:2 so isRetryableS3Error is evaluated on attempt 0
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 2, baseDelayMs: 0 } },
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
// $metadata with no httpStatusCode → ?? 0 branch → 0 < 500 → not retryable
|
||||
throw new S3ServiceException({
|
||||
name: "UnknownError",
|
||||
message: "unknown",
|
||||
$fault: "client",
|
||||
$metadata: {}, // no httpStatusCode
|
||||
});
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
await expect(s.get("k")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("isRetryableS3Error: null $metadata → not retryable", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// maxAttempts:2 so isRetryableS3Error is evaluated on attempt 0
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 2, baseDelayMs: 0 } },
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
const e = new S3ServiceException({
|
||||
name: "UnknownError",
|
||||
message: "unknown",
|
||||
$fault: "client",
|
||||
$metadata: {},
|
||||
});
|
||||
// Force $metadata to null → covers the ?. null branch
|
||||
Object.defineProperty(e, "$metadata", { value: null });
|
||||
throw e;
|
||||
}
|
||||
return {};
|
||||
})
|
||||
);
|
||||
await expect(s.get("k")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
// ── getWithMetadata: value=undefined after meta found ─────────────────────
|
||||
|
||||
it("getWithMetadata: returns null when get returns undefined despite meta existing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let headCount = 0;
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof HeadObjectCommand) {
|
||||
headCount++;
|
||||
if (headCount === 1)
|
||||
return { ContentLength: 1, LastModified: new Date() };
|
||||
// Second HeadObject call (from get's internal HEAD) → 404
|
||||
const e = new S3ServiceException({
|
||||
name: "NoSuchKey",
|
||||
message: "not found",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
if (cmd instanceof GetObjectCommand) return { Body: undefined }; // no Body → get returns undefined
|
||||
return {};
|
||||
})
|
||||
);
|
||||
const result = await s.getWithMetadata("k");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
// ── copy() with AbortSignal ────────────────────────────────────────────────
|
||||
|
||||
it("copy: passes AbortSignal to command", { timeout: 5000 }, async () => {
|
||||
const ctrl = new AbortController();
|
||||
const s = new S3(
|
||||
BASE_OPTS,
|
||||
fakeFactory(async (cmd) => {
|
||||
if (cmd instanceof CopyObjectCommand) return {};
|
||||
return {};
|
||||
})
|
||||
);
|
||||
await expect(s.copy("src", "dst", ctrl.signal)).resolves.toBeUndefined();
|
||||
});
|
||||
140
packages/persistence/storage/storage-s3/src/s3/s3.md
Normal file
140
packages/persistence/storage/storage-s3/src/s3/s3.md
Normal file
@@ -0,0 +1,140 @@
|
||||
# S3
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Implements the `AbstractStorage` interface from `@suaveplan/storage` backed by the AWS SDK v3 `@aws-sdk/client-s3`. The `S3` class provides full lifecycle management for objects in any S3-compatible bucket, including AWS S3, Cloudflare R2, MinIO, and Garage. It adds enterprise capabilities not present in the main-branch reference: structured error codes, presigned URL generation, multipart upload with automatic part splitting and abort-on-failure, injectable retry strategy, AbortSignal propagation, and OpenTelemetry span/counter/histogram instrumentation.
|
||||
|
||||
The class is intentionally thin: it translates between `AbstractStorage`'s key-value contract and S3's object model, adds a key prefix namespace, and delegates all batching, pattern-listing, and hook machinery to the base class. The `S3ClientFactory` interface enables full testing via in-memory fakes without touching real AWS infrastructure.
|
||||
|
||||
## Features
|
||||
|
||||
- Implements all `AbstractStorage` abstract methods: `get`, `set`, `delete`, `exists`, `list`, `clear`, `getWithMetadata`, `getMetadata`, `getSize`, `getPartial`, `getItemSize`, `createReadStream`, `createWriteStream`
|
||||
- S3-specific extensions: `presign` (GET/PUT), `copy`, `move`, `setMultipart`
|
||||
- Key prefix namespacing: `keyPrefix` option prepends a path segment to all keys
|
||||
- Retry with exponential backoff for retryable S3 errors (5xx, SlowDown, RequestTimeout)
|
||||
- AbortSignal propagation to all AWS SDK calls
|
||||
- OpenTelemetry spans, counters, and histograms for every operation
|
||||
- Structured `S3StorageError` with stable `S3_CODES` for all failure modes
|
||||
- Injectable `S3ClientFactory` for testability
|
||||
- AWS credential format validation via `validateCredentials`
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { S3, createS3 } from "@suaveplan/storage-s3";
|
||||
|
||||
const storage = createS3({
|
||||
bucket: "my-bucket",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
});
|
||||
|
||||
await storage.set("hello.txt", new TextEncoder().encode("hello"));
|
||||
const data = await storage.get("hello.txt");
|
||||
console.log(new TextDecoder().decode(data)); // "hello"
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { S3 } from "@suaveplan/storage-s3";
|
||||
import type { S3ClientFactory, S3Options } from "@suaveplan/storage-s3";
|
||||
|
||||
// Inject a fake client for testing
|
||||
const fakeClient = new FakeS3Client(); // your in-memory implementation
|
||||
|
||||
const fakeFactory: S3ClientFactory = {
|
||||
create: () => fakeClient as unknown as S3Client,
|
||||
};
|
||||
|
||||
const storage = new S3(
|
||||
{
|
||||
bucket: "test-bucket",
|
||||
region: "us-east-1",
|
||||
endpoint: "http://localhost:9000",
|
||||
keyPrefix: "uploads/v2",
|
||||
retry: { maxAttempts: 3, baseDelayMs: 50 },
|
||||
},
|
||||
fakeFactory
|
||||
);
|
||||
|
||||
// Multipart upload
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array(10 * 1024 * 1024)); // 10 MiB
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await storage.setMultipart("large-file.bin", stream, { partSize: 5 * 1024 * 1024 });
|
||||
|
||||
// Presigned URL
|
||||
const url = await storage.presign("large-file.bin", { expiresIn: 300, method: "GET" });
|
||||
console.log(url); // https://... signed URL
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `new S3(opts: S3Options, clientFactory?: S3ClientFactory)`
|
||||
|
||||
Creates an S3 adapter.
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
| `opts` | `S3Options` | Bucket, region, credentials, endpoint, prefix, retry, logger |
|
||||
| `clientFactory` | `S3ClientFactory?` | Injectable factory; defaults to real AWS SDK client |
|
||||
|
||||
**Throws**: `S3StorageError(S3.CREDENTIALS_INVALID)` if AWS credential format is invalid.
|
||||
|
||||
### `createS3(opts, clientFactory?): S3`
|
||||
|
||||
Factory function equivalent to `new S3(opts, clientFactory)`.
|
||||
|
||||
### `get(key, signal?): Promise<Uint8Array | undefined>`
|
||||
|
||||
Downloads an object. Returns `undefined` if the key does not exist.
|
||||
|
||||
### `set(key, value, options?, signal?): Promise<void>`
|
||||
|
||||
Uploads a `Uint8Array` as an object. Throws `S3StorageError(S3.UPLOAD_FAILED)` on failure.
|
||||
|
||||
### `delete(key, signal?): Promise<boolean>`
|
||||
|
||||
Deletes an object. Returns `false` if it did not exist, `true` if deleted.
|
||||
|
||||
### `exists(key, signal?): Promise<boolean>`
|
||||
|
||||
Checks existence via `HeadObject`.
|
||||
|
||||
### `list(options?, signal?): Promise<StorageListResult>`
|
||||
|
||||
Lists objects under an optional prefix. Follows continuation tokens internally.
|
||||
|
||||
### `clear(signal?): Promise<void>`
|
||||
|
||||
Deletes all objects under the configured key prefix.
|
||||
|
||||
### `presign(key, opts?): Promise<string>`
|
||||
|
||||
Returns a presigned URL (GET by default). Throws `S3StorageError(S3.PRESIGN_FAILED)`.
|
||||
|
||||
### `copy(source, destination, signal?): Promise<void>`
|
||||
|
||||
Copies an object within the same bucket. Throws `S3StorageError(S3.UPLOAD_FAILED)`.
|
||||
|
||||
### `move(source, destination, signal?): Promise<void>`
|
||||
|
||||
Moves an object: copy then delete source.
|
||||
|
||||
### `setMultipart(key, stream, opts?): Promise<void>`
|
||||
|
||||
Uploads a `ReadableStream` in parts. Automatically aborts the multipart session on error or signal. Throws `S3StorageError(S3.MULTIPART_FAILED)`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The `withRetry` helper is a lightweight internal implementation rather than using `@suaveplan/retry` directly, to avoid the need for the retry package to be a hard runtime dep on the critical storage path. The `RetryOptions` type is a minimal subset, and consumers can opt in by passing `retry: { maxAttempts: 3 }`. Retryable errors are detected by checking `S3ServiceException.$retryable.throttling`, `name` for `SlowDown`/`RequestTimeout`, and the HTTP status code ≥ 500.
|
||||
|
||||
For `setMultipart`, the stream is read in a single pass, buffering chunks until a full `partSize` accumulates. The final part may be smaller than `partSize`. AWS requires each part except the last to be at least 5 MiB, so the default `partSize` matches that minimum. If the multipart session fails after `CreateMultipartUpload`, the session is aborted via `AbortMultipartUploadCommand` to avoid incurring storage costs for incomplete uploads.
|
||||
747
packages/persistence/storage/storage-s3/src/s3/s3.test.ts
Normal file
747
packages/persistence/storage/storage-s3/src/s3/s3.test.ts
Normal file
@@ -0,0 +1,747 @@
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CopyObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3ServiceException,
|
||||
UploadPartCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { beforeEach, describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { S3_CODES, S3StorageError } from "../error-codes.js";
|
||||
import type { S3ClientFactory } from "../types/s3.types.js";
|
||||
import { createS3, S3 } from "./s3.js";
|
||||
|
||||
// ── In-memory S3 fake ──────────────────────────────────────────────────────
|
||||
|
||||
interface FakeObject {
|
||||
body: Uint8Array;
|
||||
lastModified: Date;
|
||||
}
|
||||
|
||||
class FakeS3Client {
|
||||
readonly store = new Map<string, FakeObject>();
|
||||
readonly multiparts = new Map<string, { parts: Map<number, Uint8Array> }>();
|
||||
|
||||
async send(command: any): Promise<any> {
|
||||
if (command instanceof PutObjectCommand) {
|
||||
const { Key, Body } = command.input;
|
||||
this.store.set(Key, {
|
||||
body:
|
||||
Body instanceof Uint8Array
|
||||
? Body
|
||||
: new TextEncoder().encode(String(Body)),
|
||||
lastModified: new Date(),
|
||||
});
|
||||
return {};
|
||||
}
|
||||
if (command instanceof GetObjectCommand) {
|
||||
const obj = this.store.get(command.input.Key);
|
||||
if (!obj) {
|
||||
const e = new S3ServiceException({
|
||||
name: "NoSuchKey",
|
||||
message: "Not found",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
const body = obj.body;
|
||||
return {
|
||||
Body: {
|
||||
[Symbol.asyncIterator]() {
|
||||
let done = false;
|
||||
return {
|
||||
async next() {
|
||||
if (done)
|
||||
return { value: undefined, done: true };
|
||||
done = true;
|
||||
return { value: body, done: false };
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (command instanceof HeadObjectCommand) {
|
||||
const obj = this.store.get(command.input.Key);
|
||||
if (!obj) {
|
||||
const e = new S3ServiceException({
|
||||
name: "NotFound",
|
||||
message: "Not found",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 404 },
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
return {
|
||||
ContentLength: obj.body.byteLength,
|
||||
LastModified: obj.lastModified,
|
||||
};
|
||||
}
|
||||
if (command instanceof DeleteObjectCommand) {
|
||||
this.store.delete(command.input.Key);
|
||||
return {};
|
||||
}
|
||||
if (command instanceof ListObjectsV2Command) {
|
||||
const prefix = command.input.Prefix ?? "";
|
||||
const contents = [...this.store.entries()]
|
||||
.filter(([k]) => k.startsWith(prefix))
|
||||
.map(([k, v]) => ({ Key: k, Size: v.body.byteLength }));
|
||||
return { Contents: contents };
|
||||
}
|
||||
if (command instanceof CopyObjectCommand) {
|
||||
const srcKey =
|
||||
command.input.CopySource?.split("/").slice(1).join("/") ?? "";
|
||||
const obj = this.store.get(srcKey);
|
||||
if (!obj) throw new Error("Source not found");
|
||||
this.store.set(command.input.Key, { ...obj });
|
||||
return {};
|
||||
}
|
||||
if (command instanceof CreateMultipartUploadCommand) {
|
||||
const uploadId = `upload-${Date.now()}`;
|
||||
this.multiparts.set(uploadId, { parts: new Map() });
|
||||
return { UploadId: uploadId };
|
||||
}
|
||||
if (command instanceof UploadPartCommand) {
|
||||
const { UploadId, PartNumber, Body } = command.input;
|
||||
const mp = this.multiparts.get(UploadId ?? "");
|
||||
if (!mp) throw new Error("Upload not found");
|
||||
mp.parts.set(
|
||||
PartNumber ?? 1,
|
||||
Body instanceof Uint8Array ? Body : new Uint8Array()
|
||||
);
|
||||
return { ETag: `etag-${PartNumber}` };
|
||||
}
|
||||
if (command instanceof CompleteMultipartUploadCommand) {
|
||||
const { UploadId, Key } = command.input;
|
||||
const mp = this.multiparts.get(UploadId ?? "");
|
||||
if (!mp) throw new Error("Upload not found");
|
||||
const sorted = [...mp.parts.entries()].sort((a, b) => a[0] - b[0]);
|
||||
const total = sorted.reduce((s, [, v]) => s + v.byteLength, 0);
|
||||
const merged = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const [, part] of sorted) {
|
||||
merged.set(part, offset);
|
||||
offset += part.byteLength;
|
||||
}
|
||||
this.store.set(Key, { body: merged, lastModified: new Date() });
|
||||
this.multiparts.delete(UploadId ?? "");
|
||||
return {};
|
||||
}
|
||||
if (command instanceof AbortMultipartUploadCommand) {
|
||||
this.multiparts.delete(command.input.UploadId ?? "");
|
||||
return {};
|
||||
}
|
||||
throw new Error(`Unknown command: ${command.constructor.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
function makeFakeFactory(fake: FakeS3Client): S3ClientFactory {
|
||||
return {
|
||||
create: () => fake as unknown as S3Client,
|
||||
};
|
||||
}
|
||||
|
||||
const BASE_OPTS = {
|
||||
bucket: "test-bucket",
|
||||
region: "us-east-1",
|
||||
endpoint: "http://localhost:9000",
|
||||
} as const;
|
||||
|
||||
describe("S3", () => {
|
||||
let fake: FakeS3Client;
|
||||
let storage: S3;
|
||||
|
||||
beforeEach(() => {
|
||||
fake = new FakeS3Client();
|
||||
storage = new S3(BASE_OPTS, makeFakeFactory(fake));
|
||||
});
|
||||
|
||||
describe("set / get", () => {
|
||||
it("stores and retrieves a value", { timeout: 5000 }, async () => {
|
||||
const data = new TextEncoder().encode("hello world");
|
||||
await storage.set("file.txt", data);
|
||||
const result = await storage.get("file.txt");
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
|
||||
it("returns undefined for missing key", { timeout: 5000 }, async () => {
|
||||
const result = await storage.get("missing.txt");
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it("overwrites existing value", { timeout: 5000 }, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
await storage.set("k", new Uint8Array([2]));
|
||||
const result = await storage.get("k");
|
||||
expect(result).toEqual(new Uint8Array([2]));
|
||||
});
|
||||
});
|
||||
|
||||
describe("delete", () => {
|
||||
it("deletes an existing key", { timeout: 5000 }, async () => {
|
||||
await storage.set("x", new Uint8Array([42]));
|
||||
const deleted = await storage.delete("x");
|
||||
expect(deleted).toBe(true);
|
||||
expect(await storage.get("x")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns false for missing key", { timeout: 5000 }, async () => {
|
||||
const deleted = await storage.delete("nonexistent");
|
||||
expect(deleted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("exists", () => {
|
||||
it("returns true for existing key", { timeout: 5000 }, async () => {
|
||||
await storage.set("ex", new Uint8Array([1]));
|
||||
expect(await storage.exists("ex")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for missing key", { timeout: 5000 }, async () => {
|
||||
expect(await storage.exists("missing")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("list", () => {
|
||||
it("lists all keys", { timeout: 5000 }, async () => {
|
||||
await storage.set("a", new Uint8Array([1]));
|
||||
await storage.set("b", new Uint8Array([2]));
|
||||
const result = await storage.list();
|
||||
expect(result.keys).toContain("a");
|
||||
expect(result.keys).toContain("b");
|
||||
});
|
||||
|
||||
it("lists keys with prefix", { timeout: 5000 }, async () => {
|
||||
await storage.set("foo/a", new Uint8Array([1]));
|
||||
await storage.set("bar/b", new Uint8Array([2]));
|
||||
const result = await storage.list({ prefix: "foo/" });
|
||||
expect(result.keys).toContain("foo/a");
|
||||
expect(result.keys).not.toContain("bar/b");
|
||||
});
|
||||
});
|
||||
|
||||
describe("clear", () => {
|
||||
it("removes all keys", { timeout: 5000 }, async () => {
|
||||
await storage.set("a", new Uint8Array([1]));
|
||||
await storage.set("b", new Uint8Array([2]));
|
||||
await storage.clear();
|
||||
const result = await storage.list();
|
||||
expect(result.keys).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getMetadata", () => {
|
||||
it("returns metadata for existing key", { timeout: 5000 }, async () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
await storage.set("meta-test", data);
|
||||
const meta = await storage.getMetadata("meta-test");
|
||||
expect(meta).not.toBeNull();
|
||||
expect(meta?.size).toBe(3);
|
||||
});
|
||||
|
||||
it("returns null for missing key", { timeout: 5000 }, async () => {
|
||||
const meta = await storage.getMetadata("missing");
|
||||
expect(meta).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getWithMetadata", () => {
|
||||
it("returns value and metadata", { timeout: 5000 }, async () => {
|
||||
const data = new Uint8Array([10, 20]);
|
||||
await storage.set("wm", data);
|
||||
const result = await storage.getWithMetadata("wm");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.value).toEqual(data);
|
||||
expect(result?.metadata.size).toBe(2);
|
||||
});
|
||||
|
||||
it("returns null for missing key", { timeout: 5000 }, async () => {
|
||||
const result = await storage.getWithMetadata("missing");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getPartial", () => {
|
||||
it("returns a slice of the value", { timeout: 5000 }, async () => {
|
||||
await storage.set("partial", new Uint8Array([1, 2, 3, 4, 5]));
|
||||
const slice = await storage.getPartial("partial", 1, 3);
|
||||
expect(slice).toEqual(new Uint8Array([2, 3, 4]));
|
||||
});
|
||||
|
||||
it("returns null for missing key", { timeout: 5000 }, async () => {
|
||||
const slice = await storage.getPartial("missing", 0, 10);
|
||||
expect(slice).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("setPartial", () => {
|
||||
it("writes bytes at offset and creates sparse zero-fill", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.setPartial("sp", 4, new Uint8Array([0xaa, 0xbb]));
|
||||
const got = await storage.get("sp");
|
||||
expect(got).toEqual(
|
||||
new Uint8Array([0x00, 0x00, 0x00, 0x00, 0xaa, 0xbb])
|
||||
);
|
||||
});
|
||||
|
||||
it("overwrites overlap and preserves surrounding bytes", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("sp2", new Uint8Array([1, 2, 3, 4, 5]));
|
||||
await storage.setPartial("sp2", 1, new Uint8Array([0xff, 0xff]));
|
||||
expect(await storage.get("sp2")).toEqual(
|
||||
new Uint8Array([1, 0xff, 0xff, 4, 5])
|
||||
);
|
||||
});
|
||||
|
||||
it("throws on aborted signal", { timeout: 5000 }, async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
await expect(
|
||||
storage.setPartial("sp3", 0, new Uint8Array([1]), ac.signal)
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("capabilities flag partialWrites is false (multipart wrapper deferred)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(storage.capabilities.partialWrites).toBe(false);
|
||||
expect(storage.capabilities.partialReads).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getItemSize", () => {
|
||||
it("returns size for existing key", { timeout: 5000 }, async () => {
|
||||
await storage.set("sz", new Uint8Array(7));
|
||||
expect(await storage.getItemSize("sz")).toBe(7);
|
||||
});
|
||||
|
||||
it("returns null for missing key", { timeout: 5000 }, async () => {
|
||||
expect(await storage.getItemSize("missing")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getSize", () => {
|
||||
it("returns total size of all objects", { timeout: 5000 }, async () => {
|
||||
await storage.set("a", new Uint8Array(3));
|
||||
await storage.set("b", new Uint8Array(5));
|
||||
expect(await storage.getSize()).toBe(8);
|
||||
});
|
||||
});
|
||||
|
||||
describe("copy", () => {
|
||||
it("copies object to new key", { timeout: 5000 }, async () => {
|
||||
await storage.set("src", new Uint8Array([99]));
|
||||
await storage.copy("src", "dst");
|
||||
expect(await storage.get("dst")).toEqual(new Uint8Array([99]));
|
||||
expect(await storage.get("src")).toEqual(new Uint8Array([99]));
|
||||
});
|
||||
});
|
||||
|
||||
describe("move", () => {
|
||||
it("moves object: dst exists, src deleted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("mv-src", new Uint8Array([77]));
|
||||
await storage.move("mv-src", "mv-dst");
|
||||
expect(await storage.get("mv-dst")).toEqual(new Uint8Array([77]));
|
||||
expect(await storage.get("mv-src")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("createReadStream / createWriteStream", () => {
|
||||
it("streams round-trip", { timeout: 5000 }, async () => {
|
||||
await storage.set(
|
||||
"stream-test",
|
||||
new TextEncoder().encode("streamed")
|
||||
);
|
||||
const rs = storage.createReadStream("stream-test");
|
||||
const reader = rs.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
|
||||
const merged = new Uint8Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
merged.set(c, off);
|
||||
off += c.byteLength;
|
||||
}
|
||||
expect(new TextDecoder().decode(merged)).toBe("streamed");
|
||||
});
|
||||
|
||||
it("write stream stores data", { timeout: 5000 }, async () => {
|
||||
const ws = storage.createWriteStream("ws-test");
|
||||
const writer = ws.getWriter();
|
||||
await writer.write(new TextEncoder().encode("written"));
|
||||
await writer.close();
|
||||
const result = await storage.get("ws-test");
|
||||
expect(new TextDecoder().decode(result)).toBe("written");
|
||||
});
|
||||
});
|
||||
|
||||
describe("setMultipart", () => {
|
||||
it("uploads stream in parts", { timeout: 5000 }, async () => {
|
||||
const data = new Uint8Array(12 * 1024 * 1024); // 12 MiB
|
||||
data.fill(42);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(data);
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await storage.setMultipart("large.bin", stream, {
|
||||
partSize: 5 * 1024 * 1024,
|
||||
});
|
||||
const result = await storage.get("large.bin");
|
||||
expect(result?.byteLength).toBe(data.byteLength);
|
||||
});
|
||||
|
||||
it("aborts multipart on signal abort", { timeout: 5000 }, async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array(6 * 1024 * 1024));
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
storage.setMultipart("aborted.bin", stream, {
|
||||
signal: ac.signal,
|
||||
})
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("presign", () => {
|
||||
it("presign throws S3StorageError on error", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// With FakeS3Client, getSignedUrl will fail because it's not a real client
|
||||
// We wrap the test to verify that presign throws S3StorageError
|
||||
const brokenFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async () => {
|
||||
throw new Error("broken");
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const brokenStorage = new S3({ ...BASE_OPTS }, brokenFactory);
|
||||
// presign uses getSignedUrl which requires real client methods
|
||||
// The error should be wrapped
|
||||
const result = await brokenStorage
|
||||
.presign("file.txt")
|
||||
.catch((e) => e);
|
||||
expect(result).toBeInstanceOf(S3StorageError);
|
||||
expect((result as S3StorageError).code).toBe(
|
||||
S3_CODES.PRESIGN_FAILED
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("keyPrefix", () => {
|
||||
it("applies prefix to all keys", { timeout: 5000 }, async () => {
|
||||
const prefixed = new S3(
|
||||
{ ...BASE_OPTS, keyPrefix: "prefix" },
|
||||
makeFakeFactory(fake)
|
||||
);
|
||||
await prefixed.set("foo", new Uint8Array([1]));
|
||||
// The fake store should have "prefix/foo"
|
||||
expect(fake.store.has("prefix/foo")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("AbortSignal", () => {
|
||||
it("throws when signal is already aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
await expect(storage.get("k", ac.signal)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
await expect(
|
||||
storage.set("k", new Uint8Array(), undefined, ac.signal)
|
||||
).rejects.toThrow(S3StorageError);
|
||||
await expect(storage.delete("k", ac.signal)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
await expect(storage.exists("k", ac.signal)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
await expect(storage.list(undefined, ac.signal)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
await expect(storage.clear(ac.signal)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createS3 factory", () => {
|
||||
it("creates S3 instance", { timeout: 5000 }, () => {
|
||||
const inst = createS3({ ...BASE_OPTS }, makeFakeFactory(fake));
|
||||
expect(inst).toBeInstanceOf(S3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("S3StorageError wrapping", () => {
|
||||
it("wraps non-404 get errors as S3StorageError", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const errorFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async (cmd: any) => {
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
const e = new S3ServiceException({
|
||||
name: "InternalError",
|
||||
message: "Server error",
|
||||
$fault: "server",
|
||||
$metadata: { httpStatusCode: 500 },
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
if (cmd instanceof HeadObjectCommand) {
|
||||
return {
|
||||
ContentLength: 5,
|
||||
LastModified: new Date(),
|
||||
};
|
||||
}
|
||||
return {};
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, errorFactory);
|
||||
await expect(s.get("k")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("wraps copy errors as S3StorageError", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const errorFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async () => {
|
||||
throw new Error("copy failed");
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, errorFactory);
|
||||
await expect(s.copy("a", "b")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("wraps list errors as S3StorageError", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const errorFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async () => {
|
||||
throw new Error("list failed");
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, errorFactory);
|
||||
await expect(s.list()).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("wraps delete errors as S3StorageError", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const errorFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async (cmd: any) => {
|
||||
if (cmd instanceof HeadObjectCommand) {
|
||||
return {
|
||||
ContentLength: 0,
|
||||
LastModified: new Date(),
|
||||
};
|
||||
}
|
||||
throw new Error("delete failed");
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, errorFactory);
|
||||
await expect(s.delete("k")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("wraps exists errors as S3StorageError for non-404", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const errorFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async () => {
|
||||
const e = new S3ServiceException({
|
||||
name: "AccessDenied",
|
||||
message: "Denied",
|
||||
$fault: "client",
|
||||
$metadata: { httpStatusCode: 403 },
|
||||
});
|
||||
throw e;
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, errorFactory);
|
||||
await expect(s.exists("k")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("wraps getMetadata errors as S3StorageError for non-404", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const errorFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async () => {
|
||||
const e = new S3ServiceException({
|
||||
name: "InternalError",
|
||||
message: "Server error",
|
||||
$fault: "server",
|
||||
$metadata: { httpStatusCode: 500 },
|
||||
});
|
||||
throw e;
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, errorFactory);
|
||||
await expect(s.getMetadata("k")).rejects.toThrow(S3StorageError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retry", () => {
|
||||
it("retries retryable errors", { timeout: 30000 }, async () => {
|
||||
let calls = 0;
|
||||
const retryFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async (cmd: any) => {
|
||||
if (cmd instanceof HeadObjectCommand)
|
||||
return {
|
||||
ContentLength: 1,
|
||||
LastModified: new Date(),
|
||||
};
|
||||
if (cmd instanceof GetObjectCommand) {
|
||||
calls++;
|
||||
if (calls < 2) {
|
||||
const e = new S3ServiceException({
|
||||
name: "SlowDown",
|
||||
message: "SlowDown",
|
||||
$fault: "server",
|
||||
$metadata: { httpStatusCode: 503 },
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
return {
|
||||
Body: {
|
||||
[Symbol.asyncIterator]() {
|
||||
let done = false;
|
||||
return {
|
||||
async next() {
|
||||
if (done)
|
||||
return {
|
||||
value: undefined,
|
||||
done: true,
|
||||
};
|
||||
done = true;
|
||||
return {
|
||||
value: new Uint8Array([
|
||||
1,
|
||||
]),
|
||||
done: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3(
|
||||
{ ...BASE_OPTS, retry: { maxAttempts: 3, baseDelayMs: 1 } },
|
||||
retryFactory
|
||||
);
|
||||
const result = await s.get("k");
|
||||
expect(result).toEqual(new Uint8Array([1]));
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("multipart error handling", () => {
|
||||
it("wraps multipart errors and aborts", { timeout: 5000 }, async () => {
|
||||
let uploadId: string | undefined;
|
||||
let aborted = false;
|
||||
const mpFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async (cmd: any) => {
|
||||
if (cmd instanceof CreateMultipartUploadCommand) {
|
||||
uploadId = "up-1";
|
||||
return { UploadId: uploadId };
|
||||
}
|
||||
if (cmd instanceof UploadPartCommand) {
|
||||
throw new Error("part upload failed");
|
||||
}
|
||||
if (cmd instanceof AbortMultipartUploadCommand) {
|
||||
aborted = true;
|
||||
return {};
|
||||
}
|
||||
return {};
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, mpFactory);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array(6 * 1024 * 1024));
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await expect(s.setMultipart("fail.bin", stream)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
expect(aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("throws when no UploadId returned", { timeout: 5000 }, async () => {
|
||||
const mpFactory: S3ClientFactory = {
|
||||
create: () =>
|
||||
({
|
||||
send: async (cmd: any) => {
|
||||
if (cmd instanceof CreateMultipartUploadCommand) {
|
||||
return {}; // no UploadId
|
||||
}
|
||||
return {};
|
||||
},
|
||||
}) as any,
|
||||
};
|
||||
const s = new S3({ ...BASE_OPTS }, mpFactory);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(ctrl) {
|
||||
ctrl.enqueue(new Uint8Array([1]));
|
||||
ctrl.close();
|
||||
},
|
||||
});
|
||||
await expect(s.setMultipart("noid.bin", stream)).rejects.toThrow(
|
||||
S3StorageError
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
712
packages/persistence/storage/storage-s3/src/s3/s3.ts
Normal file
712
packages/persistence/storage/storage-s3/src/s3/s3.ts
Normal file
@@ -0,0 +1,712 @@
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CopyObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
S3ServiceException,
|
||||
UploadPartCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
import { AbstractStorage } from "@suaveplan/storage/abstract";
|
||||
import type {
|
||||
StorageCapabilities,
|
||||
StorageListOptions,
|
||||
StorageListResult,
|
||||
StorageMetadata,
|
||||
StorageOptions,
|
||||
StorageResult,
|
||||
} from "@suaveplan/storage/types";
|
||||
import {
|
||||
DEFAULT_PART_SIZE,
|
||||
DEFAULT_PRESIGN_EXPIRES,
|
||||
} from "../constants/index.js";
|
||||
import { S3_CODES, S3StorageError } from "../error-codes.js";
|
||||
import type {
|
||||
MultipartUploadOptions,
|
||||
PresignOptions,
|
||||
S3ClientFactory,
|
||||
S3Options,
|
||||
} from "../types/s3.types.js";
|
||||
import { validateCredentials } from "../utils/credential-validator.js";
|
||||
|
||||
const meter = metrics.getMeter("@suaveplan/storage-s3");
|
||||
const tracer = trace.getTracer("@suaveplan/storage-s3");
|
||||
|
||||
const opCounter = meter.createCounter("s3.ops.total", {
|
||||
description: "Total S3 storage operations",
|
||||
});
|
||||
const durationHistogram = meter.createHistogram("s3.op.duration_ms", {
|
||||
description: "S3 operation latency in milliseconds",
|
||||
unit: "ms",
|
||||
});
|
||||
|
||||
/**
|
||||
* Default S3 client factory — creates a real AWS SDK S3Client.
|
||||
*/
|
||||
const defaultClientFactory: S3ClientFactory = {
|
||||
create(opts: S3Options): S3Client {
|
||||
return new S3Client({
|
||||
region: opts.region,
|
||||
...(opts.endpoint
|
||||
? { endpoint: opts.endpoint, forcePathStyle: true }
|
||||
: {}),
|
||||
...(opts.accessKeyId && opts.secretAccessKey
|
||||
? {
|
||||
credentials: {
|
||||
accessKeyId: opts.accessKeyId,
|
||||
secretAccessKey: opts.secretAccessKey,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
function isRetryableS3Error(err: unknown): boolean {
|
||||
if (!(err instanceof S3ServiceException)) return false;
|
||||
const code = err.name ?? "";
|
||||
return (
|
||||
err.$retryable?.throttling === true ||
|
||||
code === "SlowDown" ||
|
||||
code === "RequestTimeout" ||
|
||||
(err.$metadata?.httpStatusCode ?? 0) >= 500
|
||||
);
|
||||
}
|
||||
|
||||
function buildKey(prefix: string | undefined, key: string): string {
|
||||
if (!prefix) return key;
|
||||
return `${prefix.replace(/\/$/, "")}/${key}`;
|
||||
}
|
||||
|
||||
function stripPrefix(prefix: string | undefined, fullKey: string): string {
|
||||
if (!prefix) return fullKey;
|
||||
const p = `${prefix.replace(/\/$/, "")}/`;
|
||||
// S3 always returns keys with the requested prefix; the else branch is defensive
|
||||
return fullKey.startsWith(p)
|
||||
? fullKey.slice(p.length)
|
||||
: /* istanbul ignore next */ fullKey;
|
||||
}
|
||||
|
||||
async function withRetry<T>(fn: () => Promise<T>, opts: S3Options): Promise<T> {
|
||||
const maxAttempts = Math.max(1, opts.retry?.maxAttempts ?? 1);
|
||||
const baseDelay = opts.retry?.baseDelayMs ?? 100;
|
||||
const maxDelay = opts.retry?.maxDelayMs ?? 30000;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
if (attempt + 1 >= maxAttempts || !isRetryableS3Error(err)) {
|
||||
throw err;
|
||||
}
|
||||
const delay = Math.min(baseDelay * 2 ** attempt, maxDelay);
|
||||
await new Promise<void>((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* S3-compatible object storage adapter extending AbstractStorage.
|
||||
*
|
||||
* Backed by @aws-sdk/client-s3 with presigned URL generation,
|
||||
* multipart upload, retry, and OpenTelemetry instrumentation.
|
||||
*/
|
||||
export class S3 extends AbstractStorage {
|
||||
protected readonly options: S3Options;
|
||||
protected readonly client: S3Client;
|
||||
|
||||
public override readonly capabilities: StorageCapabilities = {
|
||||
partialReads: true,
|
||||
partialWrites: false,
|
||||
nativeStreams: true,
|
||||
atomicWrites: true,
|
||||
contentAddressable: false,
|
||||
};
|
||||
|
||||
public constructor(opts: S3Options, clientFactory?: S3ClientFactory) {
|
||||
super();
|
||||
validateCredentials(opts);
|
||||
this.options = opts;
|
||||
this.client = (clientFactory ?? defaultClientFactory).create(opts);
|
||||
}
|
||||
|
||||
protected prefixedKey(key: string): string {
|
||||
return buildKey(this.options.keyPrefix, key);
|
||||
}
|
||||
|
||||
protected strippedKey(fullKey: string): string {
|
||||
return stripPrefix(this.options.keyPrefix, fullKey);
|
||||
}
|
||||
|
||||
protected checkSignal(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) {
|
||||
throw new S3StorageError(S3_CODES.ABORTED, "Operation aborted");
|
||||
}
|
||||
}
|
||||
|
||||
protected async time<T>(op: string, fn: () => Promise<T>): Promise<T> {
|
||||
const start = Date.now();
|
||||
opCounter.add(1, { "s3.op": op });
|
||||
return tracer.startActiveSpan(`s3.${op}`, async (span) => {
|
||||
try {
|
||||
const result = await fn();
|
||||
durationHistogram.record(Date.now() - start, { "s3.op": op });
|
||||
span.end();
|
||||
return result;
|
||||
} catch (err) {
|
||||
span.recordException(err as Error);
|
||||
span.end();
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ── StorageLike implementation ─────────────────────────────────────────────
|
||||
|
||||
async get(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<Uint8Array | undefined> {
|
||||
this.checkSignal(signal);
|
||||
return this.time("get", async () => {
|
||||
try {
|
||||
const resp = await withRetry(
|
||||
() =>
|
||||
this.client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
}),
|
||||
{
|
||||
...(signal !== undefined
|
||||
? { abortSignal: signal }
|
||||
: {}),
|
||||
}
|
||||
),
|
||||
this.options
|
||||
);
|
||||
const stream = resp.Body;
|
||||
if (!stream) return undefined;
|
||||
const chunks: Uint8Array[] = [];
|
||||
// biome-ignore lint/suspicious/noExplicitAny: AWS SDK stream typing
|
||||
const readable = stream as unknown as AsyncIterable<any>;
|
||||
for await (const chunk of readable) {
|
||||
chunks.push(
|
||||
chunk instanceof Uint8Array
|
||||
? chunk
|
||||
: /* istanbul ignore next */ new Uint8Array(chunk)
|
||||
);
|
||||
}
|
||||
const total = chunks.reduce((s, c) => s + c.byteLength, 0);
|
||||
const result = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
result.set(c, offset);
|
||||
offset += c.byteLength;
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof S3ServiceException &&
|
||||
err.$metadata?.httpStatusCode === 404
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
throw new S3StorageError(
|
||||
S3_CODES.UPLOAD_FAILED,
|
||||
`S3 get failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async set(
|
||||
key: string,
|
||||
value: Uint8Array,
|
||||
_options?: StorageOptions,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
this.checkSignal(signal);
|
||||
return this.time("set", async () => {
|
||||
try {
|
||||
await withRetry(
|
||||
() =>
|
||||
this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
Body: value,
|
||||
ContentLength: value.byteLength,
|
||||
}),
|
||||
{
|
||||
...(signal !== undefined
|
||||
? { abortSignal: signal }
|
||||
: {}),
|
||||
}
|
||||
),
|
||||
this.options
|
||||
);
|
||||
this.options.logger?.debug?.("s3.set", {
|
||||
key,
|
||||
size: value.byteLength,
|
||||
});
|
||||
} catch (err) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.UPLOAD_FAILED,
|
||||
`S3 set failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async delete(key: string, signal?: AbortSignal): Promise<boolean> {
|
||||
this.checkSignal(signal);
|
||||
return this.time("delete", async () => {
|
||||
try {
|
||||
const exists = await this.exists(key, signal);
|
||||
if (!exists) return false;
|
||||
await withRetry(
|
||||
() =>
|
||||
this.client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
}),
|
||||
{
|
||||
...(signal !== undefined
|
||||
? { abortSignal: signal }
|
||||
: {}),
|
||||
}
|
||||
),
|
||||
this.options
|
||||
);
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (err instanceof S3StorageError) throw err;
|
||||
throw new S3StorageError(
|
||||
S3_CODES.DELETE_FAILED,
|
||||
`S3 delete failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async exists(key: string, signal?: AbortSignal): Promise<boolean> {
|
||||
this.checkSignal(signal);
|
||||
try {
|
||||
await this.client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
}),
|
||||
{ ...(signal !== undefined ? { abortSignal: signal } : {}) }
|
||||
);
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof S3ServiceException &&
|
||||
err.$metadata?.httpStatusCode === 404
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
throw new S3StorageError(
|
||||
S3_CODES.UPLOAD_FAILED,
|
||||
`S3 exists check failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async list(
|
||||
options?: StorageListOptions,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageListResult> {
|
||||
this.checkSignal(signal);
|
||||
return this.time("list", async () => {
|
||||
try {
|
||||
const prefix = options?.prefix
|
||||
? buildKey(this.options.keyPrefix, options.prefix)
|
||||
: this.options.keyPrefix;
|
||||
const keys: string[] = [];
|
||||
let continuationToken: string | undefined;
|
||||
do {
|
||||
this.checkSignal(signal);
|
||||
const resp = await withRetry(
|
||||
() =>
|
||||
this.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.options.bucket,
|
||||
Prefix: prefix ?? undefined,
|
||||
ContinuationToken: continuationToken,
|
||||
MaxKeys: options?.limit ?? 1000,
|
||||
}),
|
||||
{
|
||||
...(signal !== undefined
|
||||
? { abortSignal: signal }
|
||||
: {}),
|
||||
}
|
||||
),
|
||||
this.options
|
||||
);
|
||||
for (const obj of resp.Contents ?? []) {
|
||||
if (obj.Key) keys.push(this.strippedKey(obj.Key));
|
||||
}
|
||||
continuationToken = resp.NextContinuationToken;
|
||||
} while (continuationToken && !options?.continuationToken);
|
||||
|
||||
return {
|
||||
keys,
|
||||
total: keys.length,
|
||||
hasMore: false,
|
||||
continuationToken: undefined,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof S3StorageError) throw err;
|
||||
throw new S3StorageError(
|
||||
S3_CODES.LIST_FAILED,
|
||||
`S3 list failed: ${(err as Error).message}`,
|
||||
{
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async clear(signal?: AbortSignal): Promise<void> {
|
||||
this.checkSignal(signal);
|
||||
const { keys } = await this.list(undefined, signal);
|
||||
for (const key of keys) {
|
||||
this.checkSignal(signal);
|
||||
await this.delete(key, signal);
|
||||
}
|
||||
}
|
||||
|
||||
async getWithMetadata(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageResult<Uint8Array> | null> {
|
||||
this.checkSignal(signal);
|
||||
const meta = await this.getMetadata(key, signal);
|
||||
if (!meta) return null;
|
||||
const value = await this.get(key, signal);
|
||||
if (!value) return null;
|
||||
return { value, metadata: meta };
|
||||
}
|
||||
|
||||
async getMetadata(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageMetadata | null> {
|
||||
this.checkSignal(signal);
|
||||
try {
|
||||
const resp = await this.client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
}),
|
||||
{ ...(signal !== undefined ? { abortSignal: signal } : {}) }
|
||||
);
|
||||
const ts = resp.LastModified
|
||||
? resp.LastModified.getTime()
|
||||
: Date.now();
|
||||
return {
|
||||
size: resp.ContentLength ?? 0,
|
||||
createdAt: ts,
|
||||
modifiedAt: ts,
|
||||
};
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof S3ServiceException &&
|
||||
err.$metadata?.httpStatusCode === 404
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw new S3StorageError(
|
||||
S3_CODES.UPLOAD_FAILED,
|
||||
`S3 metadata failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async getSize(signal?: AbortSignal): Promise<number> {
|
||||
this.checkSignal(signal);
|
||||
const { keys } = await this.list(undefined, signal);
|
||||
let total = 0;
|
||||
for (const key of keys) {
|
||||
const meta = await this.getMetadata(key, signal);
|
||||
total += meta?.size ?? 0;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
async getPartial(
|
||||
key: string,
|
||||
offset: number,
|
||||
length: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<Uint8Array | null> {
|
||||
this.checkSignal(signal);
|
||||
const data = await this.get(key, signal);
|
||||
if (!data) return null;
|
||||
return this.defaultGetPartial(data, offset, length);
|
||||
}
|
||||
|
||||
async setPartial(
|
||||
key: string,
|
||||
offset: number,
|
||||
bytes: Uint8Array,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
this.checkSignal(signal);
|
||||
await this.defaultSetPartial(key, offset, bytes, signal);
|
||||
}
|
||||
|
||||
async getItemSize(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<number | null> {
|
||||
this.checkSignal(signal);
|
||||
const meta = await this.getMetadata(key, signal);
|
||||
return meta?.size ?? null;
|
||||
}
|
||||
|
||||
createReadStream(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): ReadableStream<Uint8Array> {
|
||||
return this.defaultCreateReadStream(key, signal);
|
||||
}
|
||||
|
||||
createWriteStream(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): WritableStream<Uint8Array> {
|
||||
return this.defaultCreateWriteStream(key, signal);
|
||||
}
|
||||
|
||||
// ── S3-specific extensions ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Copy an object within the same bucket.
|
||||
*/
|
||||
async copy(
|
||||
source: string,
|
||||
destination: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
this.checkSignal(signal);
|
||||
return this.time("copy", async () => {
|
||||
try {
|
||||
await withRetry(
|
||||
() =>
|
||||
this.client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
CopySource: `${this.options.bucket}/${this.prefixedKey(source)}`,
|
||||
Key: this.prefixedKey(destination),
|
||||
}),
|
||||
{
|
||||
...(signal !== undefined
|
||||
? { abortSignal: signal }
|
||||
: {}),
|
||||
}
|
||||
),
|
||||
this.options
|
||||
);
|
||||
} catch (err) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.UPLOAD_FAILED,
|
||||
`S3 copy failed: ${(err as Error).message}`,
|
||||
{
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Move an object by copying it then deleting the source.
|
||||
*/
|
||||
async move(
|
||||
source: string,
|
||||
destination: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
this.checkSignal(signal);
|
||||
await this.copy(source, destination, signal);
|
||||
await this.delete(source, signal);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a presigned URL for GET or PUT.
|
||||
*/
|
||||
async presign(key: string, opts?: PresignOptions): Promise<string> {
|
||||
return this.time("presign", async () => {
|
||||
try {
|
||||
const expiresIn = opts?.expiresIn ?? DEFAULT_PRESIGN_EXPIRES;
|
||||
const method = opts?.method ?? "GET";
|
||||
const command =
|
||||
method === "PUT"
|
||||
? new PutObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
})
|
||||
: new GetObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.prefixedKey(key),
|
||||
});
|
||||
return await getSignedUrl(this.client, command, { expiresIn });
|
||||
} catch (err) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.PRESIGN_FAILED,
|
||||
`S3 presign failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Multipart upload for large objects.
|
||||
*/
|
||||
async setMultipart(
|
||||
key: string,
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
opts?: MultipartUploadOptions
|
||||
): Promise<void> {
|
||||
const signal = opts?.signal;
|
||||
this.checkSignal(signal);
|
||||
const partSize = opts?.partSize ?? DEFAULT_PART_SIZE;
|
||||
const fullKey = this.prefixedKey(key);
|
||||
let uploadId: string | undefined;
|
||||
return this.time("multipart", async () => {
|
||||
try {
|
||||
const createResp = await this.client.send(
|
||||
new CreateMultipartUploadCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: fullKey,
|
||||
})
|
||||
);
|
||||
uploadId = createResp.UploadId;
|
||||
if (!uploadId) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.MULTIPART_FAILED,
|
||||
"S3 multipart: no UploadId returned"
|
||||
);
|
||||
}
|
||||
|
||||
const parts: { ETag: string; PartNumber: number }[] = [];
|
||||
let partNumber = 1;
|
||||
let buffer = new Uint8Array(0);
|
||||
const reader = stream.getReader();
|
||||
|
||||
const flushPart = async (
|
||||
data: Uint8Array,
|
||||
num: number
|
||||
): Promise<void> => {
|
||||
this.checkSignal(signal);
|
||||
const resp = await withRetry(
|
||||
() =>
|
||||
this.client.send(
|
||||
new UploadPartCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: fullKey,
|
||||
UploadId: uploadId,
|
||||
PartNumber: num,
|
||||
Body: data,
|
||||
ContentLength: data.byteLength,
|
||||
})
|
||||
),
|
||||
this.options
|
||||
);
|
||||
parts.push({ ETag: resp.ETag ?? "", PartNumber: num });
|
||||
};
|
||||
|
||||
// Collect chunks and flush parts
|
||||
while (true) {
|
||||
this.checkSignal(signal);
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const combined = new Uint8Array(
|
||||
buffer.byteLength + value.byteLength
|
||||
);
|
||||
combined.set(buffer);
|
||||
combined.set(value, buffer.byteLength);
|
||||
buffer = combined;
|
||||
while (buffer.byteLength >= partSize) {
|
||||
const part = buffer.slice(0, partSize);
|
||||
buffer = buffer.slice(partSize);
|
||||
await flushPart(part, partNumber++);
|
||||
}
|
||||
}
|
||||
if (buffer.byteLength > 0) {
|
||||
await flushPart(buffer, partNumber++);
|
||||
}
|
||||
|
||||
await this.client.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: fullKey,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: parts },
|
||||
})
|
||||
);
|
||||
} catch (err) {
|
||||
if (uploadId) {
|
||||
await this.client
|
||||
.send(
|
||||
new AbortMultipartUploadCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: fullKey,
|
||||
UploadId: uploadId,
|
||||
})
|
||||
)
|
||||
.catch(() => void 0);
|
||||
}
|
||||
if (err instanceof S3StorageError) throw err;
|
||||
throw new S3StorageError(
|
||||
S3_CODES.MULTIPART_FAILED,
|
||||
`S3 multipart upload failed: ${(err as Error).message}`,
|
||||
{
|
||||
key,
|
||||
cause: err as Error,
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function createS3(opts: S3Options, clientFactory?: S3ClientFactory): S3 {
|
||||
return new S3(opts, clientFactory);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export type { RetryOptions } from "./retry-compat.types.js";
|
||||
export type {
|
||||
MultipartUploadOptions,
|
||||
PresignOptions,
|
||||
S3ClientFactory,
|
||||
S3Options,
|
||||
} from "./s3.types.js";
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Minimal retry options compatible with @suaveplan/retry RetryStrategyOptions.
|
||||
*/
|
||||
export interface RetryOptions {
|
||||
/** Maximum number of attempts. Default: 3. */
|
||||
maxAttempts?: number;
|
||||
/** Base delay in milliseconds. Default: 100. */
|
||||
baseDelayMs?: number;
|
||||
/** Maximum delay in milliseconds. Default: 30000. */
|
||||
maxDelayMs?: number;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
# s3.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the public TypeScript types for the `@suaveplan/storage-s3` package. These types govern configuration, presigned URL generation, multipart upload, and the injectable S3 client factory interface. They are designed to be stable, serialization-friendly, and usable without importing the main `S3` class.
|
||||
|
||||
The types decouple configuration concerns from implementation, enabling dependency injection and testability. The `S3ClientFactory` interface in particular allows tests to inject a `FakeS3Client` without touching real AWS infrastructure.
|
||||
|
||||
## Features
|
||||
|
||||
- `S3Options` — complete configuration for the S3 adapter including credentials, endpoint, key prefix, retry strategy, and logger injection
|
||||
- `PresignOptions` — controls presigned URL generation: expiry and HTTP method
|
||||
- `MultipartUploadOptions` — controls multipart upload: part size, concurrency, and AbortSignal
|
||||
- `S3ClientFactory` — injectable factory interface for the underlying `S3Client`
|
||||
- `RetryOptions` — minimal subset of `@suaveplan/retry` `RetryStrategyOptions` for forward-compatible configuration
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type { S3Options } from "@suaveplan/storage-s3";
|
||||
|
||||
const options: S3Options = {
|
||||
bucket: "my-bucket",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { S3Options, S3ClientFactory } from "@suaveplan/storage-s3";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import type { LoggerLike } from "@suaveplan/types";
|
||||
|
||||
const logger: LoggerLike = console;
|
||||
|
||||
const options: S3Options = {
|
||||
bucket: "my-bucket",
|
||||
region: "auto",
|
||||
endpoint: "https://my-garage-instance.example.com",
|
||||
keyPrefix: "uploads/",
|
||||
retry: { maxAttempts: 5, baseDelayMs: 200 },
|
||||
logger,
|
||||
};
|
||||
|
||||
// Custom factory for testing
|
||||
const fakeFactory: S3ClientFactory = {
|
||||
create: (_opts) => new S3Client({ region: "us-east-1" }),
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `S3Options`
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `bucket` | `string` | Yes | S3 bucket name |
|
||||
| `region` | `string` | Yes | AWS region or "auto" |
|
||||
| `accessKeyId` | `string` | No | AWS access key ID |
|
||||
| `secretAccessKey` | `string` | No | AWS secret access key |
|
||||
| `endpoint` | `string` | No | Custom S3-compatible endpoint |
|
||||
| `keyPrefix` | `string` | No | Prefix prepended to all keys |
|
||||
| `retry` | `RetryOptions` | No | Retry strategy configuration |
|
||||
| `logger` | `LoggerLike` | No | Structured logger |
|
||||
|
||||
### `PresignOptions`
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `expiresIn` | `number` | No | Expiry in seconds (default: 3600) |
|
||||
| `method` | `"GET" \| "PUT"` | No | HTTP method (default: "GET") |
|
||||
|
||||
### `MultipartUploadOptions`
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `partSize` | `number` | No | Part size in bytes (default: 5 MiB) |
|
||||
| `concurrency` | `number` | No | Max concurrent part uploads (default: 4) |
|
||||
| `signal` | `AbortSignal` | No | Cancellation signal |
|
||||
|
||||
### `S3ClientFactory`
|
||||
|
||||
| Method | Params | Returns | Description |
|
||||
|---|---|---|---|
|
||||
| `create` | `options: S3Options` | `S3Client` | Creates an AWS SDK S3Client from options |
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`RetryOptions` is intentionally a minimal subset of `@suaveplan/retry`'s full options to avoid a circular type dependency. The `S3` class maps these to the full retry strategy internally. `S3ClientFactory` returns the AWS SDK `S3Client` type directly; consumers who want to mock S3 should implement this interface and return an object with matching method signatures rather than extending the SDK class.
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import type { LoggerLike } from "@suaveplan/types";
|
||||
import type { RetryOptions } from "../types/retry-compat.types.js";
|
||||
|
||||
/**
|
||||
* Options for configuring the S3 storage adapter.
|
||||
*/
|
||||
export interface S3Options {
|
||||
/** S3 bucket name. */
|
||||
bucket: string;
|
||||
/** AWS region (e.g., "us-east-1"). */
|
||||
region: string;
|
||||
/** AWS access key ID. */
|
||||
accessKeyId?: string;
|
||||
/** AWS secret access key. */
|
||||
secretAccessKey?: string;
|
||||
/** Custom endpoint for S3-compatible storage (MinIO, Garage, R2). */
|
||||
endpoint?: string;
|
||||
/** Object key prefix applied to all operations. */
|
||||
keyPrefix?: string;
|
||||
/** Retry strategy options for S3 API calls. */
|
||||
retry?: RetryOptions;
|
||||
/** Logger for structured operation logging. */
|
||||
logger?: LoggerLike;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for presigned URL generation.
|
||||
*/
|
||||
export interface PresignOptions {
|
||||
/** URL expiry in seconds. Default: 3600. */
|
||||
expiresIn?: number;
|
||||
/** HTTP method for the presigned URL. Default: "GET". */
|
||||
method?: "GET" | "PUT";
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for multipart upload.
|
||||
*/
|
||||
export interface MultipartUploadOptions {
|
||||
/** Size of each part in bytes. Default: 5 MiB. */
|
||||
partSize?: number;
|
||||
/** Maximum number of concurrent part uploads. Default: 4. */
|
||||
concurrency?: number;
|
||||
/** AbortSignal to cancel the upload. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Injectable factory for creating S3Client instances.
|
||||
*/
|
||||
export interface S3ClientFactory {
|
||||
/** Create an S3Client from the provided options. */
|
||||
create(options: S3Options): S3Client;
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
# credential-validator
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides AWS S3 credential format validation and endpoint classification for the `@suaveplan/storage-s3` package. It guards against misconfigured credentials before they hit the AWS API, producing structured `S3StorageError` instances rather than opaque SDK errors. For S3-compatible endpoints (MinIO, Garage, Cloudflare R2) it skips AWS-specific format checks since those providers use different credential formats.
|
||||
|
||||
The utility is also used in property-based tests to verify that `isAwsS3` correctly classifies arbitrary endpoint URLs.
|
||||
|
||||
## Features
|
||||
|
||||
- `isAwsS3(endpoint?)` — classifies an endpoint as AWS S3 proper vs. S3-compatible; uses `URL.hostname` for robustness
|
||||
- `validateCredentials(options)` — validates access key ID (AKIA prefix + 20 chars) and secret key (≥ 40 chars) only for AWS S3 endpoints
|
||||
- Allows environment/instance credentials (IAM roles, EC2 instance profiles) when neither key is provided
|
||||
- Throws `S3StorageError` with stable `S3_CODES` for all failure modes
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { isAwsS3, validateCredentials } from "@suaveplan/storage-s3/utils/credential-validator";
|
||||
|
||||
// Classify endpoint
|
||||
isAwsS3(); // true (no endpoint = AWS)
|
||||
isAwsS3("https://s3.amazonaws.com"); // true
|
||||
isAwsS3("https://my-garage.example.com"); // false
|
||||
|
||||
// Validate credentials
|
||||
validateCredentials({ bucket: "b", region: "us-east-1" }); // OK (instance creds)
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { validateCredentials } from "@suaveplan/storage-s3/utils/credential-validator";
|
||||
import { S3StorageError } from "@suaveplan/storage-s3";
|
||||
|
||||
try {
|
||||
validateCredentials({
|
||||
bucket: "my-bucket",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "BADKEY",
|
||||
secretAccessKey: "short",
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof S3StorageError) {
|
||||
console.log(err.code); // "S3.CREDENTIALS_INVALID"
|
||||
}
|
||||
}
|
||||
|
||||
// S3-compatible endpoint: skip validation
|
||||
validateCredentials({
|
||||
bucket: "my-bucket",
|
||||
region: "garage",
|
||||
endpoint: "https://my-garage.internal",
|
||||
accessKeyId: "garage-key", // non-AKIA format is fine
|
||||
secretAccessKey: "s", // short is fine too
|
||||
}); // does not throw
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `isAwsS3(endpoint?: string): boolean`
|
||||
|
||||
Returns `true` if `endpoint` is undefined (defaults to AWS) or its hostname ends with `"amazonaws.com"`. Returns `false` for all other URLs, including malformed ones.
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
| `endpoint` | `string?` | Custom endpoint URL; omit for AWS S3 proper |
|
||||
|
||||
**Returns**: `boolean`
|
||||
|
||||
### `validateCredentials(options: S3Options): void`
|
||||
|
||||
Validates credential format when targeting AWS S3. No-op for S3-compatible endpoints.
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
| `options` | `S3Options` | Full S3 options including credentials and endpoint |
|
||||
|
||||
**Throws**:
|
||||
|
||||
- `S3StorageError(S3.CREDENTIALS_MISSING)` — when one of the pair is provided but not the other
|
||||
- `S3StorageError(S3.CREDENTIALS_INVALID)` — when the access key ID or secret key fails format checks
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
AWS credentials may be sourced from environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`), IAM instance profiles, or ECS task roles — these are handled by the AWS SDK transparently. This validator only checks explicitly provided credentials. The `URL` constructor is used for endpoint parsing because string matching is fragile; any malformed URL string returns `false` from `isAwsS3` rather than throwing.
|
||||
@@ -0,0 +1,139 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { S3_CODES, S3StorageError } from "../error-codes.js";
|
||||
import { isAwsS3, validateCredentials } from "./credential-validator.js";
|
||||
|
||||
describe("isAwsS3", () => {
|
||||
it("returns true when no endpoint provided", { timeout: 5000 }, () => {
|
||||
expect(isAwsS3()).toBe(true);
|
||||
expect(isAwsS3(undefined)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for amazonaws.com endpoints", { timeout: 5000 }, () => {
|
||||
expect(isAwsS3("https://s3.amazonaws.com")).toBe(true);
|
||||
expect(isAwsS3("https://s3.us-east-1.amazonaws.com")).toBe(true);
|
||||
expect(isAwsS3("https://mybucket.s3.eu-west-1.amazonaws.com")).toBe(
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
it("returns false for S3-compatible endpoints", { timeout: 5000 }, () => {
|
||||
expect(isAwsS3("https://minio.example.com")).toBe(false);
|
||||
expect(isAwsS3("http://localhost:9000")).toBe(false);
|
||||
expect(isAwsS3("https://garage.internal:3900")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for malformed URLs", { timeout: 5000 }, () => {
|
||||
expect(isAwsS3("not-a-url")).toBe(false);
|
||||
expect(isAwsS3("://broken")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateCredentials", () => {
|
||||
it("allows no credentials for AWS (instance credentials)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
validateCredentials({ bucket: "b", region: "us-east-1" })
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("allows full valid AWS credentials", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
})
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws CREDENTIALS_MISSING when only one key provided", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err1 = (() => {
|
||||
try {
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
});
|
||||
} catch (e) {
|
||||
return e;
|
||||
}
|
||||
})();
|
||||
expect(err1).toBeInstanceOf(S3StorageError);
|
||||
expect((err1 as S3StorageError).code).toBe(
|
||||
S3_CODES.CREDENTIALS_MISSING
|
||||
);
|
||||
|
||||
const err2 = (() => {
|
||||
try {
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
});
|
||||
} catch (e) {
|
||||
return e;
|
||||
}
|
||||
})();
|
||||
expect(err2).toBeInstanceOf(S3StorageError);
|
||||
expect((err2 as S3StorageError).code).toBe(
|
||||
S3_CODES.CREDENTIALS_MISSING
|
||||
);
|
||||
});
|
||||
|
||||
it("throws CREDENTIALS_INVALID for bad access key ID prefix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "BADKEYIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
})
|
||||
).toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("throws CREDENTIALS_INVALID for wrong-length access key ID", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7", // too short
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
})
|
||||
).toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("throws CREDENTIALS_INVALID for short secret key", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "short",
|
||||
})
|
||||
).toThrow(S3StorageError);
|
||||
});
|
||||
|
||||
it("skips validation for S3-compatible endpoints", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
validateCredentials({
|
||||
bucket: "b",
|
||||
region: "garage",
|
||||
endpoint: "https://my-garage.example.com",
|
||||
accessKeyId: "NOTAKIA",
|
||||
secretAccessKey: "s",
|
||||
})
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import {
|
||||
AWS_ACCESS_KEY_LENGTH,
|
||||
AWS_ACCESS_KEY_PREFIX,
|
||||
AWS_SECRET_KEY_MIN_LENGTH,
|
||||
} from "../constants/index.js";
|
||||
import { S3_CODES, S3StorageError } from "../error-codes.js";
|
||||
import type { S3Options } from "../types/s3.types.js";
|
||||
|
||||
/**
|
||||
* Returns true when the given endpoint points to AWS S3 proper (not
|
||||
* an S3-compatible provider like MinIO, Garage, or Cloudflare R2).
|
||||
*/
|
||||
export function isAwsS3(endpoint?: string): boolean {
|
||||
if (!endpoint) return true;
|
||||
try {
|
||||
const { hostname } = new URL(endpoint);
|
||||
return hostname.endsWith("amazonaws.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates AWS credential format when targeting real AWS S3.
|
||||
*
|
||||
* For S3-compatible endpoints (non-amazonaws.com), credential format
|
||||
* validation is skipped — the provider may use different formats.
|
||||
*
|
||||
* @throws S3StorageError with code S3.CREDENTIALS_MISSING when credentials
|
||||
* are absent for AWS S3.
|
||||
* @throws S3StorageError with code S3.CREDENTIALS_INVALID when the access key
|
||||
* ID or secret key format is invalid for AWS S3.
|
||||
*/
|
||||
export function validateCredentials(options: S3Options): void {
|
||||
if (!isAwsS3(options.endpoint)) {
|
||||
// S3-compatible provider: skip AWS-specific format checks
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.accessKeyId && !options.secretAccessKey) {
|
||||
// Allow environment/instance credentials when neither is provided
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.accessKeyId || !options.secretAccessKey) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.CREDENTIALS_MISSING,
|
||||
"Both accessKeyId and secretAccessKey must be provided together"
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
!options.accessKeyId.startsWith(AWS_ACCESS_KEY_PREFIX) ||
|
||||
options.accessKeyId.length !== AWS_ACCESS_KEY_LENGTH
|
||||
) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.CREDENTIALS_INVALID,
|
||||
`AWS access key ID must start with "${AWS_ACCESS_KEY_PREFIX}" and be ${AWS_ACCESS_KEY_LENGTH} characters long`
|
||||
);
|
||||
}
|
||||
|
||||
if (options.secretAccessKey.length < AWS_SECRET_KEY_MIN_LENGTH) {
|
||||
throw new S3StorageError(
|
||||
S3_CODES.CREDENTIALS_INVALID,
|
||||
`AWS secret access key must be at least ${AWS_SECRET_KEY_MIN_LENGTH} characters long`
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("dep-graph", () => {
|
||||
it("package has correct name", { timeout: 5000 }, async () => {
|
||||
const pkg = await import("../package.json", { with: { type: "json" } });
|
||||
expect(pkg.default.name).toBe("@suaveplan/storage-s3");
|
||||
});
|
||||
|
||||
it("exports S3 class", { timeout: 30000 }, async () => {
|
||||
const { S3 } = await import("../src/index.js");
|
||||
expect(typeof S3).toBe("function");
|
||||
});
|
||||
|
||||
it("exports S3StorageError", { timeout: 30000 }, async () => {
|
||||
const { S3StorageError } = await import("../src/index.js");
|
||||
expect(typeof S3StorageError).toBe("function");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Vitest global-setup module — drives the Garage S3 fixture for storage-s3
|
||||
* integration tests.
|
||||
*
|
||||
* Uses the public `createIntegrationSetup()` factory from
|
||||
* `@suaveplan/testing-integration`. The Bun-NAPI shim that lets testcontainers
|
||||
* load under Bun is auto-installed by the re-exported
|
||||
* `@suaveplan/testing-integration/setup/global-setup` module — no per-consumer
|
||||
* wiring is needed.
|
||||
*/
|
||||
import { createIntegrationSetup } from "@suaveplan/testing-integration";
|
||||
import { GarageS3Fixture } from "@suaveplan/testing-integration-garage-s3";
|
||||
|
||||
// Default `ghcr.io/deuxfleurs/garage:v2.0.0` returns 403 for unauthenticated
|
||||
// pulls — mirror to Docker Hub.
|
||||
export const fixtures = {
|
||||
"garage-s3": new GarageS3Fixture({ image: "dxflrs/garage:v2.0.0" }),
|
||||
};
|
||||
|
||||
export const { inject } = createIntegrationSetup(fixtures);
|
||||
|
||||
// Re-export Vitest's setup/teardown contract from testing-integration's
|
||||
// public global-setup module. Vitest discovers them by name when this file
|
||||
// is wired as `test.globalSetup` in `vitest.integration.config.ts`.
|
||||
export {
|
||||
setup,
|
||||
teardown,
|
||||
} from "@suaveplan/testing-integration/setup/global-setup";
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Garage-backed integration test for `S3.setPartial`.
|
||||
*
|
||||
* Pins STORAGE-01..03 (set-partial semantics) against a real S3 wire so the
|
||||
* read-modify-write fallback survives AWS-SDK body encoding, content-length,
|
||||
* and range-read round-trips that an in-memory FakeS3Client cannot exercise.
|
||||
*
|
||||
* Skips the suite when the GarageS3Fixture failed to start (no Docker /
|
||||
* env-bridge incomplete). Set `GARAGE_S3_ENDPOINT`,
|
||||
* `GARAGE_S3_ADMIN_ENDPOINT`, `GARAGE_S3_ACCESS_KEY_ID`,
|
||||
* `GARAGE_S3_SECRET_ACCESS_KEY` to bypass container startup and use an
|
||||
* existing Garage instance.
|
||||
*/
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
inject as vitestInject,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { S3 } from "../../src/s3/s3.js";
|
||||
import { inject } from "./setup.js";
|
||||
|
||||
const failed =
|
||||
(vitestInject as (k: string) => string[] | null | undefined)(
|
||||
"__integration_failed__"
|
||||
) ?? [];
|
||||
const garageUnavailable = failed.includes("garage-s3");
|
||||
|
||||
describe.skipIf(garageUnavailable)("S3.setPartial — Garage backed", () => {
|
||||
let storage: S3;
|
||||
let key: string;
|
||||
|
||||
beforeEach(() => {
|
||||
const cfg = inject("garage-s3");
|
||||
storage = new S3({
|
||||
bucket: cfg.bucket,
|
||||
region: cfg.region,
|
||||
endpoint: cfg.endpoint,
|
||||
accessKeyId: cfg.accessKeyId,
|
||||
secretAccessKey: cfg.secretAccessKey,
|
||||
});
|
||||
key = `set-partial/${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await storage.delete(key).catch(() => undefined);
|
||||
});
|
||||
|
||||
it("creates a sparse zero-filled object when key does not exist", {
|
||||
timeout: 60_000,
|
||||
}, async () => {
|
||||
await storage.setPartial(key, 4, new Uint8Array([0xaa, 0xbb]));
|
||||
const got = await storage.get(key);
|
||||
expect(got).toEqual(
|
||||
new Uint8Array([0x00, 0x00, 0x00, 0x00, 0xaa, 0xbb])
|
||||
);
|
||||
});
|
||||
|
||||
it("extends an existing shorter object with zero gap", {
|
||||
timeout: 60_000,
|
||||
}, async () => {
|
||||
await storage.set(key, new Uint8Array([0x01, 0x02]));
|
||||
await storage.setPartial(key, 5, new Uint8Array([0xcc]));
|
||||
const got = await storage.get(key);
|
||||
expect(got).toEqual(
|
||||
new Uint8Array([0x01, 0x02, 0x00, 0x00, 0x00, 0xcc])
|
||||
);
|
||||
});
|
||||
|
||||
it("overwrites overlap and preserves surrounding bytes", {
|
||||
timeout: 60_000,
|
||||
}, async () => {
|
||||
await storage.set(key, new Uint8Array([1, 2, 3, 4, 5, 6]));
|
||||
await storage.setPartial(key, 2, new Uint8Array([0xff, 0xff]));
|
||||
const got = await storage.get(key);
|
||||
expect(got).toEqual(new Uint8Array([1, 2, 0xff, 0xff, 5, 6]));
|
||||
});
|
||||
|
||||
it("zero-length writes are a no-op", { timeout: 60_000 }, async () => {
|
||||
await storage.set(key, new Uint8Array([0x01, 0x02, 0x03]));
|
||||
await storage.setPartial(key, 1, new Uint8Array(0));
|
||||
const got = await storage.get(key);
|
||||
expect(got).toEqual(new Uint8Array([0x01, 0x02, 0x03]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Integration-test wiring for storage-s3.
|
||||
*
|
||||
* - `globalSetup` — absolute path to `./global-setup.ts`. Vitest loads this
|
||||
* file as the global-setup module; it imports the bun-napi-shim before
|
||||
* testcontainers loads, then registers the Garage fixture via the public
|
||||
* `createIntegrationSetup()` factory.
|
||||
* - `inject` — typed accessor for the published fixture configs. Re-exported
|
||||
* from `./global-setup.js` (where the typed registry was constructed).
|
||||
*/
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export { inject } from "./global-setup.js";
|
||||
|
||||
// Vitest's `globalSetup` MUST point at storage-s3's own `global-setup.ts`
|
||||
// (which loads `bun-napi-shim` first), NOT the bare path returned by
|
||||
// `createIntegrationSetup` — the testing-integration core global-setup has
|
||||
// no shim and would crash Bun before testcontainers starts.
|
||||
export const globalSetup = fileURLToPath(
|
||||
new URL("./global-setup.ts", import.meta.url)
|
||||
);
|
||||
@@ -0,0 +1 @@
|
||||
// Polyfills for test environment
|
||||
12
packages/persistence/storage/storage-s3/tsconfig.lib.json
Normal file
12
packages/persistence/storage/storage-s3/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
41
packages/persistence/storage/storage-s3/vite.config.ts
Normal file
41
packages/persistence/storage/storage-s3/vite.config.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/storage-s3",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@aws-sdk/client-s3",
|
||||
"@aws-sdk/s3-request-presigner",
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/retry",
|
||||
/^@suaveplan\/storage/,
|
||||
"@suaveplan/types",
|
||||
"tslib",
|
||||
"zod",
|
||||
"@opentelemetry/api",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
30
packages/persistence/storage/storage-s3/vitest.config.ts
Normal file
30
packages/persistence/storage/storage-s3/vitest.config.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: [
|
||||
"dist/**",
|
||||
"node_modules/**",
|
||||
"tests/integration/**",
|
||||
],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { globalSetup } from "./tests/integration/setup.js";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
fileParallelism: false,
|
||||
// bun-napi-shim is now auto-installed by @suaveplan/testing-integration's
|
||||
// global-setup module (loaded as a side-effect there). No per-consumer
|
||||
// wiring needed; this list keeps only the polyfill setup.
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["tests/integration/**/*.integration.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
globalSetup: [globalSetup],
|
||||
testTimeout: 60_000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user