S3

S3 providers expose portable object storage to executable apps. Gestalt models the de facto S3-compatible object API rather than a filesystem API, so the same app code can target AWS S3, MinIO, GCS XML interoperability mode, Cloudflare R2, and other compatible backends.

Using `S3` from a plugin

The SDK exposes a generated S3 client in every supported language. The client connects to one configured S3 binding by name; the backing bucket comes from provider configuration, so object references carry only key and an optional version_id. Object reads and writes are streamed: writes open with object metadata and then send byte chunks, and reads return object metadata plus a byte-chunk stream.

Go


import (
  "context"
  "io"
  "log"
 
  "github.com/valon-technologies/gestalt/sdk/go/client"
)
 
ctx := context.Background()
 
s3, err := client.ConnectS3(ctx, "uploads")
if err != nil {
  log.Fatal(err)
}
 
avatar := &client.S3ObjectRef{Key: "avatars/user-123.png"}
 
stream, err := s3.WriteObject(ctx, &client.WriteObjectOpen{
  Ref:         avatar,
  ContentType: "image/png",
})
if err != nil {
  log.Fatal(err)
}
if err := stream.Send(pngBytes); err != nil {
  log.Fatal(err)
}
if _, err := stream.CloseAndRecv(); err != nil {
  log.Fatal(err)
}
 
meta, data, err := s3.ReadObject(ctx, &client.ReadObjectRequest{Ref: avatar})
if err != nil {
  log.Fatal(err)
}
var body []byte
for {
  chunk, err := data.Recv()
  if err == io.EOF {
    break
  }
  if err != nil {
    log.Fatal(err)
  }
  body = append(body, chunk...)
}
_ = meta
_ = body

Python


from gestalt import ReadObjectRequest, S3, S3ObjectRef, WriteObjectOpen
 
s3 = S3.connect("uploads")
avatar = S3ObjectRef(key="avatars/user-123.png")
 
s3.write_object(
    WriteObjectOpen(ref=avatar, content_type="image/png"),
    [png_bytes],
)
 
meta, chunks = s3.read_object(ReadObjectRequest(ref=avatar))
body = b"".join(chunks)

Rust


use gestalt::s3::{ReadObjectRequest, S3ObjectRef, WriteObjectOpen};
use gestalt::S3;
 
let mut s3 = S3::connect_named("uploads").await?;
let avatar = S3ObjectRef {
    key: "avatars/user-123.png".to_string(),
    ..Default::default()
};
 
s3.write_object(
    WriteObjectOpen {
        r#ref: Some(avatar.clone()),
        content_type: "image/png".to_string(),
        ..Default::default()
    },
    tokio_stream::once(png_bytes),
)
.await?;
 
let (meta, mut data) = s3
    .read_object(ReadObjectRequest {
        r#ref: Some(avatar),
        ..Default::default()
    })
    .await?;
let mut body = Vec::new();
while let Some(chunk) = data.recv().await? {
    body.extend_from_slice(&chunk);
}

TypeScript


import { S3 } from "@valon-technologies/gestalt";
 
const s3 = S3.connect({ name: "uploads" });
const avatar = { key: "avatars/user-123.png" };
 
await s3.writeObject(
  { ref: avatar, contentType: "image/png" },
  (async function* () {
    yield pngBytes;
  })(),
);
 
const { meta, data } = await s3.readObject({ ref: avatar });
const chunks: Uint8Array[] = [];
for await (const chunk of data) {
  chunks.push(chunk);
}

Building your own S3 provider

Manifest

An S3 provider manifest declares kind: s3:


kind: s3
source: github.com/acme/gestalt-providers/s3/minio
version: 0.0.1-alpha.1
displayName: MinIO S3
description: S3-compatible object storage provider backed by MinIO.
spec:
  configSchemaPath: ./config.schema.json

Provider interface

Implement the SDK’s authored S3Provider surface. The SDK adapts this typed interface to the underlying gRPC service:

Method	Purpose
Head object	Return metadata for one object reference.
Read object	Stream one object body with its metadata.
Write object	Accept a body stream and return committed metadata.
Delete object	Delete one object reference.
List objects	List objects in the configured bucket by prefix, delimiter, and continuation token.
Copy object	Copy an object server-side between object references.
Presign object	Produce a presigned request URL plus required headers.

Go


package s3provider
 
import (
  "bytes"
  "context"
  "io"
  "time"
 
  gestalt "github.com/valon-technologies/gestalt/sdk/go"
)
 
type Provider struct{}
 
func New() *Provider { return &Provider{} }
 
func (p *Provider) Configure(_ context.Context, _ string, config map[string]any) error {
  _ = config
  return nil
}
 
func (p *Provider) HeadObject(ctx context.Context, ref gestalt.ObjectRef) (gestalt.ObjectMeta, error) {
  _ = ctx
  return gestalt.ObjectMeta{
    Ref:          ref,
    ETag:         "",
    Size:         0,
    ContentType:  "application/octet-stream",
    LastModified: time.Now(),
    Metadata:     map[string]string{},
  }, nil
}
 
func (p *Provider) ReadObject(ctx context.Context, req gestalt.ReadRequest) (gestalt.ReadResult, error) {
  meta, err := p.HeadObject(ctx, req.Ref)
  if err != nil {
    return gestalt.ReadResult{}, err
  }
  return gestalt.ReadResult{
    Meta: meta,
    Body: io.NopCloser(bytes.NewReader(nil)),
  }, nil
}
 
func (p *Provider) WriteObject(ctx context.Context, req gestalt.WriteRequest) (gestalt.ObjectMeta, error) {
  _ = req.Body
  return p.HeadObject(ctx, req.Ref)
}
 
func (p *Provider) DeleteObject(context.Context, gestalt.ObjectRef) error {
  return nil
}
 
func (p *Provider) ListObjects(context.Context, gestalt.ListRequest) (gestalt.ListPage, error) {
  return gestalt.ListPage{}, nil
}
 
func (p *Provider) CopyObject(ctx context.Context, req gestalt.CopyRequest) (gestalt.ObjectMeta, error) {
  _ = req.Source
  return p.HeadObject(ctx, req.Destination)
}
 
func (p *Provider) PresignObject(context.Context, gestalt.PresignRequest) (gestalt.PresignResult, error) {
  return gestalt.PresignResult{
    URL:       "https://example.invalid/object",
    Method:    gestalt.PresignMethodGet,
    ExpiresAt: time.Now().Add(time.Minute),
    Headers:   map[string]string{},
  }, nil
}

Python


from collections.abc import Iterable
 
from gestalt import (
    CopyObjectRequest,
    CopyObjectResponse,
    DeleteObjectRequest,
    HeadObjectRequest,
    HeadObjectResponse,
    ListObjectsRequest,
    ListObjectsResponse,
    PresignMethodValues,
    PresignObjectRequest,
    PresignObjectResponse,
    ProviderReadResult,
    ReadObjectRequest,
    S3ObjectMeta,
    S3Provider,
    WriteObjectOpen,
    WriteObjectResponse,
)
 
class Provider(S3Provider):
    def head_object(self, request: HeadObjectRequest) -> HeadObjectResponse:
        return HeadObjectResponse(
            meta=S3ObjectMeta(
                ref=request.ref,
                etag="",
                size=0,
                content_type="application/octet-stream",
            )
        )
 
    def read_object(self, request: ReadObjectRequest) -> ProviderReadResult:
        head = self.head_object(HeadObjectRequest(ref=request.ref))
        return ProviderReadResult(meta=head.meta, body=iter(()))
 
    def write_object(
        self,
        open: WriteObjectOpen,
        body: Iterable[bytes],
    ) -> WriteObjectResponse:
        for _chunk in body:
            pass
        head = self.head_object(HeadObjectRequest(ref=open.ref))
        return WriteObjectResponse(meta=head.meta)
 
    def delete_object(self, request: DeleteObjectRequest) -> None:
        pass
 
    def list_objects(self, request: ListObjectsRequest) -> ListObjectsResponse:
        return ListObjectsResponse()
 
    def copy_object(self, request: CopyObjectRequest) -> CopyObjectResponse:
        head = self.head_object(HeadObjectRequest(ref=request.destination))
        return CopyObjectResponse(meta=head.meta)
 
    def presign_object(
        self, request: PresignObjectRequest
    ) -> PresignObjectResponse:
        key = request.ref.key if request.ref else ""
        return PresignObjectResponse(
            url=f"https://example.invalid/{key}",
            method=request.method or PresignMethodValues.GET,
            headers=dict(request.headers),
        )
 
provider = Provider()

Rust


use gestalt::s3_provider::{
    CopyObjectRequest, CopyObjectResponse, DeleteObjectRequest,
    HeadObjectRequest, HeadObjectResponse, ListObjectsRequest,
    ListObjectsResponse, PresignObjectRequest, PresignObjectResponse,
    ReadObjectRequest, WriteObjectResponse,
};
use gestalt::{S3Provider, S3ReadObjectStream, S3WriteObjectStream};
 
#[derive(Default)]
pub struct MinioS3;
 
#[gestalt::async_trait]
impl S3Provider for MinioS3 {
    async fn configure(
        &self,
        _name: &str,
        _config: serde_json::Map<String, serde_json::Value>,
    ) -> gestalt::Result<()> {
        Ok(())
    }
 
    async fn head_object(
        &self,
        _request: HeadObjectRequest,
    ) -> gestalt::Result<HeadObjectResponse> {
        Err(gestalt::Error::unimplemented("todo"))
    }
 
    async fn read_object(
        &self,
        _request: ReadObjectRequest,
    ) -> gestalt::Result<S3ReadObjectStream> {
        Err(gestalt::Error::unimplemented("todo"))
    }
 
    async fn write_object(
        &self,
        _request: S3WriteObjectStream,
    ) -> gestalt::Result<WriteObjectResponse> {
        Err(gestalt::Error::unimplemented("todo"))
    }
 
    async fn delete_object(
        &self,
        _request: DeleteObjectRequest,
    ) -> gestalt::Result<()> {
        Err(gestalt::Error::unimplemented("todo"))
    }
 
    async fn list_objects(
        &self,
        _request: ListObjectsRequest,
    ) -> gestalt::Result<ListObjectsResponse> {
        Err(gestalt::Error::unimplemented("todo"))
    }
 
    async fn copy_object(
        &self,
        _request: CopyObjectRequest,
    ) -> gestalt::Result<CopyObjectResponse> {
        Err(gestalt::Error::unimplemented("todo"))
    }
 
    async fn presign_object(
        &self,
        _request: PresignObjectRequest,
    ) -> gestalt::Result<PresignObjectResponse> {
        Err(gestalt::Error::unimplemented("todo"))
    }
}
 
gestalt::export_s3_provider!(constructor = MinioS3::default);

TypeScript


import { PresignMethod, defineS3Provider } from "@valon-technologies/gestalt";
 
export const provider = defineS3Provider({
  async configure(name, config) {
    void name;
    void config.endpoint;
  },
 
  async headObject(ref) {
    return {
      ref,
      etag: "",
      size: 0n,
      contentType: "",
      lastModified: new Date(),
      metadata: {},
      storageClass: "",
    };
  },
 
  async readObject(ref) {
    return {
      meta: await this.headObject(ref),
      body: new Uint8Array(),
    };
  },
 
  async writeObject(ref, body, options) {
    void body;
    void options;
    return await this.headObject(ref);
  },
 
  async deleteObject(ref) {
    void ref;
  },
 
  async listObjects(options) {
    void options;
    return {
      objects: [],
      commonPrefixes: [],
      nextContinuationToken: "",
      hasMore: false,
    };
  },
 
  async copyObject(source, destination, options) {
    void source;
    void options;
    return await this.headObject(destination);
  },
 
  async presignObject(ref, options) {
    return {
      url: `https://example.invalid/${encodeURIComponent(ref.key)}`,
      method: options?.method ?? PresignMethod.GET,
      headers: { ...(options?.headers ?? {}) },
      expiresAt: new Date(Date.now() + 60_000),
    };
  },
});

gestalt.S3Provider adds runtime lifecycle wiring and a serve() helper for languages that expose the server-side S3 provider surface.

App access

Plugin-side S3 usage is documented in Using S3 from a plugin. Provider authors implement the S3 service surface; Gestalt handles configured plugin bindings, SDK sockets, and host-mediated object access URLs.

Deploying the provider

Custom S3 providers are wired into Gestalt the same way as first-party ones:


providers:
  s3:
    assets:
      source: ./providers/s3/custom/manifest.yaml
      config:
        endpoint: http://127.0.0.1:9000
        region: us-east-1
 
apps:
  media:
    source: ./apps/media/manifest.yaml
    s3:
      - assets

For packaging and published release URLs, see Releasing provider packages.

Implementation notes

Treat object references as {key, version_id} values scoped to the provider’s configured bucket, not filesystem paths.
Keep reads and writes streaming. Do not require the caller to buffer the full object in memory.
Preserve backend metadata that fits the portable model: etag, size, content_type, last_modified, metadata, and storage_class.
CopyObject conditionals apply to the source object, not the destination object.
Map missing objects to NotFound, conditional failures to FailedPrecondition, and invalid ranges to OutOfRange so SDK error mapping stays portable.
PresignObject should return only caller-required headers. Do not leak transport-generated headers such as Host.
Implement multipart upload and multipart copy internally when the backend needs them, or reject above the single-request S3 limit explicitly. Do not silently rely on backend-specific PutObject / CopyObject failures.

S3

Using S3 from a plugin

Go

Python

Rust

TypeScript

Building your own S3 provider

Manifest

Provider interface

Go

Python

Rust

TypeScript

App access

Deploying the provider

Implementation notes

Using `S3` from a plugin