From 5e7a4ebae552e671ecb49069c06390d9e8548b52 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 3 Apr 2026 16:59:08 +0300 Subject: [PATCH] Update docs --- CHANGELOG.md | 5 +++++ docs/configuration/storage.md | 31 ++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 271c23be..44945886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog ## [Unreleased] +### Added + +- **S3/Object storage support**: Connect to LanceDB on S3, GCS, Azure Blob, or HDFS via `lancedb.uri` and `storage_options` config. Supports S3-compatible stores (MinIO, Tigris) with custom endpoints. +- **Remote skill generation**: `create-skill` now supports remote databases — omit `--db` and provide `--config-file` to generate skills that connect to object storage at runtime instead of bundling the database. + ## [0.38.0] - 2026-04-07 ### Added diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index 0dec6fa1..a2dd1592 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -32,26 +32,47 @@ lancedb: # Amazon S3 lancedb: uri: s3://my-bucket/my-table -# Use AWS credentials or IAM roles + storage_options: + region: us-east-1 + +# Amazon S3 with explicit credentials +lancedb: + uri: s3://my-bucket/my-table + storage_options: + aws_access_key_id: YOUR_ACCESS_KEY + aws_secret_access_key: YOUR_SECRET_KEY + region: us-east-1 + +# S3-compatible (MinIO, Tigris, etc.) +lancedb: + uri: s3://my-bucket/my-table + storage_options: + endpoint: http://localhost:9000 + aws_access_key_id: minioadmin + aws_secret_access_key: minioadmin + region: us-east-1 + allow_http: "true" # Azure Blob Storage lancedb: uri: az://my-container/my-table -# Use Azure credentials # Google Cloud Storage lancedb: uri: gs://my-bucket/my-table -# Use GCP credentials # HDFS lancedb: uri: hdfs://namenode:port/path/to/table ``` -Authentication is handled through standard cloud provider credentials (AWS CLI, Azure CLI, gcloud, etc.) or by setting `api_key` for LanceDB Cloud. +- **LanceDB Cloud** (`db://`): Requires `api_key` and `region`. Table optimization and indexing are managed server-side. +- **Object storage** (`s3://`, `gs://`, `az://`, `hdfs://`): Uses `storage_options` for credentials and endpoint configuration. Authentication can also be provided via environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc.) or cloud provider SDK defaults (AWS CLI, Azure CLI, gcloud). +- **S3-compatible stores** (MinIO, Tigris, etc.): Set `endpoint` in `storage_options`. When using `http://` endpoints, also set `allow_http: "true"`. -**Note:** Table optimization is automatically handled by LanceDB Cloud (`db://` URIs) and is disabled for better performance. For object storage backends (S3, Azure, GCS), optimization is still performed locally. +The `storage_options` keys are case-insensitive and passed directly to the underlying object store library. Available keys depend on the backend — see the [LanceDB storage docs](https://lancedb.com/docs/storage/) for details. + +**Note:** Table optimization is automatically handled by LanceDB Cloud (`db://` URIs) and is disabled for better performance. For object storage backends (S3, Azure, GCS), optimization and vector indexing are still performed normally. ## Database Creation