Setting the file. One moment.
Subchapter 1.2
references/connectors.mdMarkdown7 KBView on GitHub
Quick reference for connecting DataStore to any data source. After connecting, all sources share the same pandas API.
DataStore.from_file(path, format=None, structure=None, compression=None, **kwargs)Format is auto-detected by extension: .parquet, .csv, .tsv, .json, .jsonl, .arrow, .orc, .avro, .xml.
from datastore import DataStore
ds = DataStore.from_file("sales.parquet")
ds = DataStore.from_file("data.csv")
ds = DataStore.from_file("events.jsonl")
ds = DataStore.from_file("logs/*.csv") # glob pattern
ds = DataStore.from_file("data/2024-*/events.parquet") # nested glob
ds = DataStore.from_file("data.csv.gz") # compressed, auto-detected
ds = DataStore.from_file("data.tsv", format="TabSeparatedWithNames") # explicit formatNotes:
*, **) work for querying multiple files at once.gz, .zst, .bz2, .xz, .lz4) is auto-detected from extensionstructure parameter to specify column types: structure="id UInt64, name String"DataStore.from_s3(url, access_key_id=None, secret_access_key=None, format=None, nosign=False, **kwargs)# Public bucket (no auth)
ds = DataStore.from_s3("s3://public-data/dataset.parquet", nosign=True)
# Private bucket
ds = DataStore.from_s3("s3://my-bucket/data.parquet",
access_key_id="AKIA...", secret_access_key="secret...")
# Glob pattern
ds = DataStore.from_s3("s3://bucket/logs/2024-*.parquet", nosign=True)DataStore.from_gcs(url, hmac_key=None, hmac_secret=None, format=None, nosign=False, **kwargs)ds = DataStore.from_gcs("gs://my-bucket/data.parquet", nosign=True)
ds = DataStore.from_gcs("gs://private/data.parquet", hmac_key="KEY", hmac_secret="SECRET")DataStore.from_azure(connection_string, container, path="", format=None, **kwargs)ds = DataStore.from_azure(
connection_string="DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...",
container="data", path="analytics/events.parquet")DataStore.from_hdfs(uri, format=None, structure=None, **kwargs)ds = DataStore.from_hdfs("hdfs://namenode:9000/warehouse/events/*.parquet")DataStore.from_url(url, format=None, structure=None, headers=None, **kwargs)ds = DataStore.from_url("https://example.com/data.csv")DataStore.from_mysql(host, database=None, table=None, user=None, password="", port=None, **kwargs)ds = DataStore.from_mysql(
host="db.example.com:3306", database="shop",
table="orders", user="root", password="pass")Note: Port must be included in host string (e.g., "db:3306") or passed via port parameter.
DataStore.from_postgresql(host, database=None, table=None, user=None, password="", port=None, **kwargs)ds = DataStore.from_postgresql(
host="pg:5432", database="analytics",
table="events", user="user", password="pass")DataStore.from_clickhouse(host, database=None, table=None, user="default", password="", secure=False, port=None, **kwargs)ds = DataStore.from_clickhouse(host="ch:9000", database="logs", table="access_log")
ds = DataStore.from_clickhouse(host="ch:9440", database="logs", table="hits",
user="reader", password="pass", secure=True)DataStore.from_mongodb(host, database, collection, user, password="", **kwargs)ds = DataStore.from_mongodb(
host="mongo:27017", database="app",
collection="users", user="user", password="pass")DataStore.from_sqlite(database_path, table, **kwargs)ds = DataStore.from_sqlite("/data/local.db", "users")DataStore.from_redis(host, key, structure, password=None, db_index=0, **kwargs)ds = DataStore.from_redis("localhost:6379", key="mydata",
structure="id UInt64, name String, value Float64")DataStore.from_iceberg(url, access_key_id=None, secret_access_key=None, **kwargs)ds = DataStore.from_iceberg("s3://warehouse/iceberg/events",
access_key_id="KEY", secret_access_key="SECRET")DataStore.from_delta(url, access_key_id=None, secret_access_key=None, **kwargs)ds = DataStore.from_delta("s3://warehouse/delta/transactions",
access_key_id="KEY", secret_access_key="SECRET")DataStore.from_hudi(url, access_key_id=None, secret_access_key=None, **kwargs)ds = DataStore.from_hudi("s3://warehouse/hudi/logs",
access_key_id="KEY", secret_access_key="SECRET")DataStore.uri(uri_string, **kwargs)Universal one-liner that auto-detects source type from the URI scheme:
| Scheme | Example |
|---|---|
| (path) | sales.parquet, /data/file.csv |
file | file:///data/file.csv |
s3, s3a, s3n | s3://bucket/key?nosign=true |
gs, gcs | gs://bucket/path |
az, azure, wasb | az://container/blob?account_name=X&account_key=Y |
hdfs | hdfs://namenode:9000/path |
http, https | https://example.com/data.json |
mysql | mysql://user:pass@host:port/db/table |
postgresql, postgres | postgresql://user:pass@host:port/db/table |
clickhouse | clickhouse://host:port/db/table?user=X&password=Y |
mongodb, mongo | mongodb://user:pass@host:port/db.collection |
sqlite | sqlite:///path/to/db.db?table=name |
redis | redis://host:port/db?key=mykey&password=pass |
iceberg | iceberg://catalog/namespace/table |
deltalake, delta | deltalake:///path/to/table |
hudi | hudi:///path/to/table |
from datastore import DataStore
ds = DataStore.uri("s3://public-data/dataset.parquet?nosign=true")
ds = DataStore.uri("mysql://root:pass@localhost:3306/shop/orders")
ds = DataStore.uri("postgresql://analyst:pass@pg:5432/analytics/events")
ds = DataStore.uri("clickhouse://ch:9440/analytics/hits?user=reader&password=pass")
ds = DataStore.uri("mongodb://user:pass@mongo:27017/logs.app_events")
ds = DataStore.uri("sqlite:///data/local.db?table=users")
ds = DataStore.uri("deltalake:///data/delta/events")ds = DataStore({"name": ["Alice", "Bob"], "age": [25, 30]})ds = DataStore(df)
ds = DataStore.from_df(df, name="my_data")ds = DataStore.from_numbers(100) # 0..99
ds = DataStore.from_numbers(10, start=5, step=2) # 5, 7, 9, ...ds = DataStore.from_random(
structure="id UInt64, name String, value Float64",
random_seed=42, max_string_length=10)