AWS S3 Exports

Commands use Linux syntax. See Examples & Recipes for the Windows equivalents and the arguments every sync needs.

PostgreSQL to S3

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

PostgreSQL to S3 (OVH)

Uses OVH S3-compatible storage with custom endpoint via AWS profile:

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_02 \
    --generate_metadata

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

credentials.json:

{
  "s3_02": {
    "ds_type": "s3",
    "auth_mode": "profile",
    "info": {
      "directory": "s3://my-ovh-bucket/exports",
      "profile": "ovh"
    }
  }
}

~/.aws/config:

[profile ovh]
endpoint_url = https://s3.gra.io.cloud.ovh.net
region = gra

~/.aws/credentials:

[ovh]
aws_access_key_id = YOUR_OVH_ACCESS_KEY
aws_secret_access_key = YOUR_OVH_SECRET_KEY

PostgreSQL to S3 with Sub-Path and Table Filter

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --sub_path staging/data \
    --include "nation%" \
    --generate_metadata

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

MySQL to S3

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mysql \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --sub_path mysql_export \
    --include "nation%" \
    --generate_metadata

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

MariaDB to S3

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mariadb \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --sub_path mariadb_export \
    --include "nation%" \
    --generate_metadata

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>