Version: 0.4 (Latest)

Snowflake Publishing

Commands use Linux syntax. See Examples & Recipes for the Windows equivalents and the arguments every sync needs.

Export to S3, then create Snowflake tables.

Snowflake External Tables

Data stays in S3; queryable via Snowflake.

External Tables with Views (Default)

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path snowflake_ext \
    --include "nation%" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

External Tables without Views

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path snowflake_ext \
    --include "nation%" \
    --publish_target snowflake_prod \
    --no_views

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

Snowflake Internal Tables

Data loaded into Snowflake storage.

Basic Internal Tables

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name "tpch_1%" \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path snowflake_int \
    --include "nation%" \
    --publish_method internal \
    --publish_schema_pattern "INT_{schema}" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

Internal Tables with Custom Naming

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path production \
    --publish_method internal \
    --publish_schema_pattern "INT_{subpath}_{date}" \
    --publish_table_pattern "{schema}_{table}" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

Internal Tables with Primary Key Constraints

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_postgres \
    --source_db_name tpch \
    --source_schema_name tpch_1 \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path production \
    --publish_method internal \
    --publish_schema_pattern "INT_{subpath}_{date}" \
    --publish_table_pattern "{schema}_{table}" \
    --publish_target snowflake_prod \
    --pk_constraints

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

SQL Server to Snowflake Internal

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mssql \
    --source_db_name adventureworksdw \
    --source_schema_name dbo \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --publish_method internal \
    --publish_schema_pattern "adventureworksdw" \
    --publish_table_pattern "{table}" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

MySQL to Snowflake (External)

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mysql \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path mysql_data \
    --include "nation%" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

MySQL to Snowflake (Internal)

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mysql \
    --source_db_name tpch \
    --source_schema_name tpch \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path mysql_data \
    --include "nation%" \
    --publish_method internal \
    --publish_schema_pattern "INT_{schema}" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

MariaDB to Snowflake (External)

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mariadb \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path mariadb_data \
    --include "nation%" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

MariaDB to Snowflake (Internal)

./LakeXpress config create \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --source_db_auth_id source_mariadb \
    --source_db_name tpch_mariadb \
    --source_schema_name tpch_mariadb \
    --fastbcp_p 2 \
    --n_jobs 4 \
    --target_storage_id s3_01 \
    --generate_metadata \
    --sub_path mariadb_data \
    --include "nation%" \
    --publish_method internal \
    --publish_schema_pattern "INT_{schema}" \
    --publish_target snowflake_prod

./LakeXpress sync \
    -a credentials.json \
    --lxdb_auth_id lxdb_mssql \
    --sync_id <SYNC_ID>

More Examples

The credentials entries used below, and the Snowflake setup they rely on, are described in Snowflake Publishing.

Example 1: External Tables with Custom Patterns

./LakeXpress config create -a credentials.json \
       --lxdb_auth_id lxdb_mssql \
       --source_db_auth_id source_postgres \
       --source_db_name tpch \
       --source_schema_name tpch_1 \
       --fastbcp_p 2 \
       --n_jobs 4 \
       --target_storage_id s3_01 \
       --generate_metadata \
       --sub_path toto \
       --publish_target snowflake_prod \
       --publish_schema_pattern "EXT_{subpath}_{date}" \
       --publish_table_pattern "{schema}_{table}" \
       --publish_method external

Result:

Schema: EXT_toto_20251124
Tables: tpch_1_customer, tpch_1_orders, etc.
Views: V_tpch_1_customer, V_tpch_1_orders, etc.

Example 2: Internal Tables with Primary Keys

./LakeXpress config create -a credentials.json \
       --lxdb_auth_id lxdb_mssql \
       --source_db_auth_id source_postgres \
       --source_db_name tpch \
       --source_schema_name tpch_1 \
       --fastbcp_p 2 \
       --n_jobs 4 \
       --target_storage_id s3_01 \
       --generate_metadata \
       --sub_path toto \
       --publish_method internal \
       --publish_target snowflake_prod \
       --pk_constraints

Result:

Schema: tpch_1
Tables: customer, orders, etc. (with data loaded)
PRIMARY KEY constraints applied where available

Example 3: Single Table

./LakeXpress config create \
  -a credentials.json \
  --lxdb_auth_id lxdb_mssql \
  --source_db_auth_id source_postgres \
  --target_storage_id s3_01 \
  --source_schema_name tpch_1 \
  -i nation \
  --publish_target snowflake_prod

Example 4: Full Schema

./LakeXpress config create \
  -a credentials.json \
  --lxdb_auth_id lxdb_mssql \
  --source_db_auth_id source_postgres \
  --target_storage_id s3_01 \
  --source_schema_name tpch_1 \
  --publish_target snowflake_prod

Snowflake External Tables​

Snowflake Internal Tables​

More Examples​

Example 1: External Tables with Custom Patterns​

Example 2: Internal Tables with Primary Keys​

Example 3: Single Table​

Example 4: Full Schema​