Configuring your connectionsConnection configurations

Databricks

Connection

Description

Configuration

NAMETYPEDESCRIPTIONREQUIREDREADONLY
server_hostnamestringtruefalse
portintegertruefalse
access_tokenstringtruefalse
http_pathstringtruefalse
cloud_providerstringfalsefalse
auth_modestringHow to authenticate with AWS. Defaults to Access Key and Secrettruefalse
iam_role_arnstring(required if AuthMode is “iam_role”)falsefalse
storage_credential_namestringfalsefalse
external_idstringExternal ID for the IAM rolefalsefalse
aws_access_key_idstringSee https://docs.polytomic.com/docs/databricks-connections#writing-to-databricks (required if AuthMode is “aws_access_key_id”)falsefalse
aws_secret_access_keystring(required if AuthMode is “aws_access_key_id”)falsefalse
aws_userstringfalsefalse
s3_bucket_namestringName of bucket used for staging data load files (required if CloudProvider is “aws”)falsefalse
s3_bucket_regionstring(required if CloudProvider is “aws”)falsefalse
azure_account_namestringThe account name of the storage account (required if CloudProvider is “azure”)falsefalse
azure_access_keystringThe access key associated with this storage account (required if CloudProvider is “azure”)falsefalse
container_namestringThe container which we will stage files in (required if CloudProvider is “azure”)falsefalse
unity_catalog_enabledbooleanfalsefalse
enable_delta_uniformbooleanfalsefalse
enforce_query_limitbooleanfalsefalse
concurrent_queriesintegerfalsefalse

Example

1{
2 "name": "databricks connection",
3 "type": "databricks",
4 "configuration": {
5 "access_token": "isoz8af6zvp8067gu68gvrp0oftevn",
6 "auth_mode": "access_key_and_secret",
7 "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
8 "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
9 "aws_user": "",
10 "azure_access_key": "abcdefghijklmnopqrstuvwxyz0123456789/+ABCDEabcdefghijklmnopqrstuvwxyz0123456789/+ABCDE==",
11 "azure_account_name": "account",
12 "cloud_provider": "aws",
13 "concurrent_queries": 0,
14 "container_name": "container",
15 "enable_delta_uniform": false,
16 "enforce_query_limit": false,
17 "external_id": "",
18 "http_path": "/sql",
19 "iam_role_arn": "",
20 "port": 443,
21 "s3_bucket_name": "s3://polytomic-databricks-results/customer-dataset",
22 "s3_bucket_region": "",
23 "server_hostname": "dbc-1234dsafas-d0001.cloud.databricks.com",
24 "storage_credential_name": "",
25 "unity_catalog_enabled": false
26 }
27}

Model Sync

Source

Configuration

NAMETYPEDESCRIPTIONREQUIREDREADONLY
catalogstringCatalogfalsefalse
schemastringSchemafalsefalse
tablestringTablefalsefalse
querystringQueryfalsefalse

Example

1{
2 ...
3 "configuration": {
4 "catalog": "samples",
5 "query": "SELECT * FROM samples.nyctaxi.trips",
6 "schema": "nyctaxi",
7 "table": "trips"
8 }
9}

Target

Configuration

NAMETYPEDESCRIPTIONREQUIREDREADONLY
preserve_table_on_resyncbooleanPreserve destination table when resyncingfalsefalse

Example

1{
2 ...
3 "target": {
4 "configuration": {
5 "preserve_table_on_resync": false
6 }
7 }
8}

Bulk Sync

Source

N/A

Destination

Configuration

NAMETYPEDESCRIPTIONREQUIREDREADONLY
advancedobjectfalsefalse
catalogstringCatalogfalsefalse
schemastringOutput schemafalsefalse
mirror_schemasbooleanMirror schemasfalsefalse
external_location_namestringExternal locationfalsefalse

Example

1{
2 ...
3 "destination_configuration": {
4 "advanced": {
5 "hard_deletes": false,
6 "table_prefix": "",
7 "truncate_existing": false
8 },
9 "catalog": "samples",
10 "external_location_name": "",
11 "mirror_schemas": false,
12 "schema": "nyctaxi"
13 }
14}