Databricks (2022-12-12) — Polytomic Docs

Connection

Configuration

NAME	TYPE	DESCRIPTION	REQUIRED	READONLY
server_hostname	string		true	false
port	integer		true	false
access_token	string		true	false
http_path	string		true	false
cloud_provider	string	Accepted Values: aws, azure	false	false
auth_mode	string	How to authenticate with AWS. Defaults to Access Key and Secret. Accepted Values: access_key_and_secret, iam_role	true	false
iam_role_arn	string	(required if `auth_mode` is “iam_role”)	false	false
storage_credential_name	string		false	false
external_id	string	External ID for the IAM role	false	false
aws_access_key_id	string	See https://docs.polytomic.com/docs/databricks-connections#writing-to-databricks (required if `auth_mode` is “aws_access_key_id”)	false	false
aws_secret_access_key	string	(required if `auth_mode` is “aws_access_key_id”)	false	false
aws_user	string		false	false
s3_bucket_name	string	Name of bucket used for staging data load files (required if `cloud_provider` is “aws”)	false	false
s3_bucket_region	string	(required if `cloud_provider` is “aws”)	false	false
azure_account_name	string	The account name of the storage account (required if `cloud_provider` is “azure”)	false	false
azure_access_key	string	The access key associated with this storage account (required if `cloud_provider` is “azure”)	false	false
container_name	string	The container which we will stage files in (required if `cloud_provider` is “azure”)	false	false
unity_catalog_enabled	boolean		false	false
enable_delta_uniform	boolean		false	false
enforce_query_limit	boolean		false	false
concurrent_queries	integer		false	false
set_retention_properties	boolean		false	false
log_file_retention_days	integer		false	false
deleted_file_retention_days	integer		false	false

Example

1 {
2   "name": "databricks connection",
3   "type": "databricks",
4   "configuration": {
5     "access_token": "isoz8af6zvp8067gu68gvrp0oftevn",
6     "auth_mode": "access_key_and_secret",
7     "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
8     "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
9     "aws_user": "",
10     "azure_access_key": "abcdefghijklmnopqrstuvwxyz0123456789/+ABCDEabcdefghijklmnopqrstuvwxyz0123456789/+ABCDE==",
11     "azure_account_name": "account",
12     "cloud_provider": "aws",
13     "concurrent_queries": 0,
14     "container_name": "container",
15     "deleted_file_retention_days": 0,
16     "enable_delta_uniform": false,
17     "enforce_query_limit": false,
18     "external_id": "",
19     "http_path": "/sql",
20     "iam_role_arn": "",
21     "log_file_retention_days": 0,
22     "port": 443,
23     "s3_bucket_name": "s3://polytomic-databricks-results/customer-dataset",
24     "s3_bucket_region": "",
25     "server_hostname": "dbc-1234dsafas-d0001.cloud.databricks.com",
26     "set_retention_properties": false,
27     "storage_credential_name": "",
28     "unity_catalog_enabled": false
29   }
30 }

Model Sync

Source

Configuration

NAME	TYPE	DESCRIPTION	REQUIRED	READONLY
catalog	string	Catalog	false	false
schema	string	Schema	false	false
table	string	Table	false	false
query	string	Query	false	false

Example

1 {
2   ...
3   "configuration": {
4     "catalog": "samples",
5     "query": "SELECT * FROM samples.nyctaxi.trips",
6     "schema": "nyctaxi",
7     "table": "trips"
8   }
9 }

Target

Configuration

NAME	TYPE	DESCRIPTION	REQUIRED	READONLY
preserve_table_on_resync	boolean	Preserve destination table when resyncing	false	false
write_record_timestamps	boolean	Write row timestamp metadata	false	false
created_column	string	’Created at’ timestamp column	false	false
updated_column	string	’Updated at’ timestamp column	false	false

Example

1 {
2   ...
3   "target": {
4       "configuration": {
5         "created_column": "",
6         "preserve_table_on_resync": false,
7         "updated_column": "",
8         "write_record_timestamps": false
9     }
10   }
11 }

Bulk Sync

Destination

Configuration

NAME	TYPE	DESCRIPTION	REQUIRED	READONLY
advanced	object		false	false
external_location_name	string	External location	false	false
catalog	string	Catalog	false	false
schema	string	Output schema	false	false
mirror_schemas	boolean	Mirror schemas	false	false

Example

1 {
2   ...
3   "destination_configuration": {
4     "advanced": {
5         "deleted_file_retention_days": 0,
6         "hard_deletes": false,
7         "log_file_retention_days": 0,
8         "set_retention_properties": false,
9         "table_prefix": "",
10         "truncate_existing": false
11     },
12     "catalog": "samples",
13     "external_location_name": "",
14     "mirror_schemas": false,
15     "schema": "nyctaxi"
16   }
17 }

Type handling

Destination types

POLYTOMIC TYPE	DATABRICKS TYPE
`array<>`	`ARRAY<>`
`bigint`	`BIGINT`
`boolean`	`BOOLEAN`
`date`	`DATE`
`datetime`	`TIMESTAMP`
`decimal(precision, scale)`	`DECIMAL(precision,scale)`
`double`	`DOUBLE`
`int`	`INT`
`json`	`STRING`
`jsonarray`	`STRING`
`number`	`DECIMAL(38,18)`
`object{}`	`STRUCT<>`
`single`	`FLOAT`
`smallint`	`SMALLINT`
`string`	`STRING`
`time`	`TIMESTAMP`

Source types

DATABRICKS TYPE	POLYTOMIC TYPE
`ARRAY<>`	`array<>`
`BIGINT`	`bigint`
`DATE`	`date`
`DECIMAL(precision, scale)`	`decimal(precision, scale)`
`DOUBLE`	`double`
`FLOAT`	`single`
`INT`	`int`
`INTERVAL`	`string`
`MAP<>`	`object{}`
`SMALLINT`	`smallint`
`STRUCT<>`	`object{}`
`TIMESTAMP`	`datetime`
`TIMESTAMP_NTZ`	`datetime`
`TINYINT`	`smallint`
`VARCHAR`	`string`

1	{
2	"name": "databricks connection",
3	"type": "databricks",
4	"configuration": {
5	"access_token": "isoz8af6zvp8067gu68gvrp0oftevn",
6	"auth_mode": "access_key_and_secret",
7	"aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
8	"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
9	"aws_user": "",
10	"azure_access_key": "abcdefghijklmnopqrstuvwxyz0123456789/+ABCDEabcdefghijklmnopqrstuvwxyz0123456789/+ABCDE==",
11	"azure_account_name": "account",
12	"cloud_provider": "aws",
13	"concurrent_queries": 0,
14	"container_name": "container",
15	"deleted_file_retention_days": 0,
16	"enable_delta_uniform": false,
17	"enforce_query_limit": false,
18	"external_id": "",
19	"http_path": "/sql",
20	"iam_role_arn": "",
21	"log_file_retention_days": 0,
22	"port": 443,
23	"s3_bucket_name": "s3://polytomic-databricks-results/customer-dataset",
24	"s3_bucket_region": "",
25	"server_hostname": "dbc-1234dsafas-d0001.cloud.databricks.com",
26	"set_retention_properties": false,
27	"storage_credential_name": "",
28	"unity_catalog_enabled": false
29	}
30	}

1	{
2	...
3	"configuration": {
4	"catalog": "samples",
5	"query": "SELECT * FROM samples.nyctaxi.trips",
6	"schema": "nyctaxi",
7	"table": "trips"
8	}
9	}

1	{
2	...
3	"target": {
4	"configuration": {
5	"created_column": "",
6	"preserve_table_on_resync": false,
7	"updated_column": "",
8	"write_record_timestamps": false
9	}
10	}
11	}

1	{
2	...
3	"destination_configuration": {
4	"advanced": {
5	"deleted_file_retention_days": 0,
6	"hard_deletes": false,
7	"log_file_retention_days": 0,
8	"set_retention_properties": false,
9	"table_prefix": "",
10	"truncate_existing": false
11	},
12	"catalog": "samples",
13	"external_location_name": "",
14	"mirror_schemas": false,
15	"schema": "nyctaxi"
16	}
17	}