Zakaria
69d80c60a3
CI / checks (pull_request) Successful in 28m15s
Create and delete Glue databases, and describe a folder of CSV, JSON-lines or Parquet files in S3 as a table: its columns, and for CSV the delimiter and whether the first line is a header. Tables are written with the storage settings Athena expects, so TheCloud's Athena can query one as soon as it exists. Glue jobs and crawlers are not offered: the runtime stores them but never runs them.
69 lines
3.5 KiB
TypeScript
69 lines
3.5 KiB
TypeScript
import type {CloudProvider, FieldSchema, ServiceSchema, TableColumnSchema} from './types'
|
|
|
|
/**
|
|
* Glue's rule for database and table names. Glue lowercases them, and Athena
|
|
* needs them to start with a letter or underscore to query them unquoted.
|
|
*/
|
|
export const CATALOG_NAME_PATTERN = '^[a-z_][a-z0-9_]{0,254}$'
|
|
/** The data formats TheCloud's Athena engine reads from a table's S3 location. */
|
|
export const CATALOG_TABLE_FORMATS = ['csv', 'json', 'parquet'] as const
|
|
/** Hive column types Athena understands and the console offers. */
|
|
export const CATALOG_COLUMN_TYPES = ['string', 'int', 'bigint', 'double', 'float', 'boolean', 'date', 'timestamp', 'decimal(10,2)'] as const
|
|
|
|
const databaseColumns: TableColumnSchema[] = [
|
|
{name: 'name', label: 'Name'},
|
|
{name: 'tables', label: 'Tables', path: 'metadata.tableCount', emptyText: '—'},
|
|
{name: 'description', label: 'Description', path: 'metadata.description', emptyText: '—'},
|
|
{name: 'location', label: 'Location', path: 'metadata.locationUri', emptyText: '—'},
|
|
{name: 'createdAt', label: 'Created', format: 'datetime'},
|
|
]
|
|
|
|
const databaseFilters: FieldSchema[] = [{name: 'search', label: 'Search', type: 'text', required: false}]
|
|
|
|
export function awsDataCatalogSchema(): ServiceSchema {
|
|
return {
|
|
cloud: 'aws',
|
|
service: 'datacatalog',
|
|
displayName: 'Glue Data Catalog databases',
|
|
fields: [
|
|
{
|
|
name: 'name',
|
|
label: 'Name',
|
|
type: 'text',
|
|
required: true,
|
|
description: 'Lowercase letters, numbers and underscores, starting with a letter or underscore, so Athena can query it.',
|
|
validation: {pattern: CATALOG_NAME_PATTERN, message: 'Use lowercase letters, numbers and underscores, starting with a letter or underscore'},
|
|
},
|
|
{name: 'description', label: 'Description', type: 'text', required: false, validation: {maxLength: 2048}},
|
|
{
|
|
name: 'locationUri',
|
|
label: 'Location',
|
|
type: 'text',
|
|
required: false,
|
|
span: true,
|
|
description: 'Optional S3 location for the database, such as s3://my-bucket/warehouse/.',
|
|
validation: {pattern: '^s3://[a-z0-9.-]{3,63}(/.*)?$', message: 'Use an S3 URI such as s3://my-bucket/warehouse/'},
|
|
},
|
|
],
|
|
actions: ['list', 'create', 'delete', 'inspect'],
|
|
capabilities: {
|
|
resourceActions: [
|
|
{name: 'list', label: 'List databases', enabled: true, status: 'available', runtimeRequired: true},
|
|
{name: 'create', label: 'Create database', enabled: true, status: 'available', runtimeRequired: true},
|
|
{name: 'delete', label: 'Delete database', enabled: true, status: 'available', runtimeRequired: true},
|
|
{name: 'inspect', label: 'Inspect database', enabled: true, status: 'available', runtimeRequired: true},
|
|
],
|
|
dataCatalogActions: [
|
|
{name: 'createTable', label: 'Create table', enabled: true, status: 'available', runtimeRequired: true},
|
|
{name: 'deleteTable', label: 'Delete table', enabled: true, status: 'available', runtimeRequired: true},
|
|
],
|
|
},
|
|
filters: databaseFilters,
|
|
columns: databaseColumns,
|
|
}
|
|
}
|
|
|
|
export function dataCatalogSchemaFor(cloud: CloudProvider): ServiceSchema | null {
|
|
return cloud === 'aws' ? awsDataCatalogSchema() : null
|
|
}
|