feat(api): add the Glue Data Catalog to the cloud explorer

Create and delete Glue databases, and describe a folder of CSV, JSON-lines or
Parquet files in S3 as a table: its columns, and for CSV the delimiter and
whether the first line is a header. Tables are written with the storage
settings Athena expects, so TheCloud's Athena can query one as soon as it
exists. Glue jobs and crawlers are not offered: the runtime stores them but
never runs them.

packages/api/src/cloud-spi/dataCatalogSchema.ts

@@ -0,0 +1,68 @@
+import type {CloudProvider, FieldSchema, ServiceSchema, TableColumnSchema} from './types'
+
+/**
+ * Glue's rule for database and table names. Glue lowercases them, and Athena
+ * needs them to start with a letter or underscore to query them unquoted.
+ */
+export const CATALOG_NAME_PATTERN = '^[a-z_][a-z0-9_]{0,254}$'
+/** The data formats TheCloud's Athena engine reads from a table's S3 location. */
+export const CATALOG_TABLE_FORMATS = ['csv', 'json', 'parquet'] as const
+/** Hive column types Athena understands and the console offers. */
+export const CATALOG_COLUMN_TYPES = ['string', 'int', 'bigint', 'double', 'float', 'boolean', 'date', 'timestamp', 'decimal(10,2)'] as const
+
+const databaseColumns: TableColumnSchema[] = [
+    {name: 'name', label: 'Name'},
+    {name: 'tables', label: 'Tables', path: 'metadata.tableCount', emptyText: '—'},
+    {name: 'description', label: 'Description', path: 'metadata.description', emptyText: '—'},
+    {name: 'location', label: 'Location', path: 'metadata.locationUri', emptyText: '—'},
+    {name: 'createdAt', label: 'Created', format: 'datetime'},
+]
+
+const databaseFilters: FieldSchema[] = [{name: 'search', label: 'Search', type: 'text', required: false}]
+
+export function awsDataCatalogSchema(): ServiceSchema {
+    return {
+        cloud: 'aws',
+        service: 'datacatalog',
+        displayName: 'Glue Data Catalog databases',
+        fields: [
+            {
+                name: 'name',
+                label: 'Name',
+                type: 'text',
+                required: true,
+                description: 'Lowercase letters, numbers and underscores, starting with a letter or underscore, so Athena can query it.',
+                validation: {pattern: CATALOG_NAME_PATTERN, message: 'Use lowercase letters, numbers and underscores, starting with a letter or underscore'},
+            },
+            {name: 'description', label: 'Description', type: 'text', required: false, validation: {maxLength: 2048}},
+            {
+                name: 'locationUri',
+                label: 'Location',
+                type: 'text',
+                required: false,
+                span: true,
+                description: 'Optional S3 location for the database, such as s3://my-bucket/warehouse/.',
+                validation: {pattern: '^s3://[a-z0-9.-]{3,63}(/.*)?$', message: 'Use an S3 URI such as s3://my-bucket/warehouse/'},
+            },
+        ],
+        actions: ['list', 'create', 'delete', 'inspect'],
+        capabilities: {
+            resourceActions: [
+                {name: 'list', label: 'List databases', enabled: true, status: 'available', runtimeRequired: true},
+                {name: 'create', label: 'Create database', enabled: true, status: 'available', runtimeRequired: true},
+                {name: 'delete', label: 'Delete database', enabled: true, status: 'available', runtimeRequired: true},
+                {name: 'inspect', label: 'Inspect database', enabled: true, status: 'available', runtimeRequired: true},
+            ],
+            dataCatalogActions: [
+                {name: 'createTable', label: 'Create table', enabled: true, status: 'available', runtimeRequired: true},
+                {name: 'deleteTable', label: 'Delete table', enabled: true, status: 'available', runtimeRequired: true},
+            ],
+        },
+        filters: databaseFilters,
+        columns: databaseColumns,
+    }
+}
+
+export function dataCatalogSchemaFor(cloud: CloudProvider): ServiceSchema | null {
+    return cloud === 'aws' ? awsDataCatalogSchema() : null
+}

packages/api/src/cloud-spi/serviceCatalog.ts

@@ -222,6 +222,13 @@ export const SERVICE_CATALOG = {
         group: 'Analytics',
         order: 18,
     },
+    datacatalog: {
+        displayName: 'Data Catalog',
+        displayNameByCloud: {aws: 'Glue Data Catalog'},
+        iconKey: 'datacatalog',
+        group: 'Analytics',
+        order: 20,
+    },
     identity: {displayName: 'Identity', iconKey: 'iam', group: 'Security', order: 5},
     apigateway: {displayName: 'API Gateway', iconKey: 'apigateway', group: 'Integration', order: 10},
     email: {

packages/api/src/cloud-spi/types.ts

@@ -188,6 +188,7 @@ export type UserPoolActionName =
 export type DistributionActionName = 'setEnabled' | 'createInvalidation'
 export type AutoScalingActionName = 'setCapacity' | 'terminateInstance'
 export type CertificateActionName = 'download' | 'export' | 'revoke'
+export type DataCatalogActionName = 'createTable' | 'deleteTable'
 export type FileSystemActionName = 'createMountTarget' | 'deleteMountTarget' | 'createAccessPoint' | 'deleteAccessPoint' | 'attachFunction' | 'detachFunction'
 export type MonitoringActionName = 'listMetrics' | 'getMetricData' | 'alarmHistory' | 'setAlarmState' | 'setAlarmActions'
 export type HttpApiActionName =
@@ -251,6 +252,7 @@ export interface ServiceSchema {
         autoScalingActions?: CapabilitySchema<AutoScalingActionName>[]
         certificateActions?: CapabilitySchema<CertificateActionName>[]
         fileSystemActions?: CapabilitySchema<FileSystemActionName>[]
+        dataCatalogActions?: CapabilitySchema<DataCatalogActionName>[]
         monitoringActions?: CapabilitySchema<MonitoringActionName>[]
         /** API Gateway v2 (HTTP and WebSocket APIs): what the routes, stages and security panel can do. */
         httpApiActions?: CapabilitySchema<HttpApiActionName>[]
@@ -558,6 +560,24 @@ export interface AccessPointInput {
     permissions: string
 }
 
+export interface CatalogColumnInput {
+    name: string
+    type: string
+}
+
+export interface CatalogTableInput {
+    name: string
+    description?: string
+    /** The S3 prefix holding the table's files, such as s3://bucket/orders/. */
+    location: string
+    format: 'csv' | 'json' | 'parquet'
+    /** CSV only: the field separator, a comma unless given. */
+    delimiter?: string
+    /** CSV only: whether the first line of each file is a header to skip. */
+    headerRow?: boolean
+    columns: CatalogColumnInput[]
+}
+
 export interface FileSystemAttachmentInput {
     accessPointId: string
     functionName: string
@@ -1235,6 +1255,8 @@ export interface CloudServiceAdapter {
     /** With `replace`, the group launches a new instance in its place; otherwise desired capacity drops by one. */
     terminateAutoScalingInstance?(groupName: string, instanceId: string, replace: boolean): Promise<void>
     getCertificatePem?(certificateId: string): Promise<CertificatePem>
+    createCatalogTable?(databaseName: string, input: CatalogTableInput): Promise<CloudResource>
+    deleteCatalogTable?(databaseName: string, tableName: string): Promise<CloudResource>
     createMountTarget?(fileSystemId: string, subnetId: string): Promise<CloudResource>
     deleteMountTarget?(fileSystemId: string, mountTargetId: string): Promise<CloudResource>
     createAccessPoint?(fileSystemId: string, input: AccessPointInput): Promise<CloudResource>